• source navigation  • diff markup  • identifier search  • freetext search  • 

Sources/uclient/uclient-fetch.c

  1 /*
  2  * uclient - ustream based protocol client library
  3  *
  4  * Copyright (C) 2014 Felix Fietkau <nbd@openwrt.org>
  5  *
  6  * Permission to use, copy, modify, and/or distribute this software for any
  7  * purpose with or without fee is hereby granted, provided that the above
  8  * copyright notice and this permission notice appear in all copies.
  9  *
 10  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 11  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 12  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 13  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 14  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 15  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 16  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 17  */
 18 
 19 #define _GNU_SOURCE
 20 #include <sys/stat.h>
 21 #include <sys/socket.h>
 22 #include <unistd.h>
 23 #include <stdio.h>
 24 #include <dlfcn.h>
 25 #include <getopt.h>
 26 #include <fcntl.h>
 27 #include <glob.h>
 28 #include <stdint.h>
 29 #include <inttypes.h>
 30 #include <signal.h>
 31 
 32 #include <libubox/blobmsg.h>
 33 
 34 #include "progress.h"
 35 #include "uclient.h"
 36 #include "uclient-utils.h"
 37 
 38 #ifdef __APPLE__
 39 #define LIB_EXT "dylib"
 40 #else
 41 #define LIB_EXT "so"
 42 #endif
 43 
 44 static const char *user_agent = "uclient-fetch";
 45 static const char *post_data;
 46 static struct ustream_ssl_ctx *ssl_ctx;
 47 static const struct ustream_ssl_ops *ssl_ops;
 48 static int quiet = false;
 49 static bool verify = true;
 50 static bool proxy = true;
 51 static bool default_certs = false;
 52 static bool no_output;
 53 static const char *opt_output_file;
 54 static int output_fd = -1;
 55 static int error_ret;
 56 static off_t out_offset;
 57 static off_t out_bytes;
 58 static off_t out_len;
 59 static char *auth_str;
 60 static char **urls;
 61 static int n_urls;
 62 static int timeout;
 63 static bool resume, cur_resume;
 64 
 65 static struct progress pmt;
 66 static struct uloop_timeout pmt_timer;
 67 
 68 static int init_request(struct uclient *cl);
 69 static void request_done(struct uclient *cl);
 70 
 71 static void pmt_update(struct uloop_timeout *t)
 72 {
 73         progress_update(&pmt, out_offset, out_bytes, out_len);
 74         uloop_timeout_set(t, 1000);
 75 }
 76 
 77 static const char *
 78 get_proxy_url(char *url)
 79 {
 80         char prefix[16];
 81         char *sep;
 82 
 83         if (!proxy)
 84                 return NULL;
 85 
 86         sep = strchr(url, ':');
 87         if (!sep)
 88                 return NULL;
 89 
 90         if (sep - url > 5)
 91                 return NULL;
 92 
 93         memcpy(prefix, url, sep - url);
 94         strcpy(prefix + (sep - url), "_proxy");
 95         return getenv(prefix);
 96 }
 97 
 98 static int open_output_file(const char *path, uint64_t resume_offset)
 99 {
100         const char *output_file = opt_output_file;
101         char *filename = NULL;
102         int flags;
103         int ret;
104 
105         if (cur_resume)
106                 flags = O_RDWR;
107         else
108                 flags = O_WRONLY | O_TRUNC;
109 
110         if (!cur_resume && !output_file)
111                 flags |= O_EXCL;
112 
113         flags |= O_CREAT;
114 
115         if (output_file) {
116                 if (!strcmp(output_file, "-")) {
117                         if (!quiet)
118                                 fprintf(stderr, "Writing to stdout\n");
119 
120                         ret = STDOUT_FILENO;
121                         goto done;
122                 }
123         } else {
124                 filename = uclient_get_url_filename(path, "index.html");
125                 if (!filename) {
126                         ret = -ENOMEM;
127                         goto out;
128                 }
129 
130                 output_file = filename;
131         }
132 
133         if (!quiet)
134                 fprintf(stderr, "Writing to '%s'\n", output_file);
135         ret = open(output_file, flags, 0644);
136         if (ret < 0)
137                 goto free;
138 
139         if (resume_offset &&
140             lseek(ret, resume_offset, SEEK_SET) < 0) {
141                 if (!quiet)
142                         fprintf(stderr, "Failed to seek %"PRIu64" bytes in output file\n", resume_offset);
143                 close(ret);
144                 ret = -1;
145                 goto free;
146         }
147 
148         out_offset = resume_offset;
149         out_bytes += resume_offset;
150 done:
151         if (!quiet) {
152                 progress_init(&pmt, output_file);
153                 pmt_timer.cb = pmt_update;
154                 pmt_timer.cb(&pmt_timer);
155         }
156 
157 free:
158         free(filename);
159 out:
160         return ret;
161 }
162 
163 static void header_done_cb(struct uclient *cl)
164 {
165         enum {
166                 H_RANGE,
167                 H_LEN,
168                 __H_MAX
169         };
170         static const struct blobmsg_policy policy[__H_MAX] = {
171                 [H_RANGE] = { .name = "content-range", .type = BLOBMSG_TYPE_STRING },
172                 [H_LEN] = { .name = "content-length", .type = BLOBMSG_TYPE_STRING },
173         };
174         struct blob_attr *tb[__H_MAX];
175         uint64_t resume_offset = 0, resume_end, resume_size;
176         static int retries;
177 
178         if (retries < 10) {
179                 int ret = uclient_http_redirect(cl);
180                 if (ret < 0) {
181                         if (!quiet)
182                                 fprintf(stderr, "Failed to redirect to %s on %s\n", cl->url->location, cl->url->host);
183                         error_ret = 8;
184                         request_done(cl);
185                         return;
186                 }
187                 if (ret > 0) {
188                         if (!quiet)
189                                 fprintf(stderr, "Redirected to %s on %s\n", cl->url->location, cl->url->host);
190 
191                         retries++;
192                         return;
193                 }
194         }
195 
196         if (cl->status_code == 204 && cur_resume) {
197                 /* Resume attempt failed, try normal download */
198                 cur_resume = false;
199                 init_request(cl);
200                 return;
201         }
202 
203         blobmsg_parse(policy, __H_MAX, tb, blob_data(cl->meta), blob_len(cl->meta));
204 
205         switch (cl->status_code) {
206         case 416:
207                 if (!quiet)
208                         fprintf(stderr, "File download already fully retrieved; nothing to do.\n");
209                 request_done(cl);
210                 break;
211         case 206:
212                 if (!cur_resume) {
213                         if (!quiet)
214                                 fprintf(stderr, "Error: Partial content received, full content requested\n");
215                         error_ret = 8;
216                         request_done(cl);
217                         break;
218                 }
219 
220                 if (!tb[H_RANGE]) {
221                         if (!quiet)
222                                 fprintf(stderr, "Content-Range header is missing\n");
223                         error_ret = 8;
224                         break;
225                 }
226 
227                 if (sscanf(blobmsg_get_string(tb[H_RANGE]),
228                            "bytes %"PRIu64"-%"PRIu64"/%"PRIu64,
229                            &resume_offset, &resume_end, &resume_size) != 3) {
230                         if (!quiet)
231                                 fprintf(stderr, "Content-Range header is invalid\n");
232                         error_ret = 8;
233                         break;
234                 }
235         case 204:
236         case 200:
237                 if (no_output)
238                         break;
239 
240                 if (tb[H_LEN])
241                         out_len = strtoul(blobmsg_get_string(tb[H_LEN]), NULL, 10);
242 
243                 output_fd = open_output_file(cl->url->location, resume_offset);
244                 if (output_fd < 0) {
245                         if (!quiet)
246                                 perror("Cannot open output file");
247                         error_ret = 3;
248                         request_done(cl);
249                 }
250                 break;
251 
252         default:
253                 if (!quiet)
254                         fprintf(stderr, "HTTP error %d\n", cl->status_code);
255                 request_done(cl);
256                 error_ret = 8;
257                 break;
258         }
259 }
260 
261 static void read_data_cb(struct uclient *cl)
262 {
263         char buf[256];
264         ssize_t n;
265         int len;
266 
267         if (!no_output && output_fd < 0)
268                 return;
269 
270         while (1) {
271                 len = uclient_read(cl, buf, sizeof(buf));
272                 if (len <= 0)
273                         return;
274 
275                 out_bytes += len;
276                 if (!no_output) {
277                         n = write(output_fd, buf, len);
278                         if (n < 0)
279                                 return;
280                 }
281         }
282 }
283 
284 static void msg_connecting(struct uclient *cl)
285 {
286         char addr[INET6_ADDRSTRLEN];
287         int port;
288 
289         if (quiet)
290                 return;
291 
292         uclient_get_addr(addr, &port, &cl->remote_addr);
293         fprintf(stderr, "Connecting to %s:%d\n", addr, port);
294 }
295 
296 static void check_resume_offset(struct uclient *cl)
297 {
298         char range_str[64];
299         struct stat st;
300         char *file;
301         int ret;
302 
303         file = uclient_get_url_filename(cl->url->location, "index.html");
304         if (!file)
305                 return;
306 
307         ret = stat(file, &st);
308         free(file);
309         if (ret)
310                 return;
311 
312         if (!st.st_size)
313                 return;
314 
315         snprintf(range_str, sizeof(range_str), "bytes=%"PRIu64"-", (uint64_t) st.st_size);
316         uclient_http_set_header(cl, "Range", range_str);
317 }
318 
319 static int init_request(struct uclient *cl)
320 {
321         int rc;
322 
323         out_offset = 0;
324         out_bytes = 0;
325         out_len = 0;
326         uclient_http_set_ssl_ctx(cl, ssl_ops, ssl_ctx, verify);
327 
328         if (timeout)
329                 cl->timeout_msecs = timeout * 1000;
330 
331         rc = uclient_connect(cl);
332         if (rc)
333                 return rc;
334 
335         msg_connecting(cl);
336 
337         rc = uclient_http_set_request_type(cl, post_data ? "POST" : "GET");
338         if (rc)
339                 return rc;
340 
341         uclient_http_reset_headers(cl);
342         uclient_http_set_header(cl, "User-Agent", user_agent);
343         if (cur_resume)
344                 check_resume_offset(cl);
345 
346         if (post_data) {
347                 uclient_http_set_header(cl, "Content-Type", "application/x-www-form-urlencoded");
348                 uclient_write(cl, post_data, strlen(post_data));
349         }
350 
351         rc = uclient_request(cl);
352         if (rc)
353                 return rc;
354 
355         return 0;
356 }
357 
358 static void request_done(struct uclient *cl)
359 {
360         const char *proxy_url;
361 
362         if (n_urls) {
363                 proxy_url = get_proxy_url(*urls);
364                 if (proxy_url) {
365                         uclient_set_url(cl, proxy_url, NULL);
366                         uclient_set_proxy_url(cl, *urls, auth_str);
367                 } else {
368                         uclient_set_url(cl, *urls, auth_str);
369                 }
370                 n_urls--;
371                 cur_resume = resume;
372                 error_ret = init_request(cl);
373                 if (error_ret == 0)
374                         return;
375         }
376 
377         if (output_fd >= 0 && !opt_output_file) {
378                 close(output_fd);
379                 output_fd = -1;
380         }
381         uclient_disconnect(cl);
382         uloop_end();
383 }
384 
385 
386 static void eof_cb(struct uclient *cl)
387 {
388         if (!quiet) {
389                 pmt_update(&pmt_timer);
390                 uloop_timeout_cancel(&pmt_timer);
391                 fprintf(stderr, "\n");
392         }
393 
394         if (!cl->data_eof) {
395                 if (!quiet)
396                         fprintf(stderr, "Connection reset prematurely\n");
397                 error_ret = 4;
398         } else if (!quiet) {
399                 fprintf(stderr, "Download completed (%"PRIu64" bytes)\n", (uint64_t) out_bytes);
400         }
401         request_done(cl);
402 }
403 
404 static void handle_uclient_error(struct uclient *cl, int code)
405 {
406         const char *type = "Unknown error";
407         bool ignore = false;
408 
409         switch(code) {
410         case UCLIENT_ERROR_CONNECT:
411                 type = "Connection failed";
412                 error_ret = 4;
413                 break;
414         case UCLIENT_ERROR_TIMEDOUT:
415                 type = "Connection timed out";
416                 error_ret = 4;
417                 break;
418         case UCLIENT_ERROR_SSL_INVALID_CERT:
419                 type = "Invalid SSL certificate";
420                 ignore = !verify;
421                 error_ret = 5;
422                 break;
423         case UCLIENT_ERROR_SSL_CN_MISMATCH:
424                 type = "Server hostname does not match SSL certificate";
425                 ignore = !verify;
426                 error_ret = 5;
427                 break;
428         default:
429                 error_ret = 1;
430                 break;
431         }
432 
433         if (!quiet)
434                 fprintf(stderr, "Connection error: %s%s\n", type, ignore ? " (ignored)" : "");
435 
436         if (ignore)
437                 error_ret = 0;
438         else
439                 request_done(cl);
440 }
441 
442 static const struct uclient_cb cb = {
443         .header_done = header_done_cb,
444         .data_read = read_data_cb,
445         .data_eof = eof_cb,
446         .error = handle_uclient_error,
447 };
448 
449 static int usage(const char *progname)
450 {
451         fprintf(stderr,
452                 "Usage: %s [options] <URL>\n"
453                 "Options:\n"
454                 "       -4                              Use IPv4 only\n"
455                 "       -6                              Use IPv6 only\n"
456                 "       -q                              Turn off status messages\n"
457                 "       -O <file>                       Redirect output to file (use \"-\" for stdout)\n"
458                 "       -P <dir>                        Set directory for output files\n"
459                 "       --user=<user>                   HTTP authentication username\n"
460                 "       --password=<password>           HTTP authentication password\n"
461                 "       --user-agent|-U <str>           Set HTTP user agent\n"
462                 "       --post-data=STRING              use the POST method; send STRING as the data\n"
463                 "       --spider|-s                     Spider mode - only check file existence\n"
464                 "       --timeout=N|-T N                Set connect/request timeout to N seconds\n"
465                 "       --proxy=on|off|-Y on|off        Enable/disable env var configured proxy\n"
466                 "\n"
467                 "HTTPS options:\n"
468                 "       --ca-certificate=<cert>         Load CA certificates from file <cert>\n"
469                 "       --no-check-certificate          don't validate the server's certificate\n"
470                 "       --ciphers=<cipherlist>          Set the cipher list string\n"
471                 "\n", progname);
472         return 1;
473 }
474 
475 static void init_ca_cert(void)
476 {
477         glob_t gl;
478         int i;
479 
480         glob("/etc/ssl/certs/*.crt", 0, NULL, &gl);
481         for (i = 0; i < gl.gl_pathc; i++)
482                 ssl_ops->context_add_ca_crt_file(ssl_ctx, gl.gl_pathv[i]);
483 }
484 
485 static void init_ustream_ssl(void)
486 {
487         void *dlh;
488 
489         dlh = dlopen("libustream-ssl." LIB_EXT, RTLD_LAZY | RTLD_LOCAL);
490         if (!dlh)
491                 return;
492 
493         ssl_ops = dlsym(dlh, "ustream_ssl_ops");
494         if (!ssl_ops)
495                 return;
496 
497         ssl_ctx = ssl_ops->context_new(false);
498 }
499 
500 static int no_ssl(const char *progname)
501 {
502         fprintf(stderr,
503                 "%s: SSL support not available, please install one of the "
504                 "libustream-.*[ssl|tls] packages as well as the ca-bundle and "
505                 "ca-certificates packages.\n",
506                 progname);
507 
508         return 1;
509 }
510 
511 enum {
512         L_NO_CHECK_CERTIFICATE,
513         L_CA_CERTIFICATE,
514         L_CIPHERS,
515         L_USER,
516         L_PASSWORD,
517         L_USER_AGENT,
518         L_POST_DATA,
519         L_SPIDER,
520         L_TIMEOUT,
521         L_CONTINUE,
522         L_PROXY,
523         L_NO_PROXY,
524         L_QUIET,
525 };
526 
527 static const struct option longopts[] = {
528         [L_NO_CHECK_CERTIFICATE] = { "no-check-certificate", no_argument },
529         [L_CA_CERTIFICATE] = { "ca-certificate", required_argument },
530         [L_CIPHERS] = { "ciphers", required_argument },
531         [L_USER] = { "user", required_argument },
532         [L_PASSWORD] = { "password", required_argument },
533         [L_USER_AGENT] = { "user-agent", required_argument },
534         [L_POST_DATA] = { "post-data", required_argument },
535         [L_SPIDER] = { "spider", no_argument },
536         [L_TIMEOUT] = { "timeout", required_argument },
537         [L_CONTINUE] = { "continue", no_argument },
538         [L_PROXY] = { "proxy", required_argument },
539         [L_NO_PROXY] = { "no-proxy", no_argument },
540         [L_QUIET] = { "quiet", no_argument },
541         {}
542 };
543 
544 
545 
546 int main(int argc, char **argv)
547 {
548         const char *progname = argv[0];
549         const char *proxy_url;
550         char *username = NULL;
551         char *password = NULL;
552         struct uclient *cl;
553         int longopt_idx = 0;
554         bool has_cert = false;
555         int i, ch;
556         int rc;
557         int af = -1;
558 
559         signal(SIGPIPE, SIG_IGN);
560         init_ustream_ssl();
561 
562         while ((ch = getopt_long(argc, argv, "46cO:P:qsT:U:Y:", longopts, &longopt_idx)) != -1) {
563                 switch(ch) {
564                 case 0:
565                         switch (longopt_idx) {
566                         case L_NO_CHECK_CERTIFICATE:
567                                 verify = false;
568                                 break;
569                         case L_CA_CERTIFICATE:
570                                 has_cert = true;
571                                 if (ssl_ctx)
572                                         ssl_ops->context_add_ca_crt_file(ssl_ctx, optarg);
573                                 break;
574                         case L_CIPHERS:
575                                 if (ssl_ctx) {
576                                         if (ssl_ops->context_set_ciphers(ssl_ctx, optarg)) {
577                                                 if (!quiet)
578                                                         fprintf(stderr, "No recognized ciphers in cipher list\n");
579                                                 exit(1);
580                                         }
581                                 }
582                                 break;
583                         case L_USER:
584                                 if (!strlen(optarg))
585                                         break;
586                                 username = strdup(optarg);
587                                 memset(optarg, '*', strlen(optarg));
588                                 break;
589                         case L_PASSWORD:
590                                 if (!strlen(optarg))
591                                         break;
592                                 password = strdup(optarg);
593                                 memset(optarg, '*', strlen(optarg));
594                                 break;
595                         case L_USER_AGENT:
596                                 user_agent = optarg;
597                                 break;
598                         case L_POST_DATA:
599                                 post_data = optarg;
600                                 break;
601                         case L_SPIDER:
602                                 no_output = true;
603                                 break;
604                         case L_TIMEOUT:
605                                 timeout = atoi(optarg);
606                                 break;
607                         case L_CONTINUE:
608                                 resume = true;
609                                 break;
610                         case L_PROXY:
611                                 if (strcmp(optarg, "on") != 0)
612                                         proxy = false;
613                                 break;
614                         case L_NO_PROXY:
615                                 proxy = false;
616                                 break;
617                         case L_QUIET:
618                                 quiet = true;
619                                 break;
620                         default:
621                                 return usage(progname);
622                         }
623                         break;
624                 case '4':
625                         af = AF_INET;
626                         break;
627                 case '6':
628                         af = AF_INET6;
629                         break;
630                 case 'c':
631                         resume = true;
632                         break;
633                 case 'U':
634                         user_agent = optarg;
635                         break;
636                 case 'O':
637                         opt_output_file = optarg;
638                         break;
639                 case 'P':
640                         if (chdir(optarg)) {
641                                 if (!quiet)
642                                         perror("Change output directory");
643                                 exit(1);
644                         }
645                         break;
646                 case 'q':
647                         quiet = true;
648                         break;
649                 case 's':
650                         no_output = true;
651                         break;
652                 case 'T':
653                         timeout = atoi(optarg);
654                         break;
655                 case 'Y':
656                         if (strcmp(optarg, "on") != 0)
657                                 proxy = false;
658                         break;
659                 default:
660                         return usage(progname);
661                 }
662         }
663 
664         argv += optind;
665         argc -= optind;
666 
667         if (verify && !has_cert)
668                 default_certs = true;
669 
670         if (argc < 1)
671                 return usage(progname);
672 
673         if (!ssl_ctx) {
674                 for (i = 0; i < argc; i++) {
675                         if (!strncmp(argv[i], "https", 5))
676                                 return no_ssl(progname);
677                 }
678         }
679 
680         urls = argv + 1;
681         n_urls = argc - 1;
682 
683         uloop_init();
684 
685         if (username) {
686                 if (password) {
687                         rc = asprintf(&auth_str, "%s:%s", username, password);
688                         if (rc < 0)
689                                 return rc;
690                 } else
691                         auth_str = username;
692         }
693 
694         if (!quiet)
695                 fprintf(stderr, "Downloading '%s'\n", argv[0]);
696 
697         proxy_url = get_proxy_url(argv[0]);
698         if (proxy_url) {
699                 cl = uclient_new(proxy_url, auth_str, &cb);
700                 if (cl)
701                     uclient_set_proxy_url(cl, argv[0], NULL);
702         } else {
703                 cl = uclient_new(argv[0], auth_str, &cb);
704         }
705         if (!cl) {
706                 fprintf(stderr, "Failed to allocate uclient context\n");
707                 return 1;
708         }
709         if (af >= 0)
710             uclient_http_set_address_family(cl, af);
711 
712         if (ssl_ctx && default_certs)
713                 init_ca_cert();
714 
715         cur_resume = resume;
716         rc = init_request(cl);
717         if (!rc) {
718                 /* no error received, we can enter main loop */
719                 uloop_run();
720         } else {
721                 fprintf(stderr, "Failed to establish connection\n");
722                 error_ret = 4;
723         }
724 
725         uloop_done();
726 
727         uclient_free(cl);
728 
729         if (output_fd >= 0 && output_fd != STDOUT_FILENO)
730                 close(output_fd);
731 
732         if (ssl_ctx)
733                 ssl_ops->context_free(ssl_ctx);
734 
735         return error_ret;
736 }
737 

This page was automatically generated by LXR 0.3.1.  •  OpenWrt