• source navigation  • diff markup  • identifier search  • freetext search  • 

Sources/uclient/uclient-fetch.c

  1 /*
  2  * uclient - ustream based protocol client library
  3  *
  4  * Copyright (C) 2014 Felix Fietkau <nbd@openwrt.org>
  5  *
  6  * Permission to use, copy, modify, and/or distribute this software for any
  7  * purpose with or without fee is hereby granted, provided that the above
  8  * copyright notice and this permission notice appear in all copies.
  9  *
 10  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 11  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 12  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 13  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 14  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 15  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 16  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 17  */
 18 
 19 #define _GNU_SOURCE
 20 #include <sys/stat.h>
 21 #include <sys/socket.h>
 22 #include <unistd.h>
 23 #include <stdio.h>
 24 #include <dlfcn.h>
 25 #include <getopt.h>
 26 #include <fcntl.h>
 27 #include <glob.h>
 28 #include <stdint.h>
 29 #include <inttypes.h>
 30 #include <signal.h>
 31 
 32 #include <libubox/blobmsg.h>
 33 
 34 #include "progress.h"
 35 #include "uclient.h"
 36 #include "uclient-utils.h"
 37 
 38 #ifdef __APPLE__
 39 #define LIB_EXT "dylib"
 40 #else
 41 #define LIB_EXT "so"
 42 #endif
 43 
 44 static const char *user_agent = "uclient-fetch";
 45 static const char *post_data;
 46 static struct ustream_ssl_ctx *ssl_ctx;
 47 static const struct ustream_ssl_ops *ssl_ops;
 48 static int quiet = false;
 49 static bool verify = true;
 50 static bool proxy = true;
 51 static bool default_certs = false;
 52 static bool no_output;
 53 static const char *opt_output_file;
 54 static int output_fd = -1;
 55 static int error_ret;
 56 static off_t out_offset;
 57 static off_t out_bytes;
 58 static off_t out_len;
 59 static char *auth_str;
 60 static char **urls;
 61 static int n_urls;
 62 static int timeout;
 63 static bool resume, cur_resume;
 64 
 65 static struct progress pmt;
 66 static struct uloop_timeout pmt_timer;
 67 
 68 static int init_request(struct uclient *cl);
 69 static void request_done(struct uclient *cl);
 70 
 71 static void pmt_update(struct uloop_timeout *t)
 72 {
 73         progress_update(&pmt, out_offset, out_bytes, out_len);
 74         uloop_timeout_set(t, 1000);
 75 }
 76 
 77 static const char *
 78 get_proxy_url(char *url)
 79 {
 80         char prefix[16];
 81         char *sep;
 82 
 83         if (!proxy)
 84                 return NULL;
 85 
 86         sep = strchr(url, ':');
 87         if (!sep)
 88                 return NULL;
 89 
 90         if (sep - url > 5)
 91                 return NULL;
 92 
 93         memcpy(prefix, url, sep - url);
 94         strcpy(prefix + (sep - url), "_proxy");
 95         return getenv(prefix);
 96 }
 97 
 98 static int open_output_file(const char *path, uint64_t resume_offset)
 99 {
100         const char *output_file = opt_output_file;
101         char *filename = NULL;
102         int flags;
103         int ret;
104 
105         if (cur_resume)
106                 flags = O_RDWR;
107         else
108                 flags = O_WRONLY | O_TRUNC;
109 
110         if (!cur_resume && !output_file)
111                 flags |= O_EXCL;
112 
113         flags |= O_CREAT;
114 
115         if (output_file) {
116                 if (!strcmp(output_file, "-")) {
117                         if (!quiet)
118                                 fprintf(stderr, "Writing to stdout\n");
119 
120                         ret = STDOUT_FILENO;
121                         goto done;
122                 }
123         } else {
124                 filename = uclient_get_url_filename(path, "index.html");
125                 if (!filename) {
126                         ret = -ENOMEM;
127                         goto out;
128                 }
129 
130                 output_file = filename;
131         }
132 
133         if (!quiet)
134                 fprintf(stderr, "Writing to '%s'\n", output_file);
135         ret = open(output_file, flags, 0644);
136         if (ret < 0)
137                 goto free;
138 
139         if (resume_offset &&
140             lseek(ret, resume_offset, SEEK_SET) < 0) {
141                 if (!quiet)
142                         fprintf(stderr, "Failed to seek %"PRIu64" bytes in output file\n", resume_offset);
143                 close(ret);
144                 ret = -1;
145                 goto free;
146         }
147 
148         out_offset = resume_offset;
149         out_bytes += resume_offset;
150 done:
151         if (!quiet) {
152                 progress_init(&pmt, output_file);
153                 pmt_timer.cb = pmt_update;
154                 pmt_timer.cb(&pmt_timer);
155         }
156 
157 free:
158         free(filename);
159 out:
160         return ret;
161 }
162 
163 static void header_done_cb(struct uclient *cl)
164 {
165         enum {
166                 H_RANGE,
167                 H_LEN,
168                 __H_MAX
169         };
170         static const struct blobmsg_policy policy[__H_MAX] = {
171                 [H_RANGE] = { .name = "content-range", .type = BLOBMSG_TYPE_STRING },
172                 [H_LEN] = { .name = "content-length", .type = BLOBMSG_TYPE_STRING },
173         };
174         struct blob_attr *tb[__H_MAX];
175         uint64_t resume_offset = 0, resume_end, resume_size;
176         static int retries;
177 
178         if (retries < 10) {
179                 int ret = uclient_http_redirect(cl);
180                 if (ret < 0) {
181                         if (!quiet)
182                                 fprintf(stderr, "Failed to redirect to %s on %s\n", cl->url->location, cl->url->host);
183                         error_ret = 8;
184                         request_done(cl);
185                         return;
186                 }
187                 if (ret > 0) {
188                         if (!quiet)
189                                 fprintf(stderr, "Redirected to %s on %s\n", cl->url->location, cl->url->host);
190 
191                         retries++;
192                         return;
193                 }
194         }
195 
196         if (cl->status_code == 204 && cur_resume) {
197                 /* Resume attempt failed, try normal download */
198                 cur_resume = false;
199                 init_request(cl);
200                 return;
201         }
202 
203         blobmsg_parse(policy, __H_MAX, tb, blob_data(cl->meta), blob_len(cl->meta));
204 
205         switch (cl->status_code) {
206         case 416:
207                 if (!quiet)
208                         fprintf(stderr, "File download already fully retrieved; nothing to do.\n");
209                 request_done(cl);
210                 break;
211         case 206:
212                 if (!cur_resume) {
213                         if (!quiet)
214                                 fprintf(stderr, "Error: Partial content received, full content requested\n");
215                         error_ret = 8;
216                         request_done(cl);
217                         break;
218                 }
219 
220                 if (!tb[H_RANGE]) {
221                         if (!quiet)
222                                 fprintf(stderr, "Content-Range header is missing\n");
223                         error_ret = 8;
224                         break;
225                 }
226 
227                 if (sscanf(blobmsg_get_string(tb[H_RANGE]),
228                            "bytes %"PRIu64"-%"PRIu64"/%"PRIu64,
229                            &resume_offset, &resume_end, &resume_size) != 3) {
230                         if (!quiet)
231                                 fprintf(stderr, "Content-Range header is invalid\n");
232                         error_ret = 8;
233                         break;
234                 }
235         case 204:
236         case 200:
237                 if (no_output)
238                         break;
239 
240                 if (tb[H_LEN])
241                         out_len = strtoul(blobmsg_get_string(tb[H_LEN]), NULL, 10);
242 
243                 output_fd = open_output_file(cl->url->location, resume_offset);
244                 if (output_fd < 0) {
245                         if (!quiet)
246                                 perror("Cannot open output file");
247                         error_ret = 3;
248                         request_done(cl);
249                 }
250                 break;
251 
252         default:
253                 if (!quiet)
254                         fprintf(stderr, "HTTP error %d\n", cl->status_code);
255                 request_done(cl);
256                 error_ret = 8;
257                 break;
258         }
259 }
260 
261 static void read_data_cb(struct uclient *cl)
262 {
263         char buf[256];
264         ssize_t n;
265         int len;
266 
267         if (!no_output && output_fd < 0)
268                 return;
269 
270         while (1) {
271                 len = uclient_read(cl, buf, sizeof(buf));
272                 if (len <= 0)
273                         return;
274 
275                 out_bytes += len;
276                 if (!no_output) {
277                         n = write(output_fd, buf, len);
278                         if (n < 0)
279                                 return;
280                 }
281         }
282 }
283 
284 static void msg_connecting(struct uclient *cl)
285 {
286         char addr[INET6_ADDRSTRLEN];
287         int port;
288 
289         if (quiet)
290                 return;
291 
292         uclient_get_addr(addr, &port, &cl->remote_addr);
293         fprintf(stderr, "Connecting to %s:%d\n", addr, port);
294 }
295 
296 static void check_resume_offset(struct uclient *cl)
297 {
298         char range_str[64];
299         struct stat st;
300         char *file;
301         int ret;
302 
303         file = uclient_get_url_filename(cl->url->location, "index.html");
304         if (!file)
305                 return;
306 
307         ret = stat(file, &st);
308         free(file);
309         if (ret)
310                 return;
311 
312         if (!st.st_size)
313                 return;
314 
315         snprintf(range_str, sizeof(range_str), "bytes=%"PRIu64"-", (uint64_t) st.st_size);
316         uclient_http_set_header(cl, "Range", range_str);
317 }
318 
319 static int init_request(struct uclient *cl)
320 {
321         int rc;
322 
323         out_offset = 0;
324         out_bytes = 0;
325         out_len = 0;
326         uclient_http_set_ssl_ctx(cl, ssl_ops, ssl_ctx, verify);
327 
328         if (timeout)
329                 cl->timeout_msecs = timeout * 1000;
330 
331         rc = uclient_connect(cl);
332         if (rc)
333                 return rc;
334 
335         msg_connecting(cl);
336 
337         rc = uclient_http_set_request_type(cl, post_data ? "POST" : "GET");
338         if (rc)
339                 return rc;
340 
341         uclient_http_reset_headers(cl);
342         uclient_http_set_header(cl, "User-Agent", user_agent);
343         if (cur_resume)
344                 check_resume_offset(cl);
345 
346         if (post_data) {
347                 uclient_http_set_header(cl, "Content-Type", "application/x-www-form-urlencoded");
348                 uclient_write(cl, post_data, strlen(post_data));
349         }
350 
351         rc = uclient_request(cl);
352         if (rc)
353                 return rc;
354 
355         return 0;
356 }
357 
358 static void request_done(struct uclient *cl)
359 {
360         const char *proxy_url;
361 
362         if (n_urls) {
363                 proxy_url = get_proxy_url(*urls);
364                 if (proxy_url) {
365                         uclient_set_url(cl, proxy_url, NULL);
366                         uclient_set_proxy_url(cl, *urls, auth_str);
367                 } else {
368                         uclient_set_url(cl, *urls, auth_str);
369                 }
370                 n_urls--;
371                 cur_resume = resume;
372                 error_ret = init_request(cl);
373                 if (error_ret == 0)
374                         return;
375         }
376 
377         if (output_fd >= 0 && !opt_output_file) {
378                 close(output_fd);
379                 output_fd = -1;
380         }
381         uclient_disconnect(cl);
382         uloop_end();
383 }
384 
385 
386 static void eof_cb(struct uclient *cl)
387 {
388         if (!quiet) {
389                 pmt_update(&pmt_timer);
390                 uloop_timeout_cancel(&pmt_timer);
391                 fprintf(stderr, "\n");
392         }
393 
394         if (!cl->data_eof) {
395                 if (!quiet)
396                         fprintf(stderr, "Connection reset prematurely\n");
397                 error_ret = 4;
398         } else if (!quiet) {
399                 fprintf(stderr, "Download completed (%"PRIu64" bytes)\n", (uint64_t) out_bytes);
400         }
401         request_done(cl);
402 }
403 
404 static void handle_uclient_error(struct uclient *cl, int code)
405 {
406         const char *type = "Unknown error";
407         bool ignore = false;
408 
409         switch(code) {
410         case UCLIENT_ERROR_CONNECT:
411                 type = "Connection failed";
412                 error_ret = 4;
413                 break;
414         case UCLIENT_ERROR_TIMEDOUT:
415                 type = "Connection timed out";
416                 error_ret = 4;
417                 break;
418         case UCLIENT_ERROR_SSL_INVALID_CERT:
419                 type = "Invalid SSL certificate";
420                 ignore = !verify;
421                 error_ret = 5;
422                 break;
423         case UCLIENT_ERROR_SSL_CN_MISMATCH:
424                 type = "Server hostname does not match SSL certificate";
425                 ignore = !verify;
426                 error_ret = 5;
427                 break;
428         default:
429                 error_ret = 1;
430                 break;
431         }
432 
433         if (!quiet)
434                 fprintf(stderr, "Connection error: %s%s\n", type, ignore ? " (ignored)" : "");
435 
436         if (ignore)
437                 error_ret = 0;
438         else
439                 request_done(cl);
440 }
441 
442 static const struct uclient_cb cb = {
443         .header_done = header_done_cb,
444         .data_read = read_data_cb,
445         .data_eof = eof_cb,
446         .error = handle_uclient_error,
447 };
448 
449 static int usage(const char *progname)
450 {
451         fprintf(stderr,
452                 "Usage: %s [options] <URL>\n"
453                 "Options:\n"
454                 "       -4                              Use IPv4 only\n"
455                 "       -6                              Use IPv6 only\n"
456                 "       -q                              Turn off status messages\n"
457                 "       -O <file>                       Redirect output to file (use \"-\" for stdout)\n"
458                 "       -P <dir>                        Set directory for output files\n"
459                 "       --user=<user>                   HTTP authentication username\n"
460                 "       --password=<password>           HTTP authentication password\n"
461                 "       --user-agent|-U <str>           Set HTTP user agent\n"
462                 "       --post-data=STRING              use the POST method; send STRING as the data\n"
463                 "       --spider|-s                     Spider mode - only check file existence\n"
464                 "       --timeout=N|-T N                Set connect/request timeout to N seconds\n"
465                 "       --proxy=on|off|-Y on|off        Enable/disable env var configured proxy\n"
466                 "\n"
467                 "HTTPS options:\n"
468                 "       --ca-certificate=<cert>         Load CA certificates from file <cert>\n"
469                 "       --no-check-certificate          don't validate the server's certificate\n"
470                 "\n", progname);
471         return 1;
472 }
473 
474 static void init_ca_cert(void)
475 {
476         glob_t gl;
477         int i;
478 
479         glob("/etc/ssl/certs/*.crt", 0, NULL, &gl);
480         for (i = 0; i < gl.gl_pathc; i++)
481                 ssl_ops->context_add_ca_crt_file(ssl_ctx, gl.gl_pathv[i]);
482 }
483 
484 static void init_ustream_ssl(void)
485 {
486         void *dlh;
487 
488         dlh = dlopen("libustream-ssl." LIB_EXT, RTLD_LAZY | RTLD_LOCAL);
489         if (!dlh)
490                 return;
491 
492         ssl_ops = dlsym(dlh, "ustream_ssl_ops");
493         if (!ssl_ops)
494                 return;
495 
496         ssl_ctx = ssl_ops->context_new(false);
497 }
498 
499 static int no_ssl(const char *progname)
500 {
501         fprintf(stderr,
502                 "%s: SSL support not available, please install one of the "
503                 "libustream-.*[ssl|tls] packages as well as the ca-bundle and "
504                 "ca-certificates packages.\n",
505                 progname);
506 
507         return 1;
508 }
509 
510 enum {
511         L_NO_CHECK_CERTIFICATE,
512         L_CA_CERTIFICATE,
513         L_USER,
514         L_PASSWORD,
515         L_USER_AGENT,
516         L_POST_DATA,
517         L_SPIDER,
518         L_TIMEOUT,
519         L_CONTINUE,
520         L_PROXY,
521         L_NO_PROXY,
522         L_QUIET,
523 };
524 
525 static const struct option longopts[] = {
526         [L_NO_CHECK_CERTIFICATE] = { "no-check-certificate", no_argument },
527         [L_CA_CERTIFICATE] = { "ca-certificate", required_argument },
528         [L_USER] = { "user", required_argument },
529         [L_PASSWORD] = { "password", required_argument },
530         [L_USER_AGENT] = { "user-agent", required_argument },
531         [L_POST_DATA] = { "post-data", required_argument },
532         [L_SPIDER] = { "spider", no_argument },
533         [L_TIMEOUT] = { "timeout", required_argument },
534         [L_CONTINUE] = { "continue", no_argument },
535         [L_PROXY] = { "proxy", required_argument },
536         [L_NO_PROXY] = { "no-proxy", no_argument },
537         [L_QUIET] = { "quiet", no_argument },
538         {}
539 };
540 
541 
542 
543 int main(int argc, char **argv)
544 {
545         const char *progname = argv[0];
546         const char *proxy_url;
547         char *username = NULL;
548         char *password = NULL;
549         struct uclient *cl;
550         int longopt_idx = 0;
551         bool has_cert = false;
552         int i, ch;
553         int rc;
554         int af = -1;
555 
556         signal(SIGPIPE, SIG_IGN);
557         init_ustream_ssl();
558 
559         while ((ch = getopt_long(argc, argv, "46cO:P:qsT:U:Y:", longopts, &longopt_idx)) != -1) {
560                 switch(ch) {
561                 case 0:
562                         switch (longopt_idx) {
563                         case L_NO_CHECK_CERTIFICATE:
564                                 verify = false;
565                                 break;
566                         case L_CA_CERTIFICATE:
567                                 has_cert = true;
568                                 if (ssl_ctx)
569                                         ssl_ops->context_add_ca_crt_file(ssl_ctx, optarg);
570                                 break;
571                         case L_USER:
572                                 if (!strlen(optarg))
573                                         break;
574                                 username = strdup(optarg);
575                                 memset(optarg, '*', strlen(optarg));
576                                 break;
577                         case L_PASSWORD:
578                                 if (!strlen(optarg))
579                                         break;
580                                 password = strdup(optarg);
581                                 memset(optarg, '*', strlen(optarg));
582                                 break;
583                         case L_USER_AGENT:
584                                 user_agent = optarg;
585                                 break;
586                         case L_POST_DATA:
587                                 post_data = optarg;
588                                 break;
589                         case L_SPIDER:
590                                 no_output = true;
591                                 break;
592                         case L_TIMEOUT:
593                                 timeout = atoi(optarg);
594                                 break;
595                         case L_CONTINUE:
596                                 resume = true;
597                                 break;
598                         case L_PROXY:
599                                 if (strcmp(optarg, "on") != 0)
600                                         proxy = false;
601                                 break;
602                         case L_NO_PROXY:
603                                 proxy = false;
604                                 break;
605                         case L_QUIET:
606                                 quiet = true;
607                                 break;
608                         default:
609                                 return usage(progname);
610                         }
611                         break;
612                 case '4':
613                         af = AF_INET;
614                         break;
615                 case '6':
616                         af = AF_INET6;
617                         break;
618                 case 'c':
619                         resume = true;
620                         break;
621                 case 'U':
622                         user_agent = optarg;
623                         break;
624                 case 'O':
625                         opt_output_file = optarg;
626                         break;
627                 case 'P':
628                         if (chdir(optarg)) {
629                                 if (!quiet)
630                                         perror("Change output directory");
631                                 exit(1);
632                         }
633                         break;
634                 case 'q':
635                         quiet = true;
636                         break;
637                 case 's':
638                         no_output = true;
639                         break;
640                 case 'T':
641                         timeout = atoi(optarg);
642                         break;
643                 case 'Y':
644                         if (strcmp(optarg, "on") != 0)
645                                 proxy = false;
646                         break;
647                 default:
648                         return usage(progname);
649                 }
650         }
651 
652         argv += optind;
653         argc -= optind;
654 
655         if (verify && !has_cert)
656                 default_certs = true;
657 
658         if (argc < 1)
659                 return usage(progname);
660 
661         if (!ssl_ctx) {
662                 for (i = 0; i < argc; i++) {
663                         if (!strncmp(argv[i], "https", 5))
664                                 return no_ssl(progname);
665                 }
666         }
667 
668         urls = argv + 1;
669         n_urls = argc - 1;
670 
671         uloop_init();
672 
673         if (username) {
674                 if (password) {
675                         rc = asprintf(&auth_str, "%s:%s", username, password);
676                         if (rc < 0)
677                                 return rc;
678                 } else
679                         auth_str = username;
680         }
681 
682         if (!quiet)
683                 fprintf(stderr, "Downloading '%s'\n", argv[0]);
684 
685         proxy_url = get_proxy_url(argv[0]);
686         if (proxy_url) {
687                 cl = uclient_new(proxy_url, auth_str, &cb);
688                 if (cl)
689                     uclient_set_proxy_url(cl, argv[0], NULL);
690         } else {
691                 cl = uclient_new(argv[0], auth_str, &cb);
692         }
693         if (!cl) {
694                 fprintf(stderr, "Failed to allocate uclient context\n");
695                 return 1;
696         }
697         if (af >= 0)
698             uclient_http_set_address_family(cl, af);
699 
700         if (ssl_ctx && default_certs)
701                 init_ca_cert();
702 
703         cur_resume = resume;
704         rc = init_request(cl);
705         if (!rc) {
706                 /* no error received, we can enter main loop */
707                 uloop_run();
708         } else {
709                 fprintf(stderr, "Failed to establish connection\n");
710                 error_ret = 4;
711         }
712 
713         uloop_done();
714 
715         uclient_free(cl);
716 
717         if (output_fd >= 0 && output_fd != STDOUT_FILENO)
718                 close(output_fd);
719 
720         if (ssl_ctx)
721                 ssl_ops->context_free(ssl_ctx);
722 
723         return error_ret;
724 }
725 

This page was automatically generated by LXR 0.3.1.  •  OpenWrt