1 /* 2 * uclient - ustream based protocol client library 3 * 4 * Copyright (C) 2014 Felix Fietkau <nbd@openwrt.org> 5 * 6 * Permission to use, copy, modify, and/or distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #define _GNU_SOURCE 20 #include <sys/stat.h> 21 #include <sys/socket.h> 22 #include <unistd.h> 23 #include <stdio.h> 24 #include <getopt.h> 25 #include <fcntl.h> 26 #include <glob.h> 27 #include <stdint.h> 28 #include <inttypes.h> 29 #include <signal.h> 30 31 #include <libubox/blobmsg.h> 32 33 #include "progress.h" 34 #include "uclient.h" 35 #include "uclient-utils.h" 36 37 #ifndef strdupa 38 #define strdupa(x) strcpy(alloca(strlen(x)+1),x) 39 #endif 40 41 static const char *user_agent = "uclient-fetch"; 42 static const char *post_data; 43 static const char *post_file; 44 static struct ustream_ssl_ctx *ssl_ctx; 45 static const struct ustream_ssl_ops *ssl_ops; 46 static int quiet = false; 47 static bool verify = true; 48 static bool proxy = true; 49 static bool default_certs = false; 50 static bool no_output; 51 static const char *opt_output_file; 52 static int output_fd = -1; 53 static int error_ret; 54 static off_t out_offset; 55 static off_t out_bytes; 56 static off_t out_len; 57 static char *auth_str; 58 static char **urls; 59 static int n_urls; 60 static int timeout; 61 static bool resume, cur_resume; 62 63 static struct progress pmt; 64 static struct uloop_timeout pmt_timer; 65 66 static int init_request(struct uclient *cl); 67 static void request_done(struct uclient *cl); 68 69 static void pmt_update(struct uloop_timeout *t) 70 { 71 progress_update(&pmt, out_offset, out_bytes, out_len); 72 uloop_timeout_set(t, 1000); 73 } 74 75 static const char * 76 get_proxy_url(char *url) 77 { 78 char prefix[16]; 79 char *sep; 80 81 if (!proxy) 82 return NULL; 83 84 sep = strchr(url, ':'); 85 if (!sep) 86 return NULL; 87 88 if (sep - url > 5) 89 return NULL; 90 91 memcpy(prefix, url, sep - url); 92 strcpy(prefix + (sep - url), "_proxy"); 93 return getenv(prefix); 94 } 95 96 static int open_output_file(const char *path, uint64_t resume_offset) 97 { 98 const char *output_file = opt_output_file; 99 char *filename = NULL; 100 int flags; 101 int ret; 102 103 if (cur_resume) 104 flags = O_RDWR; 105 else 106 flags = O_WRONLY | O_TRUNC; 107 108 if (!cur_resume && !output_file) 109 flags |= O_EXCL; 110 111 flags |= O_CREAT; 112 113 if (output_file) { 114 if (!strcmp(output_file, "-")) { 115 if (!quiet) 116 fprintf(stderr, "Writing to stdout\n"); 117 118 ret = STDOUT_FILENO; 119 goto done; 120 } 121 } else { 122 filename = uclient_get_url_filename(path, "index.html"); 123 if (!filename) { 124 ret = -ENOMEM; 125 goto out; 126 } 127 128 output_file = filename; 129 } 130 131 if (!quiet) 132 fprintf(stderr, "Writing to '%s'\n", output_file); 133 ret = open(output_file, flags, 0644); 134 if (ret < 0) 135 goto free; 136 137 if (resume_offset && 138 lseek(ret, resume_offset, SEEK_SET) < 0) { 139 if (!quiet) 140 fprintf(stderr, "Failed to seek %"PRIu64" bytes in output file\n", resume_offset); 141 close(ret); 142 ret = -1; 143 goto free; 144 } 145 146 out_offset = resume_offset; 147 out_bytes += resume_offset; 148 done: 149 if (!quiet) { 150 progress_init(&pmt, output_file); 151 pmt_timer.cb = pmt_update; 152 pmt_timer.cb(&pmt_timer); 153 } 154 155 free: 156 free(filename); 157 out: 158 return ret; 159 } 160 161 static void header_done_cb(struct uclient *cl) 162 { 163 enum { 164 H_RANGE, 165 H_LEN, 166 __H_MAX 167 }; 168 static const struct blobmsg_policy policy[__H_MAX] = { 169 [H_RANGE] = { .name = "content-range", .type = BLOBMSG_TYPE_STRING }, 170 [H_LEN] = { .name = "content-length", .type = BLOBMSG_TYPE_STRING }, 171 }; 172 struct blob_attr *tb[__H_MAX]; 173 uint64_t resume_offset = 0, resume_end, resume_size; 174 static int retries; 175 176 if (retries < 10) { 177 int ret = uclient_http_redirect(cl); 178 if (ret < 0) { 179 if (!quiet) 180 fprintf(stderr, "Failed to redirect to %s on %s\n", cl->url->location, cl->url->host); 181 error_ret = 8; 182 request_done(cl); 183 return; 184 } 185 if (ret > 0) { 186 if (!quiet) 187 fprintf(stderr, "Redirected to %s on %s\n", cl->url->location, cl->url->host); 188 189 retries++; 190 return; 191 } 192 } 193 194 if (cl->status_code == 204 && cur_resume) { 195 /* Resume attempt failed, try normal download */ 196 cur_resume = false; 197 init_request(cl); 198 return; 199 } 200 201 blobmsg_parse(policy, __H_MAX, tb, blob_data(cl->meta), blob_len(cl->meta)); 202 203 switch (cl->status_code) { 204 case 416: 205 if (!quiet) 206 fprintf(stderr, "File download already fully retrieved; nothing to do.\n"); 207 request_done(cl); 208 break; 209 case 206: 210 if (!cur_resume) { 211 if (!quiet) 212 fprintf(stderr, "Error: Partial content received, full content requested\n"); 213 error_ret = 8; 214 request_done(cl); 215 break; 216 } 217 218 if (!tb[H_RANGE]) { 219 if (!quiet) 220 fprintf(stderr, "Content-Range header is missing\n"); 221 error_ret = 8; 222 break; 223 } 224 225 if (sscanf(blobmsg_get_string(tb[H_RANGE]), 226 "bytes %"PRIu64"-%"PRIu64"/%"PRIu64, 227 &resume_offset, &resume_end, &resume_size) != 3) { 228 if (!quiet) 229 fprintf(stderr, "Content-Range header is invalid\n"); 230 error_ret = 8; 231 break; 232 } 233 /* fall through */ 234 case 204: 235 case 200: 236 if (no_output) 237 break; 238 239 if (tb[H_LEN]) 240 out_len = strtoul(blobmsg_get_string(tb[H_LEN]), NULL, 10); 241 242 output_fd = open_output_file(cl->url->location, resume_offset); 243 if (output_fd < 0) { 244 if (!quiet) 245 perror("Cannot open output file"); 246 error_ret = 3; 247 request_done(cl); 248 } 249 break; 250 251 default: 252 if (!quiet) 253 fprintf(stderr, "HTTP error %d\n", cl->status_code); 254 request_done(cl); 255 error_ret = 8; 256 break; 257 } 258 } 259 260 static void read_data_cb(struct uclient *cl) 261 { 262 char buf[256]; 263 ssize_t n; 264 int len; 265 266 if (!no_output && output_fd < 0) 267 return; 268 269 while (1) { 270 len = uclient_read(cl, buf, sizeof(buf)); 271 if (len <= 0) 272 return; 273 274 out_bytes += len; 275 if (!no_output) { 276 n = write(output_fd, buf, len); 277 if (n < 0) 278 return; 279 } 280 } 281 } 282 283 static void msg_connecting(struct uclient *cl) 284 { 285 char addr[INET6_ADDRSTRLEN]; 286 int port; 287 288 if (quiet) 289 return; 290 291 uclient_get_addr(addr, &port, &cl->remote_addr); 292 fprintf(stderr, "Connecting to %s:%d\n", addr, port); 293 } 294 295 static void check_resume_offset(struct uclient *cl) 296 { 297 char range_str[64]; 298 struct stat st; 299 char *file; 300 int ret; 301 302 file = uclient_get_url_filename(cl->url->location, "index.html"); 303 if (!file) 304 return; 305 306 ret = stat(file, &st); 307 free(file); 308 if (ret) 309 return; 310 311 if (!st.st_size) 312 return; 313 314 snprintf(range_str, sizeof(range_str), "bytes=%"PRIu64"-", (uint64_t) st.st_size); 315 uclient_http_set_header(cl, "Range", range_str); 316 } 317 318 static int init_request(struct uclient *cl) 319 { 320 int rc; 321 322 out_offset = 0; 323 out_bytes = 0; 324 out_len = 0; 325 uclient_http_set_ssl_ctx(cl, ssl_ops, ssl_ctx, verify); 326 327 if (timeout) 328 cl->timeout_msecs = timeout * 1000; 329 330 rc = uclient_connect(cl); 331 if (rc) 332 return rc; 333 334 msg_connecting(cl); 335 336 rc = uclient_http_set_request_type(cl, post_data || post_file ? "POST" : "GET"); 337 if (rc) 338 return rc; 339 340 uclient_http_reset_headers(cl); 341 uclient_http_set_header(cl, "User-Agent", user_agent); 342 if (cur_resume) 343 check_resume_offset(cl); 344 345 if (post_data) { 346 uclient_http_set_header(cl, "Content-Type", "application/x-www-form-urlencoded"); 347 uclient_write(cl, post_data, strlen(post_data)); 348 } 349 else if(post_file) 350 { 351 FILE *input_file; 352 uclient_http_set_header(cl, "Content-Type", "application/x-www-form-urlencoded"); 353 354 input_file = fopen(post_file, "r"); 355 if (!input_file) 356 return errno; 357 358 char tbuf[1024]; 359 size_t rlen = 0; 360 do 361 { 362 rlen = fread(tbuf, 1, sizeof(tbuf), input_file); 363 uclient_write(cl, tbuf, rlen); 364 } 365 while(rlen); 366 367 fclose(input_file); 368 } 369 370 rc = uclient_request(cl); 371 if (rc) 372 return rc; 373 374 return 0; 375 } 376 377 static void request_done(struct uclient *cl) 378 { 379 const char *proxy_url; 380 381 if (n_urls) { 382 proxy_url = get_proxy_url(*urls); 383 if (proxy_url) { 384 uclient_set_url(cl, proxy_url, NULL); 385 uclient_set_proxy_url(cl, *urls, auth_str); 386 } else { 387 uclient_set_url(cl, *urls, auth_str); 388 } 389 n_urls--; 390 cur_resume = resume; 391 error_ret = init_request(cl); 392 if (error_ret == 0) 393 return; 394 } 395 396 if (output_fd >= 0 && !opt_output_file) { 397 close(output_fd); 398 output_fd = -1; 399 } 400 uclient_disconnect(cl); 401 uloop_end(); 402 } 403 404 405 static void eof_cb(struct uclient *cl) 406 { 407 if (!quiet) { 408 pmt_update(&pmt_timer); 409 uloop_timeout_cancel(&pmt_timer); 410 fprintf(stderr, "\n"); 411 } 412 413 if (!cl->data_eof) { 414 if (!quiet) 415 fprintf(stderr, "Connection reset prematurely\n"); 416 error_ret = 4; 417 } else if (!quiet) { 418 fprintf(stderr, "Download completed (%"PRIu64" bytes)\n", (uint64_t) out_bytes); 419 } 420 request_done(cl); 421 } 422 423 static void 424 handle_uclient_log_msg(struct uclient *cl, enum uclient_log_type type, const char *msg) 425 { 426 static const char * const type_str_list[] = { 427 [UCLIENT_LOG_SSL_ERROR] = "SSL error", 428 [UCLIENT_LOG_SSL_VERIFY_ERROR] = "SSL verify error", 429 }; 430 const char *type_str = NULL; 431 432 if (type < ARRAY_SIZE(type_str_list)) 433 type_str = type_str_list[type]; 434 if (!type_str) 435 type_str = "Unknown"; 436 437 fprintf(stderr, "%s: %s\n", type_str, msg); 438 } 439 440 static void handle_uclient_error(struct uclient *cl, int code) 441 { 442 const char *type = "Unknown error"; 443 bool ignore = false; 444 445 switch(code) { 446 case UCLIENT_ERROR_CONNECT: 447 type = "Connection failed"; 448 error_ret = 4; 449 break; 450 case UCLIENT_ERROR_TIMEDOUT: 451 type = "Connection timed out"; 452 error_ret = 4; 453 break; 454 case UCLIENT_ERROR_SSL_INVALID_CERT: 455 type = "Invalid SSL certificate"; 456 ignore = !verify; 457 error_ret = 5; 458 break; 459 case UCLIENT_ERROR_SSL_CN_MISMATCH: 460 type = "Server hostname does not match SSL certificate"; 461 ignore = !verify; 462 error_ret = 5; 463 break; 464 default: 465 error_ret = 1; 466 break; 467 } 468 469 if (!quiet) 470 fprintf(stderr, "Connection error: %s%s\n", type, ignore ? " (ignored)" : ""); 471 472 if (ignore) 473 error_ret = 0; 474 else 475 request_done(cl); 476 } 477 478 static const struct uclient_cb cb = { 479 .header_done = header_done_cb, 480 .data_read = read_data_cb, 481 .data_eof = eof_cb, 482 .error = handle_uclient_error, 483 .log_msg = handle_uclient_log_msg, 484 }; 485 486 static int usage(const char *progname) 487 { 488 fprintf(stderr, 489 "Usage: %s [options] <URL>\n" 490 "Options:\n" 491 " -4 Use IPv4 only\n" 492 " -6 Use IPv6 only\n" 493 " -O <file> Redirect output to file (use \"-\" for stdout)\n" 494 " -P <dir> Set directory for output files\n" 495 " --quiet | -q Turn off status messages\n" 496 " --continue | -c Continue a partially-downloaded file\n" 497 " --user=<user> HTTP authentication username\n" 498 " --password=<password> HTTP authentication password\n" 499 " --user-agent | -U <str> Set HTTP user agent\n" 500 " --post-data=STRING use the POST method; send STRING as the data\n" 501 " --post-file=FILE use the POST method; send FILE as the data\n" 502 " --spider | -s Spider mode - only check file existence\n" 503 " --timeout=N | -T N Set connect/request timeout to N seconds\n" 504 " --proxy=on | -Y on Enable interpretation of proxy env vars (default)\n" 505 " --proxy=off | -Y off |\n" 506 " --no-proxy Disable interpretation of proxy env vars\n" 507 "\n" 508 "HTTPS options:\n" 509 " --ca-certificate=<cert> Load CA certificates from file <cert>\n" 510 " --no-check-certificate don't validate the server's certificate\n" 511 " --ciphers=<cipherlist> Set the cipher list string\n" 512 "\n", progname); 513 return 1; 514 } 515 516 static void init_ca_cert(void) 517 { 518 glob_t gl; 519 unsigned int i; 520 521 glob("/etc/ssl/certs/*.crt", 0, NULL, &gl); 522 for (i = 0; i < gl.gl_pathc; i++) 523 ssl_ops->context_add_ca_crt_file(ssl_ctx, gl.gl_pathv[i]); 524 globfree(&gl); 525 } 526 527 static int no_ssl(const char *progname) 528 { 529 fprintf(stderr, 530 "%s: SSL support not available, please install one of the " 531 "libustream-.*[ssl|tls] packages as well as the ca-bundle and " 532 "ca-certificates packages.\n", 533 progname); 534 535 return 1; 536 } 537 538 static void debug_cb(void *priv, int level, const char *msg) 539 { 540 fprintf(stderr, "%s\n", msg); 541 } 542 543 enum { 544 L_NO_CHECK_CERTIFICATE, 545 L_CA_CERTIFICATE, 546 L_CIPHERS, 547 L_USER, 548 L_PASSWORD, 549 L_USER_AGENT, 550 L_POST_DATA, 551 L_POST_FILE, 552 L_SPIDER, 553 L_TIMEOUT, 554 L_CONTINUE, 555 L_PROXY, 556 L_NO_PROXY, 557 L_QUIET, 558 L_VERBOSE, 559 }; 560 561 static const struct option longopts[] = { 562 [L_NO_CHECK_CERTIFICATE] = { "no-check-certificate", no_argument, NULL, 0 }, 563 [L_CA_CERTIFICATE] = { "ca-certificate", required_argument, NULL, 0 }, 564 [L_CIPHERS] = { "ciphers", required_argument, NULL, 0 }, 565 [L_USER] = { "user", required_argument, NULL, 0 }, 566 [L_PASSWORD] = { "password", required_argument, NULL, 0 }, 567 [L_USER_AGENT] = { "user-agent", required_argument, NULL, 0 }, 568 [L_POST_DATA] = { "post-data", required_argument, NULL, 0 }, 569 [L_POST_FILE] = { "post-file", required_argument, NULL, 0 }, 570 [L_SPIDER] = { "spider", no_argument, NULL, 0 }, 571 [L_TIMEOUT] = { "timeout", required_argument, NULL, 0 }, 572 [L_CONTINUE] = { "continue", no_argument, NULL, 0 }, 573 [L_PROXY] = { "proxy", required_argument, NULL, 0 }, 574 [L_NO_PROXY] = { "no-proxy", no_argument, NULL, 0 }, 575 [L_QUIET] = { "quiet", no_argument, NULL, 0 }, 576 [L_VERBOSE] = { "verbose", no_argument, NULL, 0 }, 577 {} 578 }; 579 580 581 582 int main(int argc, char **argv) 583 { 584 const char *progname = argv[0]; 585 const char *proxy_url; 586 char *username = NULL; 587 char *password = NULL; 588 struct uclient *cl; 589 int longopt_idx = 0; 590 bool has_cert = false; 591 int i, ch; 592 int rc; 593 int af = -1; 594 int debug_level = 0; 595 596 signal(SIGPIPE, SIG_IGN); 597 ssl_ctx = uclient_new_ssl_context(&ssl_ops); 598 599 while ((ch = getopt_long(argc, argv, "46cO:P:qsT:U:vY:", longopts, &longopt_idx)) != -1) { 600 switch(ch) { 601 case 0: 602 switch (longopt_idx) { 603 case L_NO_CHECK_CERTIFICATE: 604 verify = false; 605 if (ssl_ctx) 606 ssl_ops->context_set_require_validation(ssl_ctx, verify); 607 break; 608 case L_CA_CERTIFICATE: 609 has_cert = true; 610 if (ssl_ctx) 611 ssl_ops->context_add_ca_crt_file(ssl_ctx, optarg); 612 break; 613 case L_CIPHERS: 614 if (ssl_ctx) { 615 if (ssl_ops->context_set_ciphers(ssl_ctx, optarg)) { 616 if (!quiet) 617 fprintf(stderr, "No recognized ciphers in cipher list\n"); 618 exit(1); 619 } 620 } 621 break; 622 case L_USER: 623 if (!strlen(optarg)) 624 break; 625 username = strdupa(optarg); 626 memset(optarg, '*', strlen(optarg)); 627 break; 628 case L_PASSWORD: 629 if (!strlen(optarg)) 630 break; 631 password = strdupa(optarg); 632 memset(optarg, '*', strlen(optarg)); 633 break; 634 case L_USER_AGENT: 635 user_agent = optarg; 636 break; 637 case L_POST_DATA: 638 post_data = optarg; 639 break; 640 case L_POST_FILE: 641 post_file = optarg; 642 break; 643 case L_SPIDER: 644 no_output = true; 645 break; 646 case L_TIMEOUT: 647 timeout = atoi(optarg); 648 break; 649 case L_CONTINUE: 650 resume = true; 651 break; 652 case L_PROXY: 653 if (strcmp(optarg, "on") != 0) 654 proxy = false; 655 break; 656 case L_NO_PROXY: 657 proxy = false; 658 break; 659 case L_QUIET: 660 quiet = true; 661 break; 662 case L_VERBOSE: 663 debug_level++; 664 break; 665 default: 666 return usage(progname); 667 } 668 break; 669 case '4': 670 af = AF_INET; 671 break; 672 case '6': 673 af = AF_INET6; 674 break; 675 case 'c': 676 resume = true; 677 break; 678 case 'U': 679 user_agent = optarg; 680 break; 681 case 'O': 682 opt_output_file = optarg; 683 break; 684 case 'P': 685 if (chdir(optarg)) { 686 if (!quiet) 687 perror("Change output directory"); 688 exit(1); 689 } 690 break; 691 case 'q': 692 quiet = true; 693 break; 694 case 's': 695 no_output = true; 696 break; 697 case 'T': 698 timeout = atoi(optarg); 699 break; 700 case 'v': 701 debug_level++; 702 break; 703 case 'Y': 704 if (strcmp(optarg, "on") != 0) 705 proxy = false; 706 break; 707 default: 708 return usage(progname); 709 } 710 } 711 712 argv += optind; 713 argc -= optind; 714 715 if (debug_level) 716 ssl_ops->context_set_debug(ssl_ctx, debug_level, debug_cb, NULL); 717 718 if (verify && !has_cert) 719 default_certs = true; 720 721 if (argc < 1) 722 return usage(progname); 723 724 if (!ssl_ctx) { 725 for (i = 0; i < argc; i++) { 726 if (!strncmp(argv[i], "https", 5)) 727 return no_ssl(progname); 728 } 729 } 730 731 urls = argv + 1; 732 n_urls = argc - 1; 733 734 uloop_init(); 735 736 if (username) { 737 if (password) { 738 rc = asprintf(&auth_str, "%s:%s", username, password); 739 if (rc < 0) 740 return rc; 741 } else 742 auth_str = username; 743 } 744 745 if (!quiet) 746 fprintf(stderr, "Downloading '%s'\n", argv[0]); 747 748 proxy_url = get_proxy_url(argv[0]); 749 if (proxy_url) { 750 cl = uclient_new(proxy_url, auth_str, &cb); 751 if (cl) 752 uclient_set_proxy_url(cl, argv[0], NULL); 753 } else { 754 cl = uclient_new(argv[0], auth_str, &cb); 755 } 756 if (!cl) { 757 fprintf(stderr, "Failed to allocate uclient context\n"); 758 return 1; 759 } 760 if (af >= 0) 761 uclient_http_set_address_family(cl, af); 762 763 if (ssl_ctx && default_certs) 764 init_ca_cert(); 765 766 cur_resume = resume; 767 rc = init_request(cl); 768 if (!rc) { 769 /* no error received, we can enter main loop */ 770 uloop_run(); 771 } else { 772 fprintf(stderr, "Failed to send request: %s\n", strerror(rc)); 773 error_ret = 4; 774 } 775 776 uloop_done(); 777 778 uclient_free(cl); 779 780 if (output_fd >= 0 && output_fd != STDOUT_FILENO) 781 close(output_fd); 782 783 if (ssl_ctx) 784 ssl_ops->context_free(ssl_ctx); 785 786 return error_ret; 787 } 788
This page was automatically generated by LXR 0.3.1. • OpenWrt