http-fetch.c

   1 #include "cache.h"
   2 #include "commit.h"
   3 #include "pack.h"
   4 #include "fetch.h"
   5
   6 #include <curl/curl.h>
   7 #include <curl/easy.h>
   8
   9 #if LIBCURL_VERSION_NUM >= 0x070908
  10 #define USE_CURL_MULTI
  11 #define DEFAULT_MAX_REQUESTS 5
  12 #endif
  13
  14 #if LIBCURL_VERSION_NUM < 0x070704
  15 #define curl_global_cleanup() do { /* nothing */ } while(0)
  16 #endif
  17 #if LIBCURL_VERSION_NUM < 0x070800
  18 #define curl_global_init(a) do { /* nothing */ } while(0)
  19 #endif
  20
  21 #if LIBCURL_VERSION_NUM < 0x070c04
  22 #define NO_CURL_EASY_DUPHANDLE
  23 #endif
  24
  25 #define PREV_BUF_SIZE 4096
  26 #define RANGE_HEADER_SIZE 30
  27
  28 static int got_alternates = -1;
  29 static int active_requests = 0;
  30 static int data_received;
  31
  32 #ifdef USE_CURL_MULTI
  33 static int max_requests = -1;
  34 static CURLM *curlm;
  35 #endif
  36 #ifndef NO_CURL_EASY_DUPHANDLE
  37 static CURL *curl_default;
  38 #endif
  39 static struct curl_slist *pragma_header;
  40 static struct curl_slist *no_pragma_header;
  41 static struct curl_slist *no_range_header;
  42 static char curl_errorstr[CURL_ERROR_SIZE];
  43
  44 struct alt_base
  45 {
  46         char *base;
  47         int got_indices;
  48         struct packed_git *packs;
  49         struct alt_base *next;
  50 };
  51
  52 static struct alt_base *alt = NULL;
  53
  54 enum transfer_state {
  55         WAITING,
  56         ABORTED,
  57         ACTIVE,
  58         COMPLETE,
  59 };
  60
  61 struct transfer_request
  62 {
  63         unsigned char sha1[20];
  64         struct alt_base *repo;
  65         char *url;
  66         char filename[PATH_MAX];
  67         char tmpfile[PATH_MAX];
  68         int local;
  69         enum transfer_state state;
  70         CURLcode curl_result;
  71         char errorstr[CURL_ERROR_SIZE];
  72         long http_code;
  73         unsigned char real_sha1[20];
  74         SHA_CTX c;
  75         z_stream stream;
  76         int zret;
  77         int rename;
  78         struct active_request_slot *slot;
  79         struct transfer_request *next;
  80 };
  81
  82 struct active_request_slot
  83 {
  84         CURL *curl;
  85         FILE *local;
  86         int in_use;
  87         int done;
  88         CURLcode curl_result;
  89         long http_code;
  90         void *callback_data;
  91         void (*callback_func)(void *data);
  92         struct active_request_slot *next;
  93 };
  94
  95 struct alt_request {
  96         char *base;
  97         char *url;
  98         struct buffer *buffer;
  99         struct active_request_slot *slot;
 100         int http_specific;
 101 };
 102
 103 static struct transfer_request *request_queue_head = NULL;
 104 static struct active_request_slot *active_queue_head = NULL;
 105
 106 static int curl_ssl_verify = -1;
 107 static char *ssl_cert = NULL;
 108 #if LIBCURL_VERSION_NUM >= 0x070902
 109 static char *ssl_key = NULL;
 110 #endif
 111 #if LIBCURL_VERSION_NUM >= 0x070908
 112 static char *ssl_capath = NULL;
 113 #endif
 114 static char *ssl_cainfo = NULL;
 115 static long curl_low_speed_limit = -1;
 116 static long curl_low_speed_time = -1;
 117
 118 struct buffer
 119 {
 120         size_t posn;
 121         size_t size;
 122         void *buffer;
 123 };
 124
 125 static int http_options(const char *var, const char *value)
 126 {
 127         if (!strcmp("http.sslverify", var)) {
 128                 if (curl_ssl_verify == -1) {
 129                         curl_ssl_verify = git_config_bool(var, value);
 130                 }
 131                 return 0;
 132         }
 133
 134         if (!strcmp("http.sslcert", var)) {
 135                 if (ssl_cert == NULL) {
 136                         ssl_cert = xmalloc(strlen(value)+1);
 137                         strcpy(ssl_cert, value);
 138                 }
 139                 return 0;
 140         }
 141 #if LIBCURL_VERSION_NUM >= 0x070902
 142         if (!strcmp("http.sslkey", var)) {
 143                 if (ssl_key == NULL) {
 144                         ssl_key = xmalloc(strlen(value)+1);
 145                         strcpy(ssl_key, value);
 146                 }
 147                 return 0;
 148         }
 149 #endif
 150 #if LIBCURL_VERSION_NUM >= 0x070908
 151         if (!strcmp("http.sslcapath", var)) {
 152                 if (ssl_capath == NULL) {
 153                         ssl_capath = xmalloc(strlen(value)+1);
 154                         strcpy(ssl_capath, value);
 155                 }
 156                 return 0;
 157         }
 158 #endif
 159         if (!strcmp("http.sslcainfo", var)) {
 160                 if (ssl_cainfo == NULL) {
 161                         ssl_cainfo = xmalloc(strlen(value)+1);
 162                         strcpy(ssl_cainfo, value);
 163                 }
 164                 return 0;
 165         }
 166
 167 #ifdef USE_CURL_MULTI
 168         if (!strcmp("http.maxrequests", var)) {
 169                 if (max_requests == -1)
 170                         max_requests = git_config_int(var, value);
 171                 return 0;
 172         }
 173 #endif
 174
 175         if (!strcmp("http.lowspeedlimit", var)) {
 176                 if (curl_low_speed_limit == -1)
 177                         curl_low_speed_limit = (long)git_config_int(var, value);
 178                 return 0;
 179         }
 180         if (!strcmp("http.lowspeedtime", var)) {
 181                 if (curl_low_speed_time == -1)
 182                         curl_low_speed_time = (long)git_config_int(var, value);
 183                 return 0;
 184         }
 185
 186         /* Fall back on the default ones */
 187         return git_default_config(var, value);
 188 }
 189
 190 static size_t fwrite_buffer(void *ptr, size_t eltsize, size_t nmemb,
 191                             struct buffer *buffer)
 192 {
 193         size_t size = eltsize * nmemb;
 194         if (size > buffer->size - buffer->posn)
 195                 size = buffer->size - buffer->posn;
 196         memcpy(buffer->buffer + buffer->posn, ptr, size);
 197         buffer->posn += size;
 198         data_received++;
 199         return size;
 200 }
 201
 202 static size_t fwrite_buffer_dynamic(const void *ptr, size_t eltsize,
 203                                     size_t nmemb, struct buffer *buffer)
 204 {
 205         size_t size = eltsize * nmemb;
 206         if (size > buffer->size - buffer->posn) {
 207                 buffer->size = buffer->size * 3 / 2;
 208                 if (buffer->size < buffer->posn + size)
 209                         buffer->size = buffer->posn + size;
 210                 buffer->buffer = xrealloc(buffer->buffer, buffer->size);
 211         }
 212         memcpy(buffer->buffer + buffer->posn, ptr, size);
 213         buffer->posn += size;
 214         data_received++;
 215         return size;
 216 }
 217
 218 static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
 219                                void *data)
 220 {
 221         unsigned char expn[4096];
 222         size_t size = eltsize * nmemb;
 223         int posn = 0;
 224         struct transfer_request *request = (struct transfer_request *)data;
 225         do {
 226                 ssize_t retval = write(request->local,
 227                                        ptr + posn, size - posn);
 228                 if (retval < 0)
 229                         return posn;
 230                 posn += retval;
 231         } while (posn < size);
 232
 233         request->stream.avail_in = size;
 234         request->stream.next_in = ptr;
 235         do {
 236                 request->stream.next_out = expn;
 237                 request->stream.avail_out = sizeof(expn);
 238                 request->zret = inflate(&request->stream, Z_SYNC_FLUSH);
 239                 SHA1_Update(&request->c, expn,
 240                             sizeof(expn) - request->stream.avail_out);
 241         } while (request->stream.avail_in && request->zret == Z_OK);
 242         data_received++;
 243         return size;
 244 }
 245
 246 #ifdef USE_CURL_MULTI
 247 static void process_curl_messages(void);
 248 static void process_request_queue(void);
 249 #endif
 250 static void fetch_alternates(char *base);
 251
 252 static CURL* get_curl_handle(void)
 253 {
 254         CURL* result = curl_easy_init();
 255
 256         curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, curl_ssl_verify);
 257 #if LIBCURL_VERSION_NUM >= 0x070907
 258         curl_easy_setopt(result, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
 259 #endif
 260
 261         if (ssl_cert != NULL)
 262                 curl_easy_setopt(result, CURLOPT_SSLCERT, ssl_cert);
 263 #if LIBCURL_VERSION_NUM >= 0x070902
 264         if (ssl_key != NULL)
 265                 curl_easy_setopt(result, CURLOPT_SSLKEY, ssl_key);
 266 #endif
 267 #if LIBCURL_VERSION_NUM >= 0x070908
 268         if (ssl_capath != NULL)
 269                 curl_easy_setopt(result, CURLOPT_CAPATH, ssl_capath);
 270 #endif
 271         if (ssl_cainfo != NULL)
 272                 curl_easy_setopt(result, CURLOPT_CAINFO, ssl_cainfo);
 273         curl_easy_setopt(result, CURLOPT_FAILONERROR, 1);
 274
 275         if (curl_low_speed_limit > 0 && curl_low_speed_time > 0) {
 276                 curl_easy_setopt(result, CURLOPT_LOW_SPEED_LIMIT,
 277                                  curl_low_speed_limit);
 278                 curl_easy_setopt(result, CURLOPT_LOW_SPEED_TIME,
 279                                  curl_low_speed_time);
 280         }
 281
 282         curl_easy_setopt(result, CURLOPT_FOLLOWLOCATION, 1);
 283
 284         return result;
 285 }
 286
 287 static struct active_request_slot *get_active_slot(void)
 288 {
 289         struct active_request_slot *slot = active_queue_head;
 290         struct active_request_slot *newslot;
 291
 292 #ifdef USE_CURL_MULTI
 293         int num_transfers;
 294
 295         /* Wait for a slot to open up if the queue is full */
 296         while (active_requests >= max_requests) {
 297                 curl_multi_perform(curlm, &num_transfers);
 298                 if (num_transfers < active_requests) {
 299                         process_curl_messages();
 300                 }
 301         }
 302 #endif
 303
 304         while (slot != NULL && slot->in_use) {
 305                 slot = slot->next;
 306         }
 307         if (slot == NULL) {
 308                 newslot = xmalloc(sizeof(*newslot));
 309                 newslot->curl = NULL;
 310                 newslot->in_use = 0;
 311                 newslot->next = NULL;
 312
 313                 slot = active_queue_head;
 314                 if (slot == NULL) {
 315                         active_queue_head = newslot;
 316                 } else {
 317                         while (slot->next != NULL) {
 318                                 slot = slot->next;
 319                         }
 320                         slot->next = newslot;
 321                 }
 322                 slot = newslot;
 323         }
 324
 325         if (slot->curl == NULL) {
 326 #ifdef NO_CURL_EASY_DUPHANDLE
 327                 slot->curl = get_curl_handle();
 328 #else
 329                 slot->curl = curl_easy_duphandle(curl_default);
 330 #endif
 331         }
 332
 333         active_requests++;
 334         slot->in_use = 1;
 335         slot->done = 0;
 336         slot->local = NULL;
 337         slot->callback_data = NULL;
 338         slot->callback_func = NULL;
 339         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, pragma_header);
 340         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_range_header);
 341         curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr);
 342
 343         return slot;
 344 }
 345
 346 static int start_active_slot(struct active_request_slot *slot)
 347 {
 348 #ifdef USE_CURL_MULTI
 349         CURLMcode curlm_result = curl_multi_add_handle(curlm, slot->curl);
 350
 351         if (curlm_result != CURLM_OK &&
 352             curlm_result != CURLM_CALL_MULTI_PERFORM) {
 353                 active_requests--;
 354                 slot->in_use = 0;
 355                 return 0;
 356         }
 357 #endif
 358         return 1;
 359 }
 360
 361 static void run_active_slot(struct active_request_slot *slot)
 362 {
 363 #ifdef USE_CURL_MULTI
 364         int num_transfers;
 365         long last_pos = 0;
 366         long current_pos;
 367         fd_set readfds;
 368         fd_set writefds;
 369         fd_set excfds;
 370         int max_fd;
 371         struct timeval select_timeout;
 372         CURLMcode curlm_result;
 373
 374         while (!slot->done) {
 375                 data_received = 0;
 376                 do {
 377                         curlm_result = curl_multi_perform(curlm,
 378                                                           &num_transfers);
 379                 } while (curlm_result == CURLM_CALL_MULTI_PERFORM);
 380                 if (num_transfers < active_requests) {
 381                         process_curl_messages();
 382                         process_request_queue();
 383                 }
 384
 385                 if (!data_received && slot->local != NULL) {
 386                         current_pos = ftell(slot->local);
 387                         if (current_pos > last_pos)
 388                                 data_received++;
 389                         last_pos = current_pos;
 390                 }
 391
 392                 if (!slot->done && !data_received) {
 393                         max_fd = 0;
 394                         FD_ZERO(&readfds);
 395                         FD_ZERO(&writefds);
 396                         FD_ZERO(&excfds);
 397                         select_timeout.tv_sec = 0;
 398                         select_timeout.tv_usec = 50000;
 399                         select(max_fd, &readfds, &writefds,
 400                                &excfds, &select_timeout);
 401                 }
 402         }
 403 #else
 404         slot->curl_result = curl_easy_perform(slot->curl);
 405         active_requests--;
 406 #endif
 407 }
 408
 409 static void start_request(struct transfer_request *request)
 410 {
 411         char *hex = sha1_to_hex(request->sha1);
 412         char prevfile[PATH_MAX];
 413         char *url;
 414         char *posn;
 415         int prevlocal;
 416         unsigned char prev_buf[PREV_BUF_SIZE];
 417         ssize_t prev_read = 0;
 418         long prev_posn = 0;
 419         char range[RANGE_HEADER_SIZE];
 420         struct curl_slist *range_header = NULL;
 421         struct active_request_slot *slot;
 422
 423         snprintf(prevfile, sizeof(prevfile), "%s.prev", request->filename);
 424         unlink(prevfile);
 425         rename(request->tmpfile, prevfile);
 426         unlink(request->tmpfile);
 427
 428         if (request->local != -1)
 429                 error("fd leakage in start: %d", request->local);
 430         request->local = open(request->tmpfile,
 431                               O_WRONLY | O_CREAT | O_EXCL, 0666);
 432         /* This could have failed due to the "lazy directory creation";
 433          * try to mkdir the last path component.
 434          */
 435         if (request->local < 0 && errno == ENOENT) {
 436                 char *dir = strrchr(request->tmpfile, '/');
 437                 if (dir) {
 438                         *dir = 0;
 439                         mkdir(request->tmpfile, 0777);
 440                         *dir = '/';
 441                 }
 442                 request->local = open(request->tmpfile,
 443                                       O_WRONLY | O_CREAT | O_EXCL, 0666);
 444         }
 445
 446         if (request->local < 0) {
 447                 request->state = ABORTED;
 448                 error("Couldn't create temporary file %s for %s: %s\n",
 449                       request->tmpfile, request->filename, strerror(errno));
 450                 return;
 451         }
 452
 453         memset(&request->stream, 0, sizeof(request->stream));
 454
 455         inflateInit(&request->stream);
 456
 457         SHA1_Init(&request->c);
 458
 459         url = xmalloc(strlen(request->repo->base) + 50);
 460         request->url = xmalloc(strlen(request->repo->base) + 50);
 461         strcpy(url, request->repo->base);
 462         posn = url + strlen(request->repo->base);
 463         strcpy(posn, "objects/");
 464         posn += 8;
 465         memcpy(posn, hex, 2);
 466         posn += 2;
 467         *(posn++) = '/';
 468         strcpy(posn, hex + 2);
 469         strcpy(request->url, url);
 470
 471         /* If a previous temp file is present, process what was already
 472            fetched. */
 473         prevlocal = open(prevfile, O_RDONLY);
 474         if (prevlocal != -1) {
 475                 do {
 476                         prev_read = read(prevlocal, prev_buf, PREV_BUF_SIZE);
 477                         if (prev_read>0) {
 478                                 if (fwrite_sha1_file(prev_buf,
 479                                                      1,
 480                                                      prev_read,
 481                                                      request) == prev_read) {
 482                                         prev_posn += prev_read;
 483                                 } else {
 484                                         prev_read = -1;
 485                                 }
 486                         }
 487                 } while (prev_read > 0);
 488                 close(prevlocal);
 489         }
 490         unlink(prevfile);
 491
 492         /* Reset inflate/SHA1 if there was an error reading the previous temp
 493            file; also rewind to the beginning of the local file. */
 494         if (prev_read == -1) {
 495                 memset(&request->stream, 0, sizeof(request->stream));
 496                 inflateInit(&request->stream);
 497                 SHA1_Init(&request->c);
 498                 if (prev_posn>0) {
 499                         prev_posn = 0;
 500                         lseek(request->local, SEEK_SET, 0);
 501                         ftruncate(request->local, 0);
 502                 }
 503         }
 504
 505         slot = get_active_slot();
 506         curl_easy_setopt(slot->curl, CURLOPT_FILE, request);
 507         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
 508         curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, request->errorstr);
 509         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 510         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 511
 512         /* If we have successfully processed data from a previous fetch
 513            attempt, only fetch the data we don't already have. */
 514         if (prev_posn>0) {
 515                 if (get_verbosely)
 516                         fprintf(stderr,
 517                                 "Resuming fetch of object %s at byte %ld\n",
 518                                 hex, prev_posn);
 519                 sprintf(range, "Range: bytes=%ld-", prev_posn);
 520                 range_header = curl_slist_append(range_header, range);
 521                 curl_easy_setopt(slot->curl,
 522                                  CURLOPT_HTTPHEADER, range_header);
 523         }
 524
 525         /* Try to get the request started, abort the request on error */
 526         if (!start_active_slot(slot)) {
 527                 request->state = ABORTED;
 528                 close(request->local); request->local = -1;
 529                 free(request->url);
 530                 return;
 531         }
 532
 533         request->slot = slot;
 534         request->state = ACTIVE;
 535 }
 536
 537 static void finish_request(struct transfer_request *request)
 538 {
 539         struct stat st;
 540
 541         fchmod(request->local, 0444);
 542         close(request->local); request->local = -1;
 543
 544         if (request->http_code == 416) {
 545                 fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
 546         } else if (request->curl_result != CURLE_OK) {
 547                 if (stat(request->tmpfile, &st) == 0)
 548                         if (st.st_size == 0)
 549                                 unlink(request->tmpfile);
 550                 return;
 551         }
 552
 553         inflateEnd(&request->stream);
 554         SHA1_Final(request->real_sha1, &request->c);
 555         if (request->zret != Z_STREAM_END) {
 556                 unlink(request->tmpfile);
 557                 return;
 558         }
 559         if (memcmp(request->sha1, request->real_sha1, 20)) {
 560                 unlink(request->tmpfile);
 561                 return;
 562         }
 563         request->rename =
 564                 move_temp_to_file(request->tmpfile, request->filename);
 565
 566         if (request->rename == 0)
 567                 pull_say("got %s\n", sha1_to_hex(request->sha1));
 568 }
 569
 570 static void release_request(struct transfer_request *request)
 571 {
 572         struct transfer_request *entry = request_queue_head;
 573
 574         if (request->local != -1)
 575                 error("fd leakage in release: %d", request->local);
 576         if (request == request_queue_head) {
 577                 request_queue_head = request->next;
 578         } else {
 579                 while (entry->next != NULL && entry->next != request)
 580                         entry = entry->next;
 581                 if (entry->next == request)
 582                         entry->next = entry->next->next;
 583         }
 584
 585         free(request->url);
 586         free(request);
 587 }
 588
 589 #ifdef USE_CURL_MULTI
 590 static void process_curl_messages(void)
 591 {
 592         int num_messages;
 593         struct active_request_slot *slot;
 594         struct transfer_request *request = NULL;
 595         CURLMsg *curl_message = curl_multi_info_read(curlm, &num_messages);
 596
 597         while (curl_message != NULL) {
 598                 if (curl_message->msg == CURLMSG_DONE) {
 599                         int curl_result = curl_message->data.result;
 600                         slot = active_queue_head;
 601                         while (slot != NULL &&
 602                                slot->curl != curl_message->easy_handle)
 603                                 slot = slot->next;
 604                         if (slot != NULL) {
 605                                 curl_multi_remove_handle(curlm, slot->curl);
 606                                 active_requests--;
 607                                 slot->done = 1;
 608                                 slot->in_use = 0;
 609                                 slot->curl_result = curl_result;
 610                                 curl_easy_getinfo(slot->curl,
 611                                                   CURLINFO_HTTP_CODE,
 612                                                   &slot->http_code);
 613                                 request = request_queue_head;
 614                                 while (request != NULL &&
 615                                        request->slot != slot)
 616                                         request = request->next;
 617                         } else {
 618                                 fprintf(stderr, "Received DONE message for unknown request!\n");
 619                         }
 620
 621                         /* Process slot callback if appropriate */
 622                         if (slot->callback_func != NULL) {
 623                                 slot->callback_func(slot->callback_data);
 624                         }
 625
 626                         if (request != NULL) {
 627                                 request->curl_result = curl_result;
 628                                 request->http_code = slot->http_code;
 629                                 request->slot = NULL;
 630                                 request->state = COMPLETE;
 631
 632                                 /* Use alternates if necessary */
 633                                 if (request->http_code == 404) {
 634                                         fetch_alternates(alt->base);
 635                                         if (request->repo->next != NULL) {
 636                                                 request->repo =
 637                                                         request->repo->next;
 638                                                 close(request->local);
 639                                                         request->local = -1;
 640                                                 start_request(request);
 641                                         } else {
 642                                                 finish_request(request);
 643                                         }
 644                                 } else {
 645                                         finish_request(request);
 646                                 }
 647                         }
 648                 } else {
 649                         fprintf(stderr, "Unknown CURL message received: %d\n",
 650                                 (int)curl_message->msg);
 651                 }
 652                 curl_message = curl_multi_info_read(curlm, &num_messages);
 653         }
 654 }
 655
 656 static void process_request_queue(void)
 657 {
 658         struct transfer_request *request = request_queue_head;
 659         struct active_request_slot *slot = active_queue_head;
 660         int num_transfers;
 661
 662         while (active_requests < max_requests && request != NULL) {
 663                 if (request->state == WAITING) {
 664                         if (has_sha1_file(request->sha1))
 665                                 release_request(request);
 666                         else
 667                                 start_request(request);
 668                         curl_multi_perform(curlm, &num_transfers);
 669                 }
 670                 request = request->next;
 671         }
 672
 673         while (slot != NULL) {
 674                 if (!slot->in_use && slot->curl != NULL) {
 675                         curl_easy_cleanup(slot->curl);
 676                         slot->curl = NULL;
 677                 }
 678                 slot = slot->next;
 679         }
 680 }
 681 #endif
 682
 683 void prefetch(unsigned char *sha1)
 684 {
 685         struct transfer_request *newreq;
 686         struct transfer_request *tail;
 687         char *filename = sha1_file_name(sha1);
 688
 689         newreq = xmalloc(sizeof(*newreq));
 690         memcpy(newreq->sha1, sha1, 20);
 691         newreq->repo = alt;
 692         newreq->url = NULL;
 693         newreq->local = -1;
 694         newreq->state = WAITING;
 695         snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
 696         snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
 697                  "%s.temp", filename);
 698         newreq->next = NULL;
 699
 700         if (request_queue_head == NULL) {
 701                 request_queue_head = newreq;
 702         } else {
 703                 tail = request_queue_head;
 704                 while (tail->next != NULL) {
 705                         tail = tail->next;
 706                 }
 707                 tail->next = newreq;
 708         }
 709 #ifdef USE_CURL_MULTI
 710         process_request_queue();
 711         process_curl_messages();
 712 #endif
 713 }
 714
 715 static int fetch_index(struct alt_base *repo, unsigned char *sha1)
 716 {
 717         char *hex = sha1_to_hex(sha1);
 718         char *filename;
 719         char *url;
 720         char tmpfile[PATH_MAX];
 721         long prev_posn = 0;
 722         char range[RANGE_HEADER_SIZE];
 723         struct curl_slist *range_header = NULL;
 724
 725         FILE *indexfile;
 726         struct active_request_slot *slot;
 727
 728         if (has_pack_index(sha1))
 729                 return 0;
 730
 731         if (get_verbosely)
 732                 fprintf(stderr, "Getting index for pack %s\n", hex);
 733
 734         url = xmalloc(strlen(repo->base) + 64);
 735         sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex);
 736
 737         filename = sha1_pack_index_name(sha1);
 738         snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
 739         indexfile = fopen(tmpfile, "a");
 740         if (!indexfile)
 741                 return error("Unable to open local file %s for pack index",
 742                              filename);
 743
 744         slot = get_active_slot();
 745         curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
 746         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
 747         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 748         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 749         slot->local = indexfile;
 750
 751         /* If there is data present from a previous transfer attempt,
 752            resume where it left off */
 753         prev_posn = ftell(indexfile);
 754         if (prev_posn>0) {
 755                 if (get_verbosely)
 756                         fprintf(stderr,
 757                                 "Resuming fetch of index for pack %s at byte %ld\n",
 758                                 hex, prev_posn);
 759                 sprintf(range, "Range: bytes=%ld-", prev_posn);
 760                 range_header = curl_slist_append(range_header, range);
 761                 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
 762         }
 763
 764         if (start_active_slot(slot)) {
 765                 run_active_slot(slot);
 766                 if (slot->curl_result != CURLE_OK) {
 767                         fclose(indexfile);
 768                         return error("Unable to get pack index %s\n%s", url,
 769                                      curl_errorstr);
 770                 }
 771         } else {
 772                 fclose(indexfile);
 773                 return error("Unable to start request");
 774         }
 775
 776         fclose(indexfile);
 777
 778         return move_temp_to_file(tmpfile, filename);
 779 }
 780
 781 static int setup_index(struct alt_base *repo, unsigned char *sha1)
 782 {
 783         struct packed_git *new_pack;
 784         if (has_pack_file(sha1))
 785                 return 0; // don't list this as something we can get
 786
 787         if (fetch_index(repo, sha1))
 788                 return -1;
 789
 790         new_pack = parse_pack_index(sha1);
 791         new_pack->next = repo->packs;
 792         repo->packs = new_pack;
 793         return 0;
 794 }
 795
 796 static void process_alternates(void *callback_data)
 797 {
 798         struct alt_request *alt_req = (struct alt_request *)callback_data;
 799         struct active_request_slot *slot = alt_req->slot;
 800         struct alt_base *tail = alt;
 801         char *base = alt_req->base;
 802         static const char null_byte = '\0';
 803         char *data;
 804         int i = 0;
 805
 806         if (alt_req->http_specific) {
 807                 if (slot->curl_result != CURLE_OK ||
 808                     !alt_req->buffer->posn) {
 809
 810                         /* Try reusing the slot to get non-http alternates */
 811                         alt_req->http_specific = 0;
 812                         sprintf(alt_req->url, "%s/objects/info/alternates",
 813                                 base);
 814                         curl_easy_setopt(slot->curl, CURLOPT_URL,
 815                                          alt_req->url);
 816                         active_requests++;
 817                         slot->in_use = 1;
 818                         slot->done = 0;
 819                         if (start_active_slot(slot)) {
 820                                 return;
 821                         } else {
 822                                 got_alternates = -1;
 823                                 slot->done = 1;
 824                                 return;
 825                         }
 826                 }
 827         } else if (slot->curl_result != CURLE_OK) {
 828                 if (slot->http_code != 404) {
 829                         got_alternates = -1;
 830                         return;
 831                 }
 832         }
 833
 834         fwrite_buffer_dynamic(&null_byte, 1, 1, alt_req->buffer);
 835         alt_req->buffer->posn--;
 836         data = alt_req->buffer->buffer;
 837
 838         while (i < alt_req->buffer->posn) {
 839                 int posn = i;
 840                 while (posn < alt_req->buffer->posn && data[posn] != '\n')
 841                         posn++;
 842                 if (data[posn] == '\n') {
 843                         int okay = 0;
 844                         int serverlen = 0;
 845                         struct alt_base *newalt;
 846                         char *target = NULL;
 847                         if (data[i] == '/') {
 848                                 serverlen = strchr(base + 8, '/') - base;
 849                                 okay = 1;
 850                         } else if (!memcmp(data + i, "../", 3)) {
 851                                 i += 3;
 852                                 serverlen = strlen(base);
 853                                 while (i + 2 < posn &&
 854                                        !memcmp(data + i, "../", 3)) {
 855                                         do {
 856                                                 serverlen--;
 857                                         } while (serverlen &&
 858                                                  base[serverlen - 1] != '/');
 859                                         i += 3;
 860                                 }
 861                                 // If the server got removed, give up.
 862                                 okay = strchr(base, ':') - base + 3 <
 863                                         serverlen;
 864                         } else if (alt_req->http_specific) {
 865                                 char *colon = strchr(data + i, ':');
 866                                 char *slash = strchr(data + i, '/');
 867                                 if (colon && slash && colon < data + posn &&
 868                                     slash < data + posn && colon < slash) {
 869                                         okay = 1;
 870                                 }
 871                         }
 872                         // skip 'objects' at end
 873                         if (okay) {
 874                                 target = xmalloc(serverlen + posn - i - 6);
 875                                 strncpy(target, base, serverlen);
 876                                 strncpy(target + serverlen, data + i,
 877                                         posn - i - 7);
 878                                 target[serverlen + posn - i - 7] = '\0';
 879                                 if (get_verbosely)
 880                                         fprintf(stderr,
 881                                                 "Also look at %s\n", target);
 882                                 newalt = xmalloc(sizeof(*newalt));
 883                                 newalt->next = NULL;
 884                                 newalt->base = target;
 885                                 newalt->got_indices = 0;
 886                                 newalt->packs = NULL;
 887                                 while (tail->next != NULL)
 888                                         tail = tail->next;
 889                                 tail->next = newalt;
 890                         }
 891                 }
 892                 i = posn + 1;
 893         }
 894
 895         got_alternates = 1;
 896 }
 897
 898 static void fetch_alternates(char *base)
 899 {
 900         struct buffer buffer;
 901         char *url;
 902         char *data;
 903         struct active_request_slot *slot;
 904         static struct alt_request alt_req;
 905         int num_transfers;
 906
 907         /* If another request has already started fetching alternates,
 908            wait for them to arrive and return to processing this request's
 909            curl message */
 910         while (got_alternates == 0) {
 911                 curl_multi_perform(curlm, &num_transfers);
 912                 process_curl_messages();
 913                 process_request_queue();
 914         }
 915
 916         /* Nothing to do if they've already been fetched */
 917         if (got_alternates == 1)
 918                 return;
 919
 920         /* Start the fetch */
 921         got_alternates = 0;
 922
 923         data = xmalloc(4096);
 924         buffer.size = 4096;
 925         buffer.posn = 0;
 926         buffer.buffer = data;
 927
 928         if (get_verbosely)
 929                 fprintf(stderr, "Getting alternates list for %s\n", base);
 930
 931         url = xmalloc(strlen(base) + 31);
 932         sprintf(url, "%s/objects/info/http-alternates", base);
 933
 934         /* Use a callback to process the result, since another request
 935            may fail and need to have alternates loaded before continuing */
 936         slot = get_active_slot();
 937         slot->callback_func = process_alternates;
 938         slot->callback_data = &alt_req;
 939
 940         curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 941         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
 942                          fwrite_buffer_dynamic);
 943         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 944
 945         alt_req.base = base;
 946         alt_req.url = url;
 947         alt_req.buffer = &buffer;
 948         alt_req.http_specific = 1;
 949         alt_req.slot = slot;
 950
 951         if (start_active_slot(slot))
 952                 run_active_slot(slot);
 953         else
 954                 got_alternates = -1;
 955
 956         free(data);
 957         free(url);
 958 }
 959
 960 static int fetch_indices(struct alt_base *repo)
 961 {
 962         unsigned char sha1[20];
 963         char *url;
 964         struct buffer buffer;
 965         char *data;
 966         int i = 0;
 967
 968         struct active_request_slot *slot;
 969
 970         if (repo->got_indices)
 971                 return 0;
 972
 973         data = xmalloc(4096);
 974         buffer.size = 4096;
 975         buffer.posn = 0;
 976         buffer.buffer = data;
 977
 978         if (get_verbosely)
 979                 fprintf(stderr, "Getting pack list for %s\n", repo->base);
 980
 981         url = xmalloc(strlen(repo->base) + 21);
 982         sprintf(url, "%s/objects/info/packs", repo->base);
 983
 984         slot = get_active_slot();
 985         curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 986         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
 987                          fwrite_buffer_dynamic);
 988         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 989         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
 990         if (start_active_slot(slot)) {
 991                 run_active_slot(slot);
 992                 if (slot->curl_result != CURLE_OK) {
 993                         free(buffer.buffer);
 994                         return error("%s", curl_errorstr);
 995                 }
 996         } else {
 997                 free(buffer.buffer);
 998                 return error("Unable to start request");
 999         }
1000
1001         data = buffer.buffer;
1002         while (i < buffer.posn) {
1003                 switch (data[i]) {
1004                 case 'P':
1005                         i++;
1006                         if (i + 52 < buffer.posn &&
1007                             !strncmp(data + i, " pack-", 6) &&
1008                             !strncmp(data + i + 46, ".pack\n", 6)) {
1009                                 get_sha1_hex(data + i + 6, sha1);
1010                                 setup_index(repo, sha1);
1011                                 i += 51;
1012                                 break;
1013                         }
1014                 default:
1015                         while (data[i] != '\n')
1016                                 i++;
1017                 }
1018                 i++;
1019         }
1020
1021         free(buffer.buffer);
1022         repo->got_indices = 1;
1023         return 0;
1024 }
1025
1026 static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
1027 {
1028         char *url;
1029         struct packed_git *target;
1030         struct packed_git **lst;
1031         FILE *packfile;
1032         char *filename;
1033         char tmpfile[PATH_MAX];
1034         int ret;
1035         long prev_posn = 0;
1036         char range[RANGE_HEADER_SIZE];
1037         struct curl_slist *range_header = NULL;
1038
1039         struct active_request_slot *slot;
1040
1041         if (fetch_indices(repo))
1042                 return -1;
1043         target = find_sha1_pack(sha1, repo->packs);
1044         if (!target)
1045                 return -1;
1046
1047         if (get_verbosely) {
1048                 fprintf(stderr, "Getting pack %s\n",
1049                         sha1_to_hex(target->sha1));
1050                 fprintf(stderr, " which contains %s\n",
1051                         sha1_to_hex(sha1));
1052         }
1053
1054         url = xmalloc(strlen(repo->base) + 65);
1055         sprintf(url, "%s/objects/pack/pack-%s.pack",
1056                 repo->base, sha1_to_hex(target->sha1));
1057
1058         filename = sha1_pack_name(target->sha1);
1059         snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
1060         packfile = fopen(tmpfile, "a");
1061         if (!packfile)
1062                 return error("Unable to open local file %s for pack",
1063                              filename);
1064
1065         slot = get_active_slot();
1066         curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
1067         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
1068         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
1069         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
1070         slot->local = packfile;
1071
1072         /* If there is data present from a previous transfer attempt,
1073            resume where it left off */
1074         prev_posn = ftell(packfile);
1075         if (prev_posn>0) {
1076                 if (get_verbosely)
1077                         fprintf(stderr,
1078                                 "Resuming fetch of pack %s at byte %ld\n",
1079                                 sha1_to_hex(target->sha1), prev_posn);
1080                 sprintf(range, "Range: bytes=%ld-", prev_posn);
1081                 range_header = curl_slist_append(range_header, range);
1082                 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
1083         }
1084
1085         if (start_active_slot(slot)) {
1086                 run_active_slot(slot);
1087                 if (slot->curl_result != CURLE_OK) {
1088                         fclose(packfile);
1089                         return error("Unable to get pack file %s\n%s", url,
1090                                      curl_errorstr);
1091                 }
1092         } else {
1093                 fclose(packfile);
1094                 return error("Unable to start request");
1095         }
1096
1097         fclose(packfile);
1098
1099         ret = move_temp_to_file(tmpfile, filename);
1100         if (ret)
1101                 return ret;
1102
1103         lst = &repo->packs;
1104         while (*lst != target)
1105                 lst = &((*lst)->next);
1106         *lst = (*lst)->next;
1107
1108         if (verify_pack(target, 0))
1109                 return -1;
1110         install_packed_git(target);
1111
1112         return 0;
1113 }
1114
1115 static int fetch_object(struct alt_base *repo, unsigned char *sha1)
1116 {
1117         char *hex = sha1_to_hex(sha1);
1118         int ret;
1119         struct transfer_request *request = request_queue_head;
1120
1121         while (request != NULL && memcmp(request->sha1, sha1, 20))
1122                 request = request->next;
1123         if (request == NULL)
1124                 return error("Couldn't find request for %s in the queue", hex);
1125
1126         if (has_sha1_file(request->sha1)) {
1127                 release_request(request);
1128                 return 0;
1129         }
1130
1131 #ifdef USE_CURL_MULTI
1132         while (request->state == WAITING) {
1133                 int num_transfers;
1134                 curl_multi_perform(curlm, &num_transfers);
1135                 if (num_transfers < active_requests) {
1136                         process_curl_messages();
1137                         process_request_queue();
1138                 }
1139         }
1140 #else
1141         start_request(request);
1142 #endif
1143
1144         while (request->state == ACTIVE) {
1145                 run_active_slot(request->slot);
1146 #ifndef USE_CURL_MULTI
1147                 request->curl_result = request->slot->curl_result;
1148                 request->http_code = request->slot->http_code;
1149                 request->slot = NULL;
1150
1151                 /* Use alternates if necessary */
1152                 if (request->http_code == 404) {
1153                         fetch_alternates(alt->base);
1154                         if (request->repo->next != NULL) {
1155                                 request->repo = request->repo->next;
1156                                 close(request->local); request->local = -1;
1157                                 start_request(request);
1158                         }
1159                 } else {
1160                         finish_request(request);
1161                         request->state = COMPLETE;
1162                 }
1163 #endif
1164         }
1165         if (request->local != -1) {
1166                 close(request->local); request->local = -1;
1167         }
1168
1169         if (request->state == ABORTED) {
1170                 release_request(request);
1171                 return error("Request for %s aborted", hex);
1172         }
1173
1174         if (request->curl_result != CURLE_OK && request->http_code != 416) {
1175                 if (request->http_code == 404)
1176                         ret = -1; /* Be silent, it is probably in a pack. */
1177                 else
1178                         ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
1179                                     request->errorstr, request->curl_result,
1180                                     request->http_code, hex);
1181                 release_request(request);
1182                 return ret;
1183         }
1184
1185         if (request->zret != Z_STREAM_END) {
1186                 ret = error("File %s (%s) corrupt\n", hex, request->url);
1187                 release_request(request);
1188                 return ret;
1189         }
1190
1191         if (memcmp(request->sha1, request->real_sha1, 20)) {
1192                 release_request(request);
1193                 return error("File %s has bad hash\n", hex);
1194         }
1195
1196         if (request->rename < 0) {
1197                 ret = error("unable to write sha1 filename %s: %s",
1198                             request->filename,
1199                             strerror(request->rename));
1200                 release_request(request);
1201                 return ret;
1202         }
1203
1204         release_request(request);
1205         return 0;
1206 }
1207
1208 int fetch(unsigned char *sha1)
1209 {
1210         struct alt_base *altbase = alt;
1211
1212         if (!fetch_object(altbase, sha1))
1213                 return 0;
1214         while (altbase) {
1215                 if (!fetch_pack(altbase, sha1))
1216                         return 0;
1217                 fetch_alternates(alt->base);
1218                 altbase = altbase->next;
1219         }
1220         return error("Unable to find %s under %s\n", sha1_to_hex(sha1),
1221                      alt->base);
1222 }
1223
1224 static inline int needs_quote(int ch)
1225 {
1226         switch (ch) {
1227         case '/': case '-': case '.':
1228         case 'A'...'Z': case 'a'...'z': case '0'...'9':
1229                 return 0;
1230         default:
1231                 return 1;
1232         }
1233 }
1234
1235 static inline int hex(int v)
1236 {
1237         if (v < 10) return '0' + v;
1238         else return 'A' + v - 10;
1239 }
1240
1241 static char *quote_ref_url(const char *base, const char *ref)
1242 {
1243         const char *cp;
1244         char *dp, *qref;
1245         int len, baselen, ch;
1246
1247         baselen = strlen(base);
1248         len = baselen + 6; /* "refs/" + NUL */
1249         for (cp = ref; (ch = *cp) != 0; cp++, len++)
1250                 if (needs_quote(ch))
1251                         len += 2; /* extra two hex plus replacement % */
1252         qref = xmalloc(len);
1253         memcpy(qref, base, baselen);
1254         memcpy(qref + baselen, "refs/", 5);
1255         for (cp = ref, dp = qref + baselen + 5; (ch = *cp) != 0; cp++) {
1256                 if (needs_quote(ch)) {
1257                         *dp++ = '%';
1258                         *dp++ = hex((ch >> 4) & 0xF);
1259                         *dp++ = hex(ch & 0xF);
1260                 }
1261                 else
1262                         *dp++ = ch;
1263         }
1264         *dp = 0;
1265
1266         return qref;
1267 }
1268
1269 int fetch_ref(char *ref, unsigned char *sha1)
1270 {
1271         char *url;
1272         char hex[42];
1273         struct buffer buffer;
1274         char *base = alt->base;
1275         struct active_request_slot *slot;
1276         buffer.size = 41;
1277         buffer.posn = 0;
1278         buffer.buffer = hex;
1279         hex[41] = '\0';
1280
1281         url = quote_ref_url(base, ref);
1282         slot = get_active_slot();
1283         curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
1284         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
1285         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
1286         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
1287         if (start_active_slot(slot)) {
1288                 run_active_slot(slot);
1289                 if (slot->curl_result != CURLE_OK)
1290                         return error("Couldn't get %s for %s\n%s",
1291                                      url, ref, curl_errorstr);
1292         } else {
1293                 return error("Unable to start request");
1294         }
1295
1296         hex[40] = '\0';
1297         get_sha1_hex(hex, sha1);
1298         return 0;
1299 }
1300
1301 int main(int argc, char **argv)
1302 {
1303         char *commit_id;
1304         char *url;
1305         int arg = 1;
1306         struct active_request_slot *slot;
1307         char *low_speed_limit;
1308         char *low_speed_time;
1309         char *wait_url;
1310         int rc = 0;
1311
1312         while (arg < argc && argv[arg][0] == '-') {
1313                 if (argv[arg][1] == 't') {
1314                         get_tree = 1;
1315                 } else if (argv[arg][1] == 'c') {
1316                         get_history = 1;
1317                 } else if (argv[arg][1] == 'a') {
1318                         get_all = 1;
1319                         get_tree = 1;
1320                         get_history = 1;
1321                 } else if (argv[arg][1] == 'v') {
1322                         get_verbosely = 1;
1323                 } else if (argv[arg][1] == 'w') {
1324                         write_ref = argv[arg + 1];
1325                         arg++;
1326                 } else if (!strcmp(argv[arg], "--recover")) {
1327                         get_recover = 1;
1328                 }
1329                 arg++;
1330         }
1331         if (argc < arg + 2) {
1332                 usage("git-http-fetch [-c] [-t] [-a] [-d] [-v] [--recover] [-w ref] commit-id url");
1333                 return 1;
1334         }
1335         commit_id = argv[arg];
1336         url = argv[arg + 1];
1337
1338         curl_global_init(CURL_GLOBAL_ALL);
1339
1340 #ifdef USE_CURL_MULTI
1341         {
1342                 char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS");
1343                 if (http_max_requests != NULL)
1344                         max_requests = atoi(http_max_requests);
1345         }
1346
1347         curlm = curl_multi_init();
1348         if (curlm == NULL) {
1349                 fprintf(stderr, "Error creating curl multi handle.\n");
1350                 return 1;
1351         }
1352 #endif
1353
1354         if (getenv("GIT_SSL_NO_VERIFY"))
1355                 curl_ssl_verify = 0;
1356
1357         ssl_cert = getenv("GIT_SSL_CERT");
1358 #if LIBCURL_VERSION_NUM >= 0x070902
1359         ssl_key = getenv("GIT_SSL_KEY");
1360 #endif
1361 #if LIBCURL_VERSION_NUM >= 0x070908
1362         ssl_capath = getenv("GIT_SSL_CAPATH");
1363 #endif
1364         ssl_cainfo = getenv("GIT_SSL_CAINFO");
1365
1366         low_speed_limit = getenv("GIT_HTTP_LOW_SPEED_LIMIT");
1367         if (low_speed_limit != NULL)
1368                 curl_low_speed_limit = strtol(low_speed_limit, NULL, 10);
1369         low_speed_time = getenv("GIT_HTTP_LOW_SPEED_TIME");
1370         if (low_speed_time != NULL)
1371                 curl_low_speed_time = strtol(low_speed_time, NULL, 10);
1372
1373         git_config(http_options);
1374
1375         if (curl_ssl_verify == -1)
1376                 curl_ssl_verify = 1;
1377
1378 #ifdef USE_CURL_MULTI
1379         if (max_requests < 1)
1380                 max_requests = DEFAULT_MAX_REQUESTS;
1381 #endif
1382
1383         pragma_header = curl_slist_append(pragma_header, "Pragma: no-cache");
1384         no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
1385         no_range_header = curl_slist_append(no_range_header, "Range:");
1386
1387 #ifndef NO_CURL_EASY_DUPHANDLE
1388         curl_default = get_curl_handle();
1389 #endif
1390
1391         alt = xmalloc(sizeof(*alt));
1392         alt->base = url;
1393         alt->got_indices = 0;
1394         alt->packs = NULL;
1395         alt->next = NULL;
1396
1397         if (pull(commit_id))
1398                 rc = 1;
1399
1400         curl_slist_free_all(pragma_header);
1401         curl_slist_free_all(no_pragma_header);
1402         curl_slist_free_all(no_range_header);
1403 #ifndef NO_CURL_EASY_DUPHANDLE
1404         curl_easy_cleanup(curl_default);
1405 #endif
1406         slot = active_queue_head;
1407         while (slot != NULL) {
1408                 if (slot->in_use) {
1409                         if (get_verbosely) {
1410                                 curl_easy_getinfo(slot->curl,
1411                                                   CURLINFO_EFFECTIVE_URL,
1412                                                   &wait_url);
1413                                 fprintf(stderr, "Waiting for %s\n", wait_url);
1414                         }
1415                         run_active_slot(slot);
1416                 }
1417                 if (slot->curl != NULL)
1418                         curl_easy_cleanup(slot->curl);
1419                 slot = slot->next;
1420         }
1421 #ifdef USE_CURL_MULTI
1422         curl_multi_cleanup(curlm);
1423 #endif
1424         curl_global_cleanup();
1425         return rc;
1426 }