http-fetch.c

   1 #include "cache.h"
   2 #include "commit.h"
   3 #include "pack.h"
   4 #include "fetch.h"
   5 #include "http.h"
   6
   7 #define PREV_BUF_SIZE 4096
   8 #define RANGE_HEADER_SIZE 30
   9
  10 static int got_alternates = -1;
  11
  12 static struct curl_slist *no_pragma_header;
  13
  14 struct alt_base
  15 {
  16         char *base;
  17         int got_indices;
  18         struct packed_git *packs;
  19         struct alt_base *next;
  20 };
  21
  22 static struct alt_base *alt = NULL;
  23
  24 enum transfer_state {
  25         WAITING,
  26         ABORTED,
  27         ACTIVE,
  28         COMPLETE,
  29 };
  30
  31 struct transfer_request
  32 {
  33         unsigned char sha1[20];
  34         struct alt_base *repo;
  35         char *url;
  36         char filename[PATH_MAX];
  37         char tmpfile[PATH_MAX];
  38         int local;
  39         enum transfer_state state;
  40         CURLcode curl_result;
  41         char errorstr[CURL_ERROR_SIZE];
  42         long http_code;
  43         unsigned char real_sha1[20];
  44         SHA_CTX c;
  45         z_stream stream;
  46         int zret;
  47         int rename;
  48         struct active_request_slot *slot;
  49         struct transfer_request *next;
  50 };
  51
  52 struct alt_request {
  53         char *base;
  54         char *url;
  55         struct buffer *buffer;
  56         struct active_request_slot *slot;
  57         int http_specific;
  58 };
  59
  60 static struct transfer_request *request_queue_head = NULL;
  61
  62 static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
  63                                void *data)
  64 {
  65         unsigned char expn[4096];
  66         size_t size = eltsize * nmemb;
  67         int posn = 0;
  68         struct transfer_request *request = (struct transfer_request *)data;
  69         do {
  70                 ssize_t retval = write(request->local,
  71                                        ptr + posn, size - posn);
  72                 if (retval < 0)
  73                         return posn;
  74                 posn += retval;
  75         } while (posn < size);
  76
  77         request->stream.avail_in = size;
  78         request->stream.next_in = ptr;
  79         do {
  80                 request->stream.next_out = expn;
  81                 request->stream.avail_out = sizeof(expn);
  82                 request->zret = inflate(&request->stream, Z_SYNC_FLUSH);
  83                 SHA1_Update(&request->c, expn,
  84                             sizeof(expn) - request->stream.avail_out);
  85         } while (request->stream.avail_in && request->zret == Z_OK);
  86         data_received++;
  87         return size;
  88 }
  89
  90 static void fetch_alternates(char *base);
  91
  92 static void process_object_response(void *callback_data);
  93
  94 static void start_request(struct transfer_request *request)
  95 {
  96         char *hex = sha1_to_hex(request->sha1);
  97         char prevfile[PATH_MAX];
  98         char *url;
  99         char *posn;
 100         int prevlocal;
 101         unsigned char prev_buf[PREV_BUF_SIZE];
 102         ssize_t prev_read = 0;
 103         long prev_posn = 0;
 104         char range[RANGE_HEADER_SIZE];
 105         struct curl_slist *range_header = NULL;
 106         struct active_request_slot *slot;
 107
 108         snprintf(prevfile, sizeof(prevfile), "%s.prev", request->filename);
 109         unlink(prevfile);
 110         rename(request->tmpfile, prevfile);
 111         unlink(request->tmpfile);
 112
 113         if (request->local != -1)
 114                 error("fd leakage in start: %d", request->local);
 115         request->local = open(request->tmpfile,
 116                               O_WRONLY | O_CREAT | O_EXCL, 0666);
 117         /* This could have failed due to the "lazy directory creation";
 118          * try to mkdir the last path component.
 119          */
 120         if (request->local < 0 && errno == ENOENT) {
 121                 char *dir = strrchr(request->tmpfile, '/');
 122                 if (dir) {
 123                         *dir = 0;
 124                         mkdir(request->tmpfile, 0777);
 125                         *dir = '/';
 126                 }
 127                 request->local = open(request->tmpfile,
 128                                       O_WRONLY | O_CREAT | O_EXCL, 0666);
 129         }
 130
 131         if (request->local < 0) {
 132                 request->state = ABORTED;
 133                 error("Couldn't create temporary file %s for %s: %s\n",
 134                       request->tmpfile, request->filename, strerror(errno));
 135                 return;
 136         }
 137
 138         memset(&request->stream, 0, sizeof(request->stream));
 139
 140         inflateInit(&request->stream);
 141
 142         SHA1_Init(&request->c);
 143
 144         url = xmalloc(strlen(request->repo->base) + 50);
 145         request->url = xmalloc(strlen(request->repo->base) + 50);
 146         strcpy(url, request->repo->base);
 147         posn = url + strlen(request->repo->base);
 148         strcpy(posn, "objects/");
 149         posn += 8;
 150         memcpy(posn, hex, 2);
 151         posn += 2;
 152         *(posn++) = '/';
 153         strcpy(posn, hex + 2);
 154         strcpy(request->url, url);
 155
 156         /* If a previous temp file is present, process what was already
 157            fetched. */
 158         prevlocal = open(prevfile, O_RDONLY);
 159         if (prevlocal != -1) {
 160                 do {
 161                         prev_read = read(prevlocal, prev_buf, PREV_BUF_SIZE);
 162                         if (prev_read>0) {
 163                                 if (fwrite_sha1_file(prev_buf,
 164                                                      1,
 165                                                      prev_read,
 166                                                      request) == prev_read) {
 167                                         prev_posn += prev_read;
 168                                 } else {
 169                                         prev_read = -1;
 170                                 }
 171                         }
 172                 } while (prev_read > 0);
 173                 close(prevlocal);
 174         }
 175         unlink(prevfile);
 176
 177         /* Reset inflate/SHA1 if there was an error reading the previous temp
 178            file; also rewind to the beginning of the local file. */
 179         if (prev_read == -1) {
 180                 memset(&request->stream, 0, sizeof(request->stream));
 181                 inflateInit(&request->stream);
 182                 SHA1_Init(&request->c);
 183                 if (prev_posn>0) {
 184                         prev_posn = 0;
 185                         lseek(request->local, SEEK_SET, 0);
 186                         ftruncate(request->local, 0);
 187                 }
 188         }
 189
 190         slot = get_active_slot();
 191         slot->callback_func = process_object_response;
 192         slot->callback_data = request;
 193         request->slot = slot;
 194
 195         curl_easy_setopt(slot->curl, CURLOPT_FILE, request);
 196         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
 197         curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, request->errorstr);
 198         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 199         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 200
 201         /* If we have successfully processed data from a previous fetch
 202            attempt, only fetch the data we don't already have. */
 203         if (prev_posn>0) {
 204                 if (get_verbosely)
 205                         fprintf(stderr,
 206                                 "Resuming fetch of object %s at byte %ld\n",
 207                                 hex, prev_posn);
 208                 sprintf(range, "Range: bytes=%ld-", prev_posn);
 209                 range_header = curl_slist_append(range_header, range);
 210                 curl_easy_setopt(slot->curl,
 211                                  CURLOPT_HTTPHEADER, range_header);
 212         }
 213
 214         /* Try to get the request started, abort the request on error */
 215         request->state = ACTIVE;
 216         if (!start_active_slot(slot)) {
 217                 request->state = ABORTED;
 218                 request->slot = NULL;
 219                 close(request->local); request->local = -1;
 220                 free(request->url);
 221         }
 222 }
 223
 224 static void finish_request(struct transfer_request *request)
 225 {
 226         struct stat st;
 227
 228         fchmod(request->local, 0444);
 229         close(request->local); request->local = -1;
 230
 231         if (request->http_code == 416) {
 232                 fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
 233         } else if (request->curl_result != CURLE_OK) {
 234                 if (stat(request->tmpfile, &st) == 0)
 235                         if (st.st_size == 0)
 236                                 unlink(request->tmpfile);
 237                 return;
 238         }
 239
 240         inflateEnd(&request->stream);
 241         SHA1_Final(request->real_sha1, &request->c);
 242         if (request->zret != Z_STREAM_END) {
 243                 unlink(request->tmpfile);
 244                 return;
 245         }
 246         if (memcmp(request->sha1, request->real_sha1, 20)) {
 247                 unlink(request->tmpfile);
 248                 return;
 249         }
 250         request->rename =
 251                 move_temp_to_file(request->tmpfile, request->filename);
 252
 253         if (request->rename == 0)
 254                 pull_say("got %s\n", sha1_to_hex(request->sha1));
 255 }
 256
 257 static void process_object_response(void *callback_data)
 258 {
 259         struct transfer_request *request =
 260                 (struct transfer_request *)callback_data;
 261
 262         request->curl_result = request->slot->curl_result;
 263         request->http_code = request->slot->http_code;
 264         request->slot = NULL;
 265         request->state = COMPLETE;
 266
 267         /* Use alternates if necessary */
 268         if (request->http_code == 404) {
 269                 fetch_alternates(alt->base);
 270                 if (request->repo->next != NULL) {
 271                         request->repo =
 272                                 request->repo->next;
 273                         close(request->local);
 274                         request->local = -1;
 275                         start_request(request);
 276                         return;
 277                 }
 278         }
 279
 280         finish_request(request);
 281 }
 282
 283 static void release_request(struct transfer_request *request)
 284 {
 285         struct transfer_request *entry = request_queue_head;
 286
 287         if (request->local != -1)
 288                 error("fd leakage in release: %d", request->local);
 289         if (request == request_queue_head) {
 290                 request_queue_head = request->next;
 291         } else {
 292                 while (entry->next != NULL && entry->next != request)
 293                         entry = entry->next;
 294                 if (entry->next == request)
 295                         entry->next = entry->next->next;
 296         }
 297
 298         free(request->url);
 299         free(request);
 300 }
 301
 302 #ifdef USE_CURL_MULTI
 303 void fill_active_slots(void)
 304 {
 305         struct transfer_request *request = request_queue_head;
 306         struct active_request_slot *slot = active_queue_head;
 307         int num_transfers;
 308
 309         while (active_requests < max_requests && request != NULL) {
 310                 if (request->state == WAITING) {
 311                         if (has_sha1_file(request->sha1))
 312                                 release_request(request);
 313                         else
 314                                 start_request(request);
 315                         curl_multi_perform(curlm, &num_transfers);
 316                 }
 317                 request = request->next;
 318         }
 319
 320         while (slot != NULL) {
 321                 if (!slot->in_use && slot->curl != NULL) {
 322                         curl_easy_cleanup(slot->curl);
 323                         slot->curl = NULL;
 324                 }
 325                 slot = slot->next;
 326         }
 327 }
 328 #endif
 329
 330 void prefetch(unsigned char *sha1)
 331 {
 332         struct transfer_request *newreq;
 333         struct transfer_request *tail;
 334         char *filename = sha1_file_name(sha1);
 335
 336         newreq = xmalloc(sizeof(*newreq));
 337         memcpy(newreq->sha1, sha1, 20);
 338         newreq->repo = alt;
 339         newreq->url = NULL;
 340         newreq->local = -1;
 341         newreq->state = WAITING;
 342         snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
 343         snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
 344                  "%s.temp", filename);
 345         newreq->next = NULL;
 346
 347         if (request_queue_head == NULL) {
 348                 request_queue_head = newreq;
 349         } else {
 350                 tail = request_queue_head;
 351                 while (tail->next != NULL) {
 352                         tail = tail->next;
 353                 }
 354                 tail->next = newreq;
 355         }
 356
 357 #ifdef USE_CURL_MULTI
 358         fill_active_slots();
 359         step_active_slots();
 360 #endif
 361 }
 362
 363 static int fetch_index(struct alt_base *repo, unsigned char *sha1)
 364 {
 365         char *hex = sha1_to_hex(sha1);
 366         char *filename;
 367         char *url;
 368         char tmpfile[PATH_MAX];
 369         long prev_posn = 0;
 370         char range[RANGE_HEADER_SIZE];
 371         struct curl_slist *range_header = NULL;
 372
 373         FILE *indexfile;
 374         struct active_request_slot *slot;
 375
 376         if (has_pack_index(sha1))
 377                 return 0;
 378
 379         if (get_verbosely)
 380                 fprintf(stderr, "Getting index for pack %s\n", hex);
 381
 382         url = xmalloc(strlen(repo->base) + 64);
 383         sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex);
 384
 385         filename = sha1_pack_index_name(sha1);
 386         snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
 387         indexfile = fopen(tmpfile, "a");
 388         if (!indexfile)
 389                 return error("Unable to open local file %s for pack index",
 390                              filename);
 391
 392         slot = get_active_slot();
 393         curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
 394         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
 395         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 396         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 397         slot->local = indexfile;
 398
 399         /* If there is data present from a previous transfer attempt,
 400            resume where it left off */
 401         prev_posn = ftell(indexfile);
 402         if (prev_posn>0) {
 403                 if (get_verbosely)
 404                         fprintf(stderr,
 405                                 "Resuming fetch of index for pack %s at byte %ld\n",
 406                                 hex, prev_posn);
 407                 sprintf(range, "Range: bytes=%ld-", prev_posn);
 408                 range_header = curl_slist_append(range_header, range);
 409                 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
 410         }
 411
 412         if (start_active_slot(slot)) {
 413                 run_active_slot(slot);
 414                 if (slot->curl_result != CURLE_OK) {
 415                         fclose(indexfile);
 416                         return error("Unable to get pack index %s\n%s", url,
 417                                      curl_errorstr);
 418                 }
 419         } else {
 420                 fclose(indexfile);
 421                 return error("Unable to start request");
 422         }
 423
 424         fclose(indexfile);
 425
 426         return move_temp_to_file(tmpfile, filename);
 427 }
 428
 429 static int setup_index(struct alt_base *repo, unsigned char *sha1)
 430 {
 431         struct packed_git *new_pack;
 432         if (has_pack_file(sha1))
 433                 return 0; // don't list this as something we can get
 434
 435         if (fetch_index(repo, sha1))
 436                 return -1;
 437
 438         new_pack = parse_pack_index(sha1);
 439         new_pack->next = repo->packs;
 440         repo->packs = new_pack;
 441         return 0;
 442 }
 443
 444 static void process_alternates(void *callback_data)
 445 {
 446         struct alt_request *alt_req = (struct alt_request *)callback_data;
 447         struct active_request_slot *slot = alt_req->slot;
 448         struct alt_base *tail = alt;
 449         char *base = alt_req->base;
 450         static const char null_byte = '\0';
 451         char *data;
 452         int i = 0;
 453
 454         if (alt_req->http_specific) {
 455                 if (slot->curl_result != CURLE_OK ||
 456                     !alt_req->buffer->posn) {
 457
 458                         /* Try reusing the slot to get non-http alternates */
 459                         alt_req->http_specific = 0;
 460                         sprintf(alt_req->url, "%s/objects/info/alternates",
 461                                 base);
 462                         curl_easy_setopt(slot->curl, CURLOPT_URL,
 463                                          alt_req->url);
 464                         active_requests++;
 465                         slot->in_use = 1;
 466                         if (start_active_slot(slot)) {
 467                                 return;
 468                         } else {
 469                                 got_alternates = -1;
 470                                 slot->in_use = 0;
 471                                 return;
 472                         }
 473                 }
 474         } else if (slot->curl_result != CURLE_OK) {
 475                 if (slot->http_code != 404) {
 476                         got_alternates = -1;
 477                         return;
 478                 }
 479         }
 480
 481         fwrite_buffer(&null_byte, 1, 1, alt_req->buffer);
 482         alt_req->buffer->posn--;
 483         data = alt_req->buffer->buffer;
 484
 485         while (i < alt_req->buffer->posn) {
 486                 int posn = i;
 487                 while (posn < alt_req->buffer->posn && data[posn] != '\n')
 488                         posn++;
 489                 if (data[posn] == '\n') {
 490                         int okay = 0;
 491                         int serverlen = 0;
 492                         struct alt_base *newalt;
 493                         char *target = NULL;
 494                         if (data[i] == '/') {
 495                                 serverlen = strchr(base + 8, '/') - base;
 496                                 okay = 1;
 497                         } else if (!memcmp(data + i, "../", 3)) {
 498                                 i += 3;
 499                                 serverlen = strlen(base);
 500                                 while (i + 2 < posn &&
 501                                        !memcmp(data + i, "../", 3)) {
 502                                         do {
 503                                                 serverlen--;
 504                                         } while (serverlen &&
 505                                                  base[serverlen - 1] != '/');
 506                                         i += 3;
 507                                 }
 508                                 // If the server got removed, give up.
 509                                 okay = strchr(base, ':') - base + 3 <
 510                                         serverlen;
 511                         } else if (alt_req->http_specific) {
 512                                 char *colon = strchr(data + i, ':');
 513                                 char *slash = strchr(data + i, '/');
 514                                 if (colon && slash && colon < data + posn &&
 515                                     slash < data + posn && colon < slash) {
 516                                         okay = 1;
 517                                 }
 518                         }
 519                         // skip 'objects' at end
 520                         if (okay) {
 521                                 target = xmalloc(serverlen + posn - i - 6);
 522                                 strncpy(target, base, serverlen);
 523                                 strncpy(target + serverlen, data + i,
 524                                         posn - i - 7);
 525                                 target[serverlen + posn - i - 7] = '\0';
 526                                 if (get_verbosely)
 527                                         fprintf(stderr,
 528                                                 "Also look at %s\n", target);
 529                                 newalt = xmalloc(sizeof(*newalt));
 530                                 newalt->next = NULL;
 531                                 newalt->base = target;
 532                                 newalt->got_indices = 0;
 533                                 newalt->packs = NULL;
 534                                 while (tail->next != NULL)
 535                                         tail = tail->next;
 536                                 tail->next = newalt;
 537                         }
 538                 }
 539                 i = posn + 1;
 540         }
 541
 542         got_alternates = 1;
 543 }
 544
 545 static void fetch_alternates(char *base)
 546 {
 547         struct buffer buffer;
 548         char *url;
 549         char *data;
 550         struct active_request_slot *slot;
 551         static struct alt_request alt_req;
 552
 553         /* If another request has already started fetching alternates,
 554            wait for them to arrive and return to processing this request's
 555            curl message */
 556 #ifdef USE_CURL_MULTI
 557         while (got_alternates == 0) {
 558                 step_active_slots();
 559         }
 560 #endif
 561
 562         /* Nothing to do if they've already been fetched */
 563         if (got_alternates == 1)
 564                 return;
 565
 566         /* Start the fetch */
 567         got_alternates = 0;
 568
 569         data = xmalloc(4096);
 570         buffer.size = 4096;
 571         buffer.posn = 0;
 572         buffer.buffer = data;
 573
 574         if (get_verbosely)
 575                 fprintf(stderr, "Getting alternates list for %s\n", base);
 576
 577         url = xmalloc(strlen(base) + 31);
 578         sprintf(url, "%s/objects/info/http-alternates", base);
 579
 580         /* Use a callback to process the result, since another request
 581            may fail and need to have alternates loaded before continuing */
 582         slot = get_active_slot();
 583         slot->callback_func = process_alternates;
 584         slot->callback_data = &alt_req;
 585
 586         curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 587         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 588         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 589
 590         alt_req.base = base;
 591         alt_req.url = url;
 592         alt_req.buffer = &buffer;
 593         alt_req.http_specific = 1;
 594         alt_req.slot = slot;
 595
 596         if (start_active_slot(slot))
 597                 run_active_slot(slot);
 598         else
 599                 got_alternates = -1;
 600
 601         free(data);
 602         free(url);
 603 }
 604
 605 static int fetch_indices(struct alt_base *repo)
 606 {
 607         unsigned char sha1[20];
 608         char *url;
 609         struct buffer buffer;
 610         char *data;
 611         int i = 0;
 612
 613         struct active_request_slot *slot;
 614
 615         if (repo->got_indices)
 616                 return 0;
 617
 618         data = xmalloc(4096);
 619         buffer.size = 4096;
 620         buffer.posn = 0;
 621         buffer.buffer = data;
 622
 623         if (get_verbosely)
 624                 fprintf(stderr, "Getting pack list for %s\n", repo->base);
 625
 626         url = xmalloc(strlen(repo->base) + 21);
 627         sprintf(url, "%s/objects/info/packs", repo->base);
 628
 629         slot = get_active_slot();
 630         curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 631         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 632         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 633         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
 634         if (start_active_slot(slot)) {
 635                 run_active_slot(slot);
 636                 if (slot->curl_result != CURLE_OK) {
 637                         free(buffer.buffer);
 638                         return error("%s", curl_errorstr);
 639                 }
 640         } else {
 641                 free(buffer.buffer);
 642                 return error("Unable to start request");
 643         }
 644
 645         data = buffer.buffer;
 646         while (i < buffer.posn) {
 647                 switch (data[i]) {
 648                 case 'P':
 649                         i++;
 650                         if (i + 52 < buffer.posn &&
 651                             !strncmp(data + i, " pack-", 6) &&
 652                             !strncmp(data + i + 46, ".pack\n", 6)) {
 653                                 get_sha1_hex(data + i + 6, sha1);
 654                                 setup_index(repo, sha1);
 655                                 i += 51;
 656                                 break;
 657                         }
 658                 default:
 659                         while (data[i] != '\n')
 660                                 i++;
 661                 }
 662                 i++;
 663         }
 664
 665         free(buffer.buffer);
 666         repo->got_indices = 1;
 667         return 0;
 668 }
 669
 670 static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
 671 {
 672         char *url;
 673         struct packed_git *target;
 674         struct packed_git **lst;
 675         FILE *packfile;
 676         char *filename;
 677         char tmpfile[PATH_MAX];
 678         int ret;
 679         long prev_posn = 0;
 680         char range[RANGE_HEADER_SIZE];
 681         struct curl_slist *range_header = NULL;
 682
 683         struct active_request_slot *slot;
 684
 685         if (fetch_indices(repo))
 686                 return -1;
 687         target = find_sha1_pack(sha1, repo->packs);
 688         if (!target)
 689                 return -1;
 690
 691         if (get_verbosely) {
 692                 fprintf(stderr, "Getting pack %s\n",
 693                         sha1_to_hex(target->sha1));
 694                 fprintf(stderr, " which contains %s\n",
 695                         sha1_to_hex(sha1));
 696         }
 697
 698         url = xmalloc(strlen(repo->base) + 65);
 699         sprintf(url, "%s/objects/pack/pack-%s.pack",
 700                 repo->base, sha1_to_hex(target->sha1));
 701
 702         filename = sha1_pack_name(target->sha1);
 703         snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
 704         packfile = fopen(tmpfile, "a");
 705         if (!packfile)
 706                 return error("Unable to open local file %s for pack",
 707                              filename);
 708
 709         slot = get_active_slot();
 710         curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
 711         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
 712         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 713         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 714         slot->local = packfile;
 715
 716         /* If there is data present from a previous transfer attempt,
 717            resume where it left off */
 718         prev_posn = ftell(packfile);
 719         if (prev_posn>0) {
 720                 if (get_verbosely)
 721                         fprintf(stderr,
 722                                 "Resuming fetch of pack %s at byte %ld\n",
 723                                 sha1_to_hex(target->sha1), prev_posn);
 724                 sprintf(range, "Range: bytes=%ld-", prev_posn);
 725                 range_header = curl_slist_append(range_header, range);
 726                 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
 727         }
 728
 729         if (start_active_slot(slot)) {
 730                 run_active_slot(slot);
 731                 if (slot->curl_result != CURLE_OK) {
 732                         fclose(packfile);
 733                         return error("Unable to get pack file %s\n%s", url,
 734                                      curl_errorstr);
 735                 }
 736         } else {
 737                 fclose(packfile);
 738                 return error("Unable to start request");
 739         }
 740
 741         fclose(packfile);
 742
 743         ret = move_temp_to_file(tmpfile, filename);
 744         if (ret)
 745                 return ret;
 746
 747         lst = &repo->packs;
 748         while (*lst != target)
 749                 lst = &((*lst)->next);
 750         *lst = (*lst)->next;
 751
 752         if (verify_pack(target, 0))
 753                 return -1;
 754         install_packed_git(target);
 755
 756         return 0;
 757 }
 758
 759 static int fetch_object(struct alt_base *repo, unsigned char *sha1)
 760 {
 761         char *hex = sha1_to_hex(sha1);
 762         int ret = 0;
 763         struct transfer_request *request = request_queue_head;
 764
 765         while (request != NULL && memcmp(request->sha1, sha1, 20))
 766                 request = request->next;
 767         if (request == NULL)
 768                 return error("Couldn't find request for %s in the queue", hex);
 769
 770         if (has_sha1_file(request->sha1)) {
 771                 release_request(request);
 772                 return 0;
 773         }
 774
 775 #ifdef USE_CURL_MULTI
 776         while (request->state == WAITING) {
 777                 step_active_slots();
 778         }
 779 #else
 780         start_request(request);
 781 #endif
 782
 783         while (request->state == ACTIVE) {
 784                 run_active_slot(request->slot);
 785         }
 786         if (request->local != -1) {
 787                 close(request->local); request->local = -1;
 788         }
 789
 790         if (request->state == ABORTED) {
 791                 ret = error("Request for %s aborted", hex);
 792         } else if (request->curl_result != CURLE_OK &&
 793                    request->http_code != 416) {
 794                 if (request->http_code == 404)
 795                         ret = -1; /* Be silent, it is probably in a pack. */
 796                 else
 797                         ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
 798                                     request->errorstr, request->curl_result,
 799                                     request->http_code, hex);
 800         } else if (request->zret != Z_STREAM_END) {
 801                 ret = error("File %s (%s) corrupt\n", hex, request->url);
 802         } else if (memcmp(request->sha1, request->real_sha1, 20)) {
 803                 ret = error("File %s has bad hash\n", hex);
 804         } else if (request->rename < 0) {
 805                 ret = error("unable to write sha1 filename %s: %s",
 806                             request->filename,
 807                             strerror(request->rename));
 808         }
 809
 810         release_request(request);
 811         return ret;
 812 }
 813
 814 int fetch(unsigned char *sha1)
 815 {
 816         struct alt_base *altbase = alt;
 817
 818         if (!fetch_object(altbase, sha1))
 819                 return 0;
 820         while (altbase) {
 821                 if (!fetch_pack(altbase, sha1))
 822                         return 0;
 823                 fetch_alternates(alt->base);
 824                 altbase = altbase->next;
 825         }
 826         return error("Unable to find %s under %s\n", sha1_to_hex(sha1),
 827                      alt->base);
 828 }
 829
 830 static inline int needs_quote(int ch)
 831 {
 832         switch (ch) {
 833         case '/': case '-': case '.':
 834         case 'A'...'Z': case 'a'...'z': case '0'...'9':
 835                 return 0;
 836         default:
 837                 return 1;
 838         }
 839 }
 840
 841 static inline int hex(int v)
 842 {
 843         if (v < 10) return '0' + v;
 844         else return 'A' + v - 10;
 845 }
 846
 847 static char *quote_ref_url(const char *base, const char *ref)
 848 {
 849         const char *cp;
 850         char *dp, *qref;
 851         int len, baselen, ch;
 852
 853         baselen = strlen(base);
 854         len = baselen + 6; /* "refs/" + NUL */
 855         for (cp = ref; (ch = *cp) != 0; cp++, len++)
 856                 if (needs_quote(ch))
 857                         len += 2; /* extra two hex plus replacement % */
 858         qref = xmalloc(len);
 859         memcpy(qref, base, baselen);
 860         memcpy(qref + baselen, "refs/", 5);
 861         for (cp = ref, dp = qref + baselen + 5; (ch = *cp) != 0; cp++) {
 862                 if (needs_quote(ch)) {
 863                         *dp++ = '%';
 864                         *dp++ = hex((ch >> 4) & 0xF);
 865                         *dp++ = hex(ch & 0xF);
 866                 }
 867                 else
 868                         *dp++ = ch;
 869         }
 870         *dp = 0;
 871
 872         return qref;
 873 }
 874
 875 int fetch_ref(char *ref, unsigned char *sha1)
 876 {
 877         char *url;
 878         char hex[42];
 879         struct buffer buffer;
 880         char *base = alt->base;
 881         struct active_request_slot *slot;
 882         buffer.size = 41;
 883         buffer.posn = 0;
 884         buffer.buffer = hex;
 885         hex[41] = '\0';
 886
 887         url = quote_ref_url(base, ref);
 888         slot = get_active_slot();
 889         curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 890         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 891         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
 892         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 893         if (start_active_slot(slot)) {
 894                 run_active_slot(slot);
 895                 if (slot->curl_result != CURLE_OK)
 896                         return error("Couldn't get %s for %s\n%s",
 897                                      url, ref, curl_errorstr);
 898         } else {
 899                 return error("Unable to start request");
 900         }
 901
 902         hex[40] = '\0';
 903         get_sha1_hex(hex, sha1);
 904         return 0;
 905 }
 906
 907 int main(int argc, char **argv)
 908 {
 909         char *commit_id;
 910         char *url;
 911         int arg = 1;
 912         int rc = 0;
 913
 914         while (arg < argc && argv[arg][0] == '-') {
 915                 if (argv[arg][1] == 't') {
 916                         get_tree = 1;
 917                 } else if (argv[arg][1] == 'c') {
 918                         get_history = 1;
 919                 } else if (argv[arg][1] == 'a') {
 920                         get_all = 1;
 921                         get_tree = 1;
 922                         get_history = 1;
 923                 } else if (argv[arg][1] == 'v') {
 924                         get_verbosely = 1;
 925                 } else if (argv[arg][1] == 'w') {
 926                         write_ref = argv[arg + 1];
 927                         arg++;
 928                 } else if (!strcmp(argv[arg], "--recover")) {
 929                         get_recover = 1;
 930                 }
 931                 arg++;
 932         }
 933         if (argc < arg + 2) {
 934                 usage("git-http-fetch [-c] [-t] [-a] [-d] [-v] [--recover] [-w ref] commit-id url");
 935                 return 1;
 936         }
 937         commit_id = argv[arg];
 938         url = argv[arg + 1];
 939
 940         http_init();
 941
 942         no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
 943
 944         alt = xmalloc(sizeof(*alt));
 945         alt->base = url;
 946         alt->got_indices = 0;
 947         alt->packs = NULL;
 948         alt->next = NULL;
 949
 950         if (pull(commit_id))
 951                 rc = 1;
 952
 953         curl_slist_free_all(no_pragma_header);
 954
 955         http_cleanup();
 956
 957         return rc;
 958 }