revision: --topo-order and --unpacked
[git.git] / http-fetch.c
1 #include "cache.h"
2 #include "commit.h"
3 #include "pack.h"
4 #include "fetch.h"
5 #include "http.h"
6
7 #define PREV_BUF_SIZE 4096
8 #define RANGE_HEADER_SIZE 30
9
10 static int got_alternates = -1;
11 static int corrupt_object_found = 0;
12
13 static struct curl_slist *no_pragma_header;
14
15 struct alt_base
16 {
17         char *base;
18         int got_indices;
19         struct packed_git *packs;
20         struct alt_base *next;
21 };
22
23 static struct alt_base *alt = NULL;
24
25 enum object_request_state {
26         WAITING,
27         ABORTED,
28         ACTIVE,
29         COMPLETE,
30 };
31
32 struct object_request
33 {
34         unsigned char sha1[20];
35         struct alt_base *repo;
36         char *url;
37         char filename[PATH_MAX];
38         char tmpfile[PATH_MAX];
39         int local;
40         enum object_request_state state;
41         CURLcode curl_result;
42         char errorstr[CURL_ERROR_SIZE];
43         long http_code;
44         unsigned char real_sha1[20];
45         SHA_CTX c;
46         z_stream stream;
47         int zret;
48         int rename;
49         struct active_request_slot *slot;
50         struct object_request *next;
51 };
52
53 struct alternates_request {
54         char *base;
55         char *url;
56         struct buffer *buffer;
57         struct active_request_slot *slot;
58         int http_specific;
59 };
60
61 static struct object_request *object_queue_head = NULL;
62
63 static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
64                                void *data)
65 {
66         unsigned char expn[4096];
67         size_t size = eltsize * nmemb;
68         int posn = 0;
69         struct object_request *obj_req = (struct object_request *)data;
70         do {
71                 ssize_t retval = write(obj_req->local,
72                                        ptr + posn, size - posn);
73                 if (retval < 0)
74                         return posn;
75                 posn += retval;
76         } while (posn < size);
77
78         obj_req->stream.avail_in = size;
79         obj_req->stream.next_in = ptr;
80         do {
81                 obj_req->stream.next_out = expn;
82                 obj_req->stream.avail_out = sizeof(expn);
83                 obj_req->zret = inflate(&obj_req->stream, Z_SYNC_FLUSH);
84                 SHA1_Update(&obj_req->c, expn,
85                             sizeof(expn) - obj_req->stream.avail_out);
86         } while (obj_req->stream.avail_in && obj_req->zret == Z_OK);
87         data_received++;
88         return size;
89 }
90
91 static void fetch_alternates(char *base);
92
93 static void process_object_response(void *callback_data);
94
95 static void start_object_request(struct object_request *obj_req)
96 {
97         char *hex = sha1_to_hex(obj_req->sha1);
98         char prevfile[PATH_MAX];
99         char *url;
100         char *posn;
101         int prevlocal;
102         unsigned char prev_buf[PREV_BUF_SIZE];
103         ssize_t prev_read = 0;
104         long prev_posn = 0;
105         char range[RANGE_HEADER_SIZE];
106         struct curl_slist *range_header = NULL;
107         struct active_request_slot *slot;
108
109         snprintf(prevfile, sizeof(prevfile), "%s.prev", obj_req->filename);
110         unlink(prevfile);
111         rename(obj_req->tmpfile, prevfile);
112         unlink(obj_req->tmpfile);
113
114         if (obj_req->local != -1)
115                 error("fd leakage in start: %d", obj_req->local);
116         obj_req->local = open(obj_req->tmpfile,
117                               O_WRONLY | O_CREAT | O_EXCL, 0666);
118         /* This could have failed due to the "lazy directory creation";
119          * try to mkdir the last path component.
120          */
121         if (obj_req->local < 0 && errno == ENOENT) {
122                 char *dir = strrchr(obj_req->tmpfile, '/');
123                 if (dir) {
124                         *dir = 0;
125                         mkdir(obj_req->tmpfile, 0777);
126                         *dir = '/';
127                 }
128                 obj_req->local = open(obj_req->tmpfile,
129                                       O_WRONLY | O_CREAT | O_EXCL, 0666);
130         }
131
132         if (obj_req->local < 0) {
133                 obj_req->state = ABORTED;
134                 error("Couldn't create temporary file %s for %s: %s",
135                       obj_req->tmpfile, obj_req->filename, strerror(errno));
136                 return;
137         }
138
139         memset(&obj_req->stream, 0, sizeof(obj_req->stream));
140
141         inflateInit(&obj_req->stream);
142
143         SHA1_Init(&obj_req->c);
144
145         url = xmalloc(strlen(obj_req->repo->base) + 50);
146         obj_req->url = xmalloc(strlen(obj_req->repo->base) + 50);
147         strcpy(url, obj_req->repo->base);
148         posn = url + strlen(obj_req->repo->base);
149         strcpy(posn, "objects/");
150         posn += 8;
151         memcpy(posn, hex, 2);
152         posn += 2;
153         *(posn++) = '/';
154         strcpy(posn, hex + 2);
155         strcpy(obj_req->url, url);
156
157         /* If a previous temp file is present, process what was already
158            fetched. */
159         prevlocal = open(prevfile, O_RDONLY);
160         if (prevlocal != -1) {
161                 do {
162                         prev_read = read(prevlocal, prev_buf, PREV_BUF_SIZE);
163                         if (prev_read>0) {
164                                 if (fwrite_sha1_file(prev_buf,
165                                                      1,
166                                                      prev_read,
167                                                      obj_req) == prev_read) {
168                                         prev_posn += prev_read;
169                                 } else {
170                                         prev_read = -1;
171                                 }
172                         }
173                 } while (prev_read > 0);
174                 close(prevlocal);
175         }
176         unlink(prevfile);
177
178         /* Reset inflate/SHA1 if there was an error reading the previous temp
179            file; also rewind to the beginning of the local file. */
180         if (prev_read == -1) {
181                 memset(&obj_req->stream, 0, sizeof(obj_req->stream));
182                 inflateInit(&obj_req->stream);
183                 SHA1_Init(&obj_req->c);
184                 if (prev_posn>0) {
185                         prev_posn = 0;
186                         lseek(obj_req->local, SEEK_SET, 0);
187                         ftruncate(obj_req->local, 0);
188                 }
189         }
190
191         slot = get_active_slot();
192         slot->callback_func = process_object_response;
193         slot->callback_data = obj_req;
194         obj_req->slot = slot;
195
196         curl_easy_setopt(slot->curl, CURLOPT_FILE, obj_req);
197         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
198         curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, obj_req->errorstr);
199         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
200         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
201
202         /* If we have successfully processed data from a previous fetch
203            attempt, only fetch the data we don't already have. */
204         if (prev_posn>0) {
205                 if (get_verbosely)
206                         fprintf(stderr,
207                                 "Resuming fetch of object %s at byte %ld\n",
208                                 hex, prev_posn);
209                 sprintf(range, "Range: bytes=%ld-", prev_posn);
210                 range_header = curl_slist_append(range_header, range);
211                 curl_easy_setopt(slot->curl,
212                                  CURLOPT_HTTPHEADER, range_header);
213         }
214
215         /* Try to get the request started, abort the request on error */
216         obj_req->state = ACTIVE;
217         if (!start_active_slot(slot)) {
218                 obj_req->state = ABORTED;
219                 obj_req->slot = NULL;
220                 close(obj_req->local); obj_req->local = -1;
221                 free(obj_req->url);
222                 return;
223         }
224 }
225
226 static void finish_object_request(struct object_request *obj_req)
227 {
228         struct stat st;
229
230         fchmod(obj_req->local, 0444);
231         close(obj_req->local); obj_req->local = -1;
232
233         if (obj_req->http_code == 416) {
234                 fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
235         } else if (obj_req->curl_result != CURLE_OK) {
236                 if (stat(obj_req->tmpfile, &st) == 0)
237                         if (st.st_size == 0)
238                                 unlink(obj_req->tmpfile);
239                 return;
240         }
241
242         inflateEnd(&obj_req->stream);
243         SHA1_Final(obj_req->real_sha1, &obj_req->c);
244         if (obj_req->zret != Z_STREAM_END) {
245                 unlink(obj_req->tmpfile);
246                 return;
247         }
248         if (memcmp(obj_req->sha1, obj_req->real_sha1, 20)) {
249                 unlink(obj_req->tmpfile);
250                 return;
251         }
252         obj_req->rename =
253                 move_temp_to_file(obj_req->tmpfile, obj_req->filename);
254
255         if (obj_req->rename == 0)
256                 pull_say("got %s\n", sha1_to_hex(obj_req->sha1));
257 }
258
259 static void process_object_response(void *callback_data)
260 {
261         struct object_request *obj_req =
262                 (struct object_request *)callback_data;
263
264         obj_req->curl_result = obj_req->slot->curl_result;
265         obj_req->http_code = obj_req->slot->http_code;
266         obj_req->slot = NULL;
267         obj_req->state = COMPLETE;
268
269         /* Use alternates if necessary */
270         if (obj_req->http_code == 404 ||
271             obj_req->curl_result == CURLE_FILE_COULDNT_READ_FILE) {
272                 fetch_alternates(alt->base);
273                 if (obj_req->repo->next != NULL) {
274                         obj_req->repo =
275                                 obj_req->repo->next;
276                         close(obj_req->local);
277                         obj_req->local = -1;
278                         start_object_request(obj_req);
279                         return;
280                 }
281         }
282
283         finish_object_request(obj_req);
284 }
285
286 static void release_object_request(struct object_request *obj_req)
287 {
288         struct object_request *entry = object_queue_head;
289
290         if (obj_req->local != -1)
291                 error("fd leakage in release: %d", obj_req->local);
292         if (obj_req == object_queue_head) {
293                 object_queue_head = obj_req->next;
294         } else {
295                 while (entry->next != NULL && entry->next != obj_req)
296                         entry = entry->next;
297                 if (entry->next == obj_req)
298                         entry->next = entry->next->next;
299         }
300
301         free(obj_req->url);
302         free(obj_req);
303 }
304
305 #ifdef USE_CURL_MULTI
306 void fill_active_slots(void)
307 {
308         struct object_request *obj_req = object_queue_head;
309         struct active_request_slot *slot = active_queue_head;
310         int num_transfers;
311
312         while (active_requests < max_requests && obj_req != NULL) {
313                 if (obj_req->state == WAITING) {
314                         if (has_sha1_file(obj_req->sha1))
315                                 obj_req->state = COMPLETE;
316                         else
317                                 start_object_request(obj_req);
318                         curl_multi_perform(curlm, &num_transfers);
319                 }
320                 obj_req = obj_req->next;
321         }
322
323         while (slot != NULL) {
324                 if (!slot->in_use && slot->curl != NULL) {
325                         curl_easy_cleanup(slot->curl);
326                         slot->curl = NULL;
327                 }
328                 slot = slot->next;
329         }
330 }
331 #endif
332
333 void prefetch(unsigned char *sha1)
334 {
335         struct object_request *newreq;
336         struct object_request *tail;
337         char *filename = sha1_file_name(sha1);
338
339         newreq = xmalloc(sizeof(*newreq));
340         memcpy(newreq->sha1, sha1, 20);
341         newreq->repo = alt;
342         newreq->url = NULL;
343         newreq->local = -1;
344         newreq->state = WAITING;
345         snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
346         snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
347                  "%s.temp", filename);
348         newreq->next = NULL;
349
350         if (object_queue_head == NULL) {
351                 object_queue_head = newreq;
352         } else {
353                 tail = object_queue_head;
354                 while (tail->next != NULL) {
355                         tail = tail->next;
356                 }
357                 tail->next = newreq;
358         }
359
360 #ifdef USE_CURL_MULTI
361         fill_active_slots();
362         step_active_slots();
363 #endif
364 }
365
366 static int fetch_index(struct alt_base *repo, unsigned char *sha1)
367 {
368         char *hex = sha1_to_hex(sha1);
369         char *filename;
370         char *url;
371         char tmpfile[PATH_MAX];
372         long prev_posn = 0;
373         char range[RANGE_HEADER_SIZE];
374         struct curl_slist *range_header = NULL;
375
376         FILE *indexfile;
377         struct active_request_slot *slot;
378         struct slot_results results;
379
380         if (has_pack_index(sha1))
381                 return 0;
382
383         if (get_verbosely)
384                 fprintf(stderr, "Getting index for pack %s\n", hex);
385
386         url = xmalloc(strlen(repo->base) + 64);
387         sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex);
388
389         filename = sha1_pack_index_name(sha1);
390         snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
391         indexfile = fopen(tmpfile, "a");
392         if (!indexfile)
393                 return error("Unable to open local file %s for pack index",
394                              filename);
395
396         slot = get_active_slot();
397         slot->results = &results;
398         curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
399         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
400         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
401         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
402         slot->local = indexfile;
403
404         /* If there is data present from a previous transfer attempt,
405            resume where it left off */
406         prev_posn = ftell(indexfile);
407         if (prev_posn>0) {
408                 if (get_verbosely)
409                         fprintf(stderr,
410                                 "Resuming fetch of index for pack %s at byte %ld\n",
411                                 hex, prev_posn);
412                 sprintf(range, "Range: bytes=%ld-", prev_posn);
413                 range_header = curl_slist_append(range_header, range);
414                 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
415         }
416
417         if (start_active_slot(slot)) {
418                 run_active_slot(slot);
419                 if (results.curl_result != CURLE_OK) {
420                         fclose(indexfile);
421                         return error("Unable to get pack index %s\n%s", url,
422                                      curl_errorstr);
423                 }
424         } else {
425                 fclose(indexfile);
426                 return error("Unable to start request");
427         }
428
429         fclose(indexfile);
430
431         return move_temp_to_file(tmpfile, filename);
432 }
433
434 static int setup_index(struct alt_base *repo, unsigned char *sha1)
435 {
436         struct packed_git *new_pack;
437         if (has_pack_file(sha1))
438                 return 0; // don't list this as something we can get
439
440         if (fetch_index(repo, sha1))
441                 return -1;
442
443         new_pack = parse_pack_index(sha1);
444         new_pack->next = repo->packs;
445         repo->packs = new_pack;
446         return 0;
447 }
448
449 static void process_alternates_response(void *callback_data)
450 {
451         struct alternates_request *alt_req =
452                 (struct alternates_request *)callback_data;
453         struct active_request_slot *slot = alt_req->slot;
454         struct alt_base *tail = alt;
455         char *base = alt_req->base;
456         static const char null_byte = '\0';
457         char *data;
458         int i = 0;
459
460         if (alt_req->http_specific) {
461                 if (slot->curl_result != CURLE_OK ||
462                     !alt_req->buffer->posn) {
463
464                         /* Try reusing the slot to get non-http alternates */
465                         alt_req->http_specific = 0;
466                         sprintf(alt_req->url, "%s/objects/info/alternates",
467                                 base);
468                         curl_easy_setopt(slot->curl, CURLOPT_URL,
469                                          alt_req->url);
470                         active_requests++;
471                         slot->in_use = 1;
472                         if (slot->finished != NULL)
473                                 (*slot->finished) = 0;
474                         if (!start_active_slot(slot)) {
475                                 got_alternates = -1;
476                                 slot->in_use = 0;
477                                 if (slot->finished != NULL)
478                                         (*slot->finished) = 1;
479                         }
480                         return;
481                 }
482         } else if (slot->curl_result != CURLE_OK) {
483                 if (slot->http_code != 404 &&
484                     slot->curl_result != CURLE_FILE_COULDNT_READ_FILE) {
485                         got_alternates = -1;
486                         return;
487                 }
488         }
489
490         fwrite_buffer(&null_byte, 1, 1, alt_req->buffer);
491         alt_req->buffer->posn--;
492         data = alt_req->buffer->buffer;
493
494         while (i < alt_req->buffer->posn) {
495                 int posn = i;
496                 while (posn < alt_req->buffer->posn && data[posn] != '\n')
497                         posn++;
498                 if (data[posn] == '\n') {
499                         int okay = 0;
500                         int serverlen = 0;
501                         struct alt_base *newalt;
502                         char *target = NULL;
503                         if (data[i] == '/') {
504                                 serverlen = strchr(base + 8, '/') - base;
505                                 okay = 1;
506                         } else if (!memcmp(data + i, "../", 3)) {
507                                 i += 3;
508                                 serverlen = strlen(base);
509                                 while (i + 2 < posn &&
510                                        !memcmp(data + i, "../", 3)) {
511                                         do {
512                                                 serverlen--;
513                                         } while (serverlen &&
514                                                  base[serverlen - 1] != '/');
515                                         i += 3;
516                                 }
517                                 // If the server got removed, give up.
518                                 okay = strchr(base, ':') - base + 3 <
519                                         serverlen;
520                         } else if (alt_req->http_specific) {
521                                 char *colon = strchr(data + i, ':');
522                                 char *slash = strchr(data + i, '/');
523                                 if (colon && slash && colon < data + posn &&
524                                     slash < data + posn && colon < slash) {
525                                         okay = 1;
526                                 }
527                         }
528                         // skip 'objects' at end
529                         if (okay) {
530                                 target = xmalloc(serverlen + posn - i - 6);
531                                 strncpy(target, base, serverlen);
532                                 strncpy(target + serverlen, data + i,
533                                         posn - i - 7);
534                                 target[serverlen + posn - i - 7] = '\0';
535                                 if (get_verbosely)
536                                         fprintf(stderr,
537                                                 "Also look at %s\n", target);
538                                 newalt = xmalloc(sizeof(*newalt));
539                                 newalt->next = NULL;
540                                 newalt->base = target;
541                                 newalt->got_indices = 0;
542                                 newalt->packs = NULL;
543                                 while (tail->next != NULL)
544                                         tail = tail->next;
545                                 tail->next = newalt;
546                         }
547                 }
548                 i = posn + 1;
549         }
550
551         got_alternates = 1;
552 }
553
554 static void fetch_alternates(char *base)
555 {
556         struct buffer buffer;
557         char *url;
558         char *data;
559         struct active_request_slot *slot;
560         struct alternates_request alt_req;
561
562         /* If another request has already started fetching alternates,
563            wait for them to arrive and return to processing this request's
564            curl message */
565 #ifdef USE_CURL_MULTI
566         while (got_alternates == 0) {
567                 step_active_slots();
568         }
569 #endif
570
571         /* Nothing to do if they've already been fetched */
572         if (got_alternates == 1)
573                 return;
574
575         /* Start the fetch */
576         got_alternates = 0;
577
578         data = xmalloc(4096);
579         buffer.size = 4096;
580         buffer.posn = 0;
581         buffer.buffer = data;
582
583         if (get_verbosely)
584                 fprintf(stderr, "Getting alternates list for %s\n", base);
585
586         url = xmalloc(strlen(base) + 31);
587         sprintf(url, "%s/objects/info/http-alternates", base);
588
589         /* Use a callback to process the result, since another request
590            may fail and need to have alternates loaded before continuing */
591         slot = get_active_slot();
592         slot->callback_func = process_alternates_response;
593         slot->callback_data = &alt_req;
594
595         curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
596         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
597         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
598
599         alt_req.base = base;
600         alt_req.url = url;
601         alt_req.buffer = &buffer;
602         alt_req.http_specific = 1;
603         alt_req.slot = slot;
604
605         if (start_active_slot(slot))
606                 run_active_slot(slot);
607         else
608                 got_alternates = -1;
609
610         free(data);
611         free(url);
612 }
613
614 static int fetch_indices(struct alt_base *repo)
615 {
616         unsigned char sha1[20];
617         char *url;
618         struct buffer buffer;
619         char *data;
620         int i = 0;
621
622         struct active_request_slot *slot;
623         struct slot_results results;
624
625         if (repo->got_indices)
626                 return 0;
627
628         data = xmalloc(4096);
629         buffer.size = 4096;
630         buffer.posn = 0;
631         buffer.buffer = data;
632
633         if (get_verbosely)
634                 fprintf(stderr, "Getting pack list for %s\n", repo->base);
635
636         url = xmalloc(strlen(repo->base) + 21);
637         sprintf(url, "%s/objects/info/packs", repo->base);
638
639         slot = get_active_slot();
640         slot->results = &results;
641         curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
642         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
643         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
644         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
645         if (start_active_slot(slot)) {
646                 run_active_slot(slot);
647                 if (results.curl_result != CURLE_OK) {
648                         if (results.http_code == 404 ||
649                             results.curl_result == CURLE_FILE_COULDNT_READ_FILE) {
650                                 repo->got_indices = 1;
651                                 free(buffer.buffer);
652                                 return 0;
653                         } else {
654                                 repo->got_indices = 0;
655                                 free(buffer.buffer);
656                                 return error("%s", curl_errorstr);
657                         }
658                 }
659         } else {
660                 repo->got_indices = 0;
661                 free(buffer.buffer);
662                 return error("Unable to start request");
663         }
664
665         data = buffer.buffer;
666         while (i < buffer.posn) {
667                 switch (data[i]) {
668                 case 'P':
669                         i++;
670                         if (i + 52 <= buffer.posn &&
671                             !strncmp(data + i, " pack-", 6) &&
672                             !strncmp(data + i + 46, ".pack\n", 6)) {
673                                 get_sha1_hex(data + i + 6, sha1);
674                                 setup_index(repo, sha1);
675                                 i += 51;
676                                 break;
677                         }
678                 default:
679                         while (i < buffer.posn && data[i] != '\n')
680                                 i++;
681                 }
682                 i++;
683         }
684
685         free(buffer.buffer);
686         repo->got_indices = 1;
687         return 0;
688 }
689
690 static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
691 {
692         char *url;
693         struct packed_git *target;
694         struct packed_git **lst;
695         FILE *packfile;
696         char *filename;
697         char tmpfile[PATH_MAX];
698         int ret;
699         long prev_posn = 0;
700         char range[RANGE_HEADER_SIZE];
701         struct curl_slist *range_header = NULL;
702
703         struct active_request_slot *slot;
704         struct slot_results results;
705
706         if (fetch_indices(repo))
707                 return -1;
708         target = find_sha1_pack(sha1, repo->packs);
709         if (!target)
710                 return -1;
711
712         if (get_verbosely) {
713                 fprintf(stderr, "Getting pack %s\n",
714                         sha1_to_hex(target->sha1));
715                 fprintf(stderr, " which contains %s\n",
716                         sha1_to_hex(sha1));
717         }
718
719         url = xmalloc(strlen(repo->base) + 65);
720         sprintf(url, "%s/objects/pack/pack-%s.pack",
721                 repo->base, sha1_to_hex(target->sha1));
722
723         filename = sha1_pack_name(target->sha1);
724         snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
725         packfile = fopen(tmpfile, "a");
726         if (!packfile)
727                 return error("Unable to open local file %s for pack",
728                              filename);
729
730         slot = get_active_slot();
731         slot->results = &results;
732         curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
733         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
734         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
735         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
736         slot->local = packfile;
737
738         /* If there is data present from a previous transfer attempt,
739            resume where it left off */
740         prev_posn = ftell(packfile);
741         if (prev_posn>0) {
742                 if (get_verbosely)
743                         fprintf(stderr,
744                                 "Resuming fetch of pack %s at byte %ld\n",
745                                 sha1_to_hex(target->sha1), prev_posn);
746                 sprintf(range, "Range: bytes=%ld-", prev_posn);
747                 range_header = curl_slist_append(range_header, range);
748                 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
749         }
750
751         if (start_active_slot(slot)) {
752                 run_active_slot(slot);
753                 if (results.curl_result != CURLE_OK) {
754                         fclose(packfile);
755                         return error("Unable to get pack file %s\n%s", url,
756                                      curl_errorstr);
757                 }
758         } else {
759                 fclose(packfile);
760                 return error("Unable to start request");
761         }
762
763         fclose(packfile);
764
765         ret = move_temp_to_file(tmpfile, filename);
766         if (ret)
767                 return ret;
768
769         lst = &repo->packs;
770         while (*lst != target)
771                 lst = &((*lst)->next);
772         *lst = (*lst)->next;
773
774         if (verify_pack(target, 0))
775                 return -1;
776         install_packed_git(target);
777
778         return 0;
779 }
780
781 static void abort_object_request(struct object_request *obj_req)
782 {
783         if (obj_req->local >= 0) {
784                 close(obj_req->local);
785                 obj_req->local = -1;
786         }
787         unlink(obj_req->tmpfile);
788         if (obj_req->slot) {
789                 release_active_slot(obj_req->slot);
790                 obj_req->slot = NULL;
791         }
792         release_object_request(obj_req);
793 }
794
795 static int fetch_object(struct alt_base *repo, unsigned char *sha1)
796 {
797         char *hex = sha1_to_hex(sha1);
798         int ret = 0;
799         struct object_request *obj_req = object_queue_head;
800
801         while (obj_req != NULL && memcmp(obj_req->sha1, sha1, 20))
802                 obj_req = obj_req->next;
803         if (obj_req == NULL)
804                 return error("Couldn't find request for %s in the queue", hex);
805
806         if (has_sha1_file(obj_req->sha1)) {
807                 abort_object_request(obj_req);
808                 return 0;
809         }
810
811 #ifdef USE_CURL_MULTI
812         while (obj_req->state == WAITING) {
813                 step_active_slots();
814         }
815 #else
816         start_object_request(obj_req);
817 #endif
818
819         while (obj_req->state == ACTIVE) {
820                 run_active_slot(obj_req->slot);
821         }
822         if (obj_req->local != -1) {
823                 close(obj_req->local); obj_req->local = -1;
824         }
825
826         if (obj_req->state == ABORTED) {
827                 ret = error("Request for %s aborted", hex);
828         } else if (obj_req->curl_result != CURLE_OK &&
829                    obj_req->http_code != 416) {
830                 if (obj_req->http_code == 404 ||
831                     obj_req->curl_result == CURLE_FILE_COULDNT_READ_FILE)
832                         ret = -1; /* Be silent, it is probably in a pack. */
833                 else
834                         ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
835                                     obj_req->errorstr, obj_req->curl_result,
836                                     obj_req->http_code, hex);
837         } else if (obj_req->zret != Z_STREAM_END) {
838                 corrupt_object_found++;
839                 ret = error("File %s (%s) corrupt", hex, obj_req->url);
840         } else if (memcmp(obj_req->sha1, obj_req->real_sha1, 20)) {
841                 ret = error("File %s has bad hash", hex);
842         } else if (obj_req->rename < 0) {
843                 ret = error("unable to write sha1 filename %s",
844                             obj_req->filename);
845         }
846
847         release_object_request(obj_req);
848         return ret;
849 }
850
851 int fetch(unsigned char *sha1)
852 {
853         struct alt_base *altbase = alt;
854
855         if (!fetch_object(altbase, sha1))
856                 return 0;
857         while (altbase) {
858                 if (!fetch_pack(altbase, sha1))
859                         return 0;
860                 fetch_alternates(alt->base);
861                 altbase = altbase->next;
862         }
863         return error("Unable to find %s under %s", sha1_to_hex(sha1),
864                      alt->base);
865 }
866
867 static inline int needs_quote(int ch)
868 {
869         switch (ch) {
870         case '/': case '-': case '.':
871         case 'A'...'Z': case 'a'...'z': case '0'...'9':
872                 return 0;
873         default:
874                 return 1;
875         }
876 }
877
878 static inline int hex(int v)
879 {
880         if (v < 10) return '0' + v;
881         else return 'A' + v - 10;
882 }
883
884 static char *quote_ref_url(const char *base, const char *ref)
885 {
886         const char *cp;
887         char *dp, *qref;
888         int len, baselen, ch;
889
890         baselen = strlen(base);
891         len = baselen + 6; /* "refs/" + NUL */
892         for (cp = ref; (ch = *cp) != 0; cp++, len++)
893                 if (needs_quote(ch))
894                         len += 2; /* extra two hex plus replacement % */
895         qref = xmalloc(len);
896         memcpy(qref, base, baselen);
897         memcpy(qref + baselen, "refs/", 5);
898         for (cp = ref, dp = qref + baselen + 5; (ch = *cp) != 0; cp++) {
899                 if (needs_quote(ch)) {
900                         *dp++ = '%';
901                         *dp++ = hex((ch >> 4) & 0xF);
902                         *dp++ = hex(ch & 0xF);
903                 }
904                 else
905                         *dp++ = ch;
906         }
907         *dp = 0;
908
909         return qref;
910 }
911
912 int fetch_ref(char *ref, unsigned char *sha1)
913 {
914         char *url;
915         char hex[42];
916         struct buffer buffer;
917         char *base = alt->base;
918         struct active_request_slot *slot;
919         struct slot_results results;
920         buffer.size = 41;
921         buffer.posn = 0;
922         buffer.buffer = hex;
923         hex[41] = '\0';
924
925         url = quote_ref_url(base, ref);
926         slot = get_active_slot();
927         slot->results = &results;
928         curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
929         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
930         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
931         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
932         if (start_active_slot(slot)) {
933                 run_active_slot(slot);
934                 if (results.curl_result != CURLE_OK)
935                         return error("Couldn't get %s for %s\n%s",
936                                      url, ref, curl_errorstr);
937         } else {
938                 return error("Unable to start request");
939         }
940
941         hex[40] = '\0';
942         get_sha1_hex(hex, sha1);
943         return 0;
944 }
945
946 int main(int argc, char **argv)
947 {
948         char *commit_id;
949         char *url;
950         int arg = 1;
951         int rc = 0;
952
953         setup_git_directory();
954
955         while (arg < argc && argv[arg][0] == '-') {
956                 if (argv[arg][1] == 't') {
957                         get_tree = 1;
958                 } else if (argv[arg][1] == 'c') {
959                         get_history = 1;
960                 } else if (argv[arg][1] == 'a') {
961                         get_all = 1;
962                         get_tree = 1;
963                         get_history = 1;
964                 } else if (argv[arg][1] == 'v') {
965                         get_verbosely = 1;
966                 } else if (argv[arg][1] == 'w') {
967                         write_ref = argv[arg + 1];
968                         arg++;
969                 } else if (!strcmp(argv[arg], "--recover")) {
970                         get_recover = 1;
971                 }
972                 arg++;
973         }
974         if (argc < arg + 2) {
975                 usage("git-http-fetch [-c] [-t] [-a] [-d] [-v] [--recover] [-w ref] commit-id url");
976                 return 1;
977         }
978         commit_id = argv[arg];
979         url = argv[arg + 1];
980
981         http_init();
982
983         no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
984
985         alt = xmalloc(sizeof(*alt));
986         alt->base = url;
987         alt->got_indices = 0;
988         alt->packs = NULL;
989         alt->next = NULL;
990
991         if (pull(commit_id))
992                 rc = 1;
993
994         curl_slist_free_all(no_pragma_header);
995
996         http_cleanup();
997
998         if (corrupt_object_found) {
999                 fprintf(stderr,
1000 "Some loose object were found to be corrupt, but they might be just\n"
1001 "a false '404 Not Found' error message sent with incorrect HTTP\n"
1002 "status code.  Suggest running git fsck-objects.\n");
1003         }
1004         return rc;
1005 }