7 #define PREV_BUF_SIZE 4096
8 #define RANGE_HEADER_SIZE 30
10 static int got_alternates = -1;
12 static struct curl_slist *no_pragma_header;
18 struct packed_git *packs;
19 struct alt_base *next;
22 static struct alt_base *alt = NULL;
31 struct transfer_request
33 unsigned char sha1[20];
34 struct alt_base *repo;
36 char filename[PATH_MAX];
37 char tmpfile[PATH_MAX];
39 enum transfer_state state;
41 char errorstr[CURL_ERROR_SIZE];
43 unsigned char real_sha1[20];
48 struct active_request_slot *slot;
49 struct transfer_request *next;
55 struct buffer *buffer;
56 struct active_request_slot *slot;
60 static struct transfer_request *request_queue_head = NULL;
62 static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
65 unsigned char expn[4096];
66 size_t size = eltsize * nmemb;
68 struct transfer_request *request = (struct transfer_request *)data;
70 ssize_t retval = write(request->local,
71 ptr + posn, size - posn);
75 } while (posn < size);
77 request->stream.avail_in = size;
78 request->stream.next_in = ptr;
80 request->stream.next_out = expn;
81 request->stream.avail_out = sizeof(expn);
82 request->zret = inflate(&request->stream, Z_SYNC_FLUSH);
83 SHA1_Update(&request->c, expn,
84 sizeof(expn) - request->stream.avail_out);
85 } while (request->stream.avail_in && request->zret == Z_OK);
90 static void fetch_alternates(char *base);
92 static void process_object_response(void *callback_data);
94 static void start_request(struct transfer_request *request)
96 char *hex = sha1_to_hex(request->sha1);
97 char prevfile[PATH_MAX];
101 unsigned char prev_buf[PREV_BUF_SIZE];
102 ssize_t prev_read = 0;
104 char range[RANGE_HEADER_SIZE];
105 struct curl_slist *range_header = NULL;
106 struct active_request_slot *slot;
108 snprintf(prevfile, sizeof(prevfile), "%s.prev", request->filename);
110 rename(request->tmpfile, prevfile);
111 unlink(request->tmpfile);
113 if (request->local != -1)
114 error("fd leakage in start: %d", request->local);
115 request->local = open(request->tmpfile,
116 O_WRONLY | O_CREAT | O_EXCL, 0666);
117 /* This could have failed due to the "lazy directory creation";
118 * try to mkdir the last path component.
120 if (request->local < 0 && errno == ENOENT) {
121 char *dir = strrchr(request->tmpfile, '/');
124 mkdir(request->tmpfile, 0777);
127 request->local = open(request->tmpfile,
128 O_WRONLY | O_CREAT | O_EXCL, 0666);
131 if (request->local < 0) {
132 request->state = ABORTED;
133 error("Couldn't create temporary file %s for %s: %s\n",
134 request->tmpfile, request->filename, strerror(errno));
138 memset(&request->stream, 0, sizeof(request->stream));
140 inflateInit(&request->stream);
142 SHA1_Init(&request->c);
144 url = xmalloc(strlen(request->repo->base) + 50);
145 request->url = xmalloc(strlen(request->repo->base) + 50);
146 strcpy(url, request->repo->base);
147 posn = url + strlen(request->repo->base);
148 strcpy(posn, "objects/");
150 memcpy(posn, hex, 2);
153 strcpy(posn, hex + 2);
154 strcpy(request->url, url);
156 /* If a previous temp file is present, process what was already
158 prevlocal = open(prevfile, O_RDONLY);
159 if (prevlocal != -1) {
161 prev_read = read(prevlocal, prev_buf, PREV_BUF_SIZE);
163 if (fwrite_sha1_file(prev_buf,
166 request) == prev_read) {
167 prev_posn += prev_read;
172 } while (prev_read > 0);
177 /* Reset inflate/SHA1 if there was an error reading the previous temp
178 file; also rewind to the beginning of the local file. */
179 if (prev_read == -1) {
180 memset(&request->stream, 0, sizeof(request->stream));
181 inflateInit(&request->stream);
182 SHA1_Init(&request->c);
185 lseek(request->local, SEEK_SET, 0);
186 ftruncate(request->local, 0);
190 slot = get_active_slot();
191 slot->callback_func = process_object_response;
192 slot->callback_data = request;
193 request->slot = slot;
195 curl_easy_setopt(slot->curl, CURLOPT_FILE, request);
196 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
197 curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, request->errorstr);
198 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
199 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
201 /* If we have successfully processed data from a previous fetch
202 attempt, only fetch the data we don't already have. */
206 "Resuming fetch of object %s at byte %ld\n",
208 sprintf(range, "Range: bytes=%ld-", prev_posn);
209 range_header = curl_slist_append(range_header, range);
210 curl_easy_setopt(slot->curl,
211 CURLOPT_HTTPHEADER, range_header);
214 /* Try to get the request started, abort the request on error */
215 request->state = ACTIVE;
216 if (!start_active_slot(slot)) {
217 request->state = ABORTED;
218 request->slot = NULL;
219 close(request->local); request->local = -1;
224 static void finish_request(struct transfer_request *request)
228 fchmod(request->local, 0444);
229 close(request->local); request->local = -1;
231 if (request->http_code == 416) {
232 fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
233 } else if (request->curl_result != CURLE_OK) {
234 if (stat(request->tmpfile, &st) == 0)
236 unlink(request->tmpfile);
240 inflateEnd(&request->stream);
241 SHA1_Final(request->real_sha1, &request->c);
242 if (request->zret != Z_STREAM_END) {
243 unlink(request->tmpfile);
246 if (memcmp(request->sha1, request->real_sha1, 20)) {
247 unlink(request->tmpfile);
251 move_temp_to_file(request->tmpfile, request->filename);
253 if (request->rename == 0)
254 pull_say("got %s\n", sha1_to_hex(request->sha1));
257 static void process_object_response(void *callback_data)
259 struct transfer_request *request =
260 (struct transfer_request *)callback_data;
262 request->curl_result = request->slot->curl_result;
263 request->http_code = request->slot->http_code;
264 request->slot = NULL;
265 request->state = COMPLETE;
267 /* Use alternates if necessary */
268 if (request->http_code == 404) {
269 fetch_alternates(alt->base);
270 if (request->repo->next != NULL) {
273 close(request->local);
275 start_request(request);
280 finish_request(request);
283 static void release_request(struct transfer_request *request)
285 struct transfer_request *entry = request_queue_head;
287 if (request->local != -1)
288 error("fd leakage in release: %d", request->local);
289 if (request == request_queue_head) {
290 request_queue_head = request->next;
292 while (entry->next != NULL && entry->next != request)
294 if (entry->next == request)
295 entry->next = entry->next->next;
302 #ifdef USE_CURL_MULTI
303 void fill_active_slots(void)
305 struct transfer_request *request = request_queue_head;
306 struct active_request_slot *slot = active_queue_head;
309 while (active_requests < max_requests && request != NULL) {
310 if (request->state == WAITING) {
311 if (has_sha1_file(request->sha1))
312 release_request(request);
314 start_request(request);
315 curl_multi_perform(curlm, &num_transfers);
317 request = request->next;
320 while (slot != NULL) {
321 if (!slot->in_use && slot->curl != NULL) {
322 curl_easy_cleanup(slot->curl);
330 void prefetch(unsigned char *sha1)
332 struct transfer_request *newreq;
333 struct transfer_request *tail;
334 char *filename = sha1_file_name(sha1);
336 newreq = xmalloc(sizeof(*newreq));
337 memcpy(newreq->sha1, sha1, 20);
341 newreq->state = WAITING;
342 snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
343 snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
344 "%s.temp", filename);
347 if (request_queue_head == NULL) {
348 request_queue_head = newreq;
350 tail = request_queue_head;
351 while (tail->next != NULL) {
357 #ifdef USE_CURL_MULTI
363 static int fetch_index(struct alt_base *repo, unsigned char *sha1)
365 char *hex = sha1_to_hex(sha1);
368 char tmpfile[PATH_MAX];
370 char range[RANGE_HEADER_SIZE];
371 struct curl_slist *range_header = NULL;
374 struct active_request_slot *slot;
376 if (has_pack_index(sha1))
380 fprintf(stderr, "Getting index for pack %s\n", hex);
382 url = xmalloc(strlen(repo->base) + 64);
383 sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex);
385 filename = sha1_pack_index_name(sha1);
386 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
387 indexfile = fopen(tmpfile, "a");
389 return error("Unable to open local file %s for pack index",
392 slot = get_active_slot();
393 curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
394 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
395 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
396 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
397 slot->local = indexfile;
399 /* If there is data present from a previous transfer attempt,
400 resume where it left off */
401 prev_posn = ftell(indexfile);
405 "Resuming fetch of index for pack %s at byte %ld\n",
407 sprintf(range, "Range: bytes=%ld-", prev_posn);
408 range_header = curl_slist_append(range_header, range);
409 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
412 if (start_active_slot(slot)) {
413 run_active_slot(slot);
414 if (slot->curl_result != CURLE_OK) {
416 return error("Unable to get pack index %s\n%s", url,
421 return error("Unable to start request");
426 return move_temp_to_file(tmpfile, filename);
429 static int setup_index(struct alt_base *repo, unsigned char *sha1)
431 struct packed_git *new_pack;
432 if (has_pack_file(sha1))
433 return 0; // don't list this as something we can get
435 if (fetch_index(repo, sha1))
438 new_pack = parse_pack_index(sha1);
439 new_pack->next = repo->packs;
440 repo->packs = new_pack;
444 static void process_alternates(void *callback_data)
446 struct alt_request *alt_req = (struct alt_request *)callback_data;
447 struct active_request_slot *slot = alt_req->slot;
448 struct alt_base *tail = alt;
449 char *base = alt_req->base;
450 static const char null_byte = '\0';
454 if (alt_req->http_specific) {
455 if (slot->curl_result != CURLE_OK ||
456 !alt_req->buffer->posn) {
458 /* Try reusing the slot to get non-http alternates */
459 alt_req->http_specific = 0;
460 sprintf(alt_req->url, "%s/objects/info/alternates",
462 curl_easy_setopt(slot->curl, CURLOPT_URL,
466 if (start_active_slot(slot)) {
474 } else if (slot->curl_result != CURLE_OK) {
475 if (slot->http_code != 404) {
481 fwrite_buffer(&null_byte, 1, 1, alt_req->buffer);
482 alt_req->buffer->posn--;
483 data = alt_req->buffer->buffer;
485 while (i < alt_req->buffer->posn) {
487 while (posn < alt_req->buffer->posn && data[posn] != '\n')
489 if (data[posn] == '\n') {
492 struct alt_base *newalt;
494 if (data[i] == '/') {
495 serverlen = strchr(base + 8, '/') - base;
497 } else if (!memcmp(data + i, "../", 3)) {
499 serverlen = strlen(base);
500 while (i + 2 < posn &&
501 !memcmp(data + i, "../", 3)) {
504 } while (serverlen &&
505 base[serverlen - 1] != '/');
508 // If the server got removed, give up.
509 okay = strchr(base, ':') - base + 3 <
511 } else if (alt_req->http_specific) {
512 char *colon = strchr(data + i, ':');
513 char *slash = strchr(data + i, '/');
514 if (colon && slash && colon < data + posn &&
515 slash < data + posn && colon < slash) {
519 // skip 'objects' at end
521 target = xmalloc(serverlen + posn - i - 6);
522 strncpy(target, base, serverlen);
523 strncpy(target + serverlen, data + i,
525 target[serverlen + posn - i - 7] = '\0';
528 "Also look at %s\n", target);
529 newalt = xmalloc(sizeof(*newalt));
531 newalt->base = target;
532 newalt->got_indices = 0;
533 newalt->packs = NULL;
534 while (tail->next != NULL)
545 static void fetch_alternates(char *base)
547 struct buffer buffer;
550 struct active_request_slot *slot;
551 static struct alt_request alt_req;
553 /* If another request has already started fetching alternates,
554 wait for them to arrive and return to processing this request's
556 #ifdef USE_CURL_MULTI
557 while (got_alternates == 0) {
562 /* Nothing to do if they've already been fetched */
563 if (got_alternates == 1)
566 /* Start the fetch */
569 data = xmalloc(4096);
572 buffer.buffer = data;
575 fprintf(stderr, "Getting alternates list for %s\n", base);
577 url = xmalloc(strlen(base) + 31);
578 sprintf(url, "%s/objects/info/http-alternates", base);
580 /* Use a callback to process the result, since another request
581 may fail and need to have alternates loaded before continuing */
582 slot = get_active_slot();
583 slot->callback_func = process_alternates;
584 slot->callback_data = &alt_req;
586 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
587 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
588 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
592 alt_req.buffer = &buffer;
593 alt_req.http_specific = 1;
596 if (start_active_slot(slot))
597 run_active_slot(slot);
605 static int fetch_indices(struct alt_base *repo)
607 unsigned char sha1[20];
609 struct buffer buffer;
613 struct active_request_slot *slot;
615 if (repo->got_indices)
618 data = xmalloc(4096);
621 buffer.buffer = data;
624 fprintf(stderr, "Getting pack list for %s\n", repo->base);
626 url = xmalloc(strlen(repo->base) + 21);
627 sprintf(url, "%s/objects/info/packs", repo->base);
629 slot = get_active_slot();
630 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
631 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
632 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
633 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
634 if (start_active_slot(slot)) {
635 run_active_slot(slot);
636 if (slot->curl_result != CURLE_OK) {
638 return error("%s", curl_errorstr);
642 return error("Unable to start request");
645 data = buffer.buffer;
646 while (i < buffer.posn) {
650 if (i + 52 < buffer.posn &&
651 !strncmp(data + i, " pack-", 6) &&
652 !strncmp(data + i + 46, ".pack\n", 6)) {
653 get_sha1_hex(data + i + 6, sha1);
654 setup_index(repo, sha1);
659 while (data[i] != '\n')
666 repo->got_indices = 1;
670 static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
673 struct packed_git *target;
674 struct packed_git **lst;
677 char tmpfile[PATH_MAX];
680 char range[RANGE_HEADER_SIZE];
681 struct curl_slist *range_header = NULL;
683 struct active_request_slot *slot;
685 if (fetch_indices(repo))
687 target = find_sha1_pack(sha1, repo->packs);
692 fprintf(stderr, "Getting pack %s\n",
693 sha1_to_hex(target->sha1));
694 fprintf(stderr, " which contains %s\n",
698 url = xmalloc(strlen(repo->base) + 65);
699 sprintf(url, "%s/objects/pack/pack-%s.pack",
700 repo->base, sha1_to_hex(target->sha1));
702 filename = sha1_pack_name(target->sha1);
703 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
704 packfile = fopen(tmpfile, "a");
706 return error("Unable to open local file %s for pack",
709 slot = get_active_slot();
710 curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
711 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
712 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
713 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
714 slot->local = packfile;
716 /* If there is data present from a previous transfer attempt,
717 resume where it left off */
718 prev_posn = ftell(packfile);
722 "Resuming fetch of pack %s at byte %ld\n",
723 sha1_to_hex(target->sha1), prev_posn);
724 sprintf(range, "Range: bytes=%ld-", prev_posn);
725 range_header = curl_slist_append(range_header, range);
726 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
729 if (start_active_slot(slot)) {
730 run_active_slot(slot);
731 if (slot->curl_result != CURLE_OK) {
733 return error("Unable to get pack file %s\n%s", url,
738 return error("Unable to start request");
743 ret = move_temp_to_file(tmpfile, filename);
748 while (*lst != target)
749 lst = &((*lst)->next);
752 if (verify_pack(target, 0))
754 install_packed_git(target);
759 static int fetch_object(struct alt_base *repo, unsigned char *sha1)
761 char *hex = sha1_to_hex(sha1);
763 struct transfer_request *request = request_queue_head;
765 while (request != NULL && memcmp(request->sha1, sha1, 20))
766 request = request->next;
768 return error("Couldn't find request for %s in the queue", hex);
770 if (has_sha1_file(request->sha1)) {
771 release_request(request);
775 #ifdef USE_CURL_MULTI
776 while (request->state == WAITING) {
780 start_request(request);
783 while (request->state == ACTIVE) {
784 run_active_slot(request->slot);
786 if (request->local != -1) {
787 close(request->local); request->local = -1;
790 if (request->state == ABORTED) {
791 ret = error("Request for %s aborted", hex);
792 } else if (request->curl_result != CURLE_OK &&
793 request->http_code != 416) {
794 if (request->http_code == 404)
795 ret = -1; /* Be silent, it is probably in a pack. */
797 ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
798 request->errorstr, request->curl_result,
799 request->http_code, hex);
800 } else if (request->zret != Z_STREAM_END) {
801 ret = error("File %s (%s) corrupt\n", hex, request->url);
802 } else if (memcmp(request->sha1, request->real_sha1, 20)) {
803 ret = error("File %s has bad hash\n", hex);
804 } else if (request->rename < 0) {
805 ret = error("unable to write sha1 filename %s: %s",
807 strerror(request->rename));
810 release_request(request);
814 int fetch(unsigned char *sha1)
816 struct alt_base *altbase = alt;
818 if (!fetch_object(altbase, sha1))
821 if (!fetch_pack(altbase, sha1))
823 fetch_alternates(alt->base);
824 altbase = altbase->next;
826 return error("Unable to find %s under %s\n", sha1_to_hex(sha1),
830 static inline int needs_quote(int ch)
833 case '/': case '-': case '.':
834 case 'A'...'Z': case 'a'...'z': case '0'...'9':
841 static inline int hex(int v)
843 if (v < 10) return '0' + v;
844 else return 'A' + v - 10;
847 static char *quote_ref_url(const char *base, const char *ref)
851 int len, baselen, ch;
853 baselen = strlen(base);
854 len = baselen + 6; /* "refs/" + NUL */
855 for (cp = ref; (ch = *cp) != 0; cp++, len++)
857 len += 2; /* extra two hex plus replacement % */
859 memcpy(qref, base, baselen);
860 memcpy(qref + baselen, "refs/", 5);
861 for (cp = ref, dp = qref + baselen + 5; (ch = *cp) != 0; cp++) {
862 if (needs_quote(ch)) {
864 *dp++ = hex((ch >> 4) & 0xF);
865 *dp++ = hex(ch & 0xF);
875 int fetch_ref(char *ref, unsigned char *sha1)
879 struct buffer buffer;
880 char *base = alt->base;
881 struct active_request_slot *slot;
887 url = quote_ref_url(base, ref);
888 slot = get_active_slot();
889 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
890 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
891 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
892 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
893 if (start_active_slot(slot)) {
894 run_active_slot(slot);
895 if (slot->curl_result != CURLE_OK)
896 return error("Couldn't get %s for %s\n%s",
897 url, ref, curl_errorstr);
899 return error("Unable to start request");
903 get_sha1_hex(hex, sha1);
907 int main(int argc, char **argv)
914 while (arg < argc && argv[arg][0] == '-') {
915 if (argv[arg][1] == 't') {
917 } else if (argv[arg][1] == 'c') {
919 } else if (argv[arg][1] == 'a') {
923 } else if (argv[arg][1] == 'v') {
925 } else if (argv[arg][1] == 'w') {
926 write_ref = argv[arg + 1];
928 } else if (!strcmp(argv[arg], "--recover")) {
933 if (argc < arg + 2) {
934 usage("git-http-fetch [-c] [-t] [-a] [-d] [-v] [--recover] [-w ref] commit-id url");
937 commit_id = argv[arg];
942 no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
944 alt = xmalloc(sizeof(*alt));
946 alt->got_indices = 0;
953 curl_slist_free_all(no_pragma_header);