X-Git-Url: https://git.octo.it/?a=blobdiff_plain;f=http-fetch.c;h=21cc1b960cf511c083c594fd5fc86bb84d66ede3;hb=a52e4ef877f693a69710145f56d867365a755f91;hp=ea8af1b2defaf7e11dff0a09952234f3c7626376;hpb=90279074ca5cc336a8bfffd47d19d089b291b432;p=git.git diff --git a/http-fetch.c b/http-fetch.c index ea8af1b2..21cc1b96 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -25,7 +25,7 @@ #define PREV_BUF_SIZE 4096 #define RANGE_HEADER_SIZE 30 -static int got_alternates = 0; +static int got_alternates = -1; static int active_requests = 0; static int data_received; @@ -87,9 +87,19 @@ struct active_request_slot int done; CURLcode curl_result; long http_code; + void *callback_data; + void (*callback_func)(void *data); struct active_request_slot *next; }; +struct alt_request { + char *base; + char *url; + struct buffer *buffer; + struct active_request_slot *slot; + int http_specific; +}; + static struct transfer_request *request_queue_head = NULL; static struct active_request_slot *active_queue_head = NULL; @@ -237,7 +247,7 @@ static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb, static void process_curl_messages(void); static void process_request_queue(void); #endif -static int fetch_alternates(char *base); +static void fetch_alternates(char *base); static CURL* get_curl_handle(void) { @@ -269,6 +279,8 @@ static CURL* get_curl_handle(void) curl_low_speed_time); } + curl_easy_setopt(result, CURLOPT_FOLLOWLOCATION, 1); + return result; } @@ -322,6 +334,8 @@ static struct active_request_slot *get_active_slot(void) slot->in_use = 1; slot->done = 0; slot->local = NULL; + slot->callback_data = NULL; + slot->callback_func = NULL; curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, pragma_header); curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_range_header); curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr); @@ -411,6 +425,8 @@ static void start_request(struct transfer_request *request) rename(request->tmpfile, prevfile); unlink(request->tmpfile); + if (request->local != -1) + error("fd leakage in start: %d", request->local); request->local = open(request->tmpfile, O_WRONLY | O_CREAT | O_EXCL, 0666); /* This could have failed due to the "lazy directory creation"; @@ -509,7 +525,7 @@ static void start_request(struct transfer_request *request) /* Try to get the request started, abort the request on error */ if (!start_active_slot(slot)) { request->state = ABORTED; - close(request->local); + close(request->local); request->local = -1; free(request->url); return; } @@ -523,7 +539,7 @@ static void finish_request(struct transfer_request *request) struct stat st; fchmod(request->local, 0444); - close(request->local); + close(request->local); request->local = -1; if (request->http_code == 416) { fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n"); @@ -555,6 +571,8 @@ static void release_request(struct transfer_request *request) { struct transfer_request *entry = request_queue_head; + if (request->local != -1) + error("fd leakage in release: %d", request->local); if (request == request_queue_head) { request_queue_head = request->next; } else { @@ -569,7 +587,7 @@ static void release_request(struct transfer_request *request) } #ifdef USE_CURL_MULTI -void process_curl_messages(void) +static void process_curl_messages(void) { int num_messages; struct active_request_slot *slot; @@ -599,6 +617,12 @@ void process_curl_messages(void) } else { fprintf(stderr, "Received DONE message for unknown request!\n"); } + + /* Process slot callback if appropriate */ + if (slot->callback_func != NULL) { + slot->callback_func(slot->callback_data); + } + if (request != NULL) { request->curl_result = curl_result; request->http_code = slot->http_code; @@ -611,7 +635,11 @@ void process_curl_messages(void) if (request->repo->next != NULL) { request->repo = request->repo->next; + close(request->local); + request->local = -1; start_request(request); + } else { + finish_request(request); } } else { finish_request(request); @@ -625,7 +653,7 @@ void process_curl_messages(void) } } -void process_request_queue(void) +static void process_request_queue(void) { struct transfer_request *request = request_queue_head; struct active_request_slot *slot = active_queue_head; @@ -741,6 +769,7 @@ static int fetch_index(struct alt_base *repo, unsigned char *sha1) curl_errorstr); } } else { + fclose(indexfile); return error("Unable to start request"); } @@ -764,72 +793,51 @@ static int setup_index(struct alt_base *repo, unsigned char *sha1) return 0; } -static int fetch_alternates(char *base) +static void process_alternates(void *callback_data) { - int ret = 0; - struct buffer buffer; - char *url; - char *data; - int i = 0; - int http_specific = 1; + struct alt_request *alt_req = (struct alt_request *)callback_data; + struct active_request_slot *slot = alt_req->slot; struct alt_base *tail = alt; + char *base = alt_req->base; static const char null_byte = '\0'; + char *data; + int i = 0; - struct active_request_slot *slot; - - if (got_alternates) - return 0; - - data = xmalloc(4096); - buffer.size = 4096; - buffer.posn = 0; - buffer.buffer = data; - - if (get_verbosely) - fprintf(stderr, "Getting alternates list\n"); - - url = xmalloc(strlen(base) + 31); - sprintf(url, "%s/objects/info/http-alternates", base); - - slot = get_active_slot(); - curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer); - curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, - fwrite_buffer_dynamic); - curl_easy_setopt(slot->curl, CURLOPT_URL, url); - if (start_active_slot(slot)) { - run_active_slot(slot); - if (slot->curl_result != CURLE_OK || !buffer.posn) { - http_specific = 0; - - sprintf(url, "%s/objects/info/alternates", base); - - slot = get_active_slot(); - curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer); - curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, - fwrite_buffer_dynamic); - curl_easy_setopt(slot->curl, CURLOPT_URL, url); + if (alt_req->http_specific) { + if (slot->curl_result != CURLE_OK || + !alt_req->buffer->posn) { + + /* Try reusing the slot to get non-http alternates */ + alt_req->http_specific = 0; + sprintf(alt_req->url, "%s/objects/info/alternates", + base); + curl_easy_setopt(slot->curl, CURLOPT_URL, + alt_req->url); + active_requests++; + slot->in_use = 1; + slot->done = 0; if (start_active_slot(slot)) { - run_active_slot(slot); - if (slot->curl_result != CURLE_OK) { - free(buffer.buffer); - if (slot->http_code == 404) - got_alternates = 1; - return 0; - } + return; + } else { + got_alternates = -1; + slot->done = 1; + return; } } - } else { - free(buffer.buffer); - return 0; + } else if (slot->curl_result != CURLE_OK) { + if (slot->http_code != 404) { + got_alternates = -1; + return; + } } - fwrite_buffer_dynamic(&null_byte, 1, 1, &buffer); - buffer.posn--; - data = buffer.buffer; + fwrite_buffer_dynamic(&null_byte, 1, 1, alt_req->buffer); + alt_req->buffer->posn--; + data = alt_req->buffer->buffer; - while (i < buffer.posn) { + while (i < alt_req->buffer->posn) { int posn = i; - while (posn < buffer.posn && data[posn] != '\n') + while (posn < alt_req->buffer->posn && data[posn] != '\n') posn++; if (data[posn] == '\n') { int okay = 0; @@ -853,7 +861,7 @@ static int fetch_alternates(char *base) // If the server got removed, give up. okay = strchr(base, ':') - base + 3 < serverlen; - } else if (http_specific) { + } else if (alt_req->http_specific) { char *colon = strchr(data + i, ':'); char *slash = strchr(data + i, '/'); if (colon && slash && colon < data + posn && @@ -879,15 +887,74 @@ static int fetch_alternates(char *base) while (tail->next != NULL) tail = tail->next; tail->next = newalt; - ret++; } } i = posn + 1; } got_alternates = 1; - free(buffer.buffer); - return ret; +} + +static void fetch_alternates(char *base) +{ + struct buffer buffer; + char *url; + char *data; + struct active_request_slot *slot; + static struct alt_request alt_req; + int num_transfers; + + /* If another request has already started fetching alternates, + wait for them to arrive and return to processing this request's + curl message */ + while (got_alternates == 0) { + curl_multi_perform(curlm, &num_transfers); + process_curl_messages(); + process_request_queue(); + } + + /* Nothing to do if they've already been fetched */ + if (got_alternates == 1) + return; + + /* Start the fetch */ + got_alternates = 0; + + data = xmalloc(4096); + buffer.size = 4096; + buffer.posn = 0; + buffer.buffer = data; + + if (get_verbosely) + fprintf(stderr, "Getting alternates list for %s\n", base); + + url = xmalloc(strlen(base) + 31); + sprintf(url, "%s/objects/info/http-alternates", base); + + /* Use a callback to process the result, since another request + may fail and need to have alternates loaded before continuing */ + slot = get_active_slot(); + slot->callback_func = process_alternates; + slot->callback_data = &alt_req; + + curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer); + curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, + fwrite_buffer_dynamic); + curl_easy_setopt(slot->curl, CURLOPT_URL, url); + + alt_req.base = base; + alt_req.url = url; + alt_req.buffer = &buffer; + alt_req.http_specific = 1; + alt_req.slot = slot; + + if (start_active_slot(slot)) + run_active_slot(slot); + else + got_alternates = -1; + + free(data); + free(url); } static int fetch_indices(struct alt_base *repo) @@ -909,7 +976,7 @@ static int fetch_indices(struct alt_base *repo) buffer.buffer = data; if (get_verbosely) - fprintf(stderr, "Getting pack list\n"); + fprintf(stderr, "Getting pack list for %s\n", repo->base); url = xmalloc(strlen(repo->base) + 21); sprintf(url, "%s/objects/info/packs", repo->base); @@ -1023,6 +1090,7 @@ static int fetch_pack(struct alt_base *repo, unsigned char *sha1) curl_errorstr); } } else { + fclose(packfile); return error("Unable to start request"); } @@ -1085,6 +1153,7 @@ static int fetch_object(struct alt_base *repo, unsigned char *sha1) fetch_alternates(alt->base); if (request->repo->next != NULL) { request->repo = request->repo->next; + close(request->local); request->local = -1; start_request(request); } } else { @@ -1093,6 +1162,9 @@ static int fetch_object(struct alt_base *repo, unsigned char *sha1) } #endif } + if (request->local != -1) { + close(request->local); request->local = -1; + } if (request->state == ABORTED) { release_request(request);