X-Git-Url: https://git.octo.it/?a=blobdiff_plain;f=http-fetch.c;h=a1b03cd9c842e9daa8f743939d4382faac0c534f;hb=0f8fdc3958ad0cb7c740b76189f98307eebcd64b;hp=784aedfc700674cc79915004aee4031ab8efbff0;hpb=d402d5566fdf226697a386dfb9858e5d954e9b91;p=git.git diff --git a/http-fetch.c b/http-fetch.c index 784aedfc..a1b03cd9 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -18,9 +18,14 @@ #define curl_global_init(a) do { /* nothing */ } while(0) #endif +#if LIBCURL_VERSION_NUM < 0x070c04 +#define NO_CURL_EASY_DUPHANDLE +#endif + #define PREV_BUF_SIZE 4096 #define RANGE_HEADER_SIZE 30 +static int got_alternates = 0; static int active_requests = 0; static int data_received; @@ -28,7 +33,9 @@ static int data_received; static int max_requests = -1; static CURLM *curlm; #endif +#ifndef NO_CURL_EASY_DUPHANDLE static CURL *curl_default; +#endif static struct curl_slist *pragma_header; static struct curl_slist *no_pragma_header; static struct curl_slist *no_range_header; @@ -79,6 +86,7 @@ struct active_request_slot int in_use; int done; CURLcode curl_result; + long http_code; struct active_request_slot *next; }; @@ -87,9 +95,15 @@ static struct active_request_slot *active_queue_head = NULL; static int curl_ssl_verify = -1; static char *ssl_cert = NULL; +#if LIBCURL_VERSION_NUM >= 0x070902 static char *ssl_key = NULL; +#endif +#if LIBCURL_VERSION_NUM >= 0x070908 static char *ssl_capath = NULL; +#endif static char *ssl_cainfo = NULL; +static long curl_low_speed_limit = -1; +static long curl_low_speed_time = -1; struct buffer { @@ -148,6 +162,17 @@ static int http_options(const char *var, const char *value) } #endif + if (!strcmp("http.lowspeedlimit", var)) { + if (curl_low_speed_limit == -1) + curl_low_speed_limit = (long)git_config_int(var, value); + return 0; + } + if (!strcmp("http.lowspeedtime", var)) { + if (curl_low_speed_time == -1) + curl_low_speed_time = (long)git_config_int(var, value); + return 0; + } + /* Fall back on the default ones */ return git_default_config(var, value); } @@ -209,11 +234,45 @@ static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb, } #ifdef USE_CURL_MULTI -void process_curl_messages(); -void process_request_queue(); +static void process_curl_messages(void); +static void process_request_queue(void); #endif +static int fetch_alternates(char *base); -struct active_request_slot *get_active_slot() +static CURL* get_curl_handle(void) +{ + CURL* result = curl_easy_init(); + + curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, curl_ssl_verify); +#if LIBCURL_VERSION_NUM >= 0x070907 + curl_easy_setopt(result, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); +#endif + + if (ssl_cert != NULL) + curl_easy_setopt(result, CURLOPT_SSLCERT, ssl_cert); +#if LIBCURL_VERSION_NUM >= 0x070902 + if (ssl_key != NULL) + curl_easy_setopt(result, CURLOPT_SSLKEY, ssl_key); +#endif +#if LIBCURL_VERSION_NUM >= 0x070908 + if (ssl_capath != NULL) + curl_easy_setopt(result, CURLOPT_CAPATH, ssl_capath); +#endif + if (ssl_cainfo != NULL) + curl_easy_setopt(result, CURLOPT_CAINFO, ssl_cainfo); + curl_easy_setopt(result, CURLOPT_FAILONERROR, 1); + + if (curl_low_speed_limit > 0 && curl_low_speed_time > 0) { + curl_easy_setopt(result, CURLOPT_LOW_SPEED_LIMIT, + curl_low_speed_limit); + curl_easy_setopt(result, CURLOPT_LOW_SPEED_TIME, + curl_low_speed_time); + } + + return result; +} + +static struct active_request_slot *get_active_slot(void) { struct active_request_slot *slot = active_queue_head; struct active_request_slot *newslot; @@ -235,7 +294,7 @@ struct active_request_slot *get_active_slot() } if (slot == NULL) { newslot = xmalloc(sizeof(*newslot)); - newslot->curl = curl_easy_duphandle(curl_default); + newslot->curl = NULL; newslot->in_use = 0; newslot->next = NULL; @@ -251,6 +310,14 @@ struct active_request_slot *get_active_slot() slot = newslot; } + if (slot->curl == NULL) { +#ifdef NO_CURL_EASY_DUPHANDLE + slot->curl = get_curl_handle(); +#else + slot->curl = curl_easy_duphandle(curl_default); +#endif + } + active_requests++; slot->in_use = 1; slot->done = 0; @@ -262,7 +329,7 @@ struct active_request_slot *get_active_slot() return slot; } -int start_active_slot(struct active_request_slot *slot) +static int start_active_slot(struct active_request_slot *slot) { #ifdef USE_CURL_MULTI CURLMcode curlm_result = curl_multi_add_handle(curlm, slot->curl); @@ -277,7 +344,7 @@ int start_active_slot(struct active_request_slot *slot) return 1; } -void run_active_slot(struct active_request_slot *slot) +static void run_active_slot(struct active_request_slot *slot) { #ifdef USE_CURL_MULTI int num_transfers; @@ -325,7 +392,7 @@ void run_active_slot(struct active_request_slot *slot) #endif } -void start_request(struct transfer_request *request) +static void start_request(struct transfer_request *request) { char *hex = sha1_to_hex(request->sha1); char prevfile[PATH_MAX]; @@ -451,7 +518,7 @@ void start_request(struct transfer_request *request) request->state = ACTIVE; } -void finish_request(struct transfer_request *request) +static void finish_request(struct transfer_request *request) { fchmod(request->local, 0444); close(request->local); @@ -479,7 +546,7 @@ void finish_request(struct transfer_request *request) pull_say("got %s\n", sha1_to_hex(request->sha1)); } -void release_request(struct transfer_request *request) +static void release_request(struct transfer_request *request) { struct transfer_request *entry = request_queue_head; @@ -497,7 +564,7 @@ void release_request(struct transfer_request *request) } #ifdef USE_CURL_MULTI -void process_curl_messages() +void process_curl_messages(void) { int num_messages; struct active_request_slot *slot; @@ -516,6 +583,9 @@ void process_curl_messages() slot->done = 1; slot->in_use = 0; slot->curl_result = curl_message->data.result; + curl_easy_getinfo(slot->curl, + CURLINFO_HTTP_CODE, + &slot->http_code); request = request_queue_head; while (request != NULL && request->slot != slot) @@ -526,19 +596,20 @@ void process_curl_messages() if (request != NULL) { request->curl_result = curl_message->data.result; - curl_easy_getinfo(slot->curl, - CURLINFO_HTTP_CODE, - &request->http_code); + request->http_code = slot->http_code; request->slot = NULL; + request->state = COMPLETE; /* Use alternates if necessary */ - if (request->http_code == 404 && - request->repo->next != NULL) { - request->repo = request->repo->next; - start_request(request); + if (request->http_code == 404) { + fetch_alternates(alt->base); + if (request->repo->next != NULL) { + request->repo = + request->repo->next; + start_request(request); + } } else { finish_request(request); - request->state = COMPLETE; } } } else { @@ -549,9 +620,10 @@ void process_curl_messages() } } -void process_request_queue() +void process_request_queue(void) { struct transfer_request *request = request_queue_head; + struct active_request_slot *slot = active_queue_head; int num_transfers; while (active_requests < max_requests && request != NULL) { @@ -564,6 +636,14 @@ void process_request_queue() } request = request->next; } + + while (slot != NULL) { + if (!slot->in_use && slot->curl != NULL) { + curl_easy_cleanup(slot->curl); + slot->curl = NULL; + } + slot = slot->next; + } } #endif @@ -692,6 +772,9 @@ static int fetch_alternates(char *base) struct active_request_slot *slot; + if (got_alternates) + return 0; + data = xmalloc(4096); buffer.size = 4096; buffer.posn = 0; @@ -724,6 +807,8 @@ static int fetch_alternates(char *base) run_active_slot(slot); if (slot->curl_result != CURLE_OK) { free(buffer.buffer); + if (slot->http_code == 404) + got_alternates = 1; return 0; } } @@ -795,6 +880,7 @@ static int fetch_alternates(char *base) i = posn + 1; } + got_alternates = 1; free(buffer.buffer); return ret; } @@ -970,8 +1056,8 @@ static int fetch_object(struct alt_base *repo, unsigned char *sha1) } #ifdef USE_CURL_MULTI - int num_transfers; while (request->state == WAITING) { + int num_transfers; curl_multi_perform(curlm, &num_transfers); if (num_transfers < active_requests) { process_curl_messages(); @@ -986,16 +1072,16 @@ static int fetch_object(struct alt_base *repo, unsigned char *sha1) run_active_slot(request->slot); #ifndef USE_CURL_MULTI request->curl_result = request->slot->curl_result; - curl_easy_getinfo(request->slot->curl, - CURLINFO_HTTP_CODE, - &request->http_code); + request->http_code = request->slot->http_code; request->slot = NULL; /* Use alternates if necessary */ - if (request->http_code == 404 && - request->repo->next != NULL) { - request->repo = request->repo->next; - start_request(request); + if (request->http_code == 404) { + fetch_alternates(alt->base); + if (request->repo->next != NULL) { + request->repo = request->repo->next; + start_request(request); + } } else { finish_request(request); request->state = COMPLETE; @@ -1009,7 +1095,12 @@ static int fetch_object(struct alt_base *repo, unsigned char *sha1) } if (request->curl_result != CURLE_OK && request->http_code != 416) { - ret = error("%s", request->errorstr); + if (request->http_code == 404) + ret = -1; /* Be silent, it is probably in a pack. */ + else + ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)", + request->errorstr, request->curl_result, + request->http_code, hex); release_request(request); return ret; } @@ -1046,6 +1137,7 @@ int fetch(unsigned char *sha1) while (altbase) { if (!fetch_pack(altbase, sha1)) return 0; + fetch_alternates(alt->base); altbase = altbase->next; } return error("Unable to find %s under %s\n", sha1_to_hex(sha1), @@ -1135,6 +1227,10 @@ int main(int argc, char **argv) char *url; int arg = 1; struct active_request_slot *slot; + char *low_speed_limit; + char *low_speed_time; + char *wait_url; + int rc = 0; while (arg < argc && argv[arg][0] == '-') { if (argv[arg][1] == 't') { @@ -1165,9 +1261,11 @@ int main(int argc, char **argv) curl_global_init(CURL_GLOBAL_ALL); #ifdef USE_CURL_MULTI - char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS"); - if (http_max_requests != NULL) - max_requests = atoi(http_max_requests); + { + char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS"); + if (http_max_requests != NULL) + max_requests = atoi(http_max_requests); + } curlm = curl_multi_init(); if (curlm == NULL) { @@ -1188,6 +1286,13 @@ int main(int argc, char **argv) #endif ssl_cainfo = getenv("GIT_SSL_CAINFO"); + low_speed_limit = getenv("GIT_HTTP_LOW_SPEED_LIMIT"); + if (low_speed_limit != NULL) + curl_low_speed_limit = strtol(low_speed_limit, NULL, 10); + low_speed_time = getenv("GIT_HTTP_LOW_SPEED_TIME"); + if (low_speed_time != NULL) + curl_low_speed_time = strtol(low_speed_time, NULL, 10); + git_config(http_options); if (curl_ssl_verify == -1) @@ -1202,46 +1307,43 @@ int main(int argc, char **argv) no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:"); no_range_header = curl_slist_append(no_range_header, "Range:"); - curl_default = curl_easy_init(); - - curl_easy_setopt(curl_default, CURLOPT_SSL_VERIFYPEER, curl_ssl_verify); -#if LIBCURL_VERSION_NUM >= 0x070907 - curl_easy_setopt(curl_default, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); +#ifndef NO_CURL_EASY_DUPHANDLE + curl_default = get_curl_handle(); #endif - if (ssl_cert != NULL) - curl_easy_setopt(curl_default, CURLOPT_SSLCERT, ssl_cert); - if (ssl_key != NULL) - curl_easy_setopt(curl_default, CURLOPT_SSLKEY, ssl_key); - if (ssl_capath != NULL) - curl_easy_setopt(curl_default, CURLOPT_CAPATH, ssl_capath); - if (ssl_cainfo != NULL) - curl_easy_setopt(curl_default, CURLOPT_CAINFO, ssl_cainfo); - - curl_easy_setopt(curl_default, CURLOPT_FAILONERROR, 1); - alt = xmalloc(sizeof(*alt)); alt->base = url; alt->got_indices = 0; alt->packs = NULL; alt->next = NULL; - fetch_alternates(alt->base); if (pull(commit_id)) - return 1; + rc = 1; curl_slist_free_all(pragma_header); curl_slist_free_all(no_pragma_header); curl_slist_free_all(no_range_header); +#ifndef NO_CURL_EASY_DUPHANDLE curl_easy_cleanup(curl_default); +#endif slot = active_queue_head; while (slot != NULL) { - curl_easy_cleanup(slot->curl); + if (slot->in_use) { + if (get_verbosely) { + curl_easy_getinfo(slot->curl, + CURLINFO_EFFECTIVE_URL, + &wait_url); + fprintf(stderr, "Waiting for %s\n", wait_url); + } + run_active_slot(slot); + } + if (slot->curl != NULL) + curl_easy_cleanup(slot->curl); slot = slot->next; } #ifdef USE_CURL_MULTI curl_multi_cleanup(curlm); #endif curl_global_cleanup(); - return 0; + return rc; }