X-Git-Url: https://git.octo.it/?a=blobdiff_plain;f=http-fetch.c;h=a1b03cd9c842e9daa8f743939d4382faac0c534f;hb=23d61f8343282643c830e1c446962b213dbcc09a;hp=0aba8917548b3db9a43ce9802caa1c90176c70c1;hpb=94fa447ace56bc6514bdd66cafe8f56f0e3ce936;p=git.git diff --git a/http-fetch.c b/http-fetch.c index 0aba8917..a1b03cd9 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -18,17 +18,24 @@ #define curl_global_init(a) do { /* nothing */ } while(0) #endif +#if LIBCURL_VERSION_NUM < 0x070c04 +#define NO_CURL_EASY_DUPHANDLE +#endif + #define PREV_BUF_SIZE 4096 #define RANGE_HEADER_SIZE 30 +static int got_alternates = 0; static int active_requests = 0; static int data_received; #ifdef USE_CURL_MULTI -static int max_requests = DEFAULT_MAX_REQUESTS; +static int max_requests = -1; static CURLM *curlm; #endif +#ifndef NO_CURL_EASY_DUPHANDLE static CURL *curl_default; +#endif static struct curl_slist *pragma_header; static struct curl_slist *no_pragma_header; static struct curl_slist *no_range_header; @@ -79,17 +86,24 @@ struct active_request_slot int in_use; int done; CURLcode curl_result; + long http_code; struct active_request_slot *next; }; static struct transfer_request *request_queue_head = NULL; static struct active_request_slot *active_queue_head = NULL; -static int curl_ssl_verify; -static char *ssl_cert; -static char *ssl_key; -static char *ssl_capath; -static char *ssl_cainfo; +static int curl_ssl_verify = -1; +static char *ssl_cert = NULL; +#if LIBCURL_VERSION_NUM >= 0x070902 +static char *ssl_key = NULL; +#endif +#if LIBCURL_VERSION_NUM >= 0x070908 +static char *ssl_capath = NULL; +#endif +static char *ssl_cainfo = NULL; +static long curl_low_speed_limit = -1; +static long curl_low_speed_time = -1; struct buffer { @@ -98,6 +112,71 @@ struct buffer void *buffer; }; +static int http_options(const char *var, const char *value) +{ + if (!strcmp("http.sslverify", var)) { + if (curl_ssl_verify == -1) { + curl_ssl_verify = git_config_bool(var, value); + } + return 0; + } + + if (!strcmp("http.sslcert", var)) { + if (ssl_cert == NULL) { + ssl_cert = xmalloc(strlen(value)+1); + strcpy(ssl_cert, value); + } + return 0; + } +#if LIBCURL_VERSION_NUM >= 0x070902 + if (!strcmp("http.sslkey", var)) { + if (ssl_key == NULL) { + ssl_key = xmalloc(strlen(value)+1); + strcpy(ssl_key, value); + } + return 0; + } +#endif +#if LIBCURL_VERSION_NUM >= 0x070908 + if (!strcmp("http.sslcapath", var)) { + if (ssl_capath == NULL) { + ssl_capath = xmalloc(strlen(value)+1); + strcpy(ssl_capath, value); + } + return 0; + } +#endif + if (!strcmp("http.sslcainfo", var)) { + if (ssl_cainfo == NULL) { + ssl_cainfo = xmalloc(strlen(value)+1); + strcpy(ssl_cainfo, value); + } + return 0; + } + +#ifdef USE_CURL_MULTI + if (!strcmp("http.maxrequests", var)) { + if (max_requests == -1) + max_requests = git_config_int(var, value); + return 0; + } +#endif + + if (!strcmp("http.lowspeedlimit", var)) { + if (curl_low_speed_limit == -1) + curl_low_speed_limit = (long)git_config_int(var, value); + return 0; + } + if (!strcmp("http.lowspeedtime", var)) { + if (curl_low_speed_time == -1) + curl_low_speed_time = (long)git_config_int(var, value); + return 0; + } + + /* Fall back on the default ones */ + return git_default_config(var, value); +} + static size_t fwrite_buffer(void *ptr, size_t eltsize, size_t nmemb, struct buffer *buffer) { @@ -110,6 +189,22 @@ static size_t fwrite_buffer(void *ptr, size_t eltsize, size_t nmemb, return size; } +static size_t fwrite_buffer_dynamic(const void *ptr, size_t eltsize, + size_t nmemb, struct buffer *buffer) +{ + size_t size = eltsize * nmemb; + if (size > buffer->size - buffer->posn) { + buffer->size = buffer->size * 3 / 2; + if (buffer->size < buffer->posn + size) + buffer->size = buffer->posn + size; + buffer->buffer = xrealloc(buffer->buffer, buffer->size); + } + memcpy(buffer->buffer + buffer->posn, ptr, size); + buffer->posn += size; + data_received++; + return size; +} + static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb, void *data) { @@ -139,11 +234,45 @@ static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb, } #ifdef USE_CURL_MULTI -void process_curl_messages(); -void process_request_queue(); +static void process_curl_messages(void); +static void process_request_queue(void); +#endif +static int fetch_alternates(char *base); + +static CURL* get_curl_handle(void) +{ + CURL* result = curl_easy_init(); + + curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, curl_ssl_verify); +#if LIBCURL_VERSION_NUM >= 0x070907 + curl_easy_setopt(result, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); #endif -struct active_request_slot *get_active_slot() + if (ssl_cert != NULL) + curl_easy_setopt(result, CURLOPT_SSLCERT, ssl_cert); +#if LIBCURL_VERSION_NUM >= 0x070902 + if (ssl_key != NULL) + curl_easy_setopt(result, CURLOPT_SSLKEY, ssl_key); +#endif +#if LIBCURL_VERSION_NUM >= 0x070908 + if (ssl_capath != NULL) + curl_easy_setopt(result, CURLOPT_CAPATH, ssl_capath); +#endif + if (ssl_cainfo != NULL) + curl_easy_setopt(result, CURLOPT_CAINFO, ssl_cainfo); + curl_easy_setopt(result, CURLOPT_FAILONERROR, 1); + + if (curl_low_speed_limit > 0 && curl_low_speed_time > 0) { + curl_easy_setopt(result, CURLOPT_LOW_SPEED_LIMIT, + curl_low_speed_limit); + curl_easy_setopt(result, CURLOPT_LOW_SPEED_TIME, + curl_low_speed_time); + } + + return result; +} + +static struct active_request_slot *get_active_slot(void) { struct active_request_slot *slot = active_queue_head; struct active_request_slot *newslot; @@ -165,7 +294,7 @@ struct active_request_slot *get_active_slot() } if (slot == NULL) { newslot = xmalloc(sizeof(*newslot)); - newslot->curl = curl_easy_duphandle(curl_default); + newslot->curl = NULL; newslot->in_use = 0; newslot->next = NULL; @@ -181,6 +310,14 @@ struct active_request_slot *get_active_slot() slot = newslot; } + if (slot->curl == NULL) { +#ifdef NO_CURL_EASY_DUPHANDLE + slot->curl = get_curl_handle(); +#else + slot->curl = curl_easy_duphandle(curl_default); +#endif + } + active_requests++; slot->in_use = 1; slot->done = 0; @@ -192,7 +329,7 @@ struct active_request_slot *get_active_slot() return slot; } -int start_active_slot(struct active_request_slot *slot) +static int start_active_slot(struct active_request_slot *slot) { #ifdef USE_CURL_MULTI CURLMcode curlm_result = curl_multi_add_handle(curlm, slot->curl); @@ -207,7 +344,7 @@ int start_active_slot(struct active_request_slot *slot) return 1; } -void run_active_slot(struct active_request_slot *slot) +static void run_active_slot(struct active_request_slot *slot) { #ifdef USE_CURL_MULTI int num_transfers; @@ -255,7 +392,7 @@ void run_active_slot(struct active_request_slot *slot) #endif } -void start_request(struct transfer_request *request) +static void start_request(struct transfer_request *request) { char *hex = sha1_to_hex(request->sha1); char prevfile[PATH_MAX]; @@ -381,7 +518,7 @@ void start_request(struct transfer_request *request) request->state = ACTIVE; } -void finish_request(struct transfer_request *request) +static void finish_request(struct transfer_request *request) { fchmod(request->local, 0444); close(request->local); @@ -409,7 +546,7 @@ void finish_request(struct transfer_request *request) pull_say("got %s\n", sha1_to_hex(request->sha1)); } -void release_request(struct transfer_request *request) +static void release_request(struct transfer_request *request) { struct transfer_request *entry = request_queue_head; @@ -427,7 +564,7 @@ void release_request(struct transfer_request *request) } #ifdef USE_CURL_MULTI -void process_curl_messages() +void process_curl_messages(void) { int num_messages; struct active_request_slot *slot; @@ -446,6 +583,9 @@ void process_curl_messages() slot->done = 1; slot->in_use = 0; slot->curl_result = curl_message->data.result; + curl_easy_getinfo(slot->curl, + CURLINFO_HTTP_CODE, + &slot->http_code); request = request_queue_head; while (request != NULL && request->slot != slot) @@ -456,19 +596,20 @@ void process_curl_messages() if (request != NULL) { request->curl_result = curl_message->data.result; - curl_easy_getinfo(slot->curl, - CURLINFO_HTTP_CODE, - &request->http_code); + request->http_code = slot->http_code; request->slot = NULL; + request->state = COMPLETE; /* Use alternates if necessary */ - if (request->http_code == 404 && - request->repo->next != NULL) { - request->repo = request->repo->next; - start_request(request); + if (request->http_code == 404) { + fetch_alternates(alt->base); + if (request->repo->next != NULL) { + request->repo = + request->repo->next; + start_request(request); + } } else { finish_request(request); - request->state = COMPLETE; } } } else { @@ -479,9 +620,10 @@ void process_curl_messages() } } -void process_request_queue() +void process_request_queue(void) { struct transfer_request *request = request_queue_head; + struct active_request_slot *slot = active_queue_head; int num_transfers; while (active_requests < max_requests && request != NULL) { @@ -494,6 +636,14 @@ void process_request_queue() } request = request->next; } + + while (slot != NULL) { + if (!slot->in_use && slot->curl != NULL) { + curl_easy_cleanup(slot->curl); + slot->curl = NULL; + } + slot = slot->next; + } } #endif @@ -618,11 +768,15 @@ static int fetch_alternates(char *base) int i = 0; int http_specific = 1; struct alt_base *tail = alt; + static const char null_byte = '\0'; struct active_request_slot *slot; + if (got_alternates) + return 0; + data = xmalloc(4096); - buffer.size = 4095; + buffer.size = 4096; buffer.posn = 0; buffer.buffer = data; @@ -634,7 +788,8 @@ static int fetch_alternates(char *base) slot = get_active_slot(); curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer); - curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer); + curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, + fwrite_buffer_dynamic); curl_easy_setopt(slot->curl, CURLOPT_URL, url); if (start_active_slot(slot)) { run_active_slot(slot); @@ -646,20 +801,26 @@ static int fetch_alternates(char *base) slot = get_active_slot(); curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer); curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, - fwrite_buffer); + fwrite_buffer_dynamic); curl_easy_setopt(slot->curl, CURLOPT_URL, url); if (start_active_slot(slot)) { run_active_slot(slot); if (slot->curl_result != CURLE_OK) { + free(buffer.buffer); + if (slot->http_code == 404) + got_alternates = 1; return 0; } } } } else { + free(buffer.buffer); return 0; } - data[buffer.posn] = '\0'; + fwrite_buffer_dynamic(&null_byte, 1, 1, &buffer); + buffer.posn--; + data = buffer.buffer; while (i < buffer.posn) { int posn = i; @@ -718,7 +879,9 @@ static int fetch_alternates(char *base) } i = posn + 1; } - + + got_alternates = 1; + free(buffer.buffer); return ret; } @@ -748,17 +911,22 @@ static int fetch_indices(struct alt_base *repo) slot = get_active_slot(); curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer); - curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer); + curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, + fwrite_buffer_dynamic); curl_easy_setopt(slot->curl, CURLOPT_URL, url); curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL); if (start_active_slot(slot)) { run_active_slot(slot); - if (slot->curl_result != CURLE_OK) + if (slot->curl_result != CURLE_OK) { + free(buffer.buffer); return error("%s", curl_errorstr); + } } else { + free(buffer.buffer); return error("Unable to start request"); } + data = buffer.buffer; while (i < buffer.posn) { switch (data[i]) { case 'P': @@ -778,6 +946,7 @@ static int fetch_indices(struct alt_base *repo) i++; } + free(buffer.buffer); repo->got_indices = 1; return 0; } @@ -887,8 +1056,8 @@ static int fetch_object(struct alt_base *repo, unsigned char *sha1) } #ifdef USE_CURL_MULTI - int num_transfers; while (request->state == WAITING) { + int num_transfers; curl_multi_perform(curlm, &num_transfers); if (num_transfers < active_requests) { process_curl_messages(); @@ -903,16 +1072,16 @@ static int fetch_object(struct alt_base *repo, unsigned char *sha1) run_active_slot(request->slot); #ifndef USE_CURL_MULTI request->curl_result = request->slot->curl_result; - curl_easy_getinfo(request->slot->curl, - CURLINFO_HTTP_CODE, - &request->http_code); + request->http_code = request->slot->http_code; request->slot = NULL; /* Use alternates if necessary */ - if (request->http_code == 404 && - request->repo->next != NULL) { - request->repo = request->repo->next; - start_request(request); + if (request->http_code == 404) { + fetch_alternates(alt->base); + if (request->repo->next != NULL) { + request->repo = request->repo->next; + start_request(request); + } } else { finish_request(request); request->state = COMPLETE; @@ -926,7 +1095,12 @@ static int fetch_object(struct alt_base *repo, unsigned char *sha1) } if (request->curl_result != CURLE_OK && request->http_code != 416) { - ret = error("%s", request->errorstr); + if (request->http_code == 404) + ret = -1; /* Be silent, it is probably in a pack. */ + else + ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)", + request->errorstr, request->curl_result, + request->http_code, hex); release_request(request); return ret; } @@ -963,6 +1137,7 @@ int fetch(unsigned char *sha1) while (altbase) { if (!fetch_pack(altbase, sha1)) return 0; + fetch_alternates(alt->base); altbase = altbase->next; } return error("Unable to find %s under %s\n", sha1_to_hex(sha1), @@ -1052,6 +1227,10 @@ int main(int argc, char **argv) char *url; int arg = 1; struct active_request_slot *slot; + char *low_speed_limit; + char *low_speed_time; + char *wait_url; + int rc = 0; while (arg < argc && argv[arg][0] == '-') { if (argv[arg][1] == 't') { @@ -1082,11 +1261,11 @@ int main(int argc, char **argv) curl_global_init(CURL_GLOBAL_ALL); #ifdef USE_CURL_MULTI - char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS"); - if (http_max_requests != NULL) - max_requests = atoi(http_max_requests); - if (max_requests < 1) - max_requests = DEFAULT_MAX_REQUESTS; + { + char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS"); + if (http_max_requests != NULL) + max_requests = atoi(http_max_requests); + } curlm = curl_multi_init(); if (curlm == NULL) { @@ -1094,58 +1273,77 @@ int main(int argc, char **argv) return 1; } #endif - pragma_header = curl_slist_append(pragma_header, "Pragma: no-cache"); - no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:"); - no_range_header = curl_slist_append(no_range_header, "Range:"); - - curl_default = curl_easy_init(); - curl_ssl_verify = getenv("GIT_SSL_NO_VERIFY") ? 0 : 1; - curl_easy_setopt(curl_default, CURLOPT_SSL_VERIFYPEER, curl_ssl_verify); -#if LIBCURL_VERSION_NUM >= 0x070907 - curl_easy_setopt(curl_default, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); -#endif + if (getenv("GIT_SSL_NO_VERIFY")) + curl_ssl_verify = 0; - if ((ssl_cert = getenv("GIT_SSL_CERT")) != NULL) { - curl_easy_setopt(curl_default, CURLOPT_SSLCERT, ssl_cert); - } + ssl_cert = getenv("GIT_SSL_CERT"); #if LIBCURL_VERSION_NUM >= 0x070902 - if ((ssl_key = getenv("GIT_SSL_KEY")) != NULL) { - curl_easy_setopt(curl_default, CURLOPT_SSLKEY, ssl_key); - } + ssl_key = getenv("GIT_SSL_KEY"); #endif #if LIBCURL_VERSION_NUM >= 0x070908 - if ((ssl_capath = getenv("GIT_SSL_CAPATH")) != NULL) { - curl_easy_setopt(curl_default, CURLOPT_CAPATH, ssl_capath); - } + ssl_capath = getenv("GIT_SSL_CAPATH"); +#endif + ssl_cainfo = getenv("GIT_SSL_CAINFO"); + + low_speed_limit = getenv("GIT_HTTP_LOW_SPEED_LIMIT"); + if (low_speed_limit != NULL) + curl_low_speed_limit = strtol(low_speed_limit, NULL, 10); + low_speed_time = getenv("GIT_HTTP_LOW_SPEED_TIME"); + if (low_speed_time != NULL) + curl_low_speed_time = strtol(low_speed_time, NULL, 10); + + git_config(http_options); + + if (curl_ssl_verify == -1) + curl_ssl_verify = 1; + +#ifdef USE_CURL_MULTI + if (max_requests < 1) + max_requests = DEFAULT_MAX_REQUESTS; +#endif + + pragma_header = curl_slist_append(pragma_header, "Pragma: no-cache"); + no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:"); + no_range_header = curl_slist_append(no_range_header, "Range:"); + +#ifndef NO_CURL_EASY_DUPHANDLE + curl_default = get_curl_handle(); #endif - if ((ssl_cainfo = getenv("GIT_SSL_CAINFO")) != NULL) { - curl_easy_setopt(curl_default, CURLOPT_CAINFO, ssl_cainfo); - } - curl_easy_setopt(curl_default, CURLOPT_FAILONERROR, 1); alt = xmalloc(sizeof(*alt)); alt->base = url; alt->got_indices = 0; alt->packs = NULL; alt->next = NULL; - fetch_alternates(alt->base); if (pull(commit_id)) - return 1; + rc = 1; curl_slist_free_all(pragma_header); curl_slist_free_all(no_pragma_header); curl_slist_free_all(no_range_header); +#ifndef NO_CURL_EASY_DUPHANDLE curl_easy_cleanup(curl_default); +#endif slot = active_queue_head; while (slot != NULL) { - curl_easy_cleanup(slot->curl); + if (slot->in_use) { + if (get_verbosely) { + curl_easy_getinfo(slot->curl, + CURLINFO_EFFECTIVE_URL, + &wait_url); + fprintf(stderr, "Waiting for %s\n", wait_url); + } + run_active_slot(slot); + } + if (slot->curl != NULL) + curl_easy_cleanup(slot->curl); slot = slot->next; } #ifdef USE_CURL_MULTI curl_multi_cleanup(curlm); #endif curl_global_cleanup(); - return 0; + return rc; }