X-Git-Url: https://git.octo.it/?a=blobdiff_plain;f=http-fetch.c;h=6a9eaf3a5a9afbe1573f0cb9d82627e4001f3a5b;hb=29508e1efb8d5a18f78ca0f9be45fdde49ef6b76;hp=b8aa965ea32f2956fc162d0a3cee9923808e165f;hpb=6d5410d65137dc76a504b550872614170e2d98b3;p=git.git diff --git a/http-fetch.c b/http-fetch.c index b8aa965e..6a9eaf3a 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -2,44 +2,14 @@ #include "commit.h" #include "pack.h" #include "fetch.h" - -#include -#include - -#if LIBCURL_VERSION_NUM >= 0x070908 -#define USE_CURL_MULTI -#define DEFAULT_MAX_REQUESTS 5 -#endif - -#if LIBCURL_VERSION_NUM < 0x070704 -#define curl_global_cleanup() do { /* nothing */ } while(0) -#endif -#if LIBCURL_VERSION_NUM < 0x070800 -#define curl_global_init(a) do { /* nothing */ } while(0) -#endif - -#if LIBCURL_VERSION_NUM < 0x070c04 -#define NO_CURL_EASY_DUPHANDLE -#endif +#include "http.h" #define PREV_BUF_SIZE 4096 #define RANGE_HEADER_SIZE 30 static int got_alternates = -1; -static int active_requests = 0; -static int data_received; -#ifdef USE_CURL_MULTI -static int max_requests = -1; -static CURLM *curlm; -#endif -#ifndef NO_CURL_EASY_DUPHANDLE -static CURL *curl_default; -#endif -static struct curl_slist *pragma_header; static struct curl_slist *no_pragma_header; -static struct curl_slist *no_range_header; -static char curl_errorstr[CURL_ERROR_SIZE]; struct alt_base { @@ -79,19 +49,6 @@ struct transfer_request struct transfer_request *next; }; -struct active_request_slot -{ - CURL *curl; - FILE *local; - int in_use; - int done; - CURLcode curl_result; - long http_code; - void *callback_data; - void (*callback_func)(void *data); - struct active_request_slot *next; -}; - struct alt_request { char *base; char *url; @@ -101,119 +58,6 @@ struct alt_request { }; static struct transfer_request *request_queue_head = NULL; -static struct active_request_slot *active_queue_head = NULL; - -static int curl_ssl_verify = -1; -static char *ssl_cert = NULL; -#if LIBCURL_VERSION_NUM >= 0x070902 -static char *ssl_key = NULL; -#endif -#if LIBCURL_VERSION_NUM >= 0x070908 -static char *ssl_capath = NULL; -#endif -static char *ssl_cainfo = NULL; -static long curl_low_speed_limit = -1; -static long curl_low_speed_time = -1; - -struct buffer -{ - size_t posn; - size_t size; - void *buffer; -}; - -static int http_options(const char *var, const char *value) -{ - if (!strcmp("http.sslverify", var)) { - if (curl_ssl_verify == -1) { - curl_ssl_verify = git_config_bool(var, value); - } - return 0; - } - - if (!strcmp("http.sslcert", var)) { - if (ssl_cert == NULL) { - ssl_cert = xmalloc(strlen(value)+1); - strcpy(ssl_cert, value); - } - return 0; - } -#if LIBCURL_VERSION_NUM >= 0x070902 - if (!strcmp("http.sslkey", var)) { - if (ssl_key == NULL) { - ssl_key = xmalloc(strlen(value)+1); - strcpy(ssl_key, value); - } - return 0; - } -#endif -#if LIBCURL_VERSION_NUM >= 0x070908 - if (!strcmp("http.sslcapath", var)) { - if (ssl_capath == NULL) { - ssl_capath = xmalloc(strlen(value)+1); - strcpy(ssl_capath, value); - } - return 0; - } -#endif - if (!strcmp("http.sslcainfo", var)) { - if (ssl_cainfo == NULL) { - ssl_cainfo = xmalloc(strlen(value)+1); - strcpy(ssl_cainfo, value); - } - return 0; - } - -#ifdef USE_CURL_MULTI - if (!strcmp("http.maxrequests", var)) { - if (max_requests == -1) - max_requests = git_config_int(var, value); - return 0; - } -#endif - - if (!strcmp("http.lowspeedlimit", var)) { - if (curl_low_speed_limit == -1) - curl_low_speed_limit = (long)git_config_int(var, value); - return 0; - } - if (!strcmp("http.lowspeedtime", var)) { - if (curl_low_speed_time == -1) - curl_low_speed_time = (long)git_config_int(var, value); - return 0; - } - - /* Fall back on the default ones */ - return git_default_config(var, value); -} - -static size_t fwrite_buffer(void *ptr, size_t eltsize, size_t nmemb, - struct buffer *buffer) -{ - size_t size = eltsize * nmemb; - if (size > buffer->size - buffer->posn) - size = buffer->size - buffer->posn; - memcpy(buffer->buffer + buffer->posn, ptr, size); - buffer->posn += size; - data_received++; - return size; -} - -static size_t fwrite_buffer_dynamic(const void *ptr, size_t eltsize, - size_t nmemb, struct buffer *buffer) -{ - size_t size = eltsize * nmemb; - if (size > buffer->size - buffer->posn) { - buffer->size = buffer->size * 3 / 2; - if (buffer->size < buffer->posn + size) - buffer->size = buffer->posn + size; - buffer->buffer = xrealloc(buffer->buffer, buffer->size); - } - memcpy(buffer->buffer + buffer->posn, ptr, size); - buffer->posn += size; - data_received++; - return size; -} static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb, void *data) @@ -243,168 +87,9 @@ static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb, return size; } -#ifdef USE_CURL_MULTI -static void process_curl_messages(void); -static void process_request_queue(void); -#endif static void fetch_alternates(char *base); -static CURL* get_curl_handle(void) -{ - CURL* result = curl_easy_init(); - - curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, curl_ssl_verify); -#if LIBCURL_VERSION_NUM >= 0x070907 - curl_easy_setopt(result, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); -#endif - - if (ssl_cert != NULL) - curl_easy_setopt(result, CURLOPT_SSLCERT, ssl_cert); -#if LIBCURL_VERSION_NUM >= 0x070902 - if (ssl_key != NULL) - curl_easy_setopt(result, CURLOPT_SSLKEY, ssl_key); -#endif -#if LIBCURL_VERSION_NUM >= 0x070908 - if (ssl_capath != NULL) - curl_easy_setopt(result, CURLOPT_CAPATH, ssl_capath); -#endif - if (ssl_cainfo != NULL) - curl_easy_setopt(result, CURLOPT_CAINFO, ssl_cainfo); - curl_easy_setopt(result, CURLOPT_FAILONERROR, 1); - - if (curl_low_speed_limit > 0 && curl_low_speed_time > 0) { - curl_easy_setopt(result, CURLOPT_LOW_SPEED_LIMIT, - curl_low_speed_limit); - curl_easy_setopt(result, CURLOPT_LOW_SPEED_TIME, - curl_low_speed_time); - } - - curl_easy_setopt(result, CURLOPT_FOLLOWLOCATION, 1); - - return result; -} - -static struct active_request_slot *get_active_slot(void) -{ - struct active_request_slot *slot = active_queue_head; - struct active_request_slot *newslot; - -#ifdef USE_CURL_MULTI - int num_transfers; - - /* Wait for a slot to open up if the queue is full */ - while (active_requests >= max_requests) { - curl_multi_perform(curlm, &num_transfers); - if (num_transfers < active_requests) { - process_curl_messages(); - } - } -#endif - - while (slot != NULL && slot->in_use) { - slot = slot->next; - } - if (slot == NULL) { - newslot = xmalloc(sizeof(*newslot)); - newslot->curl = NULL; - newslot->in_use = 0; - newslot->next = NULL; - - slot = active_queue_head; - if (slot == NULL) { - active_queue_head = newslot; - } else { - while (slot->next != NULL) { - slot = slot->next; - } - slot->next = newslot; - } - slot = newslot; - } - - if (slot->curl == NULL) { -#ifdef NO_CURL_EASY_DUPHANDLE - slot->curl = get_curl_handle(); -#else - slot->curl = curl_easy_duphandle(curl_default); -#endif - } - - active_requests++; - slot->in_use = 1; - slot->done = 0; - slot->local = NULL; - slot->callback_data = NULL; - slot->callback_func = NULL; - curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, pragma_header); - curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_range_header); - curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr); - - return slot; -} - -static int start_active_slot(struct active_request_slot *slot) -{ -#ifdef USE_CURL_MULTI - CURLMcode curlm_result = curl_multi_add_handle(curlm, slot->curl); - - if (curlm_result != CURLM_OK && - curlm_result != CURLM_CALL_MULTI_PERFORM) { - active_requests--; - slot->in_use = 0; - return 0; - } -#endif - return 1; -} - -static void run_active_slot(struct active_request_slot *slot) -{ -#ifdef USE_CURL_MULTI - int num_transfers; - long last_pos = 0; - long current_pos; - fd_set readfds; - fd_set writefds; - fd_set excfds; - int max_fd; - struct timeval select_timeout; - CURLMcode curlm_result; - - while (!slot->done) { - data_received = 0; - do { - curlm_result = curl_multi_perform(curlm, - &num_transfers); - } while (curlm_result == CURLM_CALL_MULTI_PERFORM); - if (num_transfers < active_requests) { - process_curl_messages(); - process_request_queue(); - } - - if (!data_received && slot->local != NULL) { - current_pos = ftell(slot->local); - if (current_pos > last_pos) - data_received++; - last_pos = current_pos; - } - - if (!slot->done && !data_received) { - max_fd = 0; - FD_ZERO(&readfds); - FD_ZERO(&writefds); - FD_ZERO(&excfds); - select_timeout.tv_sec = 0; - select_timeout.tv_usec = 50000; - select(max_fd, &readfds, &writefds, - &excfds, &select_timeout); - } - } -#else - slot->curl_result = curl_easy_perform(slot->curl); - active_requests--; -#endif -} +static void process_object_response(void *callback_data); static void start_request(struct transfer_request *request) { @@ -425,6 +110,8 @@ static void start_request(struct transfer_request *request) rename(request->tmpfile, prevfile); unlink(request->tmpfile); + if (request->local != -1) + error("fd leakage in start: %d", request->local); request->local = open(request->tmpfile, O_WRONLY | O_CREAT | O_EXCL, 0666); /* This could have failed due to the "lazy directory creation"; @@ -501,6 +188,10 @@ static void start_request(struct transfer_request *request) } slot = get_active_slot(); + slot->callback_func = process_object_response; + slot->callback_data = request; + request->slot = slot; + curl_easy_setopt(slot->curl, CURLOPT_FILE, request); curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file); curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, request->errorstr); @@ -521,15 +212,13 @@ static void start_request(struct transfer_request *request) } /* Try to get the request started, abort the request on error */ + request->state = ACTIVE; if (!start_active_slot(slot)) { request->state = ABORTED; - close(request->local); + request->slot = NULL; + close(request->local); request->local = -1; free(request->url); - return; } - - request->slot = slot; - request->state = ACTIVE; } static void finish_request(struct transfer_request *request) @@ -537,7 +226,7 @@ static void finish_request(struct transfer_request *request) struct stat st; fchmod(request->local, 0444); - close(request->local); + close(request->local); request->local = -1; if (request->http_code == 416) { fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n"); @@ -565,10 +254,38 @@ static void finish_request(struct transfer_request *request) pull_say("got %s\n", sha1_to_hex(request->sha1)); } +static void process_object_response(void *callback_data) +{ + struct transfer_request *request = + (struct transfer_request *)callback_data; + + request->curl_result = request->slot->curl_result; + request->http_code = request->slot->http_code; + request->slot = NULL; + request->state = COMPLETE; + + /* Use alternates if necessary */ + if (request->http_code == 404) { + fetch_alternates(alt->base); + if (request->repo->next != NULL) { + request->repo = + request->repo->next; + close(request->local); + request->local = -1; + start_request(request); + return; + } + } + + finish_request(request); +} + static void release_request(struct transfer_request *request) { struct transfer_request *entry = request_queue_head; + if (request->local != -1) + error("fd leakage in release: %d", request->local); if (request == request_queue_head) { request_queue_head = request->next; } else { @@ -583,71 +300,7 @@ static void release_request(struct transfer_request *request) } #ifdef USE_CURL_MULTI -static void process_curl_messages(void) -{ - int num_messages; - struct active_request_slot *slot; - struct transfer_request *request = NULL; - CURLMsg *curl_message = curl_multi_info_read(curlm, &num_messages); - - while (curl_message != NULL) { - if (curl_message->msg == CURLMSG_DONE) { - int curl_result = curl_message->data.result; - slot = active_queue_head; - while (slot != NULL && - slot->curl != curl_message->easy_handle) - slot = slot->next; - if (slot != NULL) { - curl_multi_remove_handle(curlm, slot->curl); - active_requests--; - slot->done = 1; - slot->in_use = 0; - slot->curl_result = curl_result; - curl_easy_getinfo(slot->curl, - CURLINFO_HTTP_CODE, - &slot->http_code); - request = request_queue_head; - while (request != NULL && - request->slot != slot) - request = request->next; - } else { - fprintf(stderr, "Received DONE message for unknown request!\n"); - } - - /* Process slot callback if appropriate */ - if (slot->callback_func != NULL) { - slot->callback_func(slot->callback_data); - } - - if (request != NULL) { - request->curl_result = curl_result; - request->http_code = slot->http_code; - request->slot = NULL; - request->state = COMPLETE; - - /* Use alternates if necessary */ - if (request->http_code == 404) { - fetch_alternates(alt->base); - if (request->repo->next != NULL) { - request->repo = - request->repo->next; - start_request(request); - } else { - finish_request(request); - } - } else { - finish_request(request); - } - } - } else { - fprintf(stderr, "Unknown CURL message received: %d\n", - (int)curl_message->msg); - } - curl_message = curl_multi_info_read(curlm, &num_messages); - } -} - -static void process_request_queue(void) +void fill_active_slots(void) { struct transfer_request *request = request_queue_head; struct active_request_slot *slot = active_queue_head; @@ -700,9 +353,10 @@ void prefetch(unsigned char *sha1) } tail->next = newreq; } + #ifdef USE_CURL_MULTI - process_request_queue(); - process_curl_messages(); + fill_active_slots(); + step_active_slots(); #endif } @@ -763,6 +417,7 @@ static int fetch_index(struct alt_base *repo, unsigned char *sha1) curl_errorstr); } } else { + fclose(indexfile); return error("Unable to start request"); } @@ -808,12 +463,11 @@ static void process_alternates(void *callback_data) alt_req->url); active_requests++; slot->in_use = 1; - slot->done = 0; if (start_active_slot(slot)) { return; } else { got_alternates = -1; - slot->done = 1; + slot->in_use = 0; return; } } @@ -824,7 +478,7 @@ static void process_alternates(void *callback_data) } } - fwrite_buffer_dynamic(&null_byte, 1, 1, alt_req->buffer); + fwrite_buffer(&null_byte, 1, 1, alt_req->buffer); alt_req->buffer->posn--; data = alt_req->buffer->buffer; @@ -895,16 +549,15 @@ static void fetch_alternates(char *base) char *data; struct active_request_slot *slot; static struct alt_request alt_req; - int num_transfers; /* If another request has already started fetching alternates, wait for them to arrive and return to processing this request's curl message */ +#ifdef USE_CURL_MULTI while (got_alternates == 0) { - curl_multi_perform(curlm, &num_transfers); - process_curl_messages(); - process_request_queue(); + step_active_slots(); } +#endif /* Nothing to do if they've already been fetched */ if (got_alternates == 1) @@ -931,8 +584,7 @@ static void fetch_alternates(char *base) slot->callback_data = &alt_req; curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer); - curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, - fwrite_buffer_dynamic); + curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer); curl_easy_setopt(slot->curl, CURLOPT_URL, url); alt_req.base = base; @@ -976,8 +628,7 @@ static int fetch_indices(struct alt_base *repo) slot = get_active_slot(); curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer); - curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, - fwrite_buffer_dynamic); + curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer); curl_easy_setopt(slot->curl, CURLOPT_URL, url); curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL); if (start_active_slot(slot)) { @@ -1083,6 +734,7 @@ static int fetch_pack(struct alt_base *repo, unsigned char *sha1) curl_errorstr); } } else { + fclose(packfile); return error("Unable to start request"); } @@ -1107,7 +759,7 @@ static int fetch_pack(struct alt_base *repo, unsigned char *sha1) static int fetch_object(struct alt_base *repo, unsigned char *sha1) { char *hex = sha1_to_hex(sha1); - int ret; + int ret = 0; struct transfer_request *request = request_queue_head; while (request != NULL && memcmp(request->sha1, sha1, 20)) @@ -1122,12 +774,7 @@ static int fetch_object(struct alt_base *repo, unsigned char *sha1) #ifdef USE_CURL_MULTI while (request->state == WAITING) { - int num_transfers; - curl_multi_perform(curlm, &num_transfers); - if (num_transfers < active_requests) { - process_curl_messages(); - process_request_queue(); - } + step_active_slots(); } #else start_request(request); @@ -1135,62 +782,33 @@ static int fetch_object(struct alt_base *repo, unsigned char *sha1) while (request->state == ACTIVE) { run_active_slot(request->slot); -#ifndef USE_CURL_MULTI - request->curl_result = request->slot->curl_result; - request->http_code = request->slot->http_code; - request->slot = NULL; - - /* Use alternates if necessary */ - if (request->http_code == 404) { - fetch_alternates(alt->base); - if (request->repo->next != NULL) { - request->repo = request->repo->next; - start_request(request); - } - } else { - finish_request(request); - request->state = COMPLETE; - } -#endif } - - if (request->state == ABORTED) { - release_request(request); - return error("Request for %s aborted", hex); + if (request->local != -1) { + close(request->local); request->local = -1; } - if (request->curl_result != CURLE_OK && request->http_code != 416) { + if (request->state == ABORTED) { + ret = error("Request for %s aborted", hex); + } else if (request->curl_result != CURLE_OK && + request->http_code != 416) { if (request->http_code == 404) ret = -1; /* Be silent, it is probably in a pack. */ else ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)", request->errorstr, request->curl_result, request->http_code, hex); - release_request(request); - return ret; - } - - if (request->zret != Z_STREAM_END) { + } else if (request->zret != Z_STREAM_END) { ret = error("File %s (%s) corrupt\n", hex, request->url); - release_request(request); - return ret; - } - - if (memcmp(request->sha1, request->real_sha1, 20)) { - release_request(request); - return error("File %s has bad hash\n", hex); - } - - if (request->rename < 0) { + } else if (memcmp(request->sha1, request->real_sha1, 20)) { + ret = error("File %s has bad hash\n", hex); + } else if (request->rename < 0) { ret = error("unable to write sha1 filename %s: %s", request->filename, strerror(request->rename)); - release_request(request); - return ret; } release_request(request); - return 0; + return ret; } int fetch(unsigned char *sha1) @@ -1291,10 +909,6 @@ int main(int argc, char **argv) char *commit_id; char *url; int arg = 1; - struct active_request_slot *slot; - char *low_speed_limit; - char *low_speed_time; - char *wait_url; int rc = 0; while (arg < argc && argv[arg][0] == '-') { @@ -1323,58 +937,9 @@ int main(int argc, char **argv) commit_id = argv[arg]; url = argv[arg + 1]; - curl_global_init(CURL_GLOBAL_ALL); - -#ifdef USE_CURL_MULTI - { - char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS"); - if (http_max_requests != NULL) - max_requests = atoi(http_max_requests); - } - - curlm = curl_multi_init(); - if (curlm == NULL) { - fprintf(stderr, "Error creating curl multi handle.\n"); - return 1; - } -#endif - - if (getenv("GIT_SSL_NO_VERIFY")) - curl_ssl_verify = 0; - - ssl_cert = getenv("GIT_SSL_CERT"); -#if LIBCURL_VERSION_NUM >= 0x070902 - ssl_key = getenv("GIT_SSL_KEY"); -#endif -#if LIBCURL_VERSION_NUM >= 0x070908 - ssl_capath = getenv("GIT_SSL_CAPATH"); -#endif - ssl_cainfo = getenv("GIT_SSL_CAINFO"); - - low_speed_limit = getenv("GIT_HTTP_LOW_SPEED_LIMIT"); - if (low_speed_limit != NULL) - curl_low_speed_limit = strtol(low_speed_limit, NULL, 10); - low_speed_time = getenv("GIT_HTTP_LOW_SPEED_TIME"); - if (low_speed_time != NULL) - curl_low_speed_time = strtol(low_speed_time, NULL, 10); - - git_config(http_options); - - if (curl_ssl_verify == -1) - curl_ssl_verify = 1; - -#ifdef USE_CURL_MULTI - if (max_requests < 1) - max_requests = DEFAULT_MAX_REQUESTS; -#endif + http_init(); - pragma_header = curl_slist_append(pragma_header, "Pragma: no-cache"); no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:"); - no_range_header = curl_slist_append(no_range_header, "Range:"); - -#ifndef NO_CURL_EASY_DUPHANDLE - curl_default = get_curl_handle(); -#endif alt = xmalloc(sizeof(*alt)); alt->base = url; @@ -1385,30 +950,9 @@ int main(int argc, char **argv) if (pull(commit_id)) rc = 1; - curl_slist_free_all(pragma_header); curl_slist_free_all(no_pragma_header); - curl_slist_free_all(no_range_header); -#ifndef NO_CURL_EASY_DUPHANDLE - curl_easy_cleanup(curl_default); -#endif - slot = active_queue_head; - while (slot != NULL) { - if (slot->in_use) { - if (get_verbosely) { - curl_easy_getinfo(slot->curl, - CURLINFO_EFFECTIVE_URL, - &wait_url); - fprintf(stderr, "Waiting for %s\n", wait_url); - } - run_active_slot(slot); - } - if (slot->curl != NULL) - curl_easy_cleanup(slot->curl); - slot = slot->next; - } -#ifdef USE_CURL_MULTI - curl_multi_cleanup(curlm); -#endif - curl_global_cleanup(); + + http_cleanup(); + return rc; }