X-Git-Url: https://git.octo.it/?a=blobdiff_plain;f=http-fetch.c;h=21cc1b960cf511c083c594fd5fc86bb84d66ede3;hb=a52e4ef877f693a69710145f56d867365a755f91;hp=da4d2f2c699f4265e8977db947a19399e4f06c54;hpb=f80376c5976143a1788ecc4f7ca86ac0ec8b9874;p=git.git

diff --git a/http-fetch.c b/http-fetch.c
index da4d2f2c..21cc1b96 100644
--- a/http-fetch.c
+++ b/http-fetch.c
@@ -25,6 +25,7 @@
 #define PREV_BUF_SIZE 4096
 #define RANGE_HEADER_SIZE 30
 
+static int got_alternates = -1;
 static int active_requests = 0;
 static int data_received;
 
@@ -85,9 +86,20 @@ struct active_request_slot
 	int in_use;
 	int done;
 	CURLcode curl_result;
+	long http_code;
+	void *callback_data;
+	void (*callback_func)(void *data);
 	struct active_request_slot *next;
 };
 
+struct alt_request {
+	char *base;
+	char *url;
+	struct buffer *buffer;
+	struct active_request_slot *slot;
+	int http_specific;
+};
+
 static struct transfer_request *request_queue_head = NULL;
 static struct active_request_slot *active_queue_head = NULL;
 
@@ -100,6 +112,8 @@ static char *ssl_key = NULL;
 static char *ssl_capath = NULL;
 #endif
 static char *ssl_cainfo = NULL;
+static long curl_low_speed_limit = -1;
+static long curl_low_speed_time = -1;
 
 struct buffer
 {
@@ -158,6 +172,17 @@ static int http_options(const char *var, const char *value)
 	}
 #endif
 
+	if (!strcmp("http.lowspeedlimit", var)) {
+		if (curl_low_speed_limit == -1)
+			curl_low_speed_limit = (long)git_config_int(var, value);
+		return 0;
+	}
+	if (!strcmp("http.lowspeedtime", var)) {
+		if (curl_low_speed_time == -1)
+			curl_low_speed_time = (long)git_config_int(var, value);
+		return 0;
+	}
+
 	/* Fall back on the default ones */
 	return git_default_config(var, value);
 }
@@ -219,11 +244,12 @@ static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
 }
 
 #ifdef USE_CURL_MULTI
-void process_curl_messages();
-void process_request_queue();
+static void process_curl_messages(void);
+static void process_request_queue(void);
 #endif
+static void fetch_alternates(char *base);
 
-static CURL* get_curl_handle()
+static CURL* get_curl_handle(void)
 {
 	CURL* result = curl_easy_init();
 
@@ -246,10 +272,19 @@ static CURL* get_curl_handle()
 		curl_easy_setopt(result, CURLOPT_CAINFO, ssl_cainfo);
 	curl_easy_setopt(result, CURLOPT_FAILONERROR, 1);
 
+	if (curl_low_speed_limit > 0 && curl_low_speed_time > 0) {
+		curl_easy_setopt(result, CURLOPT_LOW_SPEED_LIMIT,
+				 curl_low_speed_limit);
+		curl_easy_setopt(result, CURLOPT_LOW_SPEED_TIME,
+				 curl_low_speed_time);
+	}
+
+	curl_easy_setopt(result, CURLOPT_FOLLOWLOCATION, 1);
+
 	return result;
 }
 
-struct active_request_slot *get_active_slot()
+static struct active_request_slot *get_active_slot(void)
 {
 	struct active_request_slot *slot = active_queue_head;
 	struct active_request_slot *newslot;
@@ -271,11 +306,7 @@ struct active_request_slot *get_active_slot()
 	}
 	if (slot == NULL) {
 		newslot = xmalloc(sizeof(*newslot));
-#ifdef NO_CURL_EASY_DUPHANDLE
-		newslot->curl = get_curl_handle();
-#else
-		newslot->curl = curl_easy_duphandle(curl_default);
-#endif
+		newslot->curl = NULL;
 		newslot->in_use = 0;
 		newslot->next = NULL;
 
@@ -291,10 +322,20 @@ struct active_request_slot *get_active_slot()
 		slot = newslot;
 	}
 
+	if (slot->curl == NULL) {
+#ifdef NO_CURL_EASY_DUPHANDLE
+		slot->curl = get_curl_handle();
+#else
+		slot->curl = curl_easy_duphandle(curl_default);
+#endif
+	}
+
 	active_requests++;
 	slot->in_use = 1;
 	slot->done = 0;
 	slot->local = NULL;
+	slot->callback_data = NULL;
+	slot->callback_func = NULL;
 	curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, pragma_header);
 	curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_range_header);
 	curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr);
@@ -302,7 +343,7 @@ struct active_request_slot *get_active_slot()
 	return slot;
 }
 
-int start_active_slot(struct active_request_slot *slot)
+static int start_active_slot(struct active_request_slot *slot)
 {
 #ifdef USE_CURL_MULTI
 	CURLMcode curlm_result = curl_multi_add_handle(curlm, slot->curl);
@@ -317,7 +358,7 @@ int start_active_slot(struct active_request_slot *slot)
 	return 1;
 }
 
-void run_active_slot(struct active_request_slot *slot)
+static void run_active_slot(struct active_request_slot *slot)
 {
 #ifdef USE_CURL_MULTI
 	int num_transfers;
@@ -365,7 +406,7 @@ void run_active_slot(struct active_request_slot *slot)
 #endif
 }
 
-void start_request(struct transfer_request *request)
+static void start_request(struct transfer_request *request)
 {
 	char *hex = sha1_to_hex(request->sha1);
 	char prevfile[PATH_MAX];
@@ -384,6 +425,8 @@ void start_request(struct transfer_request *request)
 	rename(request->tmpfile, prevfile);
 	unlink(request->tmpfile);
 
+	if (request->local != -1)
+		error("fd leakage in start: %d", request->local);
 	request->local = open(request->tmpfile,
 			      O_WRONLY | O_CREAT | O_EXCL, 0666);
 	/* This could have failed due to the "lazy directory creation";
@@ -482,7 +525,7 @@ void start_request(struct transfer_request *request)
 	/* Try to get the request started, abort the request on error */
 	if (!start_active_slot(slot)) {
 		request->state = ABORTED;
-		close(request->local);
+		close(request->local); request->local = -1;
 		free(request->url);
 		return;
 	}
@@ -491,14 +534,19 @@ void start_request(struct transfer_request *request)
 	request->state = ACTIVE;
 }
 
-void finish_request(struct transfer_request *request)
+static void finish_request(struct transfer_request *request)
 {
+	struct stat st;
+
 	fchmod(request->local, 0444);
-	close(request->local);
+	close(request->local); request->local = -1;
 
 	if (request->http_code == 416) {
 		fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
 	} else if (request->curl_result != CURLE_OK) {
+		if (stat(request->tmpfile, &st) == 0)
+			if (st.st_size == 0)
+				unlink(request->tmpfile);
 		return;
 	}
 
@@ -519,10 +567,12 @@ void finish_request(struct transfer_request *request)
 		pull_say("got %s\n", sha1_to_hex(request->sha1));
 }
 
-void release_request(struct transfer_request *request)
+static void release_request(struct transfer_request *request)
 {
 	struct transfer_request *entry = request_queue_head;
 
+	if (request->local != -1)
+		error("fd leakage in release: %d", request->local);
 	if (request == request_queue_head) {
 		request_queue_head = request->next;
 	} else {
@@ -537,7 +587,7 @@ void release_request(struct transfer_request *request)
 }
 
 #ifdef USE_CURL_MULTI
-void process_curl_messages()
+static void process_curl_messages(void)
 {
 	int num_messages;
 	struct active_request_slot *slot;
@@ -546,6 +596,7 @@ void process_curl_messages()
 
 	while (curl_message != NULL) {
 		if (curl_message->msg == CURLMSG_DONE) {
+			int curl_result = curl_message->data.result;
 			slot = active_queue_head;
 			while (slot != NULL &&
 			       slot->curl != curl_message->easy_handle)
@@ -555,7 +606,10 @@ void process_curl_messages()
 				active_requests--;
 				slot->done = 1;
 				slot->in_use = 0;
-				slot->curl_result = curl_message->data.result;
+				slot->curl_result = curl_result;
+				curl_easy_getinfo(slot->curl,
+						  CURLINFO_HTTP_CODE,
+						  &slot->http_code);
 				request = request_queue_head;
 				while (request != NULL &&
 				       request->slot != slot)
@@ -563,22 +617,32 @@ void process_curl_messages()
 			} else {
 				fprintf(stderr, "Received DONE message for unknown request!\n");
 			}
+
+			/* Process slot callback if appropriate */
+			if (slot->callback_func != NULL) {
+				slot->callback_func(slot->callback_data);
+			}
+
 			if (request != NULL) {
-				request->curl_result =
-					curl_message->data.result;
-				curl_easy_getinfo(slot->curl,
-						  CURLINFO_HTTP_CODE,
-						  &request->http_code);
+				request->curl_result = curl_result;
+				request->http_code = slot->http_code;
 				request->slot = NULL;
+				request->state = COMPLETE;
 
 				/* Use alternates if necessary */
-				if (request->http_code == 404 &&
-				    request->repo->next != NULL) {
-					request->repo = request->repo->next;
-					start_request(request);
+				if (request->http_code == 404) {
+					fetch_alternates(alt->base);
+					if (request->repo->next != NULL) {
+						request->repo =
+							request->repo->next;
+						close(request->local);
+							request->local = -1;
+						start_request(request);
+					} else {
+						finish_request(request);
+					}
 				} else {
 					finish_request(request);
-					request->state = COMPLETE;
 				}
 			}
 		} else {
@@ -589,9 +653,10 @@ void process_curl_messages()
 	}
 }
 
-void process_request_queue()
+static void process_request_queue(void)
 {
 	struct transfer_request *request = request_queue_head;
+	struct active_request_slot *slot = active_queue_head;
 	int num_transfers;
 
 	while (active_requests < max_requests && request != NULL) {
@@ -604,6 +669,14 @@ void process_request_queue()
 		}
 		request = request->next;
 	}
+
+	while (slot != NULL) {
+		if (!slot->in_use && slot->curl != NULL) {
+			curl_easy_cleanup(slot->curl);
+			slot->curl = NULL;
+		}
+		slot = slot->next;
+	}				
 }
 #endif
 
@@ -696,6 +769,7 @@ static int fetch_index(struct alt_base *repo, unsigned char *sha1)
 				     curl_errorstr);
 		}
 	} else {
+		fclose(indexfile);
 		return error("Unable to start request");
 	}
 
@@ -719,67 +793,51 @@ static int setup_index(struct alt_base *repo, unsigned char *sha1)
 	return 0;
 }
 
-static int fetch_alternates(char *base)
+static void process_alternates(void *callback_data)
 {
-	int ret = 0;
-	struct buffer buffer;
-	char *url;
-	char *data;
-	int i = 0;
-	int http_specific = 1;
+	struct alt_request *alt_req = (struct alt_request *)callback_data;
+	struct active_request_slot *slot = alt_req->slot;
 	struct alt_base *tail = alt;
+	char *base = alt_req->base;
 	static const char null_byte = '\0';
+	char *data;
+	int i = 0;
 
-	struct active_request_slot *slot;
-
-	data = xmalloc(4096);
-	buffer.size = 4096;
-	buffer.posn = 0;
-	buffer.buffer = data;
-
-	if (get_verbosely)
-		fprintf(stderr, "Getting alternates list\n");
-	
-	url = xmalloc(strlen(base) + 31);
-	sprintf(url, "%s/objects/info/http-alternates", base);
-
-	slot = get_active_slot();
-	curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
-	curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
-			 fwrite_buffer_dynamic);
-	curl_easy_setopt(slot->curl, CURLOPT_URL, url);
-	if (start_active_slot(slot)) {
-		run_active_slot(slot);
-		if (slot->curl_result != CURLE_OK || !buffer.posn) {
-			http_specific = 0;
-
-			sprintf(url, "%s/objects/info/alternates", base);
-
-			slot = get_active_slot();
-			curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
-			curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
-					 fwrite_buffer_dynamic);
-			curl_easy_setopt(slot->curl, CURLOPT_URL, url);
+	if (alt_req->http_specific) {
+		if (slot->curl_result != CURLE_OK ||
+		    !alt_req->buffer->posn) {
+
+			/* Try reusing the slot to get non-http alternates */
+			alt_req->http_specific = 0;
+			sprintf(alt_req->url, "%s/objects/info/alternates",
+				base);
+			curl_easy_setopt(slot->curl, CURLOPT_URL,
+					 alt_req->url);
+			active_requests++;
+			slot->in_use = 1;
+			slot->done = 0;
 			if (start_active_slot(slot)) {
-				run_active_slot(slot);
-				if (slot->curl_result != CURLE_OK) {
-					free(buffer.buffer);
-					return 0;
-				}
+				return;
+			} else {
+				got_alternates = -1;
+				slot->done = 1;
+				return;
 			}
 		}
-	} else {
-		free(buffer.buffer);
-		return 0;
+	} else if (slot->curl_result != CURLE_OK) {
+		if (slot->http_code != 404) {
+			got_alternates = -1;
+			return;
+		}
 	}
 
-	fwrite_buffer_dynamic(&null_byte, 1, 1, &buffer);
-	buffer.posn--;
-	data = buffer.buffer;
+	fwrite_buffer_dynamic(&null_byte, 1, 1, alt_req->buffer);
+	alt_req->buffer->posn--;
+	data = alt_req->buffer->buffer;
 
-	while (i < buffer.posn) {
+	while (i < alt_req->buffer->posn) {
 		int posn = i;
-		while (posn < buffer.posn && data[posn] != '\n')
+		while (posn < alt_req->buffer->posn && data[posn] != '\n')
 			posn++;
 		if (data[posn] == '\n') {
 			int okay = 0;
@@ -803,7 +861,7 @@ static int fetch_alternates(char *base)
 				// If the server got removed, give up.
 				okay = strchr(base, ':') - base + 3 < 
 					serverlen;
-			} else if (http_specific) {
+			} else if (alt_req->http_specific) {
 				char *colon = strchr(data + i, ':');
 				char *slash = strchr(data + i, '/');
 				if (colon && slash && colon < data + posn &&
@@ -829,14 +887,74 @@ static int fetch_alternates(char *base)
 				while (tail->next != NULL)
 					tail = tail->next;
 				tail->next = newalt;
-				ret++;
 			}
 		}
 		i = posn + 1;
 	}
 
-	free(buffer.buffer);
-	return ret;
+	got_alternates = 1;
+}
+
+static void fetch_alternates(char *base)
+{
+	struct buffer buffer;
+	char *url;
+	char *data;
+	struct active_request_slot *slot;
+	static struct alt_request alt_req;
+	int num_transfers;
+
+	/* If another request has already started fetching alternates,
+	   wait for them to arrive and return to processing this request's
+	   curl message */
+	while (got_alternates == 0) {
+		curl_multi_perform(curlm, &num_transfers);
+		process_curl_messages();
+		process_request_queue();
+	}
+
+	/* Nothing to do if they've already been fetched */
+	if (got_alternates == 1)
+		return;
+
+	/* Start the fetch */
+	got_alternates = 0;
+
+	data = xmalloc(4096);
+	buffer.size = 4096;
+	buffer.posn = 0;
+	buffer.buffer = data;
+
+	if (get_verbosely)
+		fprintf(stderr, "Getting alternates list for %s\n", base);
+	
+	url = xmalloc(strlen(base) + 31);
+	sprintf(url, "%s/objects/info/http-alternates", base);
+
+	/* Use a callback to process the result, since another request
+	   may fail and need to have alternates loaded before continuing */
+	slot = get_active_slot();
+	slot->callback_func = process_alternates;
+	slot->callback_data = &alt_req;
+
+	curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
+	curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
+			 fwrite_buffer_dynamic);
+	curl_easy_setopt(slot->curl, CURLOPT_URL, url);
+
+	alt_req.base = base;
+	alt_req.url = url;
+	alt_req.buffer = &buffer;
+	alt_req.http_specific = 1;
+	alt_req.slot = slot;
+
+	if (start_active_slot(slot))
+		run_active_slot(slot);
+	else
+		got_alternates = -1;
+
+	free(data);
+	free(url);
 }
 
 static int fetch_indices(struct alt_base *repo)
@@ -858,7 +976,7 @@ static int fetch_indices(struct alt_base *repo)
 	buffer.buffer = data;
 
 	if (get_verbosely)
-		fprintf(stderr, "Getting pack list\n");
+		fprintf(stderr, "Getting pack list for %s\n", repo->base);
 	
 	url = xmalloc(strlen(repo->base) + 21);
 	sprintf(url, "%s/objects/info/packs", repo->base);
@@ -972,6 +1090,7 @@ static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
 				     curl_errorstr);
 		}
 	} else {
+		fclose(packfile);
 		return error("Unable to start request");
 	}
 
@@ -1010,8 +1129,8 @@ static int fetch_object(struct alt_base *repo, unsigned char *sha1)
 	}
 
 #ifdef USE_CURL_MULTI
-	int num_transfers;
 	while (request->state == WAITING) {
+		int num_transfers;
 		curl_multi_perform(curlm, &num_transfers);
 		if (num_transfers < active_requests) {
 			process_curl_messages();
@@ -1026,22 +1145,26 @@ static int fetch_object(struct alt_base *repo, unsigned char *sha1)
 		run_active_slot(request->slot);
 #ifndef USE_CURL_MULTI
 		request->curl_result = request->slot->curl_result;
-		curl_easy_getinfo(request->slot->curl,
-				  CURLINFO_HTTP_CODE,
-				  &request->http_code);
+		request->http_code = request->slot->http_code;
 		request->slot = NULL;
 
 		/* Use alternates if necessary */
-		if (request->http_code == 404 &&
-		    request->repo->next != NULL) {
-			request->repo = request->repo->next;
-			start_request(request);
+		if (request->http_code == 404) {
+			fetch_alternates(alt->base);
+			if (request->repo->next != NULL) {
+				request->repo = request->repo->next;
+				close(request->local); request->local = -1;
+				start_request(request);
+			}
 		} else {
 			finish_request(request);
 			request->state = COMPLETE;
 		}
 #endif
 	}
+	if (request->local != -1) {
+		close(request->local); request->local = -1;
+	}
 
 	if (request->state == ABORTED) {
 		release_request(request);
@@ -1049,9 +1172,12 @@ static int fetch_object(struct alt_base *repo, unsigned char *sha1)
 	}
 
 	if (request->curl_result != CURLE_OK && request->http_code != 416) {
-		ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
-			    request->errorstr, request->curl_result,
-			    request->http_code, hex);
+		if (request->http_code == 404)
+			ret = -1; /* Be silent, it is probably in a pack. */
+		else
+			ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
+				    request->errorstr, request->curl_result,
+				    request->http_code, hex);
 		release_request(request);
 		return ret;
 	}
@@ -1088,6 +1214,7 @@ int fetch(unsigned char *sha1)
 	while (altbase) {
 		if (!fetch_pack(altbase, sha1))
 			return 0;
+		fetch_alternates(alt->base);
 		altbase = altbase->next;
 	}
 	return error("Unable to find %s under %s\n", sha1_to_hex(sha1), 
@@ -1177,6 +1304,10 @@ int main(int argc, char **argv)
 	char *url;
 	int arg = 1;
 	struct active_request_slot *slot;
+	char *low_speed_limit;
+	char *low_speed_time;
+	char *wait_url;
+	int rc = 0;
 
 	while (arg < argc && argv[arg][0] == '-') {
 		if (argv[arg][1] == 't') {
@@ -1207,9 +1338,11 @@ int main(int argc, char **argv)
 	curl_global_init(CURL_GLOBAL_ALL);
 
 #ifdef USE_CURL_MULTI
-	char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS");
-	if (http_max_requests != NULL)
-		max_requests = atoi(http_max_requests);
+	{
+		char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS");
+		if (http_max_requests != NULL)
+			max_requests = atoi(http_max_requests);
+	}
 
 	curlm = curl_multi_init();
 	if (curlm == NULL) {
@@ -1230,6 +1363,13 @@ int main(int argc, char **argv)
 #endif
 	ssl_cainfo = getenv("GIT_SSL_CAINFO");
 
+	low_speed_limit = getenv("GIT_HTTP_LOW_SPEED_LIMIT");
+	if (low_speed_limit != NULL)
+		curl_low_speed_limit = strtol(low_speed_limit, NULL, 10);
+	low_speed_time = getenv("GIT_HTTP_LOW_SPEED_TIME");
+	if (low_speed_time != NULL)
+		curl_low_speed_time = strtol(low_speed_time, NULL, 10);
+
 	git_config(http_options);
 
 	if (curl_ssl_verify == -1)
@@ -1253,10 +1393,9 @@ int main(int argc, char **argv)
 	alt->got_indices = 0;
 	alt->packs = NULL;
 	alt->next = NULL;
-	fetch_alternates(alt->base);
 
 	if (pull(commit_id))
-		return 1;
+		rc = 1;
 
 	curl_slist_free_all(pragma_header);
 	curl_slist_free_all(no_pragma_header);
@@ -1266,12 +1405,22 @@ int main(int argc, char **argv)
 #endif
 	slot = active_queue_head;
 	while (slot != NULL) {
-		curl_easy_cleanup(slot->curl);
+		if (slot->in_use) {
+			if (get_verbosely) {
+				curl_easy_getinfo(slot->curl,
+						  CURLINFO_EFFECTIVE_URL,
+						  &wait_url);
+				fprintf(stderr, "Waiting for %s\n", wait_url);
+			}
+			run_active_slot(slot);
+		}
+		if (slot->curl != NULL)
+			curl_easy_cleanup(slot->curl);
 		slot = slot->next;
 	}
 #ifdef USE_CURL_MULTI
 	curl_multi_cleanup(curlm);
 #endif
 	curl_global_cleanup();
-	return 0;
+	return rc;
 }