git-upload-pack: More efficient usage of the has_sha1 array

[git.git] / http-fetch.c

diff --git a/http-fetch.c b/http-fetch.c
index a7dc2cc..a1b03cd 100644 (file)
--- a/http-fetch.c
+++ b/http-fetch.c
@@ -25,6 +25,7 @@
 #define PREV_BUF_SIZE 4096
 #define RANGE_HEADER_SIZE 30
 
+static int got_alternates = 0;
 static int active_requests = 0;
 static int data_received;
 
@@ -85,6 +86,7 @@ struct active_request_slot
        int in_use;
        int done;
        CURLcode curl_result;
+       long http_code;
        struct active_request_slot *next;
 };
 
@@ -235,6 +237,7 @@ static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
 static void process_curl_messages(void);
 static void process_request_queue(void);
 #endif
+static int fetch_alternates(char *base);
 
 static CURL* get_curl_handle(void)
 {
@@ -291,11 +294,7 @@ static struct active_request_slot *get_active_slot(void)
        }
        if (slot == NULL) {
                newslot = xmalloc(sizeof(*newslot));
-#ifdef NO_CURL_EASY_DUPHANDLE
-               newslot->curl = get_curl_handle();
-#else
-               newslot->curl = curl_easy_duphandle(curl_default);
-#endif
+               newslot->curl = NULL;
                newslot->in_use = 0;
                newslot->next = NULL;
 
@@ -311,6 +310,14 @@ static struct active_request_slot *get_active_slot(void)
                slot = newslot;
        }
 
+       if (slot->curl == NULL) {
+#ifdef NO_CURL_EASY_DUPHANDLE
+               slot->curl = get_curl_handle();
+#else
+               slot->curl = curl_easy_duphandle(curl_default);
+#endif
+       }
+
        active_requests++;
        slot->in_use = 1;
        slot->done = 0;
@@ -576,6 +583,9 @@ void process_curl_messages(void)
                                slot->done = 1;
                                slot->in_use = 0;
                                slot->curl_result = curl_message->data.result;
+                               curl_easy_getinfo(slot->curl,
+                                                 CURLINFO_HTTP_CODE,
+                                                 &slot->http_code);
                                request = request_queue_head;
                                while (request != NULL &&
                                       request->slot != slot)
@@ -586,19 +596,20 @@ void process_curl_messages(void)
                        if (request != NULL) {
                                request->curl_result =
                                        curl_message->data.result;
-                               curl_easy_getinfo(slot->curl,
-                                                 CURLINFO_HTTP_CODE,
-                                                 &request->http_code);
+                               request->http_code = slot->http_code;
                                request->slot = NULL;
+                               request->state = COMPLETE;
 
                                /* Use alternates if necessary */
-                               if (request->http_code == 404 &&
-                                   request->repo->next != NULL) {
-                                       request->repo = request->repo->next;
-                                       start_request(request);
+                               if (request->http_code == 404) {
+                                       fetch_alternates(alt->base);
+                                       if (request->repo->next != NULL) {
+                                               request->repo =
+                                                       request->repo->next;
+                                               start_request(request);
+                                       }
                                } else {
                                        finish_request(request);
-                                       request->state = COMPLETE;
                                }
                        }
                } else {
@@ -612,6 +623,7 @@ void process_curl_messages(void)
 void process_request_queue(void)
 {
        struct transfer_request *request = request_queue_head;
+       struct active_request_slot *slot = active_queue_head;
        int num_transfers;
 
        while (active_requests < max_requests && request != NULL) {
@@ -624,6 +636,14 @@ void process_request_queue(void)
                }
                request = request->next;
        }
+
+       while (slot != NULL) {
+               if (!slot->in_use && slot->curl != NULL) {
+                       curl_easy_cleanup(slot->curl);
+                       slot->curl = NULL;
+               }
+               slot = slot->next;
+       }                               
 }
 #endif
 
@@ -752,6 +772,9 @@ static int fetch_alternates(char *base)
 
        struct active_request_slot *slot;
 
+       if (got_alternates)
+               return 0;
+
        data = xmalloc(4096);
        buffer.size = 4096;
        buffer.posn = 0;
@@ -784,6 +807,8 @@ static int fetch_alternates(char *base)
                                run_active_slot(slot);
                                if (slot->curl_result != CURLE_OK) {
                                        free(buffer.buffer);
+                                       if (slot->http_code == 404)
+                                               got_alternates = 1;
                                        return 0;
                                }
                        }
@@ -855,6 +880,7 @@ static int fetch_alternates(char *base)
                i = posn + 1;
        }
 
+       got_alternates = 1;
        free(buffer.buffer);
        return ret;
 }
@@ -1046,16 +1072,16 @@ static int fetch_object(struct alt_base *repo, unsigned char *sha1)
                run_active_slot(request->slot);
 #ifndef USE_CURL_MULTI
                request->curl_result = request->slot->curl_result;
-               curl_easy_getinfo(request->slot->curl,
-                                 CURLINFO_HTTP_CODE,
-                                 &request->http_code);
+               request->http_code = request->slot->http_code;
                request->slot = NULL;
 
                /* Use alternates if necessary */
-               if (request->http_code == 404 &&
-                   request->repo->next != NULL) {
-                       request->repo = request->repo->next;
-                       start_request(request);
+               if (request->http_code == 404) {
+                       fetch_alternates(alt->base);
+                       if (request->repo->next != NULL) {
+                               request->repo = request->repo->next;
+                               start_request(request);
+                       }
                } else {
                        finish_request(request);
                        request->state = COMPLETE;
@@ -1069,9 +1095,12 @@ static int fetch_object(struct alt_base *repo, unsigned char *sha1)
        }
 
        if (request->curl_result != CURLE_OK && request->http_code != 416) {
-               ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
-                           request->errorstr, request->curl_result,
-                           request->http_code, hex);
+               if (request->http_code == 404)
+                       ret = -1; /* Be silent, it is probably in a pack. */
+               else
+                       ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
+                                   request->errorstr, request->curl_result,
+                                   request->http_code, hex);
                release_request(request);
                return ret;
        }
@@ -1108,6 +1137,7 @@ int fetch(unsigned char *sha1)
        while (altbase) {
                if (!fetch_pack(altbase, sha1))
                        return 0;
+               fetch_alternates(alt->base);
                altbase = altbase->next;
        }
        return error("Unable to find %s under %s\n", sha1_to_hex(sha1), 
@@ -1199,6 +1229,8 @@ int main(int argc, char **argv)
        struct active_request_slot *slot;
        char *low_speed_limit;
        char *low_speed_time;
+       char *wait_url;
+       int rc = 0;
 
        while (arg < argc && argv[arg][0] == '-') {
                if (argv[arg][1] == 't') {
@@ -1284,10 +1316,9 @@ int main(int argc, char **argv)
        alt->got_indices = 0;
        alt->packs = NULL;
        alt->next = NULL;
-       fetch_alternates(alt->base);
 
        if (pull(commit_id))
-               return 1;
+               rc = 1;
 
        curl_slist_free_all(pragma_header);
        curl_slist_free_all(no_pragma_header);
@@ -1297,12 +1328,22 @@ int main(int argc, char **argv)
 #endif
        slot = active_queue_head;
        while (slot != NULL) {
-               curl_easy_cleanup(slot->curl);
+               if (slot->in_use) {
+                       if (get_verbosely) {
+                               curl_easy_getinfo(slot->curl,
+                                                 CURLINFO_EFFECTIVE_URL,
+                                                 &wait_url);
+                               fprintf(stderr, "Waiting for %s\n", wait_url);
+                       }
+                       run_active_slot(slot);
+               }
+               if (slot->curl != NULL)
+                       curl_easy_cleanup(slot->curl);
                slot = slot->next;
        }
 #ifdef USE_CURL_MULTI
        curl_multi_cleanup(curlm);
 #endif
        curl_global_cleanup();
-       return 0;
+       return rc;
 }