X-Git-Url: https://git.octo.it/?a=blobdiff_plain;f=http-fetch.c;h=d3602b7d7d820a37b6395a7aeb96a8dc8368a646;hb=74237d6236d7e32f69469ff26df3f3bb3875f523;hp=435317342be0d31088f5aae5b337e090b40199b5;hpb=36d277c72d90d32f99616072b64a2652248f5264;p=git.git diff --git a/http-fetch.c b/http-fetch.c index 43531734..d3602b7d 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -4,16 +4,47 @@ #include "fetch.h" #include "http.h" +#ifndef NO_EXPAT +#include + +/* Definitions for DAV requests */ +#define DAV_PROPFIND "PROPFIND" +#define DAV_PROPFIND_RESP ".multistatus.response" +#define DAV_PROPFIND_NAME ".multistatus.response.href" +#define DAV_PROPFIND_COLLECTION ".multistatus.response.propstat.prop.resourcetype.collection" +#define PROPFIND_ALL_REQUEST "\n\n\n" + +/* Definitions for processing XML DAV responses */ +#ifndef XML_STATUS_OK +enum XML_Status { + XML_STATUS_OK = 1, + XML_STATUS_ERROR = 0 +}; +#define XML_STATUS_OK 1 +#define XML_STATUS_ERROR 0 +#endif + +/* Flags that control remote_ls processing */ +#define PROCESS_FILES (1u << 0) +#define PROCESS_DIRS (1u << 1) +#define RECURSIVE (1u << 2) + +/* Flags that remote_ls passes to callback functions */ +#define IS_DIR (1u << 0) +#endif + #define PREV_BUF_SIZE 4096 #define RANGE_HEADER_SIZE 30 static int got_alternates = -1; +static int corrupt_object_found = 0; static struct curl_slist *no_pragma_header; struct alt_base { char *base; + int path_len; int got_indices; struct packed_git *packs; struct alt_base *next; @@ -57,6 +88,30 @@ struct alternates_request { int http_specific; }; +#ifndef NO_EXPAT +struct xml_ctx +{ + char *name; + int len; + char *cdata; + void (*userFunc)(struct xml_ctx *ctx, int tag_closed); + void *userData; +}; + +struct remote_ls_ctx +{ + struct alt_base *repo; + char *path; + void (*userFunc)(struct remote_ls_ctx *ls); + void *userData; + int flags; + char *dentry_name; + int dentry_flags; + int rc; + struct remote_ls_ctx *parent; +}; +#endif + static struct object_request *object_queue_head = NULL; static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb, @@ -130,7 +185,7 @@ static void start_object_request(struct object_request *obj_req) if (obj_req->local < 0) { obj_req->state = ABORTED; - error("Couldn't create temporary file %s for %s: %s\n", + error("Couldn't create temporary file %s for %s: %s", obj_req->tmpfile, obj_req->filename, strerror(errno)); return; } @@ -220,7 +275,6 @@ static void start_object_request(struct object_request *obj_req) free(obj_req->url); return; } - } static void finish_object_request(struct object_request *obj_req) @@ -267,7 +321,8 @@ static void process_object_response(void *callback_data) obj_req->state = COMPLETE; /* Use alternates if necessary */ - if (obj_req->http_code == 404) { + if (obj_req->http_code == 404 || + obj_req->curl_result == CURLE_FILE_COULDNT_READ_FILE) { fetch_alternates(alt->base); if (obj_req->repo->next != NULL) { obj_req->repo = @@ -311,7 +366,7 @@ void fill_active_slots(void) while (active_requests < max_requests && obj_req != NULL) { if (obj_req->state == WAITING) { if (has_sha1_file(obj_req->sha1)) - release_object_request(obj_req); + obj_req->state = COMPLETE; else start_object_request(obj_req); curl_multi_perform(curlm, &num_transfers); @@ -325,7 +380,7 @@ void fill_active_slots(void) slot->curl = NULL; } slot = slot->next; - } + } } #endif @@ -344,6 +399,7 @@ void prefetch(unsigned char *sha1) snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename); snprintf(newreq->tmpfile, sizeof(newreq->tmpfile), "%s.temp", filename); + newreq->slot = NULL; newreq->next = NULL; if (object_queue_head == NULL) { @@ -374,16 +430,17 @@ static int fetch_index(struct alt_base *repo, unsigned char *sha1) FILE *indexfile; struct active_request_slot *slot; + struct slot_results results; if (has_pack_index(sha1)) return 0; if (get_verbosely) fprintf(stderr, "Getting index for pack %s\n", hex); - + url = xmalloc(strlen(repo->base) + 64); sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex); - + filename = sha1_pack_index_name(sha1); snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename); indexfile = fopen(tmpfile, "a"); @@ -392,6 +449,7 @@ static int fetch_index(struct alt_base *repo, unsigned char *sha1) filename); slot = get_active_slot(); + slot->results = &results; curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile); curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite); curl_easy_setopt(slot->curl, CURLOPT_URL, url); @@ -413,7 +471,7 @@ static int fetch_index(struct alt_base *repo, unsigned char *sha1) if (start_active_slot(slot)) { run_active_slot(slot); - if (slot->curl_result != CURLE_OK) { + if (results.curl_result != CURLE_OK) { fclose(indexfile); return error("Unable to get pack index %s\n%s", url, curl_errorstr); @@ -466,16 +524,19 @@ static void process_alternates_response(void *callback_data) alt_req->url); active_requests++; slot->in_use = 1; - if (start_active_slot(slot)) { - return; - } else { + if (slot->finished != NULL) + (*slot->finished) = 0; + if (!start_active_slot(slot)) { got_alternates = -1; slot->in_use = 0; - return; + if (slot->finished != NULL) + (*slot->finished) = 1; } + return; } } else if (slot->curl_result != CURLE_OK) { - if (slot->http_code != 404) { + if (slot->http_code != 404 && + slot->curl_result != CURLE_FILE_COULDNT_READ_FILE) { got_alternates = -1; return; } @@ -494,13 +555,14 @@ static void process_alternates_response(void *callback_data) int serverlen = 0; struct alt_base *newalt; char *target = NULL; + char *path; if (data[i] == '/') { serverlen = strchr(base + 8, '/') - base; okay = 1; } else if (!memcmp(data + i, "../", 3)) { i += 3; serverlen = strlen(base); - while (i + 2 < posn && + while (i + 2 < posn && !memcmp(data + i, "../", 3)) { do { serverlen--; @@ -509,7 +571,7 @@ static void process_alternates_response(void *callback_data) i += 3; } // If the server got removed, give up. - okay = strchr(base, ':') - base + 3 < + okay = strchr(base, ':') - base + 3 < serverlen; } else if (alt_req->http_specific) { char *colon = strchr(data + i, ':'); @@ -527,13 +589,20 @@ static void process_alternates_response(void *callback_data) posn - i - 7); target[serverlen + posn - i - 7] = '\0'; if (get_verbosely) - fprintf(stderr, + fprintf(stderr, "Also look at %s\n", target); newalt = xmalloc(sizeof(*newalt)); newalt->next = NULL; newalt->base = target; newalt->got_indices = 0; newalt->packs = NULL; + path = strstr(target, "//"); + if (path) { + path = strchr(path+2, '/'); + if (path) + newalt->path_len = strlen(path); + } + while (tail->next != NULL) tail = tail->next; tail->next = newalt; @@ -551,7 +620,7 @@ static void fetch_alternates(char *base) char *url; char *data; struct active_request_slot *slot; - static struct alternates_request alt_req; + struct alternates_request alt_req; /* If another request has already started fetching alternates, wait for them to arrive and return to processing this request's @@ -576,7 +645,7 @@ static void fetch_alternates(char *base) if (get_verbosely) fprintf(stderr, "Getting alternates list for %s\n", base); - + url = xmalloc(strlen(base) + 31); sprintf(url, "%s/objects/info/http-alternates", base); @@ -605,6 +674,209 @@ static void fetch_alternates(char *base) free(url); } +#ifndef NO_EXPAT +static void +xml_start_tag(void *userData, const char *name, const char **atts) +{ + struct xml_ctx *ctx = (struct xml_ctx *)userData; + const char *c = strchr(name, ':'); + int new_len; + + if (c == NULL) + c = name; + else + c++; + + new_len = strlen(ctx->name) + strlen(c) + 2; + + if (new_len > ctx->len) { + ctx->name = xrealloc(ctx->name, new_len); + ctx->len = new_len; + } + strcat(ctx->name, "."); + strcat(ctx->name, c); + + if (ctx->cdata) { + free(ctx->cdata); + ctx->cdata = NULL; + } + + ctx->userFunc(ctx, 0); +} + +static void +xml_end_tag(void *userData, const char *name) +{ + struct xml_ctx *ctx = (struct xml_ctx *)userData; + const char *c = strchr(name, ':'); + char *ep; + + ctx->userFunc(ctx, 1); + + if (c == NULL) + c = name; + else + c++; + + ep = ctx->name + strlen(ctx->name) - strlen(c) - 1; + *ep = 0; +} + +static void +xml_cdata(void *userData, const XML_Char *s, int len) +{ + struct xml_ctx *ctx = (struct xml_ctx *)userData; + if (ctx->cdata) + free(ctx->cdata); + ctx->cdata = xcalloc(len+1, 1); + strncpy(ctx->cdata, s, len); +} + +static int remote_ls(struct alt_base *repo, const char *path, int flags, + void (*userFunc)(struct remote_ls_ctx *ls), + void *userData); + +static void handle_remote_ls_ctx(struct xml_ctx *ctx, int tag_closed) +{ + struct remote_ls_ctx *ls = (struct remote_ls_ctx *)ctx->userData; + + if (tag_closed) { + if (!strcmp(ctx->name, DAV_PROPFIND_RESP) && ls->dentry_name) { + if (ls->dentry_flags & IS_DIR) { + if (ls->flags & PROCESS_DIRS) { + ls->userFunc(ls); + } + if (strcmp(ls->dentry_name, ls->path) && + ls->flags & RECURSIVE) { + ls->rc = remote_ls(ls->repo, + ls->dentry_name, + ls->flags, + ls->userFunc, + ls->userData); + } + } else if (ls->flags & PROCESS_FILES) { + ls->userFunc(ls); + } + } else if (!strcmp(ctx->name, DAV_PROPFIND_NAME) && ctx->cdata) { + ls->dentry_name = xmalloc(strlen(ctx->cdata) - + ls->repo->path_len + 1); + strcpy(ls->dentry_name, ctx->cdata + ls->repo->path_len); + } else if (!strcmp(ctx->name, DAV_PROPFIND_COLLECTION)) { + ls->dentry_flags |= IS_DIR; + } + } else if (!strcmp(ctx->name, DAV_PROPFIND_RESP)) { + if (ls->dentry_name) { + free(ls->dentry_name); + } + ls->dentry_name = NULL; + ls->dentry_flags = 0; + } +} + +static int remote_ls(struct alt_base *repo, const char *path, int flags, + void (*userFunc)(struct remote_ls_ctx *ls), + void *userData) +{ + char *url = xmalloc(strlen(repo->base) + strlen(path) + 1); + struct active_request_slot *slot; + struct slot_results results; + struct buffer in_buffer; + struct buffer out_buffer; + char *in_data; + char *out_data; + XML_Parser parser = XML_ParserCreate(NULL); + enum XML_Status result; + struct curl_slist *dav_headers = NULL; + struct xml_ctx ctx; + struct remote_ls_ctx ls; + + ls.flags = flags; + ls.repo = repo; + ls.path = strdup(path); + ls.dentry_name = NULL; + ls.dentry_flags = 0; + ls.userData = userData; + ls.userFunc = userFunc; + ls.rc = 0; + + sprintf(url, "%s%s", repo->base, path); + + out_buffer.size = strlen(PROPFIND_ALL_REQUEST); + out_data = xmalloc(out_buffer.size + 1); + snprintf(out_data, out_buffer.size + 1, PROPFIND_ALL_REQUEST); + out_buffer.posn = 0; + out_buffer.buffer = out_data; + + in_buffer.size = 4096; + in_data = xmalloc(in_buffer.size); + in_buffer.posn = 0; + in_buffer.buffer = in_data; + + dav_headers = curl_slist_append(dav_headers, "Depth: 1"); + dav_headers = curl_slist_append(dav_headers, "Content-Type: text/xml"); + + slot = get_active_slot(); + slot->results = &results; + curl_easy_setopt(slot->curl, CURLOPT_INFILE, &out_buffer); + curl_easy_setopt(slot->curl, CURLOPT_INFILESIZE, out_buffer.size); + curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, fread_buffer); + curl_easy_setopt(slot->curl, CURLOPT_FILE, &in_buffer); + curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer); + curl_easy_setopt(slot->curl, CURLOPT_URL, url); + curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 1); + curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, DAV_PROPFIND); + curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, dav_headers); + + if (start_active_slot(slot)) { + run_active_slot(slot); + if (results.curl_result == CURLE_OK) { + ctx.name = xcalloc(10, 1); + ctx.len = 0; + ctx.cdata = NULL; + ctx.userFunc = handle_remote_ls_ctx; + ctx.userData = &ls; + XML_SetUserData(parser, &ctx); + XML_SetElementHandler(parser, xml_start_tag, + xml_end_tag); + XML_SetCharacterDataHandler(parser, xml_cdata); + result = XML_Parse(parser, in_buffer.buffer, + in_buffer.posn, 1); + free(ctx.name); + + if (result != XML_STATUS_OK) { + ls.rc = error("XML error: %s", + XML_ErrorString( + XML_GetErrorCode(parser))); + } + } else { + ls.rc = -1; + } + } else { + ls.rc = error("Unable to start PROPFIND request"); + } + + free(ls.path); + free(url); + free(out_data); + free(in_buffer.buffer); + curl_slist_free_all(dav_headers); + + return ls.rc; +} + +static void process_ls_pack(struct remote_ls_ctx *ls) +{ + unsigned char sha1[20]; + + if (strlen(ls->dentry_name) == 63 && + !strncmp(ls->dentry_name, "objects/pack/pack-", 18) && + !strncmp(ls->dentry_name+58, ".pack", 5)) { + get_sha1_hex(ls->dentry_name + 18, sha1); + setup_index(ls->repo, sha1); + } +} +#endif + static int fetch_indices(struct alt_base *repo) { unsigned char sha1[20]; @@ -614,6 +886,7 @@ static int fetch_indices(struct alt_base *repo) int i = 0; struct active_request_slot *slot; + struct slot_results results; if (repo->got_indices) return 0; @@ -625,19 +898,27 @@ static int fetch_indices(struct alt_base *repo) if (get_verbosely) fprintf(stderr, "Getting pack list for %s\n", repo->base); - + +#ifndef NO_EXPAT + if (remote_ls(repo, "objects/pack/", PROCESS_FILES, + process_ls_pack, NULL) == 0) + return 0; +#endif + url = xmalloc(strlen(repo->base) + 21); sprintf(url, "%s/objects/info/packs", repo->base); slot = get_active_slot(); + slot->results = &results; curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer); curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer); curl_easy_setopt(slot->curl, CURLOPT_URL, url); curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL); if (start_active_slot(slot)) { run_active_slot(slot); - if (slot->curl_result != CURLE_OK) { - if (slot->http_code == 404) { + if (results.curl_result != CURLE_OK) { + if (results.http_code == 404 || + results.curl_result == CURLE_FILE_COULDNT_READ_FILE) { repo->got_indices = 1; free(buffer.buffer); return 0; @@ -658,7 +939,7 @@ static int fetch_indices(struct alt_base *repo) switch (data[i]) { case 'P': i++; - if (i + 52 < buffer.posn && + if (i + 52 <= buffer.posn && !strncmp(data + i, " pack-", 6) && !strncmp(data + i + 46, ".pack\n", 6)) { get_sha1_hex(data + i + 6, sha1); @@ -667,7 +948,7 @@ static int fetch_indices(struct alt_base *repo) break; } default: - while (data[i] != '\n') + while (i < buffer.posn && data[i] != '\n') i++; } i++; @@ -692,6 +973,7 @@ static int fetch_pack(struct alt_base *repo, unsigned char *sha1) struct curl_slist *range_header = NULL; struct active_request_slot *slot; + struct slot_results results; if (fetch_indices(repo)) return -1; @@ -718,6 +1000,7 @@ static int fetch_pack(struct alt_base *repo, unsigned char *sha1) filename); slot = get_active_slot(); + slot->results = &results; curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile); curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite); curl_easy_setopt(slot->curl, CURLOPT_URL, url); @@ -739,7 +1022,7 @@ static int fetch_pack(struct alt_base *repo, unsigned char *sha1) if (start_active_slot(slot)) { run_active_slot(slot); - if (slot->curl_result != CURLE_OK) { + if (results.curl_result != CURLE_OK) { fclose(packfile); return error("Unable to get pack file %s\n%s", url, curl_errorstr); @@ -767,6 +1050,20 @@ static int fetch_pack(struct alt_base *repo, unsigned char *sha1) return 0; } +static void abort_object_request(struct object_request *obj_req) +{ + if (obj_req->local >= 0) { + close(obj_req->local); + obj_req->local = -1; + } + unlink(obj_req->tmpfile); + if (obj_req->slot) { + release_active_slot(obj_req->slot); + obj_req->slot = NULL; + } + release_object_request(obj_req); +} + static int fetch_object(struct alt_base *repo, unsigned char *sha1) { char *hex = sha1_to_hex(sha1); @@ -779,7 +1076,7 @@ static int fetch_object(struct alt_base *repo, unsigned char *sha1) return error("Couldn't find request for %s in the queue", hex); if (has_sha1_file(obj_req->sha1)) { - release_object_request(obj_req); + abort_object_request(obj_req); return 0; } @@ -802,20 +1099,21 @@ static int fetch_object(struct alt_base *repo, unsigned char *sha1) ret = error("Request for %s aborted", hex); } else if (obj_req->curl_result != CURLE_OK && obj_req->http_code != 416) { - if (obj_req->http_code == 404) + if (obj_req->http_code == 404 || + obj_req->curl_result == CURLE_FILE_COULDNT_READ_FILE) ret = -1; /* Be silent, it is probably in a pack. */ else ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)", obj_req->errorstr, obj_req->curl_result, obj_req->http_code, hex); } else if (obj_req->zret != Z_STREAM_END) { - ret = error("File %s (%s) corrupt\n", hex, obj_req->url); + corrupt_object_found++; + ret = error("File %s (%s) corrupt", hex, obj_req->url); } else if (memcmp(obj_req->sha1, obj_req->real_sha1, 20)) { - ret = error("File %s has bad hash\n", hex); + ret = error("File %s has bad hash", hex); } else if (obj_req->rename < 0) { - ret = error("unable to write sha1 filename %s: %s", - obj_req->filename, - strerror(obj_req->rename)); + ret = error("unable to write sha1 filename %s", + obj_req->filename); } release_object_request(obj_req); @@ -834,7 +1132,7 @@ int fetch(unsigned char *sha1) fetch_alternates(alt->base); altbase = altbase->next; } - return error("Unable to find %s under %s\n", sha1_to_hex(sha1), + return error("Unable to find %s under %s", sha1_to_hex(sha1), alt->base); } @@ -890,20 +1188,22 @@ int fetch_ref(char *ref, unsigned char *sha1) struct buffer buffer; char *base = alt->base; struct active_request_slot *slot; + struct slot_results results; buffer.size = 41; buffer.posn = 0; buffer.buffer = hex; hex[41] = '\0'; - + url = quote_ref_url(base, ref); slot = get_active_slot(); + slot->results = &results; curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer); curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer); curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL); curl_easy_setopt(slot->curl, CURLOPT_URL, url); if (start_active_slot(slot)) { run_active_slot(slot); - if (slot->curl_result != CURLE_OK) + if (results.curl_result != CURLE_OK) return error("Couldn't get %s for %s\n%s", url, ref, curl_errorstr); } else { @@ -919,9 +1219,13 @@ int main(int argc, char **argv) { char *commit_id; char *url; + char *path; int arg = 1; int rc = 0; + setup_git_directory(); + git_config(git_default_config); + while (arg < argc && argv[arg][0] == '-') { if (argv[arg][1] == 't') { get_tree = 1; @@ -947,6 +1251,7 @@ int main(int argc, char **argv) } commit_id = argv[arg]; url = argv[arg + 1]; + write_ref_log_details = url; http_init(); @@ -957,13 +1262,25 @@ int main(int argc, char **argv) alt->got_indices = 0; alt->packs = NULL; alt->next = NULL; + path = strstr(url, "//"); + if (path) { + path = strchr(path+2, '/'); + if (path) + alt->path_len = strlen(path); + } if (pull(commit_id)) rc = 1; - curl_slist_free_all(no_pragma_header); - http_cleanup(); + curl_slist_free_all(no_pragma_header); + + if (corrupt_object_found) { + fprintf(stderr, +"Some loose object were found to be corrupt, but they might be just\n" +"a false '404 Not Found' error message sent with incorrect HTTP\n" +"status code. Suggest running git fsck-objects.\n"); + } return rc; }