X-Git-Url: https://git.octo.it/?a=blobdiff_plain;f=http-fetch.c;h=d3602b7d7d820a37b6395a7aeb96a8dc8368a646;hb=7a97de4e19757b5576f32ce67d90cb792dbb893b;hp=61b2188adbb68d94b2d5906be63c89bda879b4b0;hpb=a14c225661fa2fc271d9e0fbf262e369dc7254fc;p=git.git diff --git a/http-fetch.c b/http-fetch.c index 61b2188a..d3602b7d 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -4,16 +4,47 @@ #include "fetch.h" #include "http.h" +#ifndef NO_EXPAT +#include + +/* Definitions for DAV requests */ +#define DAV_PROPFIND "PROPFIND" +#define DAV_PROPFIND_RESP ".multistatus.response" +#define DAV_PROPFIND_NAME ".multistatus.response.href" +#define DAV_PROPFIND_COLLECTION ".multistatus.response.propstat.prop.resourcetype.collection" +#define PROPFIND_ALL_REQUEST "\n\n\n" + +/* Definitions for processing XML DAV responses */ +#ifndef XML_STATUS_OK +enum XML_Status { + XML_STATUS_OK = 1, + XML_STATUS_ERROR = 0 +}; +#define XML_STATUS_OK 1 +#define XML_STATUS_ERROR 0 +#endif + +/* Flags that control remote_ls processing */ +#define PROCESS_FILES (1u << 0) +#define PROCESS_DIRS (1u << 1) +#define RECURSIVE (1u << 2) + +/* Flags that remote_ls passes to callback functions */ +#define IS_DIR (1u << 0) +#endif + #define PREV_BUF_SIZE 4096 #define RANGE_HEADER_SIZE 30 static int got_alternates = -1; +static int corrupt_object_found = 0; static struct curl_slist *no_pragma_header; struct alt_base { char *base; + int path_len; int got_indices; struct packed_git *packs; struct alt_base *next; @@ -57,6 +88,30 @@ struct alternates_request { int http_specific; }; +#ifndef NO_EXPAT +struct xml_ctx +{ + char *name; + int len; + char *cdata; + void (*userFunc)(struct xml_ctx *ctx, int tag_closed); + void *userData; +}; + +struct remote_ls_ctx +{ + struct alt_base *repo; + char *path; + void (*userFunc)(struct remote_ls_ctx *ls); + void *userData; + int flags; + char *dentry_name; + int dentry_flags; + int rc; + struct remote_ls_ctx *parent; +}; +#endif + static struct object_request *object_queue_head = NULL; static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb, @@ -130,7 +185,7 @@ static void start_object_request(struct object_request *obj_req) if (obj_req->local < 0) { obj_req->state = ABORTED; - error("Couldn't create temporary file %s for %s: %s\n", + error("Couldn't create temporary file %s for %s: %s", obj_req->tmpfile, obj_req->filename, strerror(errno)); return; } @@ -220,7 +275,6 @@ static void start_object_request(struct object_request *obj_req) free(obj_req->url); return; } - } static void finish_object_request(struct object_request *obj_req) @@ -312,7 +366,7 @@ void fill_active_slots(void) while (active_requests < max_requests && obj_req != NULL) { if (obj_req->state == WAITING) { if (has_sha1_file(obj_req->sha1)) - release_object_request(obj_req); + obj_req->state = COMPLETE; else start_object_request(obj_req); curl_multi_perform(curlm, &num_transfers); @@ -326,7 +380,7 @@ void fill_active_slots(void) slot->curl = NULL; } slot = slot->next; - } + } } #endif @@ -345,6 +399,7 @@ void prefetch(unsigned char *sha1) snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename); snprintf(newreq->tmpfile, sizeof(newreq->tmpfile), "%s.temp", filename); + newreq->slot = NULL; newreq->next = NULL; if (object_queue_head == NULL) { @@ -375,16 +430,17 @@ static int fetch_index(struct alt_base *repo, unsigned char *sha1) FILE *indexfile; struct active_request_slot *slot; + struct slot_results results; if (has_pack_index(sha1)) return 0; if (get_verbosely) fprintf(stderr, "Getting index for pack %s\n", hex); - + url = xmalloc(strlen(repo->base) + 64); sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex); - + filename = sha1_pack_index_name(sha1); snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename); indexfile = fopen(tmpfile, "a"); @@ -393,6 +449,7 @@ static int fetch_index(struct alt_base *repo, unsigned char *sha1) filename); slot = get_active_slot(); + slot->results = &results; curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile); curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite); curl_easy_setopt(slot->curl, CURLOPT_URL, url); @@ -414,7 +471,7 @@ static int fetch_index(struct alt_base *repo, unsigned char *sha1) if (start_active_slot(slot)) { run_active_slot(slot); - if (slot->curl_result != CURLE_OK) { + if (results.curl_result != CURLE_OK) { fclose(indexfile); return error("Unable to get pack index %s\n%s", url, curl_errorstr); @@ -467,13 +524,15 @@ static void process_alternates_response(void *callback_data) alt_req->url); active_requests++; slot->in_use = 1; - if (start_active_slot(slot)) { - return; - } else { + if (slot->finished != NULL) + (*slot->finished) = 0; + if (!start_active_slot(slot)) { got_alternates = -1; slot->in_use = 0; - return; + if (slot->finished != NULL) + (*slot->finished) = 1; } + return; } } else if (slot->curl_result != CURLE_OK) { if (slot->http_code != 404 && @@ -496,13 +555,14 @@ static void process_alternates_response(void *callback_data) int serverlen = 0; struct alt_base *newalt; char *target = NULL; + char *path; if (data[i] == '/') { serverlen = strchr(base + 8, '/') - base; okay = 1; } else if (!memcmp(data + i, "../", 3)) { i += 3; serverlen = strlen(base); - while (i + 2 < posn && + while (i + 2 < posn && !memcmp(data + i, "../", 3)) { do { serverlen--; @@ -511,7 +571,7 @@ static void process_alternates_response(void *callback_data) i += 3; } // If the server got removed, give up. - okay = strchr(base, ':') - base + 3 < + okay = strchr(base, ':') - base + 3 < serverlen; } else if (alt_req->http_specific) { char *colon = strchr(data + i, ':'); @@ -529,13 +589,20 @@ static void process_alternates_response(void *callback_data) posn - i - 7); target[serverlen + posn - i - 7] = '\0'; if (get_verbosely) - fprintf(stderr, + fprintf(stderr, "Also look at %s\n", target); newalt = xmalloc(sizeof(*newalt)); newalt->next = NULL; newalt->base = target; newalt->got_indices = 0; newalt->packs = NULL; + path = strstr(target, "//"); + if (path) { + path = strchr(path+2, '/'); + if (path) + newalt->path_len = strlen(path); + } + while (tail->next != NULL) tail = tail->next; tail->next = newalt; @@ -553,7 +620,7 @@ static void fetch_alternates(char *base) char *url; char *data; struct active_request_slot *slot; - static struct alternates_request alt_req; + struct alternates_request alt_req; /* If another request has already started fetching alternates, wait for them to arrive and return to processing this request's @@ -578,7 +645,7 @@ static void fetch_alternates(char *base) if (get_verbosely) fprintf(stderr, "Getting alternates list for %s\n", base); - + url = xmalloc(strlen(base) + 31); sprintf(url, "%s/objects/info/http-alternates", base); @@ -607,6 +674,209 @@ static void fetch_alternates(char *base) free(url); } +#ifndef NO_EXPAT +static void +xml_start_tag(void *userData, const char *name, const char **atts) +{ + struct xml_ctx *ctx = (struct xml_ctx *)userData; + const char *c = strchr(name, ':'); + int new_len; + + if (c == NULL) + c = name; + else + c++; + + new_len = strlen(ctx->name) + strlen(c) + 2; + + if (new_len > ctx->len) { + ctx->name = xrealloc(ctx->name, new_len); + ctx->len = new_len; + } + strcat(ctx->name, "."); + strcat(ctx->name, c); + + if (ctx->cdata) { + free(ctx->cdata); + ctx->cdata = NULL; + } + + ctx->userFunc(ctx, 0); +} + +static void +xml_end_tag(void *userData, const char *name) +{ + struct xml_ctx *ctx = (struct xml_ctx *)userData; + const char *c = strchr(name, ':'); + char *ep; + + ctx->userFunc(ctx, 1); + + if (c == NULL) + c = name; + else + c++; + + ep = ctx->name + strlen(ctx->name) - strlen(c) - 1; + *ep = 0; +} + +static void +xml_cdata(void *userData, const XML_Char *s, int len) +{ + struct xml_ctx *ctx = (struct xml_ctx *)userData; + if (ctx->cdata) + free(ctx->cdata); + ctx->cdata = xcalloc(len+1, 1); + strncpy(ctx->cdata, s, len); +} + +static int remote_ls(struct alt_base *repo, const char *path, int flags, + void (*userFunc)(struct remote_ls_ctx *ls), + void *userData); + +static void handle_remote_ls_ctx(struct xml_ctx *ctx, int tag_closed) +{ + struct remote_ls_ctx *ls = (struct remote_ls_ctx *)ctx->userData; + + if (tag_closed) { + if (!strcmp(ctx->name, DAV_PROPFIND_RESP) && ls->dentry_name) { + if (ls->dentry_flags & IS_DIR) { + if (ls->flags & PROCESS_DIRS) { + ls->userFunc(ls); + } + if (strcmp(ls->dentry_name, ls->path) && + ls->flags & RECURSIVE) { + ls->rc = remote_ls(ls->repo, + ls->dentry_name, + ls->flags, + ls->userFunc, + ls->userData); + } + } else if (ls->flags & PROCESS_FILES) { + ls->userFunc(ls); + } + } else if (!strcmp(ctx->name, DAV_PROPFIND_NAME) && ctx->cdata) { + ls->dentry_name = xmalloc(strlen(ctx->cdata) - + ls->repo->path_len + 1); + strcpy(ls->dentry_name, ctx->cdata + ls->repo->path_len); + } else if (!strcmp(ctx->name, DAV_PROPFIND_COLLECTION)) { + ls->dentry_flags |= IS_DIR; + } + } else if (!strcmp(ctx->name, DAV_PROPFIND_RESP)) { + if (ls->dentry_name) { + free(ls->dentry_name); + } + ls->dentry_name = NULL; + ls->dentry_flags = 0; + } +} + +static int remote_ls(struct alt_base *repo, const char *path, int flags, + void (*userFunc)(struct remote_ls_ctx *ls), + void *userData) +{ + char *url = xmalloc(strlen(repo->base) + strlen(path) + 1); + struct active_request_slot *slot; + struct slot_results results; + struct buffer in_buffer; + struct buffer out_buffer; + char *in_data; + char *out_data; + XML_Parser parser = XML_ParserCreate(NULL); + enum XML_Status result; + struct curl_slist *dav_headers = NULL; + struct xml_ctx ctx; + struct remote_ls_ctx ls; + + ls.flags = flags; + ls.repo = repo; + ls.path = strdup(path); + ls.dentry_name = NULL; + ls.dentry_flags = 0; + ls.userData = userData; + ls.userFunc = userFunc; + ls.rc = 0; + + sprintf(url, "%s%s", repo->base, path); + + out_buffer.size = strlen(PROPFIND_ALL_REQUEST); + out_data = xmalloc(out_buffer.size + 1); + snprintf(out_data, out_buffer.size + 1, PROPFIND_ALL_REQUEST); + out_buffer.posn = 0; + out_buffer.buffer = out_data; + + in_buffer.size = 4096; + in_data = xmalloc(in_buffer.size); + in_buffer.posn = 0; + in_buffer.buffer = in_data; + + dav_headers = curl_slist_append(dav_headers, "Depth: 1"); + dav_headers = curl_slist_append(dav_headers, "Content-Type: text/xml"); + + slot = get_active_slot(); + slot->results = &results; + curl_easy_setopt(slot->curl, CURLOPT_INFILE, &out_buffer); + curl_easy_setopt(slot->curl, CURLOPT_INFILESIZE, out_buffer.size); + curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, fread_buffer); + curl_easy_setopt(slot->curl, CURLOPT_FILE, &in_buffer); + curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer); + curl_easy_setopt(slot->curl, CURLOPT_URL, url); + curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 1); + curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, DAV_PROPFIND); + curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, dav_headers); + + if (start_active_slot(slot)) { + run_active_slot(slot); + if (results.curl_result == CURLE_OK) { + ctx.name = xcalloc(10, 1); + ctx.len = 0; + ctx.cdata = NULL; + ctx.userFunc = handle_remote_ls_ctx; + ctx.userData = &ls; + XML_SetUserData(parser, &ctx); + XML_SetElementHandler(parser, xml_start_tag, + xml_end_tag); + XML_SetCharacterDataHandler(parser, xml_cdata); + result = XML_Parse(parser, in_buffer.buffer, + in_buffer.posn, 1); + free(ctx.name); + + if (result != XML_STATUS_OK) { + ls.rc = error("XML error: %s", + XML_ErrorString( + XML_GetErrorCode(parser))); + } + } else { + ls.rc = -1; + } + } else { + ls.rc = error("Unable to start PROPFIND request"); + } + + free(ls.path); + free(url); + free(out_data); + free(in_buffer.buffer); + curl_slist_free_all(dav_headers); + + return ls.rc; +} + +static void process_ls_pack(struct remote_ls_ctx *ls) +{ + unsigned char sha1[20]; + + if (strlen(ls->dentry_name) == 63 && + !strncmp(ls->dentry_name, "objects/pack/pack-", 18) && + !strncmp(ls->dentry_name+58, ".pack", 5)) { + get_sha1_hex(ls->dentry_name + 18, sha1); + setup_index(ls->repo, sha1); + } +} +#endif + static int fetch_indices(struct alt_base *repo) { unsigned char sha1[20]; @@ -616,6 +886,7 @@ static int fetch_indices(struct alt_base *repo) int i = 0; struct active_request_slot *slot; + struct slot_results results; if (repo->got_indices) return 0; @@ -627,20 +898,27 @@ static int fetch_indices(struct alt_base *repo) if (get_verbosely) fprintf(stderr, "Getting pack list for %s\n", repo->base); - + +#ifndef NO_EXPAT + if (remote_ls(repo, "objects/pack/", PROCESS_FILES, + process_ls_pack, NULL) == 0) + return 0; +#endif + url = xmalloc(strlen(repo->base) + 21); sprintf(url, "%s/objects/info/packs", repo->base); slot = get_active_slot(); + slot->results = &results; curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer); curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer); curl_easy_setopt(slot->curl, CURLOPT_URL, url); curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL); if (start_active_slot(slot)) { run_active_slot(slot); - if (slot->curl_result != CURLE_OK) { - if (slot->http_code == 404 || - slot->curl_result == CURLE_FILE_COULDNT_READ_FILE) { + if (results.curl_result != CURLE_OK) { + if (results.http_code == 404 || + results.curl_result == CURLE_FILE_COULDNT_READ_FILE) { repo->got_indices = 1; free(buffer.buffer); return 0; @@ -695,6 +973,7 @@ static int fetch_pack(struct alt_base *repo, unsigned char *sha1) struct curl_slist *range_header = NULL; struct active_request_slot *slot; + struct slot_results results; if (fetch_indices(repo)) return -1; @@ -721,6 +1000,7 @@ static int fetch_pack(struct alt_base *repo, unsigned char *sha1) filename); slot = get_active_slot(); + slot->results = &results; curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile); curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite); curl_easy_setopt(slot->curl, CURLOPT_URL, url); @@ -742,7 +1022,7 @@ static int fetch_pack(struct alt_base *repo, unsigned char *sha1) if (start_active_slot(slot)) { run_active_slot(slot); - if (slot->curl_result != CURLE_OK) { + if (results.curl_result != CURLE_OK) { fclose(packfile); return error("Unable to get pack file %s\n%s", url, curl_errorstr); @@ -770,6 +1050,20 @@ static int fetch_pack(struct alt_base *repo, unsigned char *sha1) return 0; } +static void abort_object_request(struct object_request *obj_req) +{ + if (obj_req->local >= 0) { + close(obj_req->local); + obj_req->local = -1; + } + unlink(obj_req->tmpfile); + if (obj_req->slot) { + release_active_slot(obj_req->slot); + obj_req->slot = NULL; + } + release_object_request(obj_req); +} + static int fetch_object(struct alt_base *repo, unsigned char *sha1) { char *hex = sha1_to_hex(sha1); @@ -782,7 +1076,7 @@ static int fetch_object(struct alt_base *repo, unsigned char *sha1) return error("Couldn't find request for %s in the queue", hex); if (has_sha1_file(obj_req->sha1)) { - release_object_request(obj_req); + abort_object_request(obj_req); return 0; } @@ -813,13 +1107,13 @@ static int fetch_object(struct alt_base *repo, unsigned char *sha1) obj_req->errorstr, obj_req->curl_result, obj_req->http_code, hex); } else if (obj_req->zret != Z_STREAM_END) { - ret = error("File %s (%s) corrupt\n", hex, obj_req->url); + corrupt_object_found++; + ret = error("File %s (%s) corrupt", hex, obj_req->url); } else if (memcmp(obj_req->sha1, obj_req->real_sha1, 20)) { - ret = error("File %s has bad hash\n", hex); + ret = error("File %s has bad hash", hex); } else if (obj_req->rename < 0) { - ret = error("unable to write sha1 filename %s: %s", - obj_req->filename, - strerror(obj_req->rename)); + ret = error("unable to write sha1 filename %s", + obj_req->filename); } release_object_request(obj_req); @@ -838,7 +1132,7 @@ int fetch(unsigned char *sha1) fetch_alternates(alt->base); altbase = altbase->next; } - return error("Unable to find %s under %s\n", sha1_to_hex(sha1), + return error("Unable to find %s under %s", sha1_to_hex(sha1), alt->base); } @@ -894,20 +1188,22 @@ int fetch_ref(char *ref, unsigned char *sha1) struct buffer buffer; char *base = alt->base; struct active_request_slot *slot; + struct slot_results results; buffer.size = 41; buffer.posn = 0; buffer.buffer = hex; hex[41] = '\0'; - + url = quote_ref_url(base, ref); slot = get_active_slot(); + slot->results = &results; curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer); curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer); curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL); curl_easy_setopt(slot->curl, CURLOPT_URL, url); if (start_active_slot(slot)) { run_active_slot(slot); - if (slot->curl_result != CURLE_OK) + if (results.curl_result != CURLE_OK) return error("Couldn't get %s for %s\n%s", url, ref, curl_errorstr); } else { @@ -923,10 +1219,12 @@ int main(int argc, char **argv) { char *commit_id; char *url; + char *path; int arg = 1; int rc = 0; setup_git_directory(); + git_config(git_default_config); while (arg < argc && argv[arg][0] == '-') { if (argv[arg][1] == 't') { @@ -953,6 +1251,7 @@ int main(int argc, char **argv) } commit_id = argv[arg]; url = argv[arg + 1]; + write_ref_log_details = url; http_init(); @@ -963,13 +1262,25 @@ int main(int argc, char **argv) alt->got_indices = 0; alt->packs = NULL; alt->next = NULL; + path = strstr(url, "//"); + if (path) { + path = strchr(path+2, '/'); + if (path) + alt->path_len = strlen(path); + } if (pull(commit_id)) rc = 1; - curl_slist_free_all(no_pragma_header); - http_cleanup(); + curl_slist_free_all(no_pragma_header); + + if (corrupt_object_found) { + fprintf(stderr, +"Some loose object were found to be corrupt, but they might be just\n" +"a false '404 Not Found' error message sent with incorrect HTTP\n" +"status code. Suggest running git fsck-objects.\n"); + } return rc; }