From: Junio C Hamano Date: Thu, 23 Feb 2006 00:38:21 +0000 (-0800) Subject: Merge master to get fixes up to GIT 1.2.3 X-Git-Tag: v1.3.0-rc1~54^2~43 X-Git-Url: https://git.octo.it/?a=commitdiff_plain;h=597888adc6006cbca02175b6f070f8ef46b014e6;hp=-c;p=git.git Merge master to get fixes up to GIT 1.2.3 --- 597888adc6006cbca02175b6f070f8ef46b014e6 diff --combined git-fetch.sh index 23d965f3,fcc24f85..de4f011e --- a/git-fetch.sh +++ b/git-fetch.sh @@@ -320,7 -320,7 +320,7 @@@ fetch_main () ( : subshell because we muck with IFS IFS=" $LF" ( - git-fetch-pack $exec $keep "$remote" $rref || echo failed "$remote" + git-fetch-pack $exec $keep --thin "$remote" $rref || echo failed "$remote" ) | while read sha1 remote_name do @@@ -368,20 -368,25 +368,25 @@@ fetch_main "$reflist # automated tag following case "$no_tags$tags" in '') - taglist=$(IFS=" " && - git-ls-remote $upload_pack --tags "$remote" | - sed -ne 's|^\([0-9a-f]*\)[ ]\(refs/tags/.*\)^{}$|\1 \2|p' | - while read sha1 name - do - test -f "$GIT_DIR/$name" && continue - git-check-ref-format "$name" || { - echo >&2 "warning: tag ${name} ignored" - continue - } - git-cat-file -t "$sha1" >/dev/null 2>&1 || continue - echo >&2 "Auto-following $name" - echo ".${name}:${name}" - done) + case "$reflist" in + *:refs/*) + # effective only when we are following remote branch + # using local tracking branch. + taglist=$(IFS=" " && + git-ls-remote $upload_pack --tags "$remote" | + sed -ne 's|^\([0-9a-f]*\)[ ]\(refs/tags/.*\)^{}$|\1 \2|p' | + while read sha1 name + do + test -f "$GIT_DIR/$name" && continue + git-check-ref-format "$name" || { + echo >&2 "warning: tag ${name} ignored" + continue + } + git-cat-file -t "$sha1" >/dev/null 2>&1 || continue + echo >&2 "Auto-following $name" + echo ".${name}:${name}" + done) + esac case "$taglist" in '') ;; ?*) diff --combined pack-objects.c index 4f8814d8,8f352aa6..32befa00 --- a/pack-objects.c +++ b/pack-objects.c @@@ -3,8 -3,8 +3,9 @@@ #include "delta.h" #include "pack.h" #include "csum-file.h" +#include "diff.h" #include + #include static const char pack_usage[] = "git-pack-objects [-q] [--no-reuse-delta] [--non-empty] [--local] [--incremental] [--window=N] [--depth=N] {--stdout | base-name} < object-list"; @@@ -27,13 -27,6 +28,13 @@@ struct object_entry struct object_entry *delta_sibling; /* other deltified objects who * uses the same base as me */ + int preferred_base; /* we do not pack this, but is encouraged to + * be used as the base objectto delta huge + * objects against. + */ + int based_on_preferred; /* current delta candidate is a preferred + * one, or delta against a preferred one. + */ }; /* @@@ -56,10 -49,11 +57,11 @@@ static int local = 0 static int incremental = 0; static struct object_entry **sorted_by_sha, **sorted_by_type; static struct object_entry *objects = NULL; -static int nr_objects = 0, nr_alloc = 0; +static int nr_objects = 0, nr_alloc = 0, nr_result = 0; static const char *base_name; static unsigned char pack_file_sha1[20]; static int progress = 1; + static volatile int progress_update = 0; /* * The object names in objects array are hashed with this hashtable, @@@ -237,8 -231,7 +239,8 @@@ static int encode_header(enum object_ty return n; } -static unsigned long write_object(struct sha1file *f, struct object_entry *entry) +static unsigned long write_object(struct sha1file *f, + struct object_entry *entry) { unsigned long size; char type[10]; @@@ -248,9 -241,6 +250,9 @@@ enum object_type obj_type; int to_reuse = 0; + if (entry->preferred_base) + return 0; + obj_type = entry->type; if (! entry->in_pack) to_reuse = 0; /* can't reuse what we don't have */ @@@ -334,30 -324,48 +336,51 @@@ static void write_pack_file(void struct sha1file *f; unsigned long offset; struct pack_header hdr; + unsigned last_percent = 999; + int do_progress = 0; if (!base_name) f = sha1fd(1, ""); - else + else { f = sha1create("%s-%s.%s", base_name, sha1_to_hex(object_list_sha1), "pack"); + do_progress = progress; + } + if (do_progress) - fprintf(stderr, "Writing %d objects.\n", nr_objects); ++ fprintf(stderr, "Writing %d objects.\n", nr_result); + hdr.hdr_signature = htonl(PACK_SIGNATURE); hdr.hdr_version = htonl(PACK_VERSION); - hdr.hdr_entries = htonl(nr_objects); + hdr.hdr_entries = htonl(nr_result); sha1write(f, &hdr, sizeof(hdr)); offset = sizeof(hdr); - for (i = 0; i < nr_objects; i++) ++ if (!nr_result) ++ goto done; + for (i = 0; i < nr_objects; i++) { offset = write_one(f, objects + i, offset); - + if (do_progress) { - unsigned percent = written * 100 / nr_objects; ++ unsigned percent = written * 100 / nr_result; + if (progress_update || percent != last_percent) { + fprintf(stderr, "%4u%% (%u/%u) done\r", - percent, written, nr_objects); ++ percent, written, nr_result); + progress_update = 0; + last_percent = percent; + } + } + } + if (do_progress) + fputc('\n', stderr); - ++ done: sha1close(f, pack_file_sha1, 1); } static void write_index_file(void) { int i; - struct sha1file *f = sha1create("%s-%s.%s", base_name, sha1_to_hex(object_list_sha1), "idx"); + struct sha1file *f = sha1create("%s-%s.%s", base_name, + sha1_to_hex(object_list_sha1), "idx"); struct object_entry **list = sorted_by_sha; - struct object_entry **last = list + nr_objects; + struct object_entry **last = list + nr_result; unsigned int array[256]; /* @@@ -382,7 -390,7 +405,7 @@@ * Write the actual SHA1 entries.. */ list = sorted_by_sha; - for (i = 0; i < nr_objects; i++) { + for (i = 0; i < nr_result; i++) { struct object_entry *entry = *list++; unsigned int offset = htonl(entry->offset); sha1write(f, &offset, 4); @@@ -392,87 -400,27 +415,87 @@@ sha1close(f, NULL, 1); } -static int add_object_entry(unsigned char *sha1, unsigned int hash) +static int locate_object_entry_hash(const unsigned char *sha1) +{ + int i; + unsigned int ui; + memcpy(&ui, sha1, sizeof(unsigned int)); + i = ui % object_ix_hashsz; + while (0 < object_ix[i]) { + if (!memcmp(sha1, objects[object_ix[i]-1].sha1, 20)) + return i; + if (++i == object_ix_hashsz) + i = 0; + } + return -1 - i; +} + +static struct object_entry *locate_object_entry(const unsigned char *sha1) +{ + int i; + + if (!object_ix_hashsz) + return NULL; + + i = locate_object_entry_hash(sha1); + if (0 <= i) + return &objects[object_ix[i]-1]; + return NULL; +} + +static void rehash_objects(void) { + int i; + struct object_entry *oe; + + object_ix_hashsz = nr_objects * 3; + if (object_ix_hashsz < 1024) + object_ix_hashsz = 1024; + object_ix = xrealloc(object_ix, sizeof(int) * object_ix_hashsz); + object_ix = memset(object_ix, 0, sizeof(int) * object_ix_hashsz); + for (i = 0, oe = objects; i < nr_objects; i++, oe++) { + int ix = locate_object_entry_hash(oe->sha1); + if (0 <= ix) + continue; + ix = -1 - ix; + object_ix[ix] = i + 1; + } +} + +static int add_object_entry(const unsigned char *sha1, const char *name, int exclude) +{ + unsigned int hash = 0; unsigned int idx = nr_objects; struct object_entry *entry; struct packed_git *p; unsigned int found_offset = 0; struct packed_git *found_pack = NULL; - - for (p = packed_git; p; p = p->next) { - struct pack_entry e; - if (find_pack_entry_one(sha1, &e, p)) { - if (incremental) - return 0; - if (local && !p->pack_local) - return 0; - if (!found_pack) { - found_offset = e.offset; - found_pack = e.p; + int ix; + + if (!exclude) { + for (p = packed_git; p; p = p->next) { + struct pack_entry e; + if (find_pack_entry_one(sha1, &e, p)) { + if (incremental) + return 0; + if (local && !p->pack_local) + return 0; + if (!found_pack) { + found_offset = e.offset; + found_pack = e.p; + } } } } + if ((entry = locate_object_entry(sha1)) != NULL) + goto already_added; + + while (*name) { + unsigned char c = *name++; + if (isspace(c)) + continue; + hash = hash * 11 + c; + } if (idx >= nr_alloc) { unsigned int needed = (idx + 1024) * 3 / 2; @@@ -480,79 -428,45 +503,83 @@@ nr_alloc = needed; } entry = objects + idx; + nr_objects = idx + 1; memset(entry, 0, sizeof(*entry)); memcpy(entry->sha1, sha1, 20); entry->hash = hash; - if (found_pack) { - entry->in_pack = found_pack; - entry->in_pack_offset = found_offset; + + if (object_ix_hashsz * 3 <= nr_objects * 4) + rehash_objects(); + else { + ix = locate_object_entry_hash(entry->sha1); + if (0 <= ix) + die("internal error in object hashing."); + object_ix[-1 - ix] = idx + 1; + } + + already_added: ++ if (progress_update) { ++ fprintf(stderr, "Counting objects...%d\r", nr_objects); ++ progress_update = 0; ++ } + if (exclude) + entry->preferred_base = 1; + else { + if (found_pack) { + entry->in_pack = found_pack; + entry->in_pack_offset = found_offset; + } } - nr_objects = idx+1; return 1; } -static int locate_object_entry_hash(unsigned char *sha1) +static void add_pbase_tree(struct tree_desc *tree) { - int i; - unsigned int ui; - memcpy(&ui, sha1, sizeof(unsigned int)); - i = ui % object_ix_hashsz; - while (0 < object_ix[i]) { - if (!memcmp(sha1, objects[object_ix[i]-1].sha1, 20)) - return i; - if (++i == object_ix_hashsz) - i = 0; + while (tree->size) { + const unsigned char *sha1; + const char *name; + unsigned mode; + unsigned long size; + char type[20]; + + sha1 = tree_entry_extract(tree, &name, &mode); + update_tree_entry(tree); + if (!has_sha1_file(sha1)) + continue; + if (sha1_object_info(sha1, type, &size)) + continue; + add_object_entry(sha1, name, 1); + if (!strcmp(type, "tree")) { + struct tree_desc sub; + void *elem; + elem = read_sha1_file(sha1, type, &sub.size); + sub.buf = elem; + if (sub.buf) { + add_pbase_tree(&sub); + free(elem); + } + } } - return -1 - i; } -static struct object_entry *locate_object_entry(unsigned char *sha1) +static void add_preferred_base(unsigned char *sha1) { - int i = locate_object_entry_hash(sha1); - if (0 <= i) - return &objects[object_ix[i]-1]; - return NULL; + struct tree_desc tree; + void *elem; + elem = read_object_with_reference(sha1, "tree", &tree.size, NULL); + tree.buf = elem; + if (!tree.buf) + return; + add_object_entry(sha1, "", 1); + add_pbase_tree(&tree); + free(elem); } static void check_object(struct object_entry *entry) { char type[20]; - if (entry->in_pack) { + if (entry->in_pack && !entry->preferred_base) { unsigned char base[20]; unsigned long size; struct object_entry *base_entry; @@@ -571,8 -485,7 +598,8 @@@ */ if (!no_reuse_delta && entry->in_pack_type == OBJ_DELTA && - (base_entry = locate_object_entry(base))) { + (base_entry = locate_object_entry(base)) && + (!base_entry->preferred_base)) { /* Depth value does not matter - find_deltas() * will never consider reused delta as the @@@ -610,6 -523,25 +637,6 @@@ sha1_to_hex(entry->sha1), type); } -static void hash_objects(void) -{ - int i; - struct object_entry *oe; - - object_ix_hashsz = nr_objects * 2; - object_ix = xcalloc(sizeof(int), object_ix_hashsz); - for (i = 0, oe = objects; i < nr_objects; i++, oe++) { - int ix = locate_object_entry_hash(oe->sha1); - if (0 <= ix) { - error("the same object '%s' added twice", - sha1_to_hex(oe->sha1)); - continue; - } - ix = -1 - ix; - object_ix[ix] = i + 1; - } -} - static unsigned int check_delta_limit(struct object_entry *me, unsigned int n) { struct object_entry *child = me->delta_child; @@@ -628,6 -560,7 +655,6 @@@ static void get_object_details(void int i; struct object_entry *entry; - hash_objects(); prepare_pack_ix(); for (i = 0, entry = objects; i < nr_objects; i++, entry++) check_object(entry); @@@ -665,24 -598,6 +692,24 @@@ static int sha1_sort(const struct objec return memcmp(a->sha1, b->sha1, 20); } +static struct object_entry **create_final_object_list() +{ + struct object_entry **list; + int i, j; + + for (i = nr_result = 0; i < nr_objects; i++) + if (!objects[i].preferred_base) + nr_result++; + list = xmalloc(nr_result * sizeof(struct object_entry *)); + for (i = j = 0; i < nr_objects; i++) { + if (!objects[i].preferred_base) + list[j++] = objects + i; + } + current_sort = sha1_sort; + qsort(list, nr_result, sizeof(struct object_entry *), sort_comparator); + return list; +} + static int type_size_sort(const struct object_entry *a, const struct object_entry *b) { if (a->type < b->type) @@@ -693,10 -608,6 +720,10 @@@ return -1; if (a->hash > b->hash) return 1; + if (a->preferred_base < b->preferred_base) + return -1; + if (a->preferred_base > b->preferred_base) + return 1; if (a->size < b->size) return -1; if (a->size > b->size) @@@ -721,8 -632,6 +748,8 @@@ static int try_delta(struct unpacked *c { struct object_entry *cur_entry = cur->entry; struct object_entry *old_entry = old->entry; + int old_preferred = (old_entry->preferred_base || + old_entry->based_on_preferred); unsigned long size, oldsize, delta_size, sizediff; long max_size; void *delta_buf; @@@ -731,15 -640,9 +758,15 @@@ if (cur_entry->type != old_entry->type) return -1; - /* If the current object is at edge, take the depth the objects - * that depend on the current object into account -- otherwise - * they would become too deep. + /* We do not compute delta to *create* objects we are not + * going to pack. + */ + if (cur_entry->preferred_base) + return -1; + + /* If the current object is at pack edge, take the depth the + * objects that depend on the current object into account -- + * otherwise they would become too deep. */ if (cur_entry->delta_child) { if (max_depth <= cur_entry->delta_limit) @@@ -748,10 -651,11 +775,10 @@@ } size = cur_entry->size; - if (size < 50) - return -1; oldsize = old_entry->size; sizediff = oldsize > size ? oldsize - size : size - oldsize; - if (sizediff > size / 8) + + if (size < 50) return -1; if (old_entry->depth >= max_depth) return 0; @@@ -764,27 -668,8 +791,27 @@@ * delete). */ max_size = size / 2 - 20; - if (cur_entry->delta) - max_size = cur_entry->delta_size-1; + if (cur_entry->delta) { + if (cur_entry->based_on_preferred) { + if (old_preferred) + max_size = cur_entry->delta_size-1; + else + /* trying with non-preferred one when we + * already have a delta based on preferred + * one is pointless. + */ + return 0; + } + else if (!old_preferred) + max_size = cur_entry->delta_size-1; + else + /* otherwise... even if delta with a + * preferred one produces a bigger result than + * what we currently have, which is based on a + * non-preferred one, it is OK. + */ + ; + } if (sizediff >= max_size) return -1; delta_buf = diff_delta(old->data, oldsize, @@@ -794,22 -679,29 +821,30 @@@ cur_entry->delta = old_entry; cur_entry->delta_size = delta_size; cur_entry->depth = old_entry->depth + 1; + cur_entry->based_on_preferred = old_preferred; free(delta_buf); return 0; } + static void progress_interval(int signum) + { + signal(SIGALRM, progress_interval); + progress_update = 1; + } + static void find_deltas(struct object_entry **list, int window, int depth) { int i, idx; unsigned int array_size = window * sizeof(struct unpacked); struct unpacked *array = xmalloc(array_size); - int eye_candy; + unsigned processed = 0; + unsigned last_percent = 999; memset(array, 0, array_size); i = nr_objects; idx = 0; - eye_candy = i - (nr_objects / 20); + if (progress) - fprintf(stderr, "Deltifying %d objects.\n", nr_objects); ++ fprintf(stderr, "Deltifying %d objects.\n", nr_result); while (--i >= 0) { struct object_entry *entry = list[i]; @@@ -818,9 -710,15 +853,17 @@@ char type[10]; int j; - if (progress && i <= eye_candy) { - eye_candy -= nr_objects / 20; - fputc('.', stderr); - processed++; ++ if (!entry->preferred_base) ++ processed++; ++ + if (progress) { - unsigned percent = processed * 100 / nr_objects; ++ unsigned percent = processed * 100 / nr_result; + if (percent != last_percent || progress_update) { + fprintf(stderr, "%4u%% (%u/%u) done\r", - percent, processed, nr_objects); ++ percent, processed, nr_result); + progress_update = 0; + last_percent = percent; + } } if (entry->delta) @@@ -852,6 -750,9 +895,9 @@@ idx = 0; } + if (progress) + fputc('\n', stderr); + for (i = 0; i < window; ++i) free(array[i].data); free(array); @@@ -859,18 -760,10 +905,10 @@@ static void prepare_pack(int window, int depth) { - if (progress) - fprintf(stderr, "Packing %d objects", nr_result); get_object_details(); - if (progress) - fputc('.', stderr); - sorted_by_type = create_sorted_list(type_size_sort); if (window && depth) find_deltas(sorted_by_type, window+1, depth); - if (progress) - fputc('\n', stderr); - write_pack_file(); } static int reuse_cached_pack(unsigned char *sha1, int pack_to_stdout) @@@ -934,10 -827,6 +972,6 @@@ int main(int argc, char **argv int window = 10, depth = 10, pack_to_stdout = 0; struct object_entry **list; int i; - struct timeval prev_tv; - int eye_candy = 0; - int eye_candy_incr = 500; - setup_git_directory(); @@@ -994,60 -883,64 +1028,60 @@@ usage(pack_usage); prepare_packed_git(); + if (progress) { + struct itimerval v; + v.it_interval.tv_sec = 1; + v.it_interval.tv_usec = 0; + v.it_value = v.it_interval; + signal(SIGALRM, progress_interval); + setitimer(ITIMER_REAL, &v, NULL); fprintf(stderr, "Generating pack...\n"); - gettimeofday(&prev_tv, NULL); } + while (fgets(line, sizeof(line), stdin) != NULL) { - unsigned int hash; - char *p; unsigned char sha1[20]; - if (progress && (eye_candy <= nr_objects)) { - if (progress_update) { -- fprintf(stderr, "Counting objects...%d\r", nr_objects); - if (eye_candy && (50 <= eye_candy_incr)) { - struct timeval tv; - int time_diff; - gettimeofday(&tv, NULL); - time_diff = (tv.tv_sec - prev_tv.tv_sec); - time_diff <<= 10; - time_diff += (tv.tv_usec - prev_tv.tv_usec); - if ((1 << 9) < time_diff) - eye_candy_incr += 50; - else if (50 < eye_candy_incr) - eye_candy_incr -= 50; - } - eye_candy += eye_candy_incr; - } - progress_update = 0; + if (line[0] == '-') { + if (get_sha1_hex(line+1, sha1)) + die("expected edge sha1, got garbage:\n %s", + line+1); + add_preferred_base(sha1); + continue; } if (get_sha1_hex(line, sha1)) die("expected sha1, got garbage:\n %s", line); - hash = 0; - p = line+40; - while (*p) { - unsigned char c = *p++; - if (isspace(c)) - continue; - hash = hash * 11 + c; - } - add_object_entry(sha1, hash); + add_object_entry(sha1, line+40, 0); } if (progress) fprintf(stderr, "Done counting %d objects.\n", nr_objects); -- if (non_empty && !nr_objects) ++ sorted_by_sha = create_final_object_list(); ++ if (non_empty && !nr_result) return 0; - sorted_by_sha = create_final_object_list(); - sorted_by_sha = create_sorted_list(sha1_sort); SHA1_Init(&ctx); list = sorted_by_sha; - for (i = 0; i < nr_objects; i++) { + for (i = 0; i < nr_result; i++) { struct object_entry *entry = *list++; SHA1_Update(&ctx, entry->sha1, 20); } SHA1_Final(object_list_sha1, &ctx); + if (progress && (nr_objects != nr_result)) + fprintf(stderr, "Result has %d objects.\n", nr_result); if (reuse_cached_pack(object_list_sha1, pack_to_stdout)) ; else { -- prepare_pack(window, depth); ++ if (nr_result) ++ prepare_pack(window, depth); + if (progress && pack_to_stdout) { + /* the other end usually displays progress itself */ + struct itimerval v = {{0,},}; + setitimer(ITIMER_REAL, &v, NULL); + signal(SIGALRM, SIG_IGN ); + progress_update = 0; + } + write_pack_file(); if (!pack_to_stdout) { write_index_file(); puts(sha1_to_hex(object_list_sha1)); @@@ -1055,6 -948,6 +1089,6 @@@ } if (progress) fprintf(stderr, "Total %d, written %d (delta %d), reused %d (delta %d)\n", - nr_objects, written, written_delta, reused, reused_delta); + nr_result, written, written_delta, reused, reused_delta); return 0; }