From: Junio C Hamano Date: Sun, 2 Apr 2006 20:54:28 +0000 (-0700) Subject: Merge branch 'lt/fix-sol-pack' into next X-Git-Tag: v1.3.0-rc4~49 X-Git-Url: https://git.octo.it/?p=git.git;a=commitdiff_plain;h=f0c979f4dbbb7d6d9565d945c540f0a5748f1b80;hp=-c Merge branch 'lt/fix-sol-pack' into next * lt/fix-sol-pack: pack-objects: be incredibly anal about stdio semantics Fix Solaris stdio signal handling stupidities Signed-off-by: Junio C Hamano --- f0c979f4dbbb7d6d9565d945c540f0a5748f1b80 diff --combined pack-objects.c index ccfaa5f6,084c2006..0ea16ad6 --- a/pack-objects.c +++ b/pack-objects.c @@@ -3,7 -3,6 +3,7 @@@ #include "delta.h" #include "pack.h" #include "csum-file.h" +#include "tree-walk.h" #include #include @@@ -28,10 -27,6 +28,10 @@@ struct object_entry struct object_entry *delta_sibling; /* other deltified objects who * uses the same base as me */ + int preferred_base; /* we do not pack this, but is encouraged to + * be used as the base objectto delta huge + * objects against. + */ }; /* @@@ -54,11 -49,11 +54,11 @@@ static int local = 0 static int incremental = 0; static struct object_entry **sorted_by_sha, **sorted_by_type; static struct object_entry *objects = NULL; -static int nr_objects = 0, nr_alloc = 0; +static int nr_objects = 0, nr_alloc = 0, nr_result = 0; static const char *base_name; static unsigned char pack_file_sha1[20]; static int progress = 1; - static volatile int progress_update = 0; + static volatile sig_atomic_t progress_update = 0; /* * The object names in objects array are hashed with this hashtable, @@@ -96,7 -91,7 +96,7 @@@ static int reused_delta = 0 static int pack_revindex_ix(struct packed_git *p) { - unsigned int ui = (unsigned int) p; + unsigned long ui = (unsigned long)p; int i; ui = ui ^ (ui >> 16); /* defeat structure alignment */ @@@ -236,8 -231,7 +236,8 @@@ static int encode_header(enum object_ty return n; } -static unsigned long write_object(struct sha1file *f, struct object_entry *entry) +static unsigned long write_object(struct sha1file *f, + struct object_entry *entry) { unsigned long size; char type[10]; @@@ -247,9 -241,6 +247,9 @@@ enum object_type obj_type; int to_reuse = 0; + if (entry->preferred_base) + return 0; + obj_type = entry->type; if (! entry->in_pack) to_reuse = 0; /* can't reuse what we don't have */ @@@ -344,22 -335,20 +344,22 @@@ static void write_pack_file(void do_progress = progress; } if (do_progress) - fprintf(stderr, "Writing %d objects.\n", nr_objects); + fprintf(stderr, "Writing %d objects.\n", nr_result); hdr.hdr_signature = htonl(PACK_SIGNATURE); hdr.hdr_version = htonl(PACK_VERSION); - hdr.hdr_entries = htonl(nr_objects); + hdr.hdr_entries = htonl(nr_result); sha1write(f, &hdr, sizeof(hdr)); offset = sizeof(hdr); + if (!nr_result) + goto done; for (i = 0; i < nr_objects; i++) { offset = write_one(f, objects + i, offset); if (do_progress) { - unsigned percent = written * 100 / nr_objects; + unsigned percent = written * 100 / nr_result; if (progress_update || percent != last_percent) { fprintf(stderr, "%4u%% (%u/%u) done\r", - percent, written, nr_objects); + percent, written, nr_result); progress_update = 0; last_percent = percent; } @@@ -367,17 -356,16 +367,17 @@@ } if (do_progress) fputc('\n', stderr); - + done: sha1close(f, pack_file_sha1, 1); } static void write_index_file(void) { int i; - struct sha1file *f = sha1create("%s-%s.%s", base_name, sha1_to_hex(object_list_sha1), "idx"); + struct sha1file *f = sha1create("%s-%s.%s", base_name, + sha1_to_hex(object_list_sha1), "idx"); struct object_entry **list = sorted_by_sha; - struct object_entry **last = list + nr_objects; + struct object_entry **last = list + nr_result; unsigned int array[256]; /* @@@ -402,7 -390,7 +402,7 @@@ * Write the actual SHA1 entries.. */ list = sorted_by_sha; - for (i = 0; i < nr_objects; i++) { + for (i = 0; i < nr_result; i++) { struct object_entry *entry = *list++; unsigned int offset = htonl(entry->offset); sha1write(f, &offset, 4); @@@ -412,139 -400,27 +412,139 @@@ sha1close(f, NULL, 1); } -static int add_object_entry(unsigned char *sha1, unsigned int hash) +static int locate_object_entry_hash(const unsigned char *sha1) +{ + int i; + unsigned int ui; + memcpy(&ui, sha1, sizeof(unsigned int)); + i = ui % object_ix_hashsz; + while (0 < object_ix[i]) { + if (!memcmp(sha1, objects[object_ix[i]-1].sha1, 20)) + return i; + if (++i == object_ix_hashsz) + i = 0; + } + return -1 - i; +} + +static struct object_entry *locate_object_entry(const unsigned char *sha1) +{ + int i; + + if (!object_ix_hashsz) + return NULL; + + i = locate_object_entry_hash(sha1); + if (0 <= i) + return &objects[object_ix[i]-1]; + return NULL; +} + +static void rehash_objects(void) +{ + int i; + struct object_entry *oe; + + object_ix_hashsz = nr_objects * 3; + if (object_ix_hashsz < 1024) + object_ix_hashsz = 1024; + object_ix = xrealloc(object_ix, sizeof(int) * object_ix_hashsz); + object_ix = memset(object_ix, 0, sizeof(int) * object_ix_hashsz); + for (i = 0, oe = objects; i < nr_objects; i++, oe++) { + int ix = locate_object_entry_hash(oe->sha1); + if (0 <= ix) + continue; + ix = -1 - ix; + object_ix[ix] = i + 1; + } +} + +struct name_path { + struct name_path *up; + const char *elem; + int len; +}; + +#define DIRBITS 12 + +static unsigned name_hash(struct name_path *path, const char *name) +{ + struct name_path *p = path; + const char *n = name + strlen(name); + unsigned hash = 0, name_hash = 0, name_done = 0; + + if (n != name && n[-1] == '\n') + n--; + while (name <= --n) { + unsigned char c = *n; + if (c == '/' && !name_done) { + name_hash = hash; + name_done = 1; + hash = 0; + } + hash = hash * 11 + c; + } + if (!name_done) { + name_hash = hash; + hash = 0; + } + for (p = path; p; p = p->up) { + hash = hash * 11 + '/'; + n = p->elem + p->len; + while (p->elem <= --n) { + unsigned char c = *n; + hash = hash * 11 + c; + } + } + /* + * Make sure "Makefile" and "t/Makefile" are hashed separately + * but close enough. + */ + hash = (name_hash<up) { + fputc('/', stderr); + n = p->elem + p->len; + while (p->elem <= --n) + fputc(*n, stderr); + } + fprintf(stderr, "\t%08x\n", hash); + } + return hash; +} + +static int add_object_entry(const unsigned char *sha1, unsigned hash, int exclude) { unsigned int idx = nr_objects; struct object_entry *entry; struct packed_git *p; unsigned int found_offset = 0; struct packed_git *found_pack = NULL; - - for (p = packed_git; p; p = p->next) { - struct pack_entry e; - if (find_pack_entry_one(sha1, &e, p)) { - if (incremental) - return 0; - if (local && !p->pack_local) - return 0; - if (!found_pack) { - found_offset = e.offset; - found_pack = e.p; + int ix, status = 0; + + if (!exclude) { + for (p = packed_git; p; p = p->next) { + struct pack_entry e; + if (find_pack_entry_one(sha1, &e, p)) { + if (incremental) + return 0; + if (local && !p->pack_local) + return 0; + if (!found_pack) { + found_offset = e.offset; + found_pack = e.p; + } } } } + if ((entry = locate_object_entry(sha1)) != NULL) + goto already_added; if (idx >= nr_alloc) { unsigned int needed = (idx + 1024) * 3 / 2; @@@ -552,94 -428,45 +552,94 @@@ nr_alloc = needed; } entry = objects + idx; + nr_objects = idx + 1; memset(entry, 0, sizeof(*entry)); memcpy(entry->sha1, sha1, 20); entry->hash = hash; - if (found_pack) { - entry->in_pack = found_pack; - entry->in_pack_offset = found_offset; + + if (object_ix_hashsz * 3 <= nr_objects * 4) + rehash_objects(); + else { + ix = locate_object_entry_hash(entry->sha1); + if (0 <= ix) + die("internal error in object hashing."); + object_ix[-1 - ix] = idx + 1; } - nr_objects = idx+1; - return 1; + status = 1; + + already_added: + if (progress_update) { + fprintf(stderr, "Counting objects...%d\r", nr_objects); + progress_update = 0; + } + if (exclude) + entry->preferred_base = 1; + else { + if (found_pack) { + entry->in_pack = found_pack; + entry->in_pack_offset = found_offset; + } + } + return status; } -static int locate_object_entry_hash(unsigned char *sha1) +static void add_pbase_tree(struct tree_desc *tree, struct name_path *up) { - int i; - unsigned int ui; - memcpy(&ui, sha1, sizeof(unsigned int)); - i = ui % object_ix_hashsz; - while (0 < object_ix[i]) { - if (!memcmp(sha1, objects[object_ix[i]-1].sha1, 20)) - return i; - if (++i == object_ix_hashsz) - i = 0; + while (tree->size) { + const unsigned char *sha1; + const char *name; + unsigned mode, hash; + unsigned long size; + char type[20]; + + sha1 = tree_entry_extract(tree, &name, &mode); + update_tree_entry(tree); + if (!has_sha1_file(sha1)) + continue; + if (sha1_object_info(sha1, type, &size)) + continue; + + hash = name_hash(up, name); + if (!add_object_entry(sha1, hash, 1)) + continue; + + if (!strcmp(type, "tree")) { + struct tree_desc sub; + void *elem; + struct name_path me; + + elem = read_sha1_file(sha1, type, &sub.size); + sub.buf = elem; + if (sub.buf) { + me.up = up; + me.elem = name; + me.len = strlen(name); + add_pbase_tree(&sub, &me); + free(elem); + } + } } - return -1 - i; } -static struct object_entry *locate_object_entry(unsigned char *sha1) +static void add_preferred_base(unsigned char *sha1) { - int i = locate_object_entry_hash(sha1); - if (0 <= i) - return &objects[object_ix[i]-1]; - return NULL; + struct tree_desc tree; + void *elem; + + elem = read_object_with_reference(sha1, "tree", &tree.size, NULL); + tree.buf = elem; + if (!tree.buf) + return; + if (add_object_entry(sha1, name_hash(NULL, ""), 1)) + add_pbase_tree(&tree, NULL); + free(elem); } static void check_object(struct object_entry *entry) { char type[20]; - if (entry->in_pack) { + if (entry->in_pack && !entry->preferred_base) { unsigned char base[20]; unsigned long size; struct object_entry *base_entry; @@@ -658,8 -485,7 +658,8 @@@ */ if (!no_reuse_delta && entry->in_pack_type == OBJ_DELTA && - (base_entry = locate_object_entry(base))) { + (base_entry = locate_object_entry(base)) && + (!base_entry->preferred_base)) { /* Depth value does not matter - find_deltas() * will never consider reused delta as the @@@ -697,6 -523,25 +697,6 @@@ sha1_to_hex(entry->sha1), type); } -static void hash_objects(void) -{ - int i; - struct object_entry *oe; - - object_ix_hashsz = nr_objects * 2; - object_ix = xcalloc(sizeof(int), object_ix_hashsz); - for (i = 0, oe = objects; i < nr_objects; i++, oe++) { - int ix = locate_object_entry_hash(oe->sha1); - if (0 <= ix) { - error("the same object '%s' added twice", - sha1_to_hex(oe->sha1)); - continue; - } - ix = -1 - ix; - object_ix[ix] = i + 1; - } -} - static unsigned int check_delta_limit(struct object_entry *me, unsigned int n) { struct object_entry *child = me->delta_child; @@@ -715,26 -560,14 +715,26 @@@ static void get_object_details(void int i; struct object_entry *entry; - hash_objects(); prepare_pack_ix(); for (i = 0, entry = objects; i < nr_objects; i++, entry++) check_object(entry); - for (i = 0, entry = objects; i < nr_objects; i++, entry++) - if (!entry->delta && entry->delta_child) - entry->delta_limit = - check_delta_limit(entry, 1); + + if (nr_objects == nr_result) { + /* + * Depth of objects that depend on the entry -- this + * is subtracted from depth-max to break too deep + * delta chain because of delta data reusing. + * However, we loosen this restriction when we know we + * are creating a thin pack -- it will have to be + * expanded on the other end anyway, so do not + * artificially cut the delta chain and let it go as + * deep as it wants. + */ + for (i = 0, entry = objects; i < nr_objects; i++, entry++) + if (!entry->delta && entry->delta_child) + entry->delta_limit = + check_delta_limit(entry, 1); + } } typedef int (*entry_sort_t)(const struct object_entry *, const struct object_entry *); @@@ -765,24 -598,6 +765,24 @@@ static int sha1_sort(const struct objec return memcmp(a->sha1, b->sha1, 20); } +static struct object_entry **create_final_object_list(void) +{ + struct object_entry **list; + int i, j; + + for (i = nr_result = 0; i < nr_objects; i++) + if (!objects[i].preferred_base) + nr_result++; + list = xmalloc(nr_result * sizeof(struct object_entry *)); + for (i = j = 0; i < nr_objects; i++) { + if (!objects[i].preferred_base) + list[j++] = objects + i; + } + current_sort = sha1_sort; + qsort(list, nr_result, sizeof(struct object_entry *), sort_comparator); + return list; +} + static int type_size_sort(const struct object_entry *a, const struct object_entry *b) { if (a->type < b->type) @@@ -793,10 -608,6 +793,10 @@@ return -1; if (a->hash > b->hash) return 1; + if (a->preferred_base < b->preferred_base) + return -1; + if (a->preferred_base > b->preferred_base) + return 1; if (a->size < b->size) return -1; if (a->size > b->size) @@@ -829,15 -640,9 +829,15 @@@ static int try_delta(struct unpacked *c if (cur_entry->type != old_entry->type) return -1; - /* If the current object is at edge, take the depth the objects - * that depend on the current object into account -- otherwise - * they would become too deep. + /* We do not compute delta to *create* objects we are not + * going to pack. + */ + if (cur_entry->preferred_base) + return -1; + + /* If the current object is at pack edge, take the depth the + * objects that depend on the current object into account -- + * otherwise they would become too deep. */ if (cur_entry->delta_child) { if (max_depth <= cur_entry->delta_limit) @@@ -846,10 -651,11 +846,10 @@@ } size = cur_entry->size; - if (size < 50) - return -1; oldsize = old_entry->size; sizediff = oldsize > size ? oldsize - size : size - oldsize; - if (sizediff > size / 8) + + if (size < 50) return -1; if (old_entry->depth >= max_depth) return 0; @@@ -879,7 -685,6 +879,6 @@@ static void progress_interval(int signum) { - signal(SIGALRM, progress_interval); progress_update = 1; } @@@ -895,7 -700,7 +894,7 @@@ static void find_deltas(struct object_e i = nr_objects; idx = 0; if (progress) - fprintf(stderr, "Deltifying %d objects.\n", nr_objects); + fprintf(stderr, "Deltifying %d objects.\n", nr_result); while (--i >= 0) { struct object_entry *entry = list[i]; @@@ -904,14 -709,12 +903,14 @@@ char type[10]; int j; - processed++; + if (!entry->preferred_base) + processed++; + if (progress) { - unsigned percent = processed * 100 / nr_objects; + unsigned percent = processed * 100 / nr_result; if (percent != last_percent || progress_update) { fprintf(stderr, "%4u%% (%u/%u) done\r", - percent, processed, nr_objects); + percent, processed, nr_result); progress_update = 0; last_percent = percent; } @@@ -941,15 -744,6 +940,15 @@@ if (try_delta(n, m, depth) < 0) break; } +#if 0 + /* if we made n a delta, and if n is already at max + * depth, leaving it in the window is pointless. we + * should evict it first. + * ... in theory only; somehow this makes things worse. + */ + if (entry->delta && depth <= entry->depth) + continue; +#endif idx++; if (idx >= window) idx = 0; @@@ -1025,6 -819,23 +1024,23 @@@ static int reuse_cached_pack(unsigned c return 1; } + static void setup_progress_signal(void) + { + struct sigaction sa; + struct itimerval v; + + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = progress_interval; + sigemptyset(&sa.sa_mask); + sa.sa_flags = SA_RESTART; + sigaction(SIGALRM, &sa, NULL); + + v.it_interval.tv_sec = 1; + v.it_interval.tv_usec = 0; + v.it_value = v.it_interval; + setitimer(ITIMER_REAL, &v, NULL); + } + int main(int argc, char **argv) { SHA_CTX ctx; @@@ -1090,50 -901,59 +1106,55 @@@ prepare_packed_git(); if (progress) { - struct itimerval v; - v.it_interval.tv_sec = 1; - v.it_interval.tv_usec = 0; - v.it_value = v.it_interval; - signal(SIGALRM, progress_interval); - setitimer(ITIMER_REAL, &v, NULL); fprintf(stderr, "Generating pack...\n"); + setup_progress_signal(); } - while (fgets(line, sizeof(line), stdin) != NULL) { + for (;;) { - unsigned int hash; - char *p; unsigned char sha1[20]; + if (!fgets(line, sizeof(line), stdin)) { + if (feof(stdin)) + break; + if (!ferror(stdin)) + die("fgets returned NULL, not EOF, not error!"); + if (errno == EINTR) + continue; + die("fgets: %s", strerror(errno)); + } + - if (progress_update) { - fprintf(stderr, "Counting objects...%d\r", nr_objects); - progress_update = 0; + if (line[0] == '-') { + if (get_sha1_hex(line+1, sha1)) + die("expected edge sha1, got garbage:\n %s", + line+1); + add_preferred_base(sha1); + continue; } if (get_sha1_hex(line, sha1)) die("expected sha1, got garbage:\n %s", line); - hash = 0; - p = line+40; - while (*p) { - unsigned char c = *p++; - if (isspace(c)) - continue; - hash = hash * 11 + c; - } - add_object_entry(sha1, hash); + add_object_entry(sha1, name_hash(NULL, line+41), 0); } if (progress) fprintf(stderr, "Done counting %d objects.\n", nr_objects); - if (non_empty && !nr_objects) + sorted_by_sha = create_final_object_list(); + if (non_empty && !nr_result) return 0; - sorted_by_sha = create_sorted_list(sha1_sort); SHA1_Init(&ctx); list = sorted_by_sha; - for (i = 0; i < nr_objects; i++) { + for (i = 0; i < nr_result; i++) { struct object_entry *entry = *list++; SHA1_Update(&ctx, entry->sha1, 20); } SHA1_Final(object_list_sha1, &ctx); + if (progress && (nr_objects != nr_result)) + fprintf(stderr, "Result has %d objects.\n", nr_result); if (reuse_cached_pack(object_list_sha1, pack_to_stdout)) ; else { - prepare_pack(window, depth); + if (nr_result) + prepare_pack(window, depth); if (progress && pack_to_stdout) { /* the other end usually displays progress itself */ struct itimerval v = {{0,},}; @@@ -1149,6 -969,6 +1170,6 @@@ } if (progress) fprintf(stderr, "Total %d, written %d (delta %d), reused %d (delta %d)\n", - nr_objects, written, written_delta, reused, reused_delta); + nr_result, written, written_delta, reused, reused_delta); return 0; }