X-Git-Url: https://git.octo.it/?a=blobdiff_plain;f=git-archimport.perl;h=980e827b27ae0fa9a07667501d4cb26b5673cedf;hb=6b17c674aa517c4b22cd88809fcf1532b8204fbf;hp=e9e6f1b7d29a1391326f6df91cbfa69931f3c5a9;hpb=215a7ad1ef790467a4cd3f0dcffbd6e5f04c38f7;p=git.git diff --git a/git-archimport.perl b/git-archimport.perl index e9e6f1b7..980e827b 100755 --- a/git-archimport.perl +++ b/git-archimport.perl @@ -6,23 +6,25 @@ # The basic idea is to walk the output of tla abrowse, # fetch the changesets and apply them. # + =head1 Invocation - git-archimport -i / [/] - [ / ] + git-archimport [ -h ] [ -v ] [ -T ] [ -t tempdir ] / [ / ] - The script expects you to provide the key roots where it can start the - import from an 'initial import' or 'tag' type of Arch commit. It will - then follow all the branching and tagging within the provided roots. +Imports a project from one or more Arch repositories. It will follow branches +and repositories within the namespaces defined by the +parameters suppplied. If it cannot find the remote branch a merge comes from +it will just import it as a regular commit. If it can find it, it will mark it +as a merge whenever possible. - It will die if it sees branches that have different roots. +See man (1) git-archimport for more details. -=head2 TODO +=head1 TODO - - keep track of merged patches, and mark a git merge when it happens - - smarter rules to parse the archive history "up" and "down" - - be able to continue an import where we left off + - create tag objects instead of ref tags - audit shell-escaping of filenames + - hide our private tags somewhere smarter + - find a way to make "cat *patches | patch" safe even when patchfiles are missing newlines =head1 Devel tricks @@ -34,7 +36,7 @@ use strict; use warnings; use Getopt::Std; use File::Spec; -use File::Temp qw(tempfile); +use File::Temp qw(tempfile tempdir); use File::Path qw(mkpath); use File::Basename qw(basename dirname); use String::ShellQuote; @@ -48,34 +50,37 @@ use IPC::Open2; $SIG{'PIPE'}="IGNORE"; $ENV{'TZ'}="UTC"; +my $git_dir = $ENV{"GIT_DIR"} || ".git"; +$ENV{"GIT_DIR"} = $git_dir; + our($opt_h,$opt_v, $opt_T, $opt_C,$opt_t); sub usage() { print STDERR <= 1 or usage(); my @arch_roots = @ARGV; -my $tmp = $opt_t; -$tmp ||= '/tmp'; -$tmp .= '/git-archimport/'; - -my $git_tree = $opt_C; -$git_tree ||= "."; - +my ($tmpdir, $tmpdirname) = tempdir('git-archimport-XXXXXX', TMPDIR => 1, CLEANUP => 1); +my $tmp = $opt_t || 1; +$tmp = tempdir('git-archimport-XXXXXX', TMPDIR => 1, CLEANUP => 1); +$opt_v && print "+ Using $tmp as temporary directory\n"; my @psets = (); # the collection +my %psets = (); # the collection, by name + +my %rptags = (); # my reverse private tags + # to map a SHA1 to a commitid foreach my $root (@arch_roots) { my ($arepo, $abranch) = split(m!/!, $root); @@ -96,6 +101,7 @@ foreach my $root (@arch_roots) { if (%ps) { my %temp = %ps; # break references push (@psets, \%temp); + $psets{$temp{id}} = \%temp; %ps = (); } @@ -158,7 +164,8 @@ foreach my $root (@arch_roots) { if (%ps) { my %temp = %ps; # break references - push (@psets, \%temp); + push (@psets, \%temp); + $psets{ $temp{id} } = \%temp; %ps = (); } close ABROWSE; @@ -174,7 +181,7 @@ foreach my $root (@arch_roots) { ## and put an initial import ## or a full tag my $import = 0; -unless (-d '.git') { # initial import +unless (-d $git_dir) { # initial import if ($psets[0]{type} eq 'i' || $psets[0]{type} eq 't') { print "Starting import from $psets[0]{id}\n"; `git-init-db`; @@ -183,6 +190,24 @@ unless (-d '.git') { # initial import } else { die "Need to start from an import or a tag -- cannot use $psets[0]{id}"; } +} else { # progressing an import + # load the rptags + opendir(DIR, "$git_dir/archimport/tags") + || die "can't opendir: $!"; + while (my $file = readdir(DIR)) { + # skip non-interesting-files + next unless -f "$git_dir/archimport/tags/$file"; + next if $file =~ m/--base-0$/; # don't care for base-0 + my $sha = ptag($file); + chomp $sha; + # reconvert the 3rd '--' sequence from the end + # into a slash + # $file = reverse $file; + # $file =~ s!^(.+?--.+?--.+?--.+?)--(.+)$!$1/$2!; + # $file = reverse $file; + $rptags{$sha} = $file; + } + closedir DIR; } # process patchsets @@ -203,10 +228,12 @@ foreach my $ps (@psets) { # skip commits already in repo # if (ptag($ps->{id})) { - $opt_v && print "Skipping already imported: $ps->{id}\n"; + $opt_v && print " * Skipping already imported: $ps->{id}\n"; next; } + print " * Starting to work on $ps->{id}\n"; + # # create the branch if needed # @@ -215,7 +242,7 @@ foreach my $ps (@psets) { } unless ($import) { # skip for import - if ( -e ".git/refs/heads/$ps->{branch}") { + if ( -e "$git_dir/refs/heads/$ps->{branch}") { # we know about this branch `git checkout $ps->{branch}`; } else { @@ -268,7 +295,7 @@ foreach my $ps (@psets) { # imports don't give us good info # on added files. Shame on them if ($ps->{type} eq 'i' || $ps->{type} eq 't') { - `find . -type f -print0 | grep -zv '^./.git' | xargs -0 -l100 git-update-index --add`; + `find . -type f -print0 | grep -zv '^./$git_dir' | xargs -0 -l100 git-update-index --add`; `git-ls-files --deleted -z | xargs --no-run-if-empty -0 -l100 git-update-index --remove`; } @@ -332,8 +359,8 @@ foreach my $ps (@psets) { # Who's your daddy? # my @par; - if ( -e ".git/refs/heads/$ps->{branch}") { - if (open HEAD, "<.git/refs/heads/$ps->{branch}") { + if ( -e "$git_dir/refs/heads/$ps->{branch}") { + if (open HEAD, "<$git_dir/refs/heads/$ps->{branch}") { my $p = ; close HEAD; chomp $p; @@ -345,6 +372,9 @@ foreach my $ps (@psets) { } } + if ($ps->{merges}) { + push @par, find_parents($ps); + } my $par = join (' ', @par); # @@ -377,11 +407,11 @@ foreach my $ps (@psets) { # # Update the branch # - open HEAD, ">.git/refs/heads/$ps->{branch}"; + open HEAD, ">$git_dir/refs/heads/$ps->{branch}"; print HEAD $commitid; close HEAD; - unlink ('.git/HEAD'); - symlink("refs/heads/$ps->{branch}",".git/HEAD"); + unlink ("$git_dir/HEAD"); + symlink("refs/heads/$ps->{branch}","$git_dir/HEAD"); # tag accordingly ptag($ps->{id}, $commitid); # private tag @@ -391,7 +421,8 @@ foreach my $ps (@psets) { print " * Committed $ps->{id}\n"; print " + tree $tree\n"; print " + commit $commitid\n"; - # print " + commit date is $ps->{date} \n"; + $opt_v && print " + commit date is $ps->{date} \n"; + $opt_v && print " + parents: $par \n"; } sub branchname { @@ -409,7 +440,7 @@ sub apply_import { `tla get -s --no-pristine -A $ps->{repo} $ps->{id} $tmp/import`; die "Cannot get import: $!" if $?; - `rsync -v --archive --delete --exclude '.git' --exclude '.arch-ids' --exclude '{arch}' $tmp/import/* ./`; + `rsync -v --archive --delete --exclude '$git_dir' --exclude '.arch-ids' --exclude '{arch}' $tmp/import/* ./`; die "Cannot rsync import:$!" if $?; `rm -fr $tmp/import`; @@ -455,7 +486,7 @@ sub apply_cset { } # bring in new files - `rsync --archive --exclude '.git' --exclude '.arch-ids' --exclude '{arch}' $tmp/changeset/new-files-archive/* ./`; + `rsync --archive --exclude '$git_dir' --exclude '.arch-ids' --exclude '{arch}' $tmp/changeset/new-files-archive/* ./`; # deleted files are hinted from the commitlog processing @@ -550,15 +581,15 @@ sub tag { $tag = shell_quote($tag); if ($commit) { - open(C,">.git/refs/tags/$tag") + open(C,">$git_dir/refs/tags/$tag") or die "Cannot create tag $tag: $!\n"; print C "$commit\n" or die "Cannot write tag $tag: $!\n"; close(C) or die "Cannot write tag $tag: $!\n"; - print "Created tag '$tag' on '$commit'\n" if $opt_v; + print " * Created tag ' $tag' on '$commit'\n" if $opt_v; } else { # read - open(C,"<.git/refs/tags/$tag") + open(C,"<$git_dir/refs/tags/$tag") or die "Cannot read tag $tag: $!\n"; $commit = ; chomp $commit; @@ -576,29 +607,181 @@ sub ptag { $tag =~ s|/|--|g; $tag = shell_quote($tag); - unless (-d '.git/archimport/tags') { - mkpath('.git/archimport/tags'); + unless (-d "$git_dir/archimport/tags") { + mkpath("$git_dir/archimport/tags"); } if ($commit) { # write - open(C,">.git/archimport/tags/$tag") + open(C,">$git_dir/archimport/tags/$tag") or die "Cannot create tag $tag: $!\n"; print C "$commit\n" or die "Cannot write tag $tag: $!\n"; close(C) or die "Cannot write tag $tag: $!\n"; + $rptags{$commit} = $tag + unless $tag =~ m/--base-0$/; } else { # read # if the tag isn't there, return 0 - unless ( -s ".git/archimport/tags/$tag") { + unless ( -s "$git_dir/archimport/tags/$tag") { return 0; } - open(C,"<.git/archimport/tags/$tag") + open(C,"<$git_dir/archimport/tags/$tag") or die "Cannot read tag $tag: $!\n"; $commit = ; chomp $commit; die "Error reading tag $tag: $!\n" unless length $commit == 40; close(C) or die "Cannot read tag $tag: $!\n"; + unless (defined $rptags{$commit}) { + $rptags{$commit} = $tag; + } return $commit; } } + +sub find_parents { + # + # Identify what branches are merging into me + # and whether we are fully merged + # git-merge-base should tell + # me what the base of the merge should be + # + my $ps = shift; + + my %branches; # holds an arrayref per branch + # the arrayref contains a list of + # merged patches between the base + # of the merge and the current head + + my @parents; # parents found for this commit + + # simple loop to split the merges + # per branch + foreach my $merge (@{$ps->{merges}}) { + my $branch = branchname($merge); + unless (defined $branches{$branch} ){ + $branches{$branch} = []; + } + push @{$branches{$branch}}, $merge; + } + + # + # foreach branch find a merge base and walk it to the + # head where we are, collecting the merged patchsets that + # Arch has recorded. Keep that in @have + # Compare that with the commits on the other branch + # between merge-base and the tip of the branch (@need) + # and see if we have a series of consecutive patches + # starting from the merge base. The tip of the series + # of consecutive patches merged is our new parent for + # that branch. + # + foreach my $branch (keys %branches) { + + # check that we actually know about the branch + next unless -e "$git_dir/refs/heads/$branch"; + + my $mergebase = `git-merge-base $branch $ps->{branch}`; + die "Cannot find merge base for $branch and $ps->{branch}" if $?; + chomp $mergebase; + + # now walk up to the mergepoint collecting what patches we have + my $branchtip = git_rev_parse($ps->{branch}); + my @ancestors = `git-rev-list --merge-order $branchtip ^$mergebase`; + my %have; # collected merges this branch has + foreach my $merge (@{$ps->{merges}}) { + $have{$merge} = 1; + } + my %ancestorshave; + foreach my $par (@ancestors) { + $par = commitid2pset($par); + if (defined $par->{merges}) { + foreach my $merge (@{$par->{merges}}) { + $ancestorshave{$merge}=1; + } + } + } + # print "++++ Merges in $ps->{id} are....\n"; + # my @have = sort keys %have; print Dumper(\@have); + + # merge what we have with what ancestors have + %have = (%have, %ancestorshave); + + # see what the remote branch has - these are the merges we + # will want to have in a consecutive series from the mergebase + my $otherbranchtip = git_rev_parse($branch); + my @needraw = `git-rev-list --merge-order $otherbranchtip ^$mergebase`; + my @need; + foreach my $needps (@needraw) { # get the psets + $needps = commitid2pset($needps); + # git-rev-list will also + # list commits merged in via earlier + # merges. we are only interested in commits + # from the branch we're looking at + if ($branch eq $needps->{branch}) { + push @need, $needps->{id}; + } + } + + # print "++++ Merges from $branch we want are....\n"; + # print Dumper(\@need); + + my $newparent; + while (my $needed_commit = pop @need) { + if ($have{$needed_commit}) { + $newparent = $needed_commit; + } else { + last; # break out of the while + } + } + if ($newparent) { + push @parents, $newparent; + } + + + } # end foreach branch + + # prune redundant parents + my %parents; + foreach my $p (@parents) { + $parents{$p} = 1; + } + foreach my $p (@parents) { + next unless exists $psets{$p}{merges}; + next unless ref $psets{$p}{merges}; + my @merges = @{$psets{$p}{merges}}; + foreach my $merge (@merges) { + if ($parents{$merge}) { + delete $parents{$merge}; + } + } + } + @parents = keys %parents; + @parents = map { " -p " . ptag($_) } @parents; + return @parents; +} + +sub git_rev_parse { + my $name = shift; + my $val = `git-rev-parse $name`; + die "Error: git-rev-parse $name" if $?; + chomp $val; + return $val; +} + +# resolve a SHA1 to a known patchset +sub commitid2pset { + my $commitid = shift; + chomp $commitid; + my $name = $rptags{$commitid} + || die "Cannot find reverse tag mapping for $commitid"; + # the keys in %rptag are slightly munged; unmunge + # reconvert the 3rd '--' sequence from the end + # into a slash + $name = reverse $name; + $name =~ s!^(.+?--.+?--.+?--.+?)--(.+)$!$1/$2!; + $name = reverse $name; + my $ps = $psets{$name} + || (print Dumper(sort keys %psets)) && die "Cannot find patchset for $name"; + return $ps; +}