filter-repo: add public method for adding objects to stream

External rewrite tools using filter-repo as a library may want to add
additional objects into the stream.  Some examples in t/t9391 did this
using an internal _output field and using syntax that did not seem so
clear.  Provide an insert() method for doing this, and convert existing
cases over to it.

Signed-off-by: Elijah Newren <newren@gmail.com>

git-filter-repo

@@ -613,6 +613,7 @@ class Commit(_GitElementWithId):
                original_id = None,
                **kwargs):
     _GitElementWithId.__init__(self)
+    self.old_id = self.id
 
     # Denote that this is a commit element
     self.type = 'commit'
@@ -1300,6 +1301,10 @@ class FastExportParser(object):
     if not command.dumped:
       command.dump(self._output)
 
+  def insert(self, obj):
+    assert not obj.dumped
+    obj.dump(self._output)
+
   def run(self, input, output):
     """
     This method filters fast export output.
@@ -2456,6 +2461,7 @@ class RepoFilter(object):
     self._fep = None  # Fast Export Process
     self._fe_orig = None  # Path to where original fast-export output stored
     self._fe_filt = None  # Path to where filtered fast-export output stored
+    self._parser = None # FastExportParser object we are working with
 
     # Defaults for output
     self._output = None
@@ -3096,13 +3102,13 @@ class RepoFilter(object):
       self._seen_refs[commit.branch] = None # was seen, doesn't need (another) reset
       if not self._prunable(commit, new_1st_parent,
                             aux_info['had_file_changes'], orig_parents):
-        commit.dump(self._output)
+        self._insert_into_stream(commit)
         self._record_remapping(commit, orig_parents)
       else:
         rewrite_to = new_1st_parent or commit.first_parent()
         commit.skip(new_id = rewrite_to)
         reset = Reset(commit.branch, rewrite_to or deleted_hash)
-        reset.dump(self._output)
+        self._insert_into_stream(reset)
         self._commit_renames[commit.original_id] = None
 
     # Show progress
@@ -3440,6 +3446,27 @@ class RepoFilter(object):
     assert self._managed_output
     self.run()
 
+  def insert(self, obj, direct_insertion = False):
+    if not direct_insertion:
+      if type(obj) == Blob:
+        self._tweak_blob(obj)
+      elif type(obj) == Commit:
+        aux_info = {'orig_parents': obj.parents,
+                    'had_file_changes': bool(obj.file_changes)}
+        self._tweak_commit(obj, aux_info)
+      elif type(obj) == Reset:
+        self._tweak_reset(obj)
+      elif type(obj) == Tag:
+        self._tweak_tag(obj)
+    self._insert_into_stream(obj)
+
+  def _insert_into_stream(self, obj):
+    if not obj.dumped:
+      if self._parser:
+        self._parser.insert(obj)
+      else:
+        obj.dump(self._output)
+
   def run(self):
     start = time.time()
     if not self._input and not self._output:
@@ -3452,12 +3479,12 @@ class RepoFilter(object):
     if self._input:
       # Create and run the filter
       self._repo_working_dir = self._args.source or b'.'
-      fef = FastExportParser(blob_callback       = self._tweak_blob,
-                             commit_callback     = self._tweak_commit,
-                             tag_callback        = self._tweak_tag,
-                             reset_callback      = self._tweak_reset,
-                             done_callback       = self._final_commands)
-      fef.run(self._input, self._output)
+      self._parser = FastExportParser(blob_callback   = self._tweak_blob,
+                                      commit_callback = self._tweak_commit,
+                                      tag_callback    = self._tweak_tag,
+                                      reset_callback  = self._tweak_reset,
+                                      done_callback   = self._final_commands)
+      self._parser.run(self._input, self._output)
       if not self._finalize_handled:
         self._final_commands()
 

t/t9391/create_fast_export_output.py

@@ -16,16 +16,14 @@ args = fr.FilteringOptions.default_options()
 out = fr.RepoFilter(args)
 out.importer_only()
 
-output = out._output
-
 world = Blob(b"Hello")
-world.dump(output)
+out.insert(world)
 
 bar = Blob(b"foo\n")
-bar.dump(output)
+out.insert(bar)
 
 master = Reset(b"refs/heads/master")
-master.dump(output)
+out.insert(master)
 
 changes = [FileChange(b'M', b'world', world.id, mode=b"100644"),
            FileChange(b'M', b'bar',   bar.id,   mode=b"100644")]
@@ -39,12 +37,12 @@ commit1 = Commit(b"refs/heads/master",
                  b"My first commit!  Wooot!\n\nLonger description",
                  changes,
                  parents = [])
-commit1.dump(output)
+out.insert(commit1)
 
 world = Blob(b"Hello\nHi")
-world.dump(output)
+out.insert(world)
 world_link = Blob(b"world")
-world_link.dump(output)
+out.insert(world_link)
 
 changes = [FileChange(b'M', b'world',  world.id,      mode=b"100644"),
            FileChange(b'M', b'planet', world_link.id, mode=b"120000")]
@@ -56,10 +54,10 @@ commit2 = Commit(b"refs/heads/master",
                  b"Make a symlink to world called planet, modify world",
                  changes,
                  parents = [commit1.id])
-commit2.dump(output)
+out.insert(commit2)
 
 script = Blob(b"#!/bin/sh\n\necho Hello")
-script.dump(output)
+out.insert(script)
 changes = [FileChange(b'M', b'runme', script.id, mode=b"100755"),
            FileChange(b'D', b'bar')]
 when_string = b"1234567890 -0700"
@@ -69,18 +67,18 @@ commit3 = Commit(b"refs/heads/master",
                  b"Add runme script, remove bar",
                  changes,
                  parents = [commit2.id])
-commit3.dump(output)
+out.insert(commit3)
 
 progress = Progress(b"Done with the master branch now...")
-progress.dump(output)
+out.insert(progress)
 checkpoint = Checkpoint()
-checkpoint.dump(output)
+out.insert(checkpoint)
 
 devel = Reset(b"refs/heads/devel", commit1.id)
-devel.dump(output)
+out.insert(devel)
 
 world = Blob(b"Hello\nGoodbye")
-world.dump(output)
+out.insert(world)
 
 changes = [FileChange(b'M', b'world', world.id, mode=b"100644")]
 when = datetime(2006, 8, 17, tzinfo=FixedTimeZone(b"+0200"))
@@ -91,10 +89,10 @@ commit4 = Commit(b"refs/heads/devel",
                  b"Modify world",
                  changes,
                  parents = [commit1.id])
-commit4.dump(output)
+out.insert(commit4)
 
 world = Blob(b"Hello\nHi\nGoodbye")
-world.dump(output)
+out.insert(world)
 when = fr.string_to_date(commit3.author_date) + timedelta(days=47)
 when_string = fr.date_to_string(when)
 # git fast-import requires file changes to be listed in terms of differences
@@ -112,11 +110,11 @@ commit5 = Commit(b"refs/heads/devel",
                  b"Merge branch 'master'\n",
                  changes,
                  parents = [commit4.id, commit3.id])
-commit5.dump(output)
+out.insert(commit5)
 
 
 mytag = Tag(b"refs/tags/v1.0", commit5.id,
             b"His R. Highness", b"royalty@my.kingdom", when_string,
             b"I bequeath to my peons this royal software")
-mytag.dump(output)
+out.insert(mytag)
 out.finish()

t/t9391/splice_repos.py

@@ -39,12 +39,15 @@ class InterleaveRepositories:
     # Splice in any extra commits needed
     if prev_letter in self.commit_map:
       new_commit = self.commit_map[prev_letter]
+      new_commit.dumped = 0
       new_commit.parents = [self.last_commit] if self.last_commit else []
-      new_commit.dump(self.out._output)
+      # direct_insertion=True to avoid weave_commit being called recursively
+      # on the same commit
+      self.out.insert(new_commit, direct_insertion = True)
       commit.parents = [new_commit.id]
 
     # Dump our commit now
-    commit.dump(self.out._output)
+    self.out.insert(commit, direct_insertion = True)
 
     # Make sure that commits that depended on new_commit.id will now depend
     # on commit.id
@@ -76,8 +79,8 @@ class InterleaveRepositories:
     i2.set_output(out)
     i2.run()
 
-    blob.dump(out._output)
-    tag.dump(out._output)
+    out.insert(blob)
+    out.insert(tag)
     out.finish()
 
 splicer = InterleaveRepositories(sys.argv[1], sys.argv[2], sys.argv[3])