From c75492ca072437c28e2786b39a6d8edf49385f4a Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Mon, 4 Feb 2019 11:10:15 -0800 Subject: [PATCH] filter-repo: get new names of commits asynchronously We have to ask fast-import for the new names of commits, but doing so immediately upon dumping out the commit related information requires context switches and waiting for fast-import to parse and handle more information. We don't need to know the new name of the commit until we run across a subsequent commit that referenced it in the commit message by its old ID. So, speed things up dramatically by waiting until we need the new name of the commit message (or the fast-import output pipe we are communicating with should be getting kind of full) before blocking on reading new commit hashes. Signed-off-by: Elijah Newren --- git-filter-repo | 53 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/git-filter-repo b/git-filter-repo index d36f251..f881ecb 100755 --- a/git-filter-repo +++ b/git-filter-repo @@ -818,6 +818,12 @@ class FastExportFilter(object): # commit became empty and was pruned or was otherwise dropped. self._commit_renames = {} + # A set of original_ids for which we have not yet gotten the + # new_ids; we use OrderedDict because we need to know the order of + # insertion, but the values are always ignored (and set to None). + # If there was an OrderedSet class, I'd use it instead. + self._pending_renames = collections.OrderedDict() + # A dict of commit_hash[1:7] -> set(commit_hashes with that prefix). # # It's common for commit messages to refer to commits by abbreviated @@ -1061,10 +1067,45 @@ class FastExportFilter(object): if not reset.dumped: reset.dump(self._output) + def _get_rename(self, old_hash): + # If we already know the rename, just return it + new_hash = self._commit_renames.get(old_hash, None) + if new_hash: + return new_hash + + # If it's not in the remaining pending renames, we don't know it + if old_hash is not None and old_hash not in self._pending_renames: + return None + + # Read through the pending renames until we find it or we've read them all, + # and return whatever we might find + self._flush_renames(old_hash) + return self._commit_renames.get(old_hash, None) + + def _flush_renames(self, old_hash=None, limit=0): + # Parse through self._pending_renames until we have read enough. We have + # read enough if: + # self._pending_renames is empty + # old_hash != None and we found a rename for old_hash + # limit > 0 and len(self._pending_renames) started less than 2*limit + # limit > 0 and len(self._pending_renames) < limit + if limit and len(self._pending_renames) < 2 * limit: + return + fi_input, fi_output = self._fast_import_pipes + while self._pending_renames: + orig_id, ignore = self._pending_renames.popitem(last=False) + new_id = fi_output.readline().rstrip() + self._commit_renames[orig_id] = new_id + if old_hash == orig_id: + return + if limit and len(self._pending_renames) < limit: + return + def _translate_commit_hash(self, matchobj): old_hash = matchobj.group(1) orig_len = len(old_hash) - if old_hash not in self._commit_renames: + new_hash = self._get_rename(old_hash) + if new_hash is None: if old_hash[0:7] not in self._commit_short_old_hashes: return old_hash possibilities = self._commit_short_old_hashes[old_hash[0:7]] @@ -1073,8 +1114,8 @@ class FastExportFilter(object): if len(matches) != 1: return old_hash old_hash = matches[0] + new_hash = self._get_rename(old_hash) - new_hash = self._commit_renames[old_hash] if new_hash is None: self._commits_referenced_but_removed.add(old_hash) return old_hash[0:orig_len] @@ -1208,6 +1249,7 @@ class FastExportFilter(object): # the new first parent has a tree matching the versions of files in # file_changes, then this new commit is empty and thus prunable. fi_input, fi_output = self._fast_import_pipes + self._flush_renames() # Avoid fi_output having other stuff present # Optimization note: we could have two loops over file_changes, the # first doing all the fi_input.write() calls, and the second doing the # rest. But I'm worried about fast-import blocking on fi_output @@ -1240,9 +1282,11 @@ class FastExportFilter(object): fi_input.write("get-mark :{}\n".format(commit.id)) fi_input.flush() orig_id = commit.original_id - new_id = fi_output.readline().rstrip() - self._commit_renames[orig_id] = new_id self._commit_short_old_hashes[orig_id[0:7]].add(orig_id) + # Note that we have queued up an id for later reading; flush a + # few of the older ones if we have too many queued up + self._pending_renames[orig_id] = None + self._flush_renames(None, limit=40) # Also, record if this was a merge commit that turned into a non-merge # commit. if len(orig_parents) >= 2 and not commit.merge_commits: @@ -1498,6 +1542,7 @@ class FastExportFilter(object): def record_metadata(self, metadata_dir, orig_refs, refs_nuked): deleted_hash = '0'*40 + self._flush_renames() with open(os.path.join(metadata_dir, 'commit-map'), 'w') as f: f.write("old new\n") for (old,new) in self._commit_renames.iteritems():