2019-04-01 23:35:49 +02:00
|
|
|
#!/usr/bin/env python3
|
2018-08-21 17:46:26 +02:00
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
"""
|
2019-03-05 20:00:11 +01:00
|
|
|
git-filter-repo filters git repositories, similar to git filter-branch, BFG
|
|
|
|
repo cleaner, and others. The basic idea is that it works by running
|
2018-08-21 17:46:26 +02:00
|
|
|
git fast-export <options> | filter | git fast-import <options>
|
|
|
|
where this program not only launches the whole pipeline but also serves as
|
|
|
|
the 'filter' in the middle. It does a few additional things on top as well
|
|
|
|
in order to make it into a well-rounded filtering tool.
|
2019-03-05 20:00:11 +01:00
|
|
|
|
|
|
|
git-filter-repo can also be used as a library for more involved filtering
|
|
|
|
operations; however:
|
|
|
|
***** API BACKWARD COMPATIBILITY CAVEAT *****
|
|
|
|
Programs using git-filter-repo as a library can reach pretty far into its
|
|
|
|
internals, but I am not prepared to guarantee backward compatibility of
|
|
|
|
all APIs. I suspect changes will be rare, but I reserve the right to
|
|
|
|
change any API. Since it is assumed that repository filtering is
|
|
|
|
something one would do very rarely, and in particular that it's a
|
|
|
|
one-shot operation, this should not be a problem in practice for anyone.
|
|
|
|
However, if you want to re-use a program you have written that uses
|
|
|
|
git-filter-repo as a library (or makes use of one of its --*-callback
|
|
|
|
arguments), you should either make sure you are using the same version of
|
|
|
|
git and git-filter-repo, or make sure to re-test it.
|
|
|
|
|
|
|
|
If there are particular pieces of the API you are concerned about, and
|
|
|
|
there is not already a testcase for it in t9391-lib-usage.sh or
|
|
|
|
t9392-python-callback.sh, please contribute a testcase. That will not
|
|
|
|
prevent me from changing the API, but it will allow you to look at the
|
|
|
|
history of a testcase to see whether and how the API changed.
|
|
|
|
***** END API BACKWARD COMPATIBILITY CAVEAT *****
|
2009-03-27 02:23:13 +01:00
|
|
|
"""
|
|
|
|
|
2018-08-21 17:46:26 +02:00
|
|
|
import argparse
|
2018-09-01 03:26:27 +02:00
|
|
|
import collections
|
2018-08-24 01:30:56 +02:00
|
|
|
import fnmatch
|
2019-03-17 07:40:01 +01:00
|
|
|
import gettext
|
2018-08-21 17:46:26 +02:00
|
|
|
import os
|
|
|
|
import re
|
2019-02-14 22:00:22 +01:00
|
|
|
import shutil
|
2019-04-01 23:35:49 +02:00
|
|
|
import io
|
2018-08-21 17:46:26 +02:00
|
|
|
import subprocess
|
|
|
|
import sys
|
2018-10-27 00:19:48 +02:00
|
|
|
import time
|
2018-10-18 03:31:26 +02:00
|
|
|
import textwrap
|
2009-03-27 02:23:13 +01:00
|
|
|
|
2009-03-13 01:14:42 +01:00
|
|
|
from datetime import tzinfo, timedelta, datetime
|
2009-02-05 07:03:48 +01:00
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
__all__ = ["Blob", "Reset", "FileChanges", "Commit", "Tag", "Progress",
|
2019-02-05 03:30:50 +01:00
|
|
|
"Checkpoint", "FastExportFilter", "ProgressWriter",
|
|
|
|
"string_to_date", "date_to_string",
|
2019-01-19 03:30:42 +01:00
|
|
|
"record_id_rename", "GitUtils", "FilteringOptions", "RepoFilter"]
|
2009-03-27 02:23:13 +01:00
|
|
|
|
2019-03-17 07:40:01 +01:00
|
|
|
def gettext_poison(msg):
|
|
|
|
if "GIT_TEST_GETTEXT_POISON" in os.environ: # pragma: no cover
|
|
|
|
return "# GETTEXT POISON #"
|
|
|
|
return gettext.gettext(msg)
|
|
|
|
|
|
|
|
_ = gettext_poison
|
|
|
|
|
|
|
|
def setup_gettext():
|
|
|
|
TEXTDOMAIN="git"
|
|
|
|
podir = os.environ.get("GIT_TEXTDOMAINDIR") or "@@LOCALEDIR@@"
|
|
|
|
if not os.path.isdir(podir): # pragma: no cover
|
|
|
|
podir = None # Python has its own fallback; use that
|
|
|
|
|
|
|
|
## This looks like the most straightforward translation of the relevant
|
|
|
|
## code in git.git:gettext.c and git.git:perl/Git/I18n.pm:
|
|
|
|
#import locale
|
|
|
|
#locale.setlocale(locale.LC_MESSAGES, "");
|
|
|
|
#locale.setlocale(locale.LC_TIME, "");
|
|
|
|
#locale.textdomain(TEXTDOMAIN);
|
|
|
|
#locale.bindtextdomain(TEXTDOMAIN, podir);
|
|
|
|
## but the python docs suggest using the gettext module (which doesn't
|
|
|
|
## have setlocale()) instead, so:
|
|
|
|
gettext.textdomain(TEXTDOMAIN);
|
|
|
|
gettext.bindtextdomain(TEXTDOMAIN, podir);
|
2018-08-21 17:46:26 +02:00
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
def _timedelta_to_seconds(delta):
|
|
|
|
"""
|
|
|
|
Converts timedelta to seconds
|
|
|
|
"""
|
|
|
|
offset = delta.days*86400 + delta.seconds + (delta.microseconds+0.0)/1000000
|
|
|
|
return round(offset)
|
2009-02-05 07:03:48 +01:00
|
|
|
|
2009-04-05 20:47:53 +02:00
|
|
|
class FixedTimeZone(tzinfo):
|
2009-03-27 02:23:13 +01:00
|
|
|
"""
|
|
|
|
Fixed offset in minutes east from UTC.
|
|
|
|
"""
|
|
|
|
|
2019-02-12 02:35:27 +01:00
|
|
|
tz_re = re.compile(r'^([-+]?)(\d\d)(\d\d)$')
|
|
|
|
|
2009-03-13 01:14:42 +01:00
|
|
|
def __init__(self, offset_string):
|
2009-03-27 02:23:13 +01:00
|
|
|
tzinfo.__init__(self)
|
2019-02-12 02:35:27 +01:00
|
|
|
sign, hh, mm = FixedTimeZone.tz_re.match(offset_string).groups()
|
2018-12-18 02:41:04 +01:00
|
|
|
factor = -1 if (sign and sign == '-') else 1
|
|
|
|
self._offset = timedelta(minutes = factor*(60*int(hh) + int(mm)))
|
2009-03-13 01:14:42 +01:00
|
|
|
self._offset_string = offset_string
|
|
|
|
|
|
|
|
def utcoffset(self, dt):
|
|
|
|
return self._offset
|
|
|
|
|
|
|
|
def tzname(self, dt):
|
|
|
|
return self._offset_string
|
|
|
|
|
|
|
|
def dst(self, dt):
|
|
|
|
return timedelta(0)
|
|
|
|
|
2019-02-05 03:30:50 +01:00
|
|
|
def string_to_date(datestring):
|
|
|
|
(unix_timestamp, tz_offset) = datestring.split()
|
|
|
|
return datetime.fromtimestamp(int(unix_timestamp),
|
|
|
|
FixedTimeZone(tz_offset))
|
|
|
|
|
|
|
|
def date_to_string(dateobj):
|
|
|
|
epoch = datetime.fromtimestamp(0, dateobj.tzinfo)
|
|
|
|
return('{} {}'.format(int(_timedelta_to_seconds(dateobj - epoch)),
|
|
|
|
dateobj.tzinfo.tzname(0)))
|
|
|
|
|
2018-12-20 02:09:17 +01:00
|
|
|
class PathQuoting:
|
|
|
|
_unescape = {'a': '\a',
|
|
|
|
'b': '\b',
|
|
|
|
'f': '\f',
|
|
|
|
'n': '\n',
|
|
|
|
'r': '\r',
|
|
|
|
't': '\t',
|
|
|
|
'v': '\v',
|
|
|
|
'"': '"',
|
|
|
|
'\\':'\\'}
|
|
|
|
_unescape_re = re.compile(r'\\([a-z"\\]|[0-9]{3})')
|
2019-04-01 23:38:02 +02:00
|
|
|
_escape = [chr(x) for x in range(127)]+['\\'+oct(x)[1:] for x in range(127,256)]
|
2018-12-20 02:09:17 +01:00
|
|
|
_reverse = dict(map(reversed, _unescape.items()))
|
|
|
|
for x in _reverse:
|
|
|
|
_escape[ord(x)] = '\\'+_reverse[x]
|
|
|
|
_special_chars = [len(x) > 1 for x in _escape]
|
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
def unescape_sequence(orig):
|
|
|
|
seq = orig.group(1)
|
|
|
|
return PathQuoting._unescape[seq] if len(seq) == 1 else chr(int(seq, 8))
|
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
def dequote(quoted_string):
|
|
|
|
if quoted_string.startswith('"'):
|
|
|
|
assert quoted_string.endswith('"')
|
|
|
|
return PathQuoting._unescape_re.sub(PathQuoting.unescape_sequence,
|
|
|
|
quoted_string[1:-1])
|
|
|
|
return quoted_string
|
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
def enquote(unquoted_string):
|
2018-12-20 20:25:44 +01:00
|
|
|
# Option 1: Quoting when fast-export would:
|
|
|
|
# pqsc = PathQuoting._special_chars
|
|
|
|
# if any(pqsc[ord(x)] for x in set(unquoted_string)):
|
|
|
|
# Option 2, perf hack: do minimal amount of quoting required by fast-import
|
|
|
|
if unquoted_string.startswith('"') or '\n' in unquoted_string:
|
2018-12-20 02:09:17 +01:00
|
|
|
pqe = PathQuoting._escape
|
|
|
|
return '"' + ''.join(pqe[ord(x)] for x in unquoted_string) + '"'
|
|
|
|
return unquoted_string
|
|
|
|
|
filter-repo: avoid merging a commit with one of its own ancestors
Pruning of empty commits can cause an entire line of history to become
empty and be pruned, resulting in a merge commit that merges some commit
with one of its ancestors. In such a case, we should remove the
unnecessary parent(s) -- which can and will often result in the merge
commit being empty so we can remove it as well.
Currently, if the side that becomes empty is the first parent side, then
we do not detect if the commit becomes empty, due to the way that
fast-export lists changes in a merge commit relative to first parent only.
A subsequent commit will address this.
Note that the callbacks could theoretically insert additional commits or
reparent our commit on top of something else, meaning that the ancestry
graph might need post-callback updates. However, in any extreme case
where that mattered, we would more or less need full updates to the
ancestry graph to be made for all the new commits from the callback as
well, and once we expect the callback to handle any ancestry graph
updates it can handle modifying it for the current commit. However, it
is hard to come up with a case where it matters, since for the most part
we just want to know whether our filtering causes commits to become
empty and knowing the source repo we are exporting from is sufficient
information without knowing any new commits inserted or reparenting that
happens elsewhere.
Signed-off-by: Elijah Newren <newren@gmail.com>
2018-09-04 18:26:05 +02:00
|
|
|
class AncestryGraph(object):
|
|
|
|
"""
|
|
|
|
A class that maintains a direct acycle graph of commits for the purpose of
|
|
|
|
determining if one commit is the ancestor of another.
|
|
|
|
"""
|
|
|
|
|
|
|
|
def __init__(self):
|
|
|
|
self.cur_value = 0
|
|
|
|
|
|
|
|
# A mapping from the external identifers given to us to the simple integers
|
|
|
|
# we use in self.graph
|
|
|
|
self.value = {}
|
|
|
|
|
|
|
|
# A tuple of (depth, list-of-ancestors). Values and keys in this graph are
|
|
|
|
# all integers from the self.value dict. The depth of a commit is one more
|
|
|
|
# than the max depth of any of its ancestors.
|
|
|
|
self.graph = {}
|
|
|
|
|
2019-03-14 07:34:47 +01:00
|
|
|
def record_external_commits(self, external_commits):
|
|
|
|
"""
|
|
|
|
Record in graph that each commit in external_commits exists, and is
|
|
|
|
treated as a root commit with no parents.
|
|
|
|
"""
|
|
|
|
for c in external_commits:
|
|
|
|
if c not in self.value:
|
|
|
|
self.cur_value += 1
|
|
|
|
self.value[c] = self.cur_value
|
|
|
|
self.graph[self.cur_value] = (1, [])
|
|
|
|
|
filter-repo: avoid merging a commit with one of its own ancestors
Pruning of empty commits can cause an entire line of history to become
empty and be pruned, resulting in a merge commit that merges some commit
with one of its ancestors. In such a case, we should remove the
unnecessary parent(s) -- which can and will often result in the merge
commit being empty so we can remove it as well.
Currently, if the side that becomes empty is the first parent side, then
we do not detect if the commit becomes empty, due to the way that
fast-export lists changes in a merge commit relative to first parent only.
A subsequent commit will address this.
Note that the callbacks could theoretically insert additional commits or
reparent our commit on top of something else, meaning that the ancestry
graph might need post-callback updates. However, in any extreme case
where that mattered, we would more or less need full updates to the
ancestry graph to be made for all the new commits from the callback as
well, and once we expect the callback to handle any ancestry graph
updates it can handle modifying it for the current commit. However, it
is hard to come up with a case where it matters, since for the most part
we just want to know whether our filtering causes commits to become
empty and knowing the source repo we are exporting from is sufficient
information without knowing any new commits inserted or reparenting that
happens elsewhere.
Signed-off-by: Elijah Newren <newren@gmail.com>
2018-09-04 18:26:05 +02:00
|
|
|
def add_commit_and_parents(self, commit, parents):
|
|
|
|
"""
|
|
|
|
Record in graph that commit has the given parents. parents _MUST_ have
|
|
|
|
been first recorded. commit _MUST_ not have been recorded yet.
|
|
|
|
"""
|
|
|
|
assert all(p in self.value for p in parents)
|
|
|
|
assert commit not in self.value
|
|
|
|
|
|
|
|
# Get values for commit and parents
|
|
|
|
self.cur_value += 1
|
|
|
|
self.value[commit] = self.cur_value
|
|
|
|
graph_parents = [self.value[x] for x in parents]
|
|
|
|
|
|
|
|
# Determine depth for commit, then insert the info into the graph
|
|
|
|
depth = 1
|
|
|
|
if parents:
|
|
|
|
depth += max(self.graph[p][0] for p in graph_parents)
|
|
|
|
self.graph[self.cur_value] = (depth, graph_parents)
|
|
|
|
|
|
|
|
def is_ancestor(self, possible_ancestor, check):
|
|
|
|
"""
|
|
|
|
Return whether possible_ancestor is an ancestor of check
|
|
|
|
"""
|
|
|
|
a, b = self.value[possible_ancestor], self.value[check]
|
|
|
|
a_depth = self.graph[a][0]
|
|
|
|
ancestors = [b]
|
|
|
|
visited = set()
|
|
|
|
while ancestors:
|
|
|
|
ancestor = ancestors.pop()
|
|
|
|
if ancestor in visited:
|
|
|
|
continue
|
|
|
|
visited.add(ancestor)
|
|
|
|
depth, more_ancestors = self.graph[ancestor]
|
|
|
|
if ancestor == a:
|
|
|
|
return True
|
|
|
|
elif depth <= a_depth:
|
|
|
|
continue
|
|
|
|
ancestors.extend(more_ancestors)
|
|
|
|
return False
|
|
|
|
|
2019-01-29 02:26:49 +01:00
|
|
|
class MailmapInfo(object):
|
|
|
|
def __init__(self, filename):
|
|
|
|
self.changes = {}
|
|
|
|
self._parse_file(filename)
|
|
|
|
|
|
|
|
def _parse_file(self, filename):
|
|
|
|
name_and_email_re = re.compile(r'(.*?)\s*<([^>]+)>\s*')
|
2019-02-12 02:35:27 +01:00
|
|
|
comment_re = re.compile(r'\s*#.*')
|
2019-01-29 02:26:49 +01:00
|
|
|
if not os.access(filename, os.R_OK):
|
2019-03-17 07:39:39 +01:00
|
|
|
raise SystemExit(_("Cannot read %s") % filename)
|
2019-01-29 02:26:49 +01:00
|
|
|
with open(filename) as f:
|
|
|
|
count = 0
|
|
|
|
for line in f:
|
|
|
|
count += 1
|
|
|
|
err = "Unparseable mailmap file: line #{} is bad: {}".format(count, line)
|
|
|
|
# Remove comments
|
2019-02-12 02:35:27 +01:00
|
|
|
line = comment_re.sub('', line)
|
2019-01-29 02:26:49 +01:00
|
|
|
# Remove leading and trailing whitespace
|
|
|
|
line = line.strip()
|
|
|
|
if not line:
|
|
|
|
continue
|
|
|
|
|
|
|
|
m = name_and_email_re.match(line)
|
|
|
|
if not m:
|
|
|
|
raise SystemExit(err)
|
|
|
|
proper_name, proper_email = m.groups()
|
|
|
|
if len(line) == m.end():
|
|
|
|
self.changes[(None, proper_email)] = (proper_name, proper_email)
|
|
|
|
continue
|
|
|
|
rest = line[m.end():]
|
|
|
|
m = name_and_email_re.match(rest)
|
|
|
|
if m:
|
|
|
|
commit_name, commit_email = m.groups()
|
|
|
|
if len(rest) != m.end():
|
|
|
|
raise SystemExit(err)
|
|
|
|
else:
|
|
|
|
commit_name, commit_email = rest, None
|
|
|
|
self.changes[(commit_name, commit_email)] = (proper_name, proper_email)
|
|
|
|
|
|
|
|
def translate(self, name, email):
|
|
|
|
''' Given a name and email, return the expected new name and email from the
|
|
|
|
mailmap if there is a translation rule for it, otherwise just return
|
|
|
|
the given name and email.'''
|
2019-04-01 23:36:57 +02:00
|
|
|
for old, new in self.changes.items():
|
2019-01-29 02:26:49 +01:00
|
|
|
old_name, old_email = old
|
|
|
|
new_name, new_email = new
|
|
|
|
if (email == old_email or not old_email) and (
|
|
|
|
name == old_name or not old_name):
|
|
|
|
return (new_name or name, new_email or email)
|
|
|
|
return (name, email)
|
|
|
|
|
2018-11-27 18:59:49 +01:00
|
|
|
class ProgressWriter(object):
|
|
|
|
def __init__(self):
|
|
|
|
self._last_progress_update = time.time()
|
|
|
|
self._last_message = None
|
|
|
|
|
|
|
|
def show(self, msg):
|
|
|
|
self._last_message = msg
|
|
|
|
now = time.time()
|
|
|
|
if now - self._last_progress_update > .1:
|
|
|
|
self._last_progress_update = now
|
|
|
|
sys.stdout.write("\r{}".format(msg))
|
|
|
|
sys.stdout.flush()
|
|
|
|
|
|
|
|
def finish(self):
|
|
|
|
self._last_progress_update = 0
|
|
|
|
if self._last_message:
|
|
|
|
self.show(self._last_message)
|
|
|
|
sys.stdout.write("\n")
|
filter-repo: avoid merging a commit with one of its own ancestors
Pruning of empty commits can cause an entire line of history to become
empty and be pruned, resulting in a merge commit that merges some commit
with one of its ancestors. In such a case, we should remove the
unnecessary parent(s) -- which can and will often result in the merge
commit being empty so we can remove it as well.
Currently, if the side that becomes empty is the first parent side, then
we do not detect if the commit becomes empty, due to the way that
fast-export lists changes in a merge commit relative to first parent only.
A subsequent commit will address this.
Note that the callbacks could theoretically insert additional commits or
reparent our commit on top of something else, meaning that the ancestry
graph might need post-callback updates. However, in any extreme case
where that mattered, we would more or less need full updates to the
ancestry graph to be made for all the new commits from the callback as
well, and once we expect the callback to handle any ancestry graph
updates it can handle modifying it for the current commit. However, it
is hard to come up with a case where it matters, since for the most part
we just want to know whether our filtering causes commits to become
empty and knowing the source repo we are exporting from is sufficient
information without knowing any new commits inserted or reparenting that
happens elsewhere.
Signed-off-by: Elijah Newren <newren@gmail.com>
2018-09-04 18:26:05 +02:00
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
class _IDs(object):
|
|
|
|
"""
|
|
|
|
A class that maintains the 'name domain' of all the 'marks' (short int
|
|
|
|
id for a blob/commit git object). The reason this mechanism is necessary
|
|
|
|
is because the text of fast-export may refer to an object using a different
|
|
|
|
mark than the mark that was assigned to that object using IDS.new(). This
|
|
|
|
class allows you to translate the fast-export marks (old) to the marks
|
|
|
|
assigned from IDS.new() (new).
|
|
|
|
|
|
|
|
Note that there are two reasons why the marks may differ: (1) The
|
|
|
|
user manually creates Blob or Commit objects (for insertion into the
|
|
|
|
stream) (2) We're reading the data from two different repositories
|
|
|
|
and trying to combine the data (git fast-export will number ids from
|
|
|
|
1...n, and having two 1's, two 2's, two 3's, causes issues).
|
|
|
|
"""
|
|
|
|
|
|
|
|
def __init__(self):
|
|
|
|
"""
|
|
|
|
Init
|
|
|
|
"""
|
|
|
|
# The id for the next created blob/commit object
|
|
|
|
self._next_id = 1
|
2009-03-13 01:14:42 +01:00
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
# A map of old-ids to new-ids (1:1 map)
|
|
|
|
self._translation = {}
|
2009-03-13 01:14:42 +01:00
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
# A map of new-ids to every old-id that points to the new-id (1:N map)
|
|
|
|
self._reverse_translation = {}
|
2009-02-05 15:12:25 +01:00
|
|
|
|
|
|
|
def new(self):
|
2009-03-27 02:23:13 +01:00
|
|
|
"""
|
|
|
|
Should be called whenever a new blob or commit object is created. The
|
|
|
|
returned value should be used as the id/mark for that object.
|
|
|
|
"""
|
|
|
|
rv = self._next_id
|
|
|
|
self._next_id += 1
|
|
|
|
return rv
|
|
|
|
|
2009-02-23 18:19:05 +01:00
|
|
|
def record_rename(self, old_id, new_id, handle_transitivity = False):
|
2009-03-27 02:23:13 +01:00
|
|
|
"""
|
|
|
|
Record that old_id is being renamed to new_id.
|
|
|
|
"""
|
2009-02-05 15:12:25 +01:00
|
|
|
if old_id != new_id:
|
2009-02-21 23:36:54 +01:00
|
|
|
# old_id -> new_id
|
2009-03-27 02:23:13 +01:00
|
|
|
self._translation[old_id] = new_id
|
2009-02-05 15:12:25 +01:00
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
# Transitivity will be needed if new commits are being inserted mid-way
|
|
|
|
# through a branch.
|
2009-02-23 18:19:05 +01:00
|
|
|
if handle_transitivity:
|
|
|
|
# Anything that points to old_id should point to new_id
|
2009-03-27 02:23:13 +01:00
|
|
|
if old_id in self._reverse_translation:
|
|
|
|
for id_ in self._reverse_translation[old_id]:
|
|
|
|
self._translation[id_] = new_id
|
2009-02-21 23:36:54 +01:00
|
|
|
|
|
|
|
# Record that new_id is pointed to by old_id
|
2009-03-27 02:23:13 +01:00
|
|
|
if new_id not in self._reverse_translation:
|
|
|
|
self._reverse_translation[new_id] = []
|
|
|
|
self._reverse_translation[new_id].append(old_id)
|
2009-02-21 23:36:54 +01:00
|
|
|
|
2009-02-05 15:12:25 +01:00
|
|
|
def translate(self, old_id):
|
2009-03-27 02:23:13 +01:00
|
|
|
"""
|
|
|
|
If old_id has been mapped to an alternate id, return the alternate id.
|
|
|
|
"""
|
|
|
|
if old_id in self._translation:
|
|
|
|
return self._translation[old_id]
|
2009-02-05 15:12:25 +01:00
|
|
|
else:
|
|
|
|
return old_id
|
2009-02-21 23:36:54 +01:00
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
def __str__(self):
|
|
|
|
"""
|
|
|
|
Convert IDs to string; used for debugging
|
|
|
|
"""
|
|
|
|
rv = "Current count: %d\nTranslation:\n" % self._next_id
|
|
|
|
for k in sorted(self._translation):
|
2019-03-13 22:22:09 +01:00
|
|
|
rv += " %d -> %s\n" % (k, self._translation[k])
|
2009-03-27 02:23:13 +01:00
|
|
|
|
|
|
|
rv += "Reverse translation:\n"
|
|
|
|
for k in sorted(self._reverse_translation):
|
|
|
|
rv += " " + str(k) + " -> " + str(self._reverse_translation[k]) + "\n"
|
|
|
|
|
|
|
|
return rv
|
|
|
|
|
|
|
|
class _GitElement(object):
|
|
|
|
"""
|
|
|
|
The base class for all git elements that we create.
|
|
|
|
"""
|
|
|
|
|
2009-02-03 01:26:17 +01:00
|
|
|
def __init__(self):
|
2009-03-27 02:23:13 +01:00
|
|
|
# A string that describes what type of Git element this is
|
2009-02-03 01:26:17 +01:00
|
|
|
self.type = None
|
2009-03-27 02:23:13 +01:00
|
|
|
|
|
|
|
# A flag telling us if this Git element has been dumped
|
|
|
|
# (i.e. printed) or skipped. Typically elements that have been
|
|
|
|
# dumped or skipped will not be dumped again.
|
2009-02-08 03:45:12 +01:00
|
|
|
self.dumped = 0
|
2009-03-27 02:23:13 +01:00
|
|
|
|
|
|
|
def dump(self, file_):
|
|
|
|
"""
|
|
|
|
This version should never be called. Derived classes need to
|
|
|
|
override! We should note that subclasses should implement this
|
|
|
|
method such that the output would match the format produced by
|
|
|
|
fast-export.
|
|
|
|
"""
|
2019-03-17 07:39:39 +01:00
|
|
|
raise SystemExit(_("Unimplemented function: %s") % type(self).__name__
|
|
|
|
+".dump()") # pragma: no cover
|
2009-03-27 02:23:13 +01:00
|
|
|
|
2018-11-01 18:29:13 +01:00
|
|
|
def __str__(self):
|
|
|
|
"""
|
|
|
|
Convert GitElement to string; used for debugging
|
|
|
|
"""
|
|
|
|
old_dumped = self.dumped
|
2019-04-01 23:35:49 +02:00
|
|
|
writeme = io.StringIO()
|
2018-11-01 18:29:13 +01:00
|
|
|
self.dump(writeme)
|
|
|
|
output_lines = writeme.getvalue().splitlines()
|
|
|
|
writeme.close()
|
|
|
|
self.dumped = old_dumped
|
|
|
|
return "{}:\n {}".format(type(self).__name__, "\n ".join(output_lines))
|
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
def skip(self, new_id=None):
|
|
|
|
"""
|
|
|
|
Ensures this element will not be written to output
|
|
|
|
"""
|
|
|
|
self.dumped = 2
|
|
|
|
|
|
|
|
class _GitElementWithId(_GitElement):
|
|
|
|
"""
|
|
|
|
The base class for Git elements that have IDs (commits and blobs)
|
|
|
|
"""
|
|
|
|
|
|
|
|
def __init__(self):
|
|
|
|
_GitElement.__init__(self)
|
|
|
|
|
|
|
|
# The mark (short, portable id) for this element
|
|
|
|
self.id = _IDS.new()
|
|
|
|
|
|
|
|
# The previous mark for this element
|
2009-02-13 01:40:51 +01:00
|
|
|
self.old_id = None
|
2009-02-03 01:26:17 +01:00
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
def skip(self, new_id=None):
|
|
|
|
"""
|
|
|
|
This element will no longer be automatically written to output. When a
|
|
|
|
commit gets skipped, it's ID will need to be translated to that of its
|
|
|
|
parent.
|
|
|
|
"""
|
|
|
|
self.dumped = 2
|
|
|
|
|
|
|
|
_IDS.record_rename(self.old_id or self.id, new_id)
|
2009-02-03 01:26:17 +01:00
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
class Blob(_GitElementWithId):
|
|
|
|
"""
|
2009-04-04 22:57:02 +02:00
|
|
|
This class defines our representation of git blob elements (i.e. our
|
|
|
|
way of representing file contents).
|
2009-03-27 02:23:13 +01:00
|
|
|
"""
|
2009-02-13 01:40:51 +01:00
|
|
|
|
2018-10-31 01:55:54 +01:00
|
|
|
def __init__(self, data, original_id = None):
|
2009-03-27 02:23:13 +01:00
|
|
|
_GitElementWithId.__init__(self)
|
|
|
|
|
|
|
|
# Denote that this is a blob
|
2009-02-03 01:26:17 +01:00
|
|
|
self.type = 'blob'
|
2009-03-27 02:23:13 +01:00
|
|
|
|
2018-10-31 01:55:54 +01:00
|
|
|
# Record original id
|
|
|
|
self.original_id = original_id
|
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
# Stores the blob's data
|
2009-02-03 01:26:17 +01:00
|
|
|
self.data = data
|
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
def dump(self, file_):
|
|
|
|
"""
|
|
|
|
Write this blob element to a file.
|
|
|
|
"""
|
2009-02-08 03:45:12 +01:00
|
|
|
self.dumped = 1
|
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
file_.write('blob\n')
|
|
|
|
file_.write('mark :%d\n' % self.id)
|
|
|
|
file_.write('data %d\n%s' % (len(self.data), self.data))
|
|
|
|
file_.write('\n')
|
2009-02-02 15:14:13 +01:00
|
|
|
|
2009-02-13 01:40:51 +01:00
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
class Reset(_GitElement):
|
|
|
|
"""
|
2009-04-04 22:57:02 +02:00
|
|
|
This class defines our representation of git reset elements. A reset
|
|
|
|
event is the creation (or recreation) of a named branch, optionally
|
|
|
|
starting from a specific revision).
|
2009-03-27 02:23:13 +01:00
|
|
|
"""
|
|
|
|
|
2009-02-03 15:53:06 +01:00
|
|
|
def __init__(self, ref, from_ref = None):
|
2009-03-27 02:23:13 +01:00
|
|
|
_GitElement.__init__(self)
|
|
|
|
|
|
|
|
# Denote that this is a reset
|
2009-02-03 15:53:06 +01:00
|
|
|
self.type = 'reset'
|
2009-03-27 02:23:13 +01:00
|
|
|
|
|
|
|
# The name of the branch being (re)created
|
2009-02-03 15:53:06 +01:00
|
|
|
self.ref = ref
|
2009-03-27 02:23:13 +01:00
|
|
|
|
|
|
|
# Some reference to the branch/commit we are resetting from
|
2009-02-03 15:53:06 +01:00
|
|
|
self.from_ref = from_ref
|
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
def dump(self, file_):
|
|
|
|
"""
|
|
|
|
Write this reset element to a file
|
|
|
|
"""
|
2009-02-08 03:45:12 +01:00
|
|
|
self.dumped = 1
|
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
file_.write('reset %s\n' % self.ref)
|
2009-02-03 15:53:06 +01:00
|
|
|
if self.from_ref:
|
2009-03-27 02:23:13 +01:00
|
|
|
file_.write('from :%d\n' % self.from_ref)
|
|
|
|
file_.write('\n')
|
|
|
|
|
|
|
|
class FileChanges(_GitElement):
|
|
|
|
"""
|
|
|
|
This class defines our representation of file change elements. File change
|
|
|
|
elements are components within a Commit element.
|
|
|
|
"""
|
|
|
|
|
|
|
|
def __init__(self, type_, filename, id_ = None, mode = None):
|
|
|
|
_GitElement.__init__(self)
|
|
|
|
|
|
|
|
# Denote the type of file-change (M for modify, D for delete, etc)
|
|
|
|
self.type = type_
|
|
|
|
|
|
|
|
# Record the name of the file being changed
|
2009-02-04 15:30:40 +01:00
|
|
|
self.filename = filename
|
2009-03-27 02:23:13 +01:00
|
|
|
|
|
|
|
# Record the mode (mode describes type of file entry (non-executable,
|
|
|
|
# executable, or symlink)).
|
2009-02-13 01:40:51 +01:00
|
|
|
self.mode = None
|
2009-03-27 02:23:13 +01:00
|
|
|
|
|
|
|
# blob_id is the id (mark) of the affected blob
|
|
|
|
self.blob_id = None
|
|
|
|
|
|
|
|
# For 'M' file changes (modify), expect to have id and mode
|
|
|
|
if type_ == 'M':
|
2009-02-13 01:40:51 +01:00
|
|
|
if mode is None:
|
2019-03-17 07:39:39 +01:00
|
|
|
raise SystemExit(_("file mode and idnum needed for %s") % filename) # pragma: no cover
|
2009-02-05 15:12:25 +01:00
|
|
|
self.mode = mode
|
2009-03-27 02:23:13 +01:00
|
|
|
self.blob_id = id_
|
|
|
|
|
filter-repo: add --analyze option
This option walks through the repository history and creates a report
with basic statistics, rename related information, and sizes of objects
and when/if those have been deleted. It primarily looks at unpacked
sizes (i.e. size of object ignoring delta-ing and compression), and
sums the size of each version of the file for each path. Additionally,
it aggregates these sums by extension and by directory, and tracks
whether paths, extensions, and directories have been deleted. This can
be very useful in determining what the big things are, and whether they
might have been considered to have been mistakes to add to the
repository in the first place.
There are numerous caveats with the determination of "deleted" and
"renamed", and can give both false positives and false negatives. But
they are only meant as a helpful heuristic to give others a starting
point for an investigation, and the information provide so far is useful.
I do want to improve the equivalence classes (rename handling), but that
is for a future commit.
Signed-off-by: Elijah Newren <newren@gmail.com>
2018-10-21 16:44:33 +02:00
|
|
|
# For 'R' file changes (rename), expect to have newname as third arg
|
2019-03-12 19:26:21 +01:00
|
|
|
elif type_ == 'R': # pragma: no cover (now avoid fast-export renames)
|
filter-repo: add --analyze option
This option walks through the repository history and creates a report
with basic statistics, rename related information, and sizes of objects
and when/if those have been deleted. It primarily looks at unpacked
sizes (i.e. size of object ignoring delta-ing and compression), and
sums the size of each version of the file for each path. Additionally,
it aggregates these sums by extension and by directory, and tracks
whether paths, extensions, and directories have been deleted. This can
be very useful in determining what the big things are, and whether they
might have been considered to have been mistakes to add to the
repository in the first place.
There are numerous caveats with the determination of "deleted" and
"renamed", and can give both false positives and false negatives. But
they are only meant as a helpful heuristic to give others a starting
point for an investigation, and the information provide so far is useful.
I do want to improve the equivalence classes (rename handling), but that
is for a future commit.
Signed-off-by: Elijah Newren <newren@gmail.com>
2018-10-21 16:44:33 +02:00
|
|
|
if id_ is None:
|
2019-03-17 07:39:39 +01:00
|
|
|
raise SystemExit(_("new name needed for rename of %s") % filename)
|
filter-repo: add --analyze option
This option walks through the repository history and creates a report
with basic statistics, rename related information, and sizes of objects
and when/if those have been deleted. It primarily looks at unpacked
sizes (i.e. size of object ignoring delta-ing and compression), and
sums the size of each version of the file for each path. Additionally,
it aggregates these sums by extension and by directory, and tracks
whether paths, extensions, and directories have been deleted. This can
be very useful in determining what the big things are, and whether they
might have been considered to have been mistakes to add to the
repository in the first place.
There are numerous caveats with the determination of "deleted" and
"renamed", and can give both false positives and false negatives. But
they are only meant as a helpful heuristic to give others a starting
point for an investigation, and the information provide so far is useful.
I do want to improve the equivalence classes (rename handling), but that
is for a future commit.
Signed-off-by: Elijah Newren <newren@gmail.com>
2018-10-21 16:44:33 +02:00
|
|
|
self.filename = (self.filename, id_)
|
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
def dump(self, file_):
|
|
|
|
"""
|
|
|
|
Write this file-change element to a file
|
|
|
|
"""
|
|
|
|
skipped_blob = (self.type == 'M' and self.blob_id is None)
|
2009-02-14 15:31:07 +01:00
|
|
|
if skipped_blob: return
|
2009-02-08 03:45:12 +01:00
|
|
|
self.dumped = 1
|
|
|
|
|
2018-12-20 02:09:17 +01:00
|
|
|
quoted_filename = PathQuoting.enquote(self.filename)
|
2018-01-16 19:53:12 +01:00
|
|
|
if self.type == 'M' and isinstance(self.blob_id, int):
|
2018-12-20 02:09:17 +01:00
|
|
|
file_.write('M %s :%d %s\n' % (self.mode, self.blob_id, quoted_filename))
|
2018-01-16 19:53:12 +01:00
|
|
|
elif self.type == 'M':
|
2018-12-20 02:09:17 +01:00
|
|
|
file_.write('M %s %s %s\n' % (self.mode, self.blob_id, quoted_filename))
|
2009-02-05 01:27:57 +01:00
|
|
|
elif self.type == 'D':
|
2018-12-20 02:09:17 +01:00
|
|
|
file_.write('D %s\n' % quoted_filename)
|
2009-02-08 17:41:50 +01:00
|
|
|
else:
|
2019-03-17 07:39:39 +01:00
|
|
|
raise SystemExit(_("Unhandled filechange type: %s") % self.type) # pragma: no cover
|
2009-02-04 15:30:40 +01:00
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
class Commit(_GitElementWithId):
|
|
|
|
"""
|
|
|
|
This class defines our representation of commit elements. Commit elements
|
|
|
|
contain all the information associated with a commit.
|
|
|
|
"""
|
|
|
|
|
2009-02-04 01:48:01 +01:00
|
|
|
def __init__(self, branch,
|
|
|
|
author_name, author_email, author_date,
|
|
|
|
committer_name, committer_email, committer_date,
|
|
|
|
message,
|
|
|
|
file_changes,
|
2019-04-26 15:28:16 +02:00
|
|
|
parents,
|
2018-10-31 01:55:54 +01:00
|
|
|
original_id = None,
|
2009-02-18 00:52:48 +01:00
|
|
|
**kwargs):
|
2009-03-27 02:23:13 +01:00
|
|
|
_GitElementWithId.__init__(self)
|
|
|
|
|
|
|
|
# Denote that this is a commit element
|
2009-02-04 01:48:01 +01:00
|
|
|
self.type = 'commit'
|
2009-03-27 02:23:13 +01:00
|
|
|
|
|
|
|
# Record the affected branch
|
2009-02-04 01:48:01 +01:00
|
|
|
self.branch = branch
|
2009-03-27 02:23:13 +01:00
|
|
|
|
2018-10-31 01:55:54 +01:00
|
|
|
# Record original id
|
|
|
|
self.original_id = original_id
|
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
# Record author's name
|
2009-02-04 01:48:01 +01:00
|
|
|
self.author_name = author_name
|
2009-03-27 02:23:13 +01:00
|
|
|
|
|
|
|
# Record author's email
|
2009-02-04 01:48:01 +01:00
|
|
|
self.author_email = author_email
|
2009-03-27 02:23:13 +01:00
|
|
|
|
|
|
|
# Record date of authoring
|
2009-02-04 01:48:01 +01:00
|
|
|
self.author_date = author_date
|
2009-03-27 02:23:13 +01:00
|
|
|
|
|
|
|
# Record committer's name
|
2009-02-04 01:48:01 +01:00
|
|
|
self.committer_name = committer_name
|
2009-03-27 02:23:13 +01:00
|
|
|
|
|
|
|
# Record committer's email
|
2009-02-04 01:48:01 +01:00
|
|
|
self.committer_email = committer_email
|
2009-03-27 02:23:13 +01:00
|
|
|
|
|
|
|
# Record date the commit was made
|
2009-02-04 01:48:01 +01:00
|
|
|
self.committer_date = committer_date
|
2009-03-27 02:23:13 +01:00
|
|
|
|
|
|
|
# Record commit message
|
2009-02-04 01:48:01 +01:00
|
|
|
self.message = message
|
2009-03-27 02:23:13 +01:00
|
|
|
|
|
|
|
# List of file-changes associated with this commit. Note that file-changes
|
|
|
|
# are also represented as git elements
|
2009-02-04 01:48:01 +01:00
|
|
|
self.file_changes = file_changes
|
2009-03-27 02:23:13 +01:00
|
|
|
|
2019-04-26 15:28:16 +02:00
|
|
|
self.parents = parents
|
2009-03-27 02:23:13 +01:00
|
|
|
|
|
|
|
def dump(self, file_):
|
|
|
|
"""
|
|
|
|
Write this commit element to a file.
|
|
|
|
"""
|
2009-02-08 03:45:12 +01:00
|
|
|
self.dumped = 1
|
|
|
|
|
2019-03-04 22:34:58 +01:00
|
|
|
# Make output to fast-import slightly easier for humans to read if the
|
|
|
|
# message has no trailing newline of its own; cosmetic, but a nice touch...
|
|
|
|
extra_newline = '\n'
|
2019-04-26 15:28:16 +02:00
|
|
|
if self.message.endswith('\n') or not (self.parents or self.file_changes):
|
2019-03-04 22:34:58 +01:00
|
|
|
extra_newline = ''
|
|
|
|
|
2019-02-12 20:49:51 +01:00
|
|
|
file_.write(('commit {}\n'
|
|
|
|
'mark :{}\n'
|
|
|
|
'author {} <{}> {}\n'
|
|
|
|
'committer {} <{}> {}\n'
|
|
|
|
'data {}\n{}{}'
|
|
|
|
).format(
|
|
|
|
self.branch, self.id,
|
|
|
|
self.author_name, self.author_email, self.author_date,
|
|
|
|
self.committer_name, self.committer_email, self.committer_date,
|
|
|
|
len(self.message), self.message,
|
2019-03-04 22:34:58 +01:00
|
|
|
extra_newline)
|
2019-02-12 20:49:51 +01:00
|
|
|
)
|
2019-04-26 15:28:16 +02:00
|
|
|
for i, parent in enumerate(self.parents):
|
|
|
|
mark = ':' if isinstance(parent, int) else ''
|
|
|
|
file_.write('from ' if i==0 else 'merge ')
|
|
|
|
file_.write('{}{}\n'.format(mark, parent))
|
2009-02-04 01:48:01 +01:00
|
|
|
for change in self.file_changes:
|
2009-03-27 02:23:13 +01:00
|
|
|
change.dump(file_)
|
2019-04-26 15:28:16 +02:00
|
|
|
if not self.parents and not self.file_changes:
|
2019-03-04 22:34:58 +01:00
|
|
|
# Workaround a bug in pre-git-2.22 versions of fast-import with
|
|
|
|
# the get-mark directive.
|
|
|
|
file_.write('\n')
|
2009-03-27 02:23:13 +01:00
|
|
|
file_.write('\n')
|
|
|
|
|
2009-03-19 14:58:41 +01:00
|
|
|
def first_parent(self):
|
2009-03-27 02:23:13 +01:00
|
|
|
"""
|
|
|
|
Return first parent commit
|
|
|
|
"""
|
2019-04-26 15:28:16 +02:00
|
|
|
if self.parents:
|
|
|
|
return self.parents[0]
|
2009-03-19 14:58:41 +01:00
|
|
|
return None
|
|
|
|
|
2018-12-18 02:43:15 +01:00
|
|
|
def skip(self, new_id=None):
|
|
|
|
_SKIPPED_COMMITS.add(self.old_id or self.id)
|
|
|
|
_GitElementWithId.skip(self, new_id)
|
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
class Tag(_GitElement):
|
|
|
|
"""
|
2009-04-04 22:57:02 +02:00
|
|
|
This class defines our representation of annotated tag elements.
|
2009-03-27 02:23:13 +01:00
|
|
|
"""
|
|
|
|
|
2009-03-21 23:49:43 +01:00
|
|
|
def __init__(self, ref, from_ref,
|
2018-10-31 01:55:54 +01:00
|
|
|
tagger_name, tagger_email, tagger_date, tag_msg,
|
|
|
|
original_id = None):
|
2009-03-27 02:23:13 +01:00
|
|
|
_GitElement.__init__(self)
|
|
|
|
|
|
|
|
# Denote that this is a tag element
|
2009-03-21 23:49:43 +01:00
|
|
|
self.type = 'tag'
|
2009-03-27 02:23:13 +01:00
|
|
|
|
|
|
|
# Store the name of the tag
|
2009-03-21 23:49:43 +01:00
|
|
|
self.ref = ref
|
2009-03-27 02:23:13 +01:00
|
|
|
|
|
|
|
# Store the entity being tagged (this should be a commit)
|
2009-03-21 23:49:43 +01:00
|
|
|
self.from_ref = from_ref
|
2009-03-27 02:23:13 +01:00
|
|
|
|
2018-10-31 01:55:54 +01:00
|
|
|
# Record original id
|
|
|
|
self.original_id = original_id
|
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
# Store the name of the tagger
|
2009-03-21 23:49:43 +01:00
|
|
|
self.tagger_name = tagger_name
|
2009-03-27 02:23:13 +01:00
|
|
|
|
|
|
|
# Store the email of the tagger
|
2009-03-21 23:49:43 +01:00
|
|
|
self.tagger_email = tagger_email
|
2009-03-27 02:23:13 +01:00
|
|
|
|
|
|
|
# Store the date
|
2009-03-21 23:49:43 +01:00
|
|
|
self.tagger_date = tagger_date
|
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
# Store the tag message
|
2019-02-16 20:44:01 +01:00
|
|
|
self.message = tag_msg
|
2009-03-21 23:49:43 +01:00
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
def dump(self, file_):
|
|
|
|
"""
|
|
|
|
Write this tag element to a file
|
|
|
|
"""
|
2009-03-21 23:49:43 +01:00
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
self.dumped = 1
|
2009-03-21 23:49:43 +01:00
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
file_.write('tag %s\n' % self.ref)
|
2018-09-12 01:37:54 +02:00
|
|
|
mark = ':' if isinstance(self.from_ref, int) else ''
|
|
|
|
file_.write('from {}{}\n'.format(mark, self.from_ref))
|
2019-02-02 05:52:02 +01:00
|
|
|
if self.tagger_name:
|
|
|
|
file_.write('tagger %s <%s> ' % (self.tagger_name, self.tagger_email))
|
2019-02-05 03:30:50 +01:00
|
|
|
file_.write(self.tagger_date)
|
2019-02-02 05:52:02 +01:00
|
|
|
file_.write('\n')
|
2019-02-16 20:44:01 +01:00
|
|
|
file_.write('data %d\n%s' % (len(self.message), self.message))
|
2009-03-27 02:23:13 +01:00
|
|
|
file_.write('\n')
|
|
|
|
|
|
|
|
class Progress(_GitElement):
|
|
|
|
"""
|
|
|
|
This class defines our representation of progress elements. The progress
|
2009-04-04 22:57:02 +02:00
|
|
|
element only contains a progress message, which is printed by fast-import
|
|
|
|
when it processes the progress output.
|
2009-03-27 02:23:13 +01:00
|
|
|
"""
|
|
|
|
|
2009-03-22 00:03:01 +01:00
|
|
|
def __init__(self, message):
|
2009-03-27 02:23:13 +01:00
|
|
|
_GitElement.__init__(self)
|
|
|
|
|
|
|
|
# Denote that this is a progress element
|
2009-03-22 00:03:01 +01:00
|
|
|
self.type = 'progress'
|
2009-03-27 02:23:13 +01:00
|
|
|
|
|
|
|
# Store the progress message
|
2009-03-22 00:03:01 +01:00
|
|
|
self.message = message
|
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
def dump(self, file_):
|
|
|
|
"""
|
|
|
|
Write this progress element to a file
|
|
|
|
"""
|
2009-03-22 00:03:01 +01:00
|
|
|
self.dumped = 1
|
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
file_.write('progress %s\n' % self.message)
|
2019-04-28 00:08:26 +02:00
|
|
|
file_.write('\n')
|
2009-03-27 02:23:13 +01:00
|
|
|
|
|
|
|
class Checkpoint(_GitElement):
|
|
|
|
"""
|
2009-04-04 22:57:02 +02:00
|
|
|
This class defines our representation of checkpoint elements. These
|
|
|
|
elements represent events which force fast-import to close the current
|
|
|
|
packfile, start a new one, and to save out all current branch refs, tags
|
|
|
|
and marks.
|
2009-03-27 02:23:13 +01:00
|
|
|
"""
|
|
|
|
|
2009-04-05 20:47:53 +02:00
|
|
|
def __init__(self):
|
2009-03-27 02:23:13 +01:00
|
|
|
_GitElement.__init__(self)
|
|
|
|
|
|
|
|
# Denote that this is a checkpoint element
|
2009-03-22 00:03:01 +01:00
|
|
|
self.type = 'checkpoint'
|
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
def dump(self, file_):
|
|
|
|
"""
|
|
|
|
Write this checkpoint element to a file
|
|
|
|
"""
|
2009-03-22 00:03:01 +01:00
|
|
|
self.dumped = 1
|
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
file_.write('checkpoint\n')
|
|
|
|
file_.write('\n')
|
2009-03-22 00:03:01 +01:00
|
|
|
|
2018-09-03 21:10:17 +02:00
|
|
|
class LiteralCommand(_GitElement):
|
|
|
|
"""
|
|
|
|
This class defines our representation of commands. The literal command
|
|
|
|
includes only a single line, and is not processed in any special way.
|
|
|
|
"""
|
|
|
|
|
|
|
|
def __init__(self, line):
|
|
|
|
_GitElement.__init__(self)
|
|
|
|
|
|
|
|
# Denote that this is a literal element
|
|
|
|
self.type = 'literal'
|
|
|
|
|
|
|
|
# Store the command
|
|
|
|
self.line = line
|
|
|
|
|
|
|
|
def dump(self, file_):
|
|
|
|
"""
|
|
|
|
Write this progress element to a file
|
|
|
|
"""
|
|
|
|
self.dumped = 1
|
|
|
|
|
|
|
|
file_.write(self.line)
|
|
|
|
|
2009-02-05 07:08:06 +01:00
|
|
|
class FastExportFilter(object):
|
2009-03-27 02:23:13 +01:00
|
|
|
"""
|
|
|
|
A class for parsing and handling the output from fast-export. This
|
|
|
|
class allows the user to register callbacks when various types of
|
|
|
|
data are encountered in the fast-export output. The basic idea is that,
|
|
|
|
FastExportFilter takes fast-export output, creates the various objects
|
|
|
|
as it encounters them, the user gets to use/modify these objects via
|
|
|
|
callbacks, and finally FastExportFilter outputs the modified objects
|
|
|
|
in fast-import format (presumably so they can be used to create a new
|
|
|
|
repo).
|
|
|
|
"""
|
|
|
|
|
2019-01-08 20:02:53 +01:00
|
|
|
def __init__(self, repo_working_dir,
|
2009-02-03 01:26:17 +01:00
|
|
|
tag_callback = None, commit_callback = None,
|
|
|
|
blob_callback = None, progress_callback = None,
|
|
|
|
reset_callback = None, checkpoint_callback = None,
|
|
|
|
everything_callback = None):
|
2019-01-08 20:02:53 +01:00
|
|
|
# Repo we are exporting
|
|
|
|
self._repo_working_dir = repo_working_dir
|
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
# Members below simply store callback functions for the various git
|
|
|
|
# elements
|
|
|
|
self._tag_callback = tag_callback
|
|
|
|
self._blob_callback = blob_callback
|
|
|
|
self._reset_callback = reset_callback
|
|
|
|
self._commit_callback = commit_callback
|
|
|
|
self._progress_callback = progress_callback
|
|
|
|
self._checkpoint_callback = checkpoint_callback
|
|
|
|
self._everything_callback = everything_callback
|
|
|
|
|
2018-09-04 04:35:25 +02:00
|
|
|
# A list of all the refs we've seen, plus any mark we need to set them
|
|
|
|
# to if the last (or even only) commit on that branch was pruned
|
|
|
|
self._seen_refs = {}
|
2018-08-21 17:46:26 +02:00
|
|
|
|
filter-repo: avoid merging a commit with one of its own ancestors
Pruning of empty commits can cause an entire line of history to become
empty and be pruned, resulting in a merge commit that merges some commit
with one of its ancestors. In such a case, we should remove the
unnecessary parent(s) -- which can and will often result in the merge
commit being empty so we can remove it as well.
Currently, if the side that becomes empty is the first parent side, then
we do not detect if the commit becomes empty, due to the way that
fast-export lists changes in a merge commit relative to first parent only.
A subsequent commit will address this.
Note that the callbacks could theoretically insert additional commits or
reparent our commit on top of something else, meaning that the ancestry
graph might need post-callback updates. However, in any extreme case
where that mattered, we would more or less need full updates to the
ancestry graph to be made for all the new commits from the callback as
well, and once we expect the callback to handle any ancestry graph
updates it can handle modifying it for the current commit. However, it
is hard to come up with a case where it matters, since for the most part
we just want to know whether our filtering causes commits to become
empty and knowing the source repo we are exporting from is sufficient
information without knowing any new commits inserted or reparenting that
happens elsewhere.
Signed-off-by: Elijah Newren <newren@gmail.com>
2018-09-04 18:26:05 +02:00
|
|
|
# A tuple of (depth, list-of-ancestors). Commits and ancestors are
|
|
|
|
# identified by their id (their 'mark' in fast-export or fast-import
|
|
|
|
# speak). The depth of a commit is one more than the max depth of any
|
|
|
|
# of its ancestors.
|
|
|
|
self._graph = AncestryGraph()
|
2019-03-01 03:12:49 +01:00
|
|
|
# Another one, for ancestry of commits in the original repo
|
|
|
|
self._orig_graph = AncestryGraph()
|
filter-repo: avoid merging a commit with one of its own ancestors
Pruning of empty commits can cause an entire line of history to become
empty and be pruned, resulting in a merge commit that merges some commit
with one of its ancestors. In such a case, we should remove the
unnecessary parent(s) -- which can and will often result in the merge
commit being empty so we can remove it as well.
Currently, if the side that becomes empty is the first parent side, then
we do not detect if the commit becomes empty, due to the way that
fast-export lists changes in a merge commit relative to first parent only.
A subsequent commit will address this.
Note that the callbacks could theoretically insert additional commits or
reparent our commit on top of something else, meaning that the ancestry
graph might need post-callback updates. However, in any extreme case
where that mattered, we would more or less need full updates to the
ancestry graph to be made for all the new commits from the callback as
well, and once we expect the callback to handle any ancestry graph
updates it can handle modifying it for the current commit. However, it
is hard to come up with a case where it matters, since for the most part
we just want to know whether our filtering causes commits to become
empty and knowing the source repo we are exporting from is sufficient
information without knowing any new commits inserted or reparenting that
happens elsewhere.
Signed-off-by: Elijah Newren <newren@gmail.com>
2018-09-04 18:26:05 +02:00
|
|
|
|
2018-10-18 03:31:26 +02:00
|
|
|
# A set of commit hash pairs (oldhash, newhash) which used to be merge
|
|
|
|
# commits but due to filtering were turned into non-merge commits.
|
|
|
|
# The commits probably have suboptimal commit messages (e.g. "Merge branch
|
|
|
|
# next into master").
|
|
|
|
self._commits_no_longer_merges = []
|
|
|
|
|
2018-09-14 03:02:41 +02:00
|
|
|
# A dict of original_ids to new_ids; filtering commits means getting
|
|
|
|
# new commit hash (sha1sums), and we record the mapping both for
|
|
|
|
# diagnostic purposes and so we can rewrite commit messages. Note that
|
|
|
|
# the new_id can be None rather than a commit hash if the original
|
|
|
|
# commit became empty and was pruned or was otherwise dropped.
|
|
|
|
self._commit_renames = {}
|
|
|
|
|
2019-02-04 20:10:15 +01:00
|
|
|
# A set of original_ids for which we have not yet gotten the
|
|
|
|
# new_ids; we use OrderedDict because we need to know the order of
|
|
|
|
# insertion, but the values are always ignored (and set to None).
|
|
|
|
# If there was an OrderedSet class, I'd use it instead.
|
|
|
|
self._pending_renames = collections.OrderedDict()
|
|
|
|
|
2018-09-14 03:02:41 +02:00
|
|
|
# A dict of commit_hash[1:7] -> set(commit_hashes with that prefix).
|
|
|
|
#
|
|
|
|
# It's common for commit messages to refer to commits by abbreviated
|
|
|
|
# commit hashes, as short as 7 characters. To facilitate translating
|
|
|
|
# such short hashes, we have a mapping of prefixes to full old hashes.
|
|
|
|
self._commit_short_old_hashes = collections.defaultdict(set)
|
|
|
|
|
|
|
|
# A set of commit hash references appearing in commit messages which
|
|
|
|
# mapped to a valid commit that was removed entirely in the filtering
|
|
|
|
# process. The commit message will continue to reference the
|
|
|
|
# now-missing commit hash, since there was nothing to map it to.
|
|
|
|
self._commits_referenced_but_removed = set()
|
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
# A handle to the input source for the fast-export data
|
|
|
|
self._input = None
|
|
|
|
|
|
|
|
# A handle to the output file for the output we generate (we call dump
|
|
|
|
# on many of the git elements we create).
|
|
|
|
self._output = None
|
|
|
|
|
2019-02-02 18:59:54 +01:00
|
|
|
# A pair of (input, output) pipes for communicating with fast import.
|
|
|
|
self._fast_import_pipes = None
|
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
# Stores the contents of the current line of input being parsed
|
|
|
|
self._currentline = ''
|
|
|
|
|
2018-10-27 00:19:48 +02:00
|
|
|
# Progress handling (number of commits parsed, etc.)
|
2018-11-27 18:59:49 +01:00
|
|
|
self._progress_writer = ProgressWriter()
|
2018-10-27 00:19:48 +02:00
|
|
|
self._num_commits = 0
|
|
|
|
self._quiet = False
|
|
|
|
|
2018-09-04 04:35:25 +02:00
|
|
|
# Whether we've run our post-processing extra commands
|
|
|
|
self._finalize_handled = False
|
|
|
|
|
2018-12-31 16:46:05 +01:00
|
|
|
# Names of files that were tweaked in any commit; such paths could lead
|
|
|
|
# to subsequent commits being empty
|
|
|
|
self._files_tweaked = set()
|
|
|
|
|
2019-02-12 02:35:27 +01:00
|
|
|
# Compile some regexes and cache those
|
|
|
|
self._mark_re = re.compile(r'mark :(\d+)\n$')
|
|
|
|
self._parent_regexes = {}
|
|
|
|
parent_regex_rules = ('{} :(\d+)\n$', '{} ([0-9a-f]{{40}})\n')
|
|
|
|
for parent_refname in ('from', 'merge'):
|
|
|
|
ans = [re.compile(x.format(parent_refname)) for x in parent_regex_rules]
|
|
|
|
self._parent_regexes[parent_refname] = ans
|
|
|
|
self._quoted_string_re = re.compile(r'"(?:[^"\\]|\\.)*"')
|
|
|
|
self._refline_regexes = {}
|
|
|
|
for refline_name in ('reset', 'commit', 'tag', 'progress'):
|
|
|
|
self._refline_regexes[refline_name] = re.compile(refline_name+' (.*)\n$')
|
|
|
|
self._user_regexes = {}
|
|
|
|
for user in ('author', 'committer', 'tagger'):
|
|
|
|
self._user_regexes[user] = re.compile(user + ' (.*?) <(.*?)> (.*)\n$')
|
|
|
|
self._hash_re = re.compile(r'(\b[0-9a-f]{7,40}\b)')
|
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
def _advance_currentline(self):
|
|
|
|
"""
|
|
|
|
Grab the next line of input
|
|
|
|
"""
|
|
|
|
self._currentline = self._input.readline()
|
|
|
|
|
2009-02-08 17:41:50 +01:00
|
|
|
def _parse_optional_mark(self):
|
2009-03-27 02:23:13 +01:00
|
|
|
"""
|
|
|
|
If the current line contains a mark, parse it and advance to the
|
|
|
|
next line; return None otherwise
|
|
|
|
"""
|
2009-02-08 17:41:50 +01:00
|
|
|
mark = None
|
2019-02-12 02:35:27 +01:00
|
|
|
matches = self._mark_re.match(self._currentline)
|
2009-02-08 17:41:50 +01:00
|
|
|
if matches:
|
filter-repo: delete complex code
Over a decade ago, I added code to deal with splitting and splicing
repositories where you weren't always dealing with first parents and
linear histories, and in particular where the mainline tended to be the
second parent (because there was no integrator or special central
gatekeeper like gerrit or github; instead, everyone pushed directly to
the main repository after locally testing, and integration happened via
everyone running 'git pull'). When attempting to splice repositories
the fact that fast-export always gave changes relative to the first
parent caused some grief with my splitting and splicing efforts.
It has been over a decade, I don't know of a good testcase of this
functionality separate from the live repositories I lost access to over
six years ago, git-subtree was released in the meantime which I'm
certain handled the task better, git-fast-export since gained a
--full-tree option which might have provided a better way to attack the
problem (though with splicing repos you often want work with additive
changes rather than recreating from scratch), and I just don't
quite understand the code anymore anyway. I think it had some
fundamental limitations that I knew my usecase avoided, but I don't
remember the details (and I'm not certain if this is true).
Even though code coverage hits all but one of the lines, I'd rather
rewrite any needed functionality if the usecase arises, and in view of
what facilities exist today rather than what I was working with a decade
ago. So, just nuke this code.
Signed-off-by: Elijah Newren <newren@gmail.com>
2019-03-15 02:26:04 +01:00
|
|
|
mark = int(matches.group(1))
|
2009-03-27 02:23:13 +01:00
|
|
|
self._advance_currentline()
|
2009-02-08 17:41:50 +01:00
|
|
|
return mark
|
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
def _parse_optional_parent_ref(self, refname):
|
|
|
|
"""
|
|
|
|
If the current line contains a reference to a parent commit, then
|
|
|
|
parse it and advance the current line; otherwise return None. Note
|
|
|
|
that the name of the reference ('from', 'merge') must match the
|
|
|
|
refname arg.
|
|
|
|
"""
|
filter-repo: modify parse_optional_parent_ref to return original parent too
commits may not have any parents at all. As such,
parse_optional_parent_ref() is used expecting that it will sometimes
return None.
Now, when commits are skipped, we have a scheme to translate anyone that
depends on such commits to instead depend on the nearest ancestor of
such commits. If the entire ancestry of a commit was skipped along with
a comit, then that commit will be translated to None, which is
indistinguishable from there having been no parent to begin with.
Sometimes our scheme needs to distinguish between a commit that started
with no parents and one which ended up with no parents, so we need a way
to tell these apart.
Also, not knowing the original parent makes it hard for us to
determine if the original had the same weird topology that the current
commit does. For example, it is possible for a merge commit to have
one parent be the ancestor of another (particularly when --no-ff is
passed to git merge), or even for a merge commit to have the same
commit used as both parents (if you use low-level commands to create
a crazy commit). There are cases where the pruning of some commits
could cause either of these situations to arise, and it's useful to be
able to distinguish between intentionally "weird" history and history
that has been made weird due to other pruning, because the latter we
may have reason to do additional pruning on.
Signed-off-by: Elijah Newren <newren@gmail.com>
2018-12-15 18:20:40 +01:00
|
|
|
orig_baseref, baseref = None, None
|
2019-02-12 02:35:27 +01:00
|
|
|
rule, altrule = self._parent_regexes[refname]
|
|
|
|
matches = rule.match(self._currentline)
|
2009-02-08 17:41:50 +01:00
|
|
|
if matches:
|
filter-repo: delete complex code
Over a decade ago, I added code to deal with splitting and splicing
repositories where you weren't always dealing with first parents and
linear histories, and in particular where the mainline tended to be the
second parent (because there was no integrator or special central
gatekeeper like gerrit or github; instead, everyone pushed directly to
the main repository after locally testing, and integration happened via
everyone running 'git pull'). When attempting to splice repositories
the fact that fast-export always gave changes relative to the first
parent caused some grief with my splitting and splicing efforts.
It has been over a decade, I don't know of a good testcase of this
functionality separate from the live repositories I lost access to over
six years ago, git-subtree was released in the meantime which I'm
certain handled the task better, git-fast-export since gained a
--full-tree option which might have provided a better way to attack the
problem (though with splicing repos you often want work with additive
changes rather than recreating from scratch), and I just don't
quite understand the code anymore anyway. I think it had some
fundamental limitations that I knew my usecase avoided, but I don't
remember the details (and I'm not certain if this is true).
Even though code coverage hits all but one of the lines, I'd rather
rewrite any needed functionality if the usecase arises, and in view of
what facilities exist today rather than what I was working with a decade
ago. So, just nuke this code.
Signed-off-by: Elijah Newren <newren@gmail.com>
2019-03-15 02:26:04 +01:00
|
|
|
orig_baseref = int(matches.group(1))
|
2009-03-27 02:23:13 +01:00
|
|
|
# We translate the parent commit mark to what it needs to be in
|
|
|
|
# our mark namespace
|
filter-repo: modify parse_optional_parent_ref to return original parent too
commits may not have any parents at all. As such,
parse_optional_parent_ref() is used expecting that it will sometimes
return None.
Now, when commits are skipped, we have a scheme to translate anyone that
depends on such commits to instead depend on the nearest ancestor of
such commits. If the entire ancestry of a commit was skipped along with
a comit, then that commit will be translated to None, which is
indistinguishable from there having been no parent to begin with.
Sometimes our scheme needs to distinguish between a commit that started
with no parents and one which ended up with no parents, so we need a way
to tell these apart.
Also, not knowing the original parent makes it hard for us to
determine if the original had the same weird topology that the current
commit does. For example, it is possible for a merge commit to have
one parent be the ancestor of another (particularly when --no-ff is
passed to git merge), or even for a merge commit to have the same
commit used as both parents (if you use low-level commands to create
a crazy commit). There are cases where the pruning of some commits
could cause either of these situations to arise, and it's useful to be
able to distinguish between intentionally "weird" history and history
that has been made weird due to other pruning, because the latter we
may have reason to do additional pruning on.
Signed-off-by: Elijah Newren <newren@gmail.com>
2018-12-15 18:20:40 +01:00
|
|
|
baseref = _IDS.translate(orig_baseref)
|
2009-03-27 02:23:13 +01:00
|
|
|
self._advance_currentline()
|
2018-09-12 01:37:54 +02:00
|
|
|
else:
|
2019-02-12 02:35:27 +01:00
|
|
|
matches = altrule.match(self._currentline)
|
2018-09-12 01:37:54 +02:00
|
|
|
if matches:
|
filter-repo: modify parse_optional_parent_ref to return original parent too
commits may not have any parents at all. As such,
parse_optional_parent_ref() is used expecting that it will sometimes
return None.
Now, when commits are skipped, we have a scheme to translate anyone that
depends on such commits to instead depend on the nearest ancestor of
such commits. If the entire ancestry of a commit was skipped along with
a comit, then that commit will be translated to None, which is
indistinguishable from there having been no parent to begin with.
Sometimes our scheme needs to distinguish between a commit that started
with no parents and one which ended up with no parents, so we need a way
to tell these apart.
Also, not knowing the original parent makes it hard for us to
determine if the original had the same weird topology that the current
commit does. For example, it is possible for a merge commit to have
one parent be the ancestor of another (particularly when --no-ff is
passed to git merge), or even for a merge commit to have the same
commit used as both parents (if you use low-level commands to create
a crazy commit). There are cases where the pruning of some commits
could cause either of these situations to arise, and it's useful to be
able to distinguish between intentionally "weird" history and history
that has been made weird due to other pruning, because the latter we
may have reason to do additional pruning on.
Signed-off-by: Elijah Newren <newren@gmail.com>
2018-12-15 18:20:40 +01:00
|
|
|
orig_baseref = matches.group(1)
|
|
|
|
baseref = orig_baseref
|
2018-09-12 01:37:54 +02:00
|
|
|
self._advance_currentline()
|
filter-repo: modify parse_optional_parent_ref to return original parent too
commits may not have any parents at all. As such,
parse_optional_parent_ref() is used expecting that it will sometimes
return None.
Now, when commits are skipped, we have a scheme to translate anyone that
depends on such commits to instead depend on the nearest ancestor of
such commits. If the entire ancestry of a commit was skipped along with
a comit, then that commit will be translated to None, which is
indistinguishable from there having been no parent to begin with.
Sometimes our scheme needs to distinguish between a commit that started
with no parents and one which ended up with no parents, so we need a way
to tell these apart.
Also, not knowing the original parent makes it hard for us to
determine if the original had the same weird topology that the current
commit does. For example, it is possible for a merge commit to have
one parent be the ancestor of another (particularly when --no-ff is
passed to git merge), or even for a merge commit to have the same
commit used as both parents (if you use low-level commands to create
a crazy commit). There are cases where the pruning of some commits
could cause either of these situations to arise, and it's useful to be
able to distinguish between intentionally "weird" history and history
that has been made weird due to other pruning, because the latter we
may have reason to do additional pruning on.
Signed-off-by: Elijah Newren <newren@gmail.com>
2018-12-15 18:20:40 +01:00
|
|
|
return orig_baseref, baseref
|
2009-02-08 17:41:50 +01:00
|
|
|
|
|
|
|
def _parse_optional_filechange(self):
|
2009-03-27 02:23:13 +01:00
|
|
|
"""
|
|
|
|
If the current line contains a file-change object, then parse it
|
|
|
|
and advance the current line; otherwise return None. We only care
|
|
|
|
about file changes of type 'M' and 'D' (these are the only types
|
|
|
|
of file-changes that fast-export will provide).
|
|
|
|
"""
|
2009-02-08 17:41:50 +01:00
|
|
|
filechange = None
|
2019-02-12 22:14:57 +01:00
|
|
|
changetype = self._currentline[0]
|
|
|
|
if changetype == 'M':
|
|
|
|
(changetype, mode, idnum, path) = self._currentline.split(None, 3)
|
|
|
|
if idnum[0] == ':':
|
|
|
|
idnum = idnum[1:]
|
|
|
|
path = path.rstrip('\n')
|
2009-03-27 02:23:13 +01:00
|
|
|
# We translate the idnum to our id system
|
2018-01-16 19:53:12 +01:00
|
|
|
if len(idnum) != 40:
|
filter-repo: delete complex code
Over a decade ago, I added code to deal with splitting and splicing
repositories where you weren't always dealing with first parents and
linear histories, and in particular where the mainline tended to be the
second parent (because there was no integrator or special central
gatekeeper like gerrit or github; instead, everyone pushed directly to
the main repository after locally testing, and integration happened via
everyone running 'git pull'). When attempting to splice repositories
the fact that fast-export always gave changes relative to the first
parent caused some grief with my splitting and splicing efforts.
It has been over a decade, I don't know of a good testcase of this
functionality separate from the live repositories I lost access to over
six years ago, git-subtree was released in the meantime which I'm
certain handled the task better, git-fast-export since gained a
--full-tree option which might have provided a better way to attack the
problem (though with splicing repos you often want work with additive
changes rather than recreating from scratch), and I just don't
quite understand the code anymore anyway. I think it had some
fundamental limitations that I knew my usecase avoided, but I don't
remember the details (and I'm not certain if this is true).
Even though code coverage hits all but one of the lines, I'd rather
rewrite any needed functionality if the usecase arises, and in view of
what facilities exist today rather than what I was working with a decade
ago. So, just nuke this code.
Signed-off-by: Elijah Newren <newren@gmail.com>
2019-03-15 02:26:04 +01:00
|
|
|
idnum = _IDS.translate( int(idnum) )
|
2009-03-12 00:51:31 +01:00
|
|
|
if idnum is not None:
|
|
|
|
if path.startswith('"'):
|
2018-12-20 02:09:17 +01:00
|
|
|
path = PathQuoting.dequote(path)
|
2009-03-12 00:51:31 +01:00
|
|
|
filechange = FileChanges('M', path, idnum, mode)
|
2009-04-04 15:47:29 +02:00
|
|
|
else:
|
|
|
|
filechange = 'skipped'
|
2009-03-27 02:23:13 +01:00
|
|
|
self._advance_currentline()
|
2019-02-12 22:14:57 +01:00
|
|
|
elif changetype == 'D':
|
|
|
|
(changetype, path) = self._currentline.split(None, 1)
|
|
|
|
path = path.rstrip('\n')
|
2009-02-08 17:41:50 +01:00
|
|
|
if path.startswith('"'):
|
2018-12-20 02:09:17 +01:00
|
|
|
path = PathQuoting.dequote(path)
|
2009-02-08 17:41:50 +01:00
|
|
|
filechange = FileChanges('D', path)
|
2009-03-27 02:23:13 +01:00
|
|
|
self._advance_currentline()
|
2019-03-12 19:26:21 +01:00
|
|
|
elif changetype == 'R': # pragma: no cover (now avoid fast-export renames)
|
filter-repo: add --analyze option
This option walks through the repository history and creates a report
with basic statistics, rename related information, and sizes of objects
and when/if those have been deleted. It primarily looks at unpacked
sizes (i.e. size of object ignoring delta-ing and compression), and
sums the size of each version of the file for each path. Additionally,
it aggregates these sums by extension and by directory, and tracks
whether paths, extensions, and directories have been deleted. This can
be very useful in determining what the big things are, and whether they
might have been considered to have been mistakes to add to the
repository in the first place.
There are numerous caveats with the determination of "deleted" and
"renamed", and can give both false positives and false negatives. But
they are only meant as a helpful heuristic to give others a starting
point for an investigation, and the information provide so far is useful.
I do want to improve the equivalence classes (rename handling), but that
is for a future commit.
Signed-off-by: Elijah Newren <newren@gmail.com>
2018-10-21 16:44:33 +02:00
|
|
|
rest = self._currentline[2:-1]
|
|
|
|
if rest.startswith('"'):
|
2019-02-12 02:35:27 +01:00
|
|
|
m = self._quoted_string_re.match(rest)
|
filter-repo: add --analyze option
This option walks through the repository history and creates a report
with basic statistics, rename related information, and sizes of objects
and when/if those have been deleted. It primarily looks at unpacked
sizes (i.e. size of object ignoring delta-ing and compression), and
sums the size of each version of the file for each path. Additionally,
it aggregates these sums by extension and by directory, and tracks
whether paths, extensions, and directories have been deleted. This can
be very useful in determining what the big things are, and whether they
might have been considered to have been mistakes to add to the
repository in the first place.
There are numerous caveats with the determination of "deleted" and
"renamed", and can give both false positives and false negatives. But
they are only meant as a helpful heuristic to give others a starting
point for an investigation, and the information provide so far is useful.
I do want to improve the equivalence classes (rename handling), but that
is for a future commit.
Signed-off-by: Elijah Newren <newren@gmail.com>
2018-10-21 16:44:33 +02:00
|
|
|
if not m:
|
2019-03-17 07:39:39 +01:00
|
|
|
raise SystemExit(_("Couldn't parse rename source"))
|
2018-12-20 02:09:17 +01:00
|
|
|
orig = PathQuoting.dequote(m.group(0))
|
filter-repo: add --analyze option
This option walks through the repository history and creates a report
with basic statistics, rename related information, and sizes of objects
and when/if those have been deleted. It primarily looks at unpacked
sizes (i.e. size of object ignoring delta-ing and compression), and
sums the size of each version of the file for each path. Additionally,
it aggregates these sums by extension and by directory, and tracks
whether paths, extensions, and directories have been deleted. This can
be very useful in determining what the big things are, and whether they
might have been considered to have been mistakes to add to the
repository in the first place.
There are numerous caveats with the determination of "deleted" and
"renamed", and can give both false positives and false negatives. But
they are only meant as a helpful heuristic to give others a starting
point for an investigation, and the information provide so far is useful.
I do want to improve the equivalence classes (rename handling), but that
is for a future commit.
Signed-off-by: Elijah Newren <newren@gmail.com>
2018-10-21 16:44:33 +02:00
|
|
|
new = rest[m.end()+1:]
|
|
|
|
else:
|
|
|
|
orig, new = rest.split(' ', 1)
|
|
|
|
if new.startswith('"'):
|
2018-12-20 02:09:17 +01:00
|
|
|
new = PathQuoting.dequote(new)
|
filter-repo: add --analyze option
This option walks through the repository history and creates a report
with basic statistics, rename related information, and sizes of objects
and when/if those have been deleted. It primarily looks at unpacked
sizes (i.e. size of object ignoring delta-ing and compression), and
sums the size of each version of the file for each path. Additionally,
it aggregates these sums by extension and by directory, and tracks
whether paths, extensions, and directories have been deleted. This can
be very useful in determining what the big things are, and whether they
might have been considered to have been mistakes to add to the
repository in the first place.
There are numerous caveats with the determination of "deleted" and
"renamed", and can give both false positives and false negatives. But
they are only meant as a helpful heuristic to give others a starting
point for an investigation, and the information provide so far is useful.
I do want to improve the equivalence classes (rename handling), but that
is for a future commit.
Signed-off-by: Elijah Newren <newren@gmail.com>
2018-10-21 16:44:33 +02:00
|
|
|
filechange = FileChanges('R', orig, new)
|
|
|
|
self._advance_currentline()
|
2009-02-08 17:41:50 +01:00
|
|
|
return filechange
|
|
|
|
|
2018-10-31 01:55:54 +01:00
|
|
|
def _parse_original_id(self):
|
|
|
|
original_id = self._currentline[len('original-oid '):].rstrip()
|
|
|
|
self._advance_currentline()
|
|
|
|
return original_id
|
|
|
|
|
2009-02-08 17:41:50 +01:00
|
|
|
def _parse_ref_line(self, refname):
|
2009-03-27 02:23:13 +01:00
|
|
|
"""
|
2009-04-04 22:57:02 +02:00
|
|
|
Parses string data (often a branch name) from current-line. The name of
|
|
|
|
the string data must match the refname arg. The program will crash if
|
|
|
|
current-line does not match, so current-line will always be advanced if
|
|
|
|
this method returns.
|
2009-03-27 02:23:13 +01:00
|
|
|
"""
|
2019-02-12 02:35:27 +01:00
|
|
|
matches = self._refline_regexes[refname].match(self._currentline)
|
2009-02-08 17:41:50 +01:00
|
|
|
if not matches:
|
2019-03-17 07:39:39 +01:00
|
|
|
raise SystemExit(_("Malformed %(refname)s line: '%(line)s'") %
|
|
|
|
({'refname': refname, 'line':self._currentline})
|
|
|
|
) # pragma: no cover
|
2009-02-08 17:41:50 +01:00
|
|
|
ref = matches.group(1)
|
2009-03-27 02:23:13 +01:00
|
|
|
self._advance_currentline()
|
2009-02-08 17:41:50 +01:00
|
|
|
return ref
|
|
|
|
|
|
|
|
def _parse_user(self, usertype):
|
2009-03-27 02:23:13 +01:00
|
|
|
"""
|
|
|
|
Get user name, email, datestamp from current-line. Current-line will
|
|
|
|
be advanced.
|
|
|
|
"""
|
2019-02-12 02:35:27 +01:00
|
|
|
user_regex = self._user_regexes[usertype]
|
|
|
|
(name, email, when) = user_regex.match(self._currentline).groups()
|
2009-03-13 01:14:42 +01:00
|
|
|
|
2019-02-05 03:30:50 +01:00
|
|
|
# TimeZone idiocy; IST is any of four timezones, so someone translated
|
|
|
|
# it to something that was totally invalid...and it got recorded that
|
|
|
|
# way. Others have suggested just using an invalid timezone that
|
|
|
|
# fast-import will not choke on. Let's do that. Note that +051800
|
|
|
|
# seems to be the only weird timezone found in the wild, by me or some
|
|
|
|
# other posts google returned on the subject...
|
|
|
|
if when.endswith('+051800'):
|
|
|
|
when = when[0:-7]+'+0261'
|
2009-03-13 01:14:42 +01:00
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
self._advance_currentline()
|
2019-02-05 03:30:50 +01:00
|
|
|
return (name, email, when)
|
2009-02-08 17:41:50 +01:00
|
|
|
|
|
|
|
def _parse_data(self):
|
2009-03-27 02:23:13 +01:00
|
|
|
"""
|
|
|
|
Reads data from _input. Current-line will be advanced until it is beyond
|
|
|
|
the data.
|
|
|
|
"""
|
2019-02-12 02:35:27 +01:00
|
|
|
fields = self._currentline.split()
|
|
|
|
assert fields[0] == 'data'
|
|
|
|
size = int(fields[1])
|
2009-03-27 02:23:13 +01:00
|
|
|
data = self._input.read(size)
|
|
|
|
self._advance_currentline()
|
|
|
|
if self._currentline == '\n':
|
|
|
|
self._advance_currentline()
|
2009-02-08 17:41:50 +01:00
|
|
|
return data
|
|
|
|
|
|
|
|
def _parse_blob(self):
|
2009-03-27 02:23:13 +01:00
|
|
|
"""
|
|
|
|
Parse input data into a Blob object. Once the Blob has been created, it
|
|
|
|
will be handed off to the appropriate callbacks. Current-line will be
|
|
|
|
advanced until it is beyond this blob's data. The Blob will be dumped
|
|
|
|
to _output once everything else is done (unless it has been skipped by
|
|
|
|
the callback).
|
|
|
|
"""
|
2009-02-08 17:41:50 +01:00
|
|
|
# Parse the Blob
|
2009-03-27 02:23:13 +01:00
|
|
|
self._advance_currentline()
|
|
|
|
id_ = self._parse_optional_mark()
|
2018-10-31 01:55:54 +01:00
|
|
|
|
|
|
|
original_id = None
|
|
|
|
if self._currentline.startswith('original-oid'):
|
|
|
|
original_id = self._parse_original_id();
|
|
|
|
|
2009-02-08 17:41:50 +01:00
|
|
|
data = self._parse_data()
|
2009-03-27 02:23:13 +01:00
|
|
|
if self._currentline == '\n':
|
|
|
|
self._advance_currentline()
|
2009-02-08 17:41:50 +01:00
|
|
|
|
|
|
|
# Create the blob
|
2018-10-31 01:55:54 +01:00
|
|
|
blob = Blob(data, original_id)
|
2009-02-03 01:26:17 +01:00
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
# If fast-export text had a mark for this blob, need to make sure this
|
|
|
|
# mark translates to the blob's true id.
|
|
|
|
if id_:
|
|
|
|
blob.old_id = id_
|
|
|
|
_IDS.record_rename(id_, blob.id)
|
|
|
|
|
|
|
|
# Call any user callback to allow them to use/modify the blob
|
|
|
|
if self._blob_callback:
|
|
|
|
self._blob_callback(blob)
|
|
|
|
if self._everything_callback:
|
2019-03-14 17:32:02 +01:00
|
|
|
self._everything_callback(blob)
|
2009-02-03 01:26:17 +01:00
|
|
|
|
2009-02-08 17:41:50 +01:00
|
|
|
# Now print the resulting blob
|
2009-02-14 15:31:07 +01:00
|
|
|
if not blob.dumped:
|
2009-03-27 02:23:13 +01:00
|
|
|
blob.dump(self._output)
|
2009-02-03 01:26:17 +01:00
|
|
|
|
2009-02-08 17:41:50 +01:00
|
|
|
def _parse_reset(self):
|
2009-03-27 02:23:13 +01:00
|
|
|
"""
|
|
|
|
Parse input data into a Reset object. Once the Reset has been created,
|
|
|
|
it will be handed off to the appropriate callbacks. Current-line will
|
|
|
|
be advanced until it is beyond the reset data. The Reset will be dumped
|
|
|
|
to _output once everything else is done (unless it has been skipped by
|
|
|
|
the callback).
|
|
|
|
"""
|
2009-02-08 17:41:50 +01:00
|
|
|
# Parse the Reset
|
|
|
|
ref = self._parse_ref_line('reset')
|
filter-repo: modify parse_optional_parent_ref to return original parent too
commits may not have any parents at all. As such,
parse_optional_parent_ref() is used expecting that it will sometimes
return None.
Now, when commits are skipped, we have a scheme to translate anyone that
depends on such commits to instead depend on the nearest ancestor of
such commits. If the entire ancestry of a commit was skipped along with
a comit, then that commit will be translated to None, which is
indistinguishable from there having been no parent to begin with.
Sometimes our scheme needs to distinguish between a commit that started
with no parents and one which ended up with no parents, so we need a way
to tell these apart.
Also, not knowing the original parent makes it hard for us to
determine if the original had the same weird topology that the current
commit does. For example, it is possible for a merge commit to have
one parent be the ancestor of another (particularly when --no-ff is
passed to git merge), or even for a merge commit to have the same
commit used as both parents (if you use low-level commands to create
a crazy commit). There are cases where the pruning of some commits
could cause either of these situations to arise, and it's useful to be
able to distinguish between intentionally "weird" history and history
that has been made weird due to other pruning, because the latter we
may have reason to do additional pruning on.
Signed-off-by: Elijah Newren <newren@gmail.com>
2018-12-15 18:20:40 +01:00
|
|
|
ignoreme, from_ref = self._parse_optional_parent_ref('from')
|
2009-03-27 02:23:13 +01:00
|
|
|
if self._currentline == '\n':
|
|
|
|
self._advance_currentline()
|
2009-02-03 01:26:17 +01:00
|
|
|
|
2019-02-14 01:55:14 +01:00
|
|
|
# fast-export likes to print extraneous resets that serve no purpose.
|
|
|
|
# While we could continue processing such resets, that is a waste of
|
|
|
|
# resources. Also, we want to avoid recording that this ref was
|
|
|
|
# seen in such cases, since this ref could be rewritten to nothing.
|
|
|
|
if not from_ref:
|
|
|
|
return
|
|
|
|
|
2009-02-08 17:41:50 +01:00
|
|
|
# Create the reset
|
2009-02-03 15:53:06 +01:00
|
|
|
reset = Reset(ref, from_ref)
|
|
|
|
|
|
|
|
# Call any user callback to allow them to modify the reset
|
2009-03-27 02:23:13 +01:00
|
|
|
if self._reset_callback:
|
|
|
|
self._reset_callback(reset)
|
|
|
|
if self._everything_callback:
|
2019-03-14 17:32:02 +01:00
|
|
|
self._everything_callback(reset)
|
2009-02-03 15:53:06 +01:00
|
|
|
|
2009-02-08 17:41:50 +01:00
|
|
|
# Now print the resulting reset
|
2018-11-09 16:22:36 +01:00
|
|
|
self._seen_refs[reset.ref] = None
|
2009-02-14 15:31:07 +01:00
|
|
|
if not reset.dumped:
|
2009-03-27 02:23:13 +01:00
|
|
|
reset.dump(self._output)
|
2009-02-03 15:53:06 +01:00
|
|
|
|
2019-02-04 20:10:15 +01:00
|
|
|
def _get_rename(self, old_hash):
|
|
|
|
# If we already know the rename, just return it
|
|
|
|
new_hash = self._commit_renames.get(old_hash, None)
|
|
|
|
if new_hash:
|
|
|
|
return new_hash
|
|
|
|
|
|
|
|
# If it's not in the remaining pending renames, we don't know it
|
|
|
|
if old_hash is not None and old_hash not in self._pending_renames:
|
|
|
|
return None
|
|
|
|
|
|
|
|
# Read through the pending renames until we find it or we've read them all,
|
|
|
|
# and return whatever we might find
|
|
|
|
self._flush_renames(old_hash)
|
|
|
|
return self._commit_renames.get(old_hash, None)
|
|
|
|
|
|
|
|
def _flush_renames(self, old_hash=None, limit=0):
|
|
|
|
# Parse through self._pending_renames until we have read enough. We have
|
|
|
|
# read enough if:
|
|
|
|
# self._pending_renames is empty
|
|
|
|
# old_hash != None and we found a rename for old_hash
|
|
|
|
# limit > 0 and len(self._pending_renames) started less than 2*limit
|
|
|
|
# limit > 0 and len(self._pending_renames) < limit
|
|
|
|
if limit and len(self._pending_renames) < 2 * limit:
|
|
|
|
return
|
|
|
|
fi_input, fi_output = self._fast_import_pipes
|
|
|
|
while self._pending_renames:
|
|
|
|
orig_id, ignore = self._pending_renames.popitem(last=False)
|
|
|
|
new_id = fi_output.readline().rstrip()
|
|
|
|
self._commit_renames[orig_id] = new_id
|
|
|
|
if old_hash == orig_id:
|
|
|
|
return
|
|
|
|
if limit and len(self._pending_renames) < limit:
|
|
|
|
return
|
|
|
|
|
2018-09-14 03:02:41 +02:00
|
|
|
def _translate_commit_hash(self, matchobj):
|
|
|
|
old_hash = matchobj.group(1)
|
|
|
|
orig_len = len(old_hash)
|
2019-02-04 20:10:15 +01:00
|
|
|
new_hash = self._get_rename(old_hash)
|
|
|
|
if new_hash is None:
|
2018-09-14 03:02:41 +02:00
|
|
|
if old_hash[0:7] not in self._commit_short_old_hashes:
|
2019-03-12 03:29:53 +01:00
|
|
|
self._commits_referenced_but_removed.add(old_hash)
|
2018-09-14 03:02:41 +02:00
|
|
|
return old_hash
|
|
|
|
possibilities = self._commit_short_old_hashes[old_hash[0:7]]
|
|
|
|
matches = [x for x in possibilities
|
|
|
|
if x[0:orig_len] == old_hash]
|
|
|
|
if len(matches) != 1:
|
2019-03-12 03:29:53 +01:00
|
|
|
self._commits_referenced_but_removed.add(old_hash)
|
2018-09-14 03:02:41 +02:00
|
|
|
return old_hash
|
|
|
|
old_hash = matches[0]
|
2019-02-04 20:10:15 +01:00
|
|
|
new_hash = self._get_rename(old_hash)
|
2018-09-14 03:02:41 +02:00
|
|
|
|
2019-03-14 23:49:13 +01:00
|
|
|
assert new_hash is not None
|
|
|
|
return new_hash[0:orig_len]
|
2018-09-14 03:02:41 +02:00
|
|
|
|
2018-12-19 18:45:24 +01:00
|
|
|
def trim_extra_parents(self, orig_parents, parents):
|
|
|
|
'''Due to pruning of empty commits, some parents could be non-existent
|
|
|
|
(None) or otherwise redundant. Remove the non-existent parents, and
|
|
|
|
remove redundant parents so long as that doesn't transform a merge
|
|
|
|
commit into a non-merge commit.
|
|
|
|
|
|
|
|
Returns a tuple:
|
|
|
|
(parents, new_first_parent_if_would_become_non_merge)'''
|
|
|
|
|
|
|
|
# Pruning of empty commits means multiple things:
|
|
|
|
# * An original parent of this commit may have been pruned causing the
|
|
|
|
# need to rewrite the reported parent to the nearest ancestor. We
|
|
|
|
# want to know when we're dealing with such a parent.
|
|
|
|
# * Further, there may be no "nearest ancestor" if the entire history
|
|
|
|
# of that parent was also pruned. (Detectable by the parent being
|
|
|
|
# 'None')
|
|
|
|
# Remove all parents rewritten to None, and keep track of which parents
|
|
|
|
# were rewritten to an ancestor.
|
2019-03-01 03:12:49 +01:00
|
|
|
tmp = zip(parents,
|
|
|
|
orig_parents,
|
|
|
|
[x in _SKIPPED_COMMITS for x in orig_parents])
|
2018-12-19 18:45:24 +01:00
|
|
|
tmp2 = [x for x in tmp if x[0] is not None]
|
2019-03-01 03:12:49 +01:00
|
|
|
if not tmp2:
|
2019-04-26 15:28:16 +02:00
|
|
|
# All ancestors have been pruned; we have no parents.
|
|
|
|
return [], None
|
2019-03-01 03:12:49 +01:00
|
|
|
parents, orig_parents, is_rewritten = [list(x) for x in zip(*tmp2)]
|
2018-12-19 18:45:24 +01:00
|
|
|
|
|
|
|
# We can't have redundant parents if we don't have at least 2 parents
|
|
|
|
if len(parents) < 2:
|
|
|
|
return parents, None
|
|
|
|
|
|
|
|
# Remove duplicate parents (if both sides of history have lots of commits
|
|
|
|
# which become empty due to pruning, the most recent ancestor on both
|
|
|
|
# sides may be the same commit), except only remove parents that have
|
|
|
|
# been rewritten due to previous empty pruning.
|
|
|
|
seen = set()
|
|
|
|
seen_add = seen.add
|
|
|
|
# Deleting duplicate rewritten parents means keeping parents if either
|
|
|
|
# they have not been seen or they are ones that have not been rewritten.
|
|
|
|
parents_copy = parents
|
2019-03-01 03:12:49 +01:00
|
|
|
uniq = [[p, orig_parents[i], is_rewritten[i]] for i, p in enumerate(parents)
|
|
|
|
if not (p in seen or seen_add(p)) or not is_rewritten[i]]
|
|
|
|
parents, orig_parents, is_rewritten = [list(x) for x in zip(*uniq)]
|
2018-12-19 18:45:24 +01:00
|
|
|
if len(parents) < 2:
|
|
|
|
return parents_copy, parents[0]
|
|
|
|
|
|
|
|
# Flatten unnecessary merges. (If one side of history is entirely
|
|
|
|
# empty commits that were pruned, we may end up attempting to
|
|
|
|
# merge a commit with its ancestor. Remove parents that are an
|
|
|
|
# ancestor of another parent.)
|
|
|
|
num_parents = len(parents)
|
|
|
|
to_remove = []
|
2019-04-01 23:38:02 +02:00
|
|
|
for cur in range(num_parents):
|
2018-12-19 18:45:24 +01:00
|
|
|
if not is_rewritten[cur]:
|
|
|
|
continue
|
2019-04-01 23:38:02 +02:00
|
|
|
for other in range(num_parents):
|
2019-03-01 03:12:49 +01:00
|
|
|
if cur == other:
|
|
|
|
continue
|
|
|
|
if not self._graph.is_ancestor(parents[cur], parents[other]):
|
|
|
|
continue
|
|
|
|
# parents[cur] is an ancestor of parents[other], so parents[cur]
|
|
|
|
# seems redundant. However, if it was intentionally redundant
|
|
|
|
# (e.g. a no-ff merge) in the original, then we want to keep it.
|
|
|
|
if self._orig_graph.is_ancestor(orig_parents[cur],
|
|
|
|
orig_parents[other]):
|
|
|
|
continue
|
|
|
|
# Okay so the cur-th parent is an ancestor of the other-th parent,
|
|
|
|
# and it wasn't that way in the original repository; mark the
|
|
|
|
# cur-th parent as removable.
|
|
|
|
to_remove.append(cur)
|
|
|
|
break # cur removed, so skip rest of others -- i.e. check cur+=1
|
2018-12-19 18:45:24 +01:00
|
|
|
for x in reversed(to_remove):
|
|
|
|
parents.pop(x)
|
|
|
|
if len(parents) < 2:
|
|
|
|
return parents_copy, parents[0]
|
|
|
|
|
|
|
|
return parents, None
|
|
|
|
|
2019-02-02 18:59:54 +01:00
|
|
|
def prunable(self, commit, new_1st_parent, had_file_changes, orig_parents):
|
2019-04-26 15:28:16 +02:00
|
|
|
parents = commit.parents
|
2018-12-19 18:45:24 +01:00
|
|
|
|
|
|
|
# For merge commits, unless there are prunable (redundant) parents, we
|
|
|
|
# do not want to prune
|
|
|
|
if len(parents) >= 2 and not new_1st_parent:
|
|
|
|
return False
|
|
|
|
|
2019-02-26 03:14:55 +01:00
|
|
|
if len(orig_parents) < 2:
|
2018-12-19 18:45:24 +01:00
|
|
|
# Special logic for commits that started empty...
|
|
|
|
if not had_file_changes:
|
2019-02-26 03:14:55 +01:00
|
|
|
had_parents_pruned = (len(parents) < len(orig_parents) or
|
|
|
|
(len(orig_parents) == 1 and
|
|
|
|
orig_parents[0] in _SKIPPED_COMMITS))
|
|
|
|
# If the commit remains empty and had parents which were pruned,
|
|
|
|
# then prune this commit; otherwise, retain it
|
|
|
|
return (not commit.file_changes and had_parents_pruned)
|
2018-12-19 18:45:24 +01:00
|
|
|
|
|
|
|
# We can only get here if the commit didn't start empty, so if it's
|
|
|
|
# empty now, it obviously became empty
|
|
|
|
if not commit.file_changes:
|
|
|
|
return True
|
|
|
|
|
|
|
|
# If there are no parents of this commit and we didn't match the case
|
|
|
|
# above, then this commit cannot be pruned. Since we have no parent(s)
|
|
|
|
# to compare to, abort now to prevent future checks from failing.
|
|
|
|
if not parents:
|
|
|
|
return False
|
|
|
|
|
|
|
|
# Similarly, we cannot handle the hard cases if we don't have a pipe
|
|
|
|
# to communicate with fast-import
|
2019-02-02 18:59:54 +01:00
|
|
|
if not self._fast_import_pipes:
|
2018-12-19 18:45:24 +01:00
|
|
|
return False
|
|
|
|
|
2018-12-31 16:46:05 +01:00
|
|
|
# non-merge commits can only be empty if blob/file-change editing caused
|
|
|
|
# all file changes in the commit to have the same file contents as
|
|
|
|
# the parent.
|
|
|
|
changed_files = set(change.filename for change in commit.file_changes)
|
|
|
|
if len(orig_parents) < 2 and changed_files - self._files_tweaked:
|
2018-12-20 20:27:29 +01:00
|
|
|
return False
|
|
|
|
|
2018-12-19 18:45:24 +01:00
|
|
|
# Finally, the hard case: due to either blob rewriting, or due to pruning
|
|
|
|
# of empty commits wiping out the first parent history back to the merge
|
|
|
|
# base, the list of file_changes we have may not actually differ from our
|
|
|
|
# (new) first parent's version of the files, i.e. this would actually be
|
|
|
|
# an empty commit. Check by comparing the contents of this commit to its
|
|
|
|
# (remaining) parent.
|
|
|
|
#
|
|
|
|
# NOTE on why this works, for the case of original first parent history
|
|
|
|
# having been pruned away due to being empty:
|
|
|
|
# The first parent history having been pruned away due to being
|
|
|
|
# empty implies the original first parent would have a tree (after
|
|
|
|
# filtering) that matched the merge base's tree. Since
|
|
|
|
# file_changes has the changes needed to go from what would have
|
|
|
|
# been the first parent to our new commit, and what would have been
|
|
|
|
# our first parent has a tree that matches the merge base, then if
|
|
|
|
# the new first parent has a tree matching the versions of files in
|
|
|
|
# file_changes, then this new commit is empty and thus prunable.
|
2019-02-02 18:59:54 +01:00
|
|
|
fi_input, fi_output = self._fast_import_pipes
|
2019-02-04 20:10:15 +01:00
|
|
|
self._flush_renames() # Avoid fi_output having other stuff present
|
2018-12-19 18:45:24 +01:00
|
|
|
# Optimization note: we could have two loops over file_changes, the
|
2019-02-21 00:46:26 +01:00
|
|
|
# first doing all the self._output.write() calls, and the second doing
|
|
|
|
# the rest. But I'm worried about fast-import blocking on fi_output
|
2018-12-19 18:45:24 +01:00
|
|
|
# buffers filling up so I instead read from it as I go.
|
|
|
|
for change in commit.file_changes:
|
2019-04-26 15:28:16 +02:00
|
|
|
parent = new_1st_parent or commit.parents[0] # exists due to above checks
|
2019-04-28 00:04:27 +02:00
|
|
|
quoted_filename = PathQuoting.enquote(change.filename)
|
|
|
|
self._output.write("ls :{} {}\n".format(parent, quoted_filename))
|
2019-02-21 00:46:26 +01:00
|
|
|
self._output.flush()
|
2018-12-19 18:45:24 +01:00
|
|
|
parent_version = fi_output.readline().split()
|
|
|
|
if change.type == 'D':
|
2019-04-28 00:04:27 +02:00
|
|
|
if parent_version != ['missing', quoted_filename]:
|
2018-12-19 18:45:24 +01:00
|
|
|
return False
|
|
|
|
else:
|
|
|
|
blob_sha = change.blob_id
|
|
|
|
if isinstance(change.blob_id, int):
|
2019-02-21 00:46:26 +01:00
|
|
|
self._output.write("get-mark :{}\n".format(change.blob_id))
|
|
|
|
self._output.flush()
|
2018-12-19 18:45:24 +01:00
|
|
|
blob_sha = fi_output.readline().rstrip()
|
2019-04-28 00:04:27 +02:00
|
|
|
if parent_version != [change.mode, 'blob', blob_sha, quoted_filename]:
|
2018-12-19 18:45:24 +01:00
|
|
|
return False
|
|
|
|
|
|
|
|
return True
|
|
|
|
|
2019-02-02 18:59:54 +01:00
|
|
|
def record_remapping(self, commit, orig_parents):
|
2018-12-19 18:45:24 +01:00
|
|
|
new_id = None
|
|
|
|
# Record the mapping of old commit hash to new one
|
2019-02-02 18:59:54 +01:00
|
|
|
if commit.original_id and self._fast_import_pipes:
|
|
|
|
fi_input, fi_output = self._fast_import_pipes
|
2019-02-21 00:46:26 +01:00
|
|
|
self._output.write("get-mark :{}\n".format(commit.id))
|
|
|
|
self._output.flush()
|
2018-12-19 18:45:24 +01:00
|
|
|
orig_id = commit.original_id
|
|
|
|
self._commit_short_old_hashes[orig_id[0:7]].add(orig_id)
|
2019-02-04 20:10:15 +01:00
|
|
|
# Note that we have queued up an id for later reading; flush a
|
|
|
|
# few of the older ones if we have too many queued up
|
|
|
|
self._pending_renames[orig_id] = None
|
|
|
|
self._flush_renames(None, limit=40)
|
2018-12-19 18:45:24 +01:00
|
|
|
# Also, record if this was a merge commit that turned into a non-merge
|
|
|
|
# commit.
|
2019-04-26 15:28:16 +02:00
|
|
|
if len(orig_parents) >= 2 and len(commit.parents) < 2:
|
2018-12-19 18:45:24 +01:00
|
|
|
self._commits_no_longer_merges.append((commit.original_id, new_id))
|
|
|
|
|
filter-repo: add --analyze option
This option walks through the repository history and creates a report
with basic statistics, rename related information, and sizes of objects
and when/if those have been deleted. It primarily looks at unpacked
sizes (i.e. size of object ignoring delta-ing and compression), and
sums the size of each version of the file for each path. Additionally,
it aggregates these sums by extension and by directory, and tracks
whether paths, extensions, and directories have been deleted. This can
be very useful in determining what the big things are, and whether they
might have been considered to have been mistakes to add to the
repository in the first place.
There are numerous caveats with the determination of "deleted" and
"renamed", and can give both false positives and false negatives. But
they are only meant as a helpful heuristic to give others a starting
point for an investigation, and the information provide so far is useful.
I do want to improve the equivalence classes (rename handling), but that
is for a future commit.
Signed-off-by: Elijah Newren <newren@gmail.com>
2018-10-21 16:44:33 +02:00
|
|
|
def num_commits_parsed(self):
|
|
|
|
return self._num_commits
|
|
|
|
|
2019-02-02 18:59:54 +01:00
|
|
|
def _parse_commit(self):
|
2009-03-27 02:23:13 +01:00
|
|
|
"""
|
|
|
|
Parse input data into a Commit object. Once the Commit has been created,
|
|
|
|
it will be handed off to the appropriate callbacks. Current-line will
|
|
|
|
be advanced until it is beyond the commit data. The Commit will be dumped
|
|
|
|
to _output once everything else is done (unless it has been skipped by
|
|
|
|
the callback OR the callback has removed all file-changes from the commit).
|
|
|
|
"""
|
|
|
|
# Parse the Commit. This may look involved, but it's pretty simple; it only
|
|
|
|
# looks bad because a commit object contains many pieces of data.
|
2009-02-08 17:41:50 +01:00
|
|
|
branch = self._parse_ref_line('commit')
|
2009-03-27 02:23:13 +01:00
|
|
|
id_ = self._parse_optional_mark()
|
2009-02-04 01:48:01 +01:00
|
|
|
|
2018-10-31 01:55:54 +01:00
|
|
|
original_id = None
|
|
|
|
if self._currentline.startswith('original-oid'):
|
|
|
|
original_id = self._parse_original_id();
|
|
|
|
|
2009-02-08 17:41:50 +01:00
|
|
|
author_name = None
|
2009-03-27 02:23:13 +01:00
|
|
|
if self._currentline.startswith('author'):
|
2009-02-08 17:41:50 +01:00
|
|
|
(author_name, author_email, author_date) = self._parse_user('author')
|
|
|
|
|
|
|
|
(committer_name, committer_email, committer_date) = \
|
|
|
|
self._parse_user('committer')
|
|
|
|
|
|
|
|
if not author_name:
|
|
|
|
(author_name, author_email, author_date) = \
|
|
|
|
(committer_name, committer_email, committer_date)
|
|
|
|
|
|
|
|
commit_msg = self._parse_data()
|
2019-02-12 02:35:27 +01:00
|
|
|
commit_msg = self._hash_re.sub(self._translate_commit_hash, commit_msg)
|
2009-02-08 17:41:50 +01:00
|
|
|
|
filter-repo: modify parse_optional_parent_ref to return original parent too
commits may not have any parents at all. As such,
parse_optional_parent_ref() is used expecting that it will sometimes
return None.
Now, when commits are skipped, we have a scheme to translate anyone that
depends on such commits to instead depend on the nearest ancestor of
such commits. If the entire ancestry of a commit was skipped along with
a comit, then that commit will be translated to None, which is
indistinguishable from there having been no parent to begin with.
Sometimes our scheme needs to distinguish between a commit that started
with no parents and one which ended up with no parents, so we need a way
to tell these apart.
Also, not knowing the original parent makes it hard for us to
determine if the original had the same weird topology that the current
commit does. For example, it is possible for a merge commit to have
one parent be the ancestor of another (particularly when --no-ff is
passed to git merge), or even for a merge commit to have the same
commit used as both parents (if you use low-level commands to create
a crazy commit). There are cases where the pruning of some commits
could cause either of these situations to arise, and it's useful to be
able to distinguish between intentionally "weird" history and history
that has been made weird due to other pruning, because the latter we
may have reason to do additional pruning on.
Signed-off-by: Elijah Newren <newren@gmail.com>
2018-12-15 18:20:40 +01:00
|
|
|
pinfo = [self._parse_optional_parent_ref('from')]
|
2018-11-10 23:13:22 +01:00
|
|
|
# Due to empty pruning, we can have real 'from' and 'merge' lines that
|
|
|
|
# due to commit rewriting map to a parent of None. We need to record
|
|
|
|
# 'from' if its non-None, and we need to parse all 'merge' lines.
|
|
|
|
while self._currentline.startswith('merge '):
|
filter-repo: modify parse_optional_parent_ref to return original parent too
commits may not have any parents at all. As such,
parse_optional_parent_ref() is used expecting that it will sometimes
return None.
Now, when commits are skipped, we have a scheme to translate anyone that
depends on such commits to instead depend on the nearest ancestor of
such commits. If the entire ancestry of a commit was skipped along with
a comit, then that commit will be translated to None, which is
indistinguishable from there having been no parent to begin with.
Sometimes our scheme needs to distinguish between a commit that started
with no parents and one which ended up with no parents, so we need a way
to tell these apart.
Also, not knowing the original parent makes it hard for us to
determine if the original had the same weird topology that the current
commit does. For example, it is possible for a merge commit to have
one parent be the ancestor of another (particularly when --no-ff is
passed to git merge), or even for a merge commit to have the same
commit used as both parents (if you use low-level commands to create
a crazy commit). There are cases where the pruning of some commits
could cause either of these situations to arise, and it's useful to be
able to distinguish between intentionally "weird" history and history
that has been made weird due to other pruning, because the latter we
may have reason to do additional pruning on.
Signed-off-by: Elijah Newren <newren@gmail.com>
2018-12-15 18:20:40 +01:00
|
|
|
pinfo.append(self._parse_optional_parent_ref('merge'))
|
2018-12-19 18:45:24 +01:00
|
|
|
orig_parents, parents = [list(tmp) for tmp in zip(*pinfo)]
|
filter-repo: avoid merging a commit with one of its own ancestors
Pruning of empty commits can cause an entire line of history to become
empty and be pruned, resulting in a merge commit that merges some commit
with one of its ancestors. In such a case, we should remove the
unnecessary parent(s) -- which can and will often result in the merge
commit being empty so we can remove it as well.
Currently, if the side that becomes empty is the first parent side, then
we do not detect if the commit becomes empty, due to the way that
fast-export lists changes in a merge commit relative to first parent only.
A subsequent commit will address this.
Note that the callbacks could theoretically insert additional commits or
reparent our commit on top of something else, meaning that the ancestry
graph might need post-callback updates. However, in any extreme case
where that mattered, we would more or less need full updates to the
ancestry graph to be made for all the new commits from the callback as
well, and once we expect the callback to handle any ancestry graph
updates it can handle modifying it for the current commit. However, it
is hard to come up with a case where it matters, since for the most part
we just want to know whether our filtering causes commits to become
empty and knowing the source repo we are exporting from is sufficient
information without knowing any new commits inserted or reparenting that
happens elsewhere.
Signed-off-by: Elijah Newren <newren@gmail.com>
2018-09-04 18:26:05 +02:00
|
|
|
|
2019-04-26 15:28:16 +02:00
|
|
|
# No parents is oddly represented as [None] instead of [], due to the
|
|
|
|
# special 'from' handling. Convert it here to a more canonical form.
|
|
|
|
if parents == [None]:
|
|
|
|
parents = []
|
2019-03-01 03:12:49 +01:00
|
|
|
if orig_parents == [None]:
|
|
|
|
orig_parents = []
|
2018-09-01 03:26:27 +02:00
|
|
|
|
2019-04-26 15:28:16 +02:00
|
|
|
# Prune parents (due to pruning of empty commits) if relevant
|
|
|
|
parents, new_1st_parent = self.trim_extra_parents(orig_parents, parents)
|
|
|
|
|
2018-12-19 18:45:24 +01:00
|
|
|
# Get the list of file changes
|
2009-02-08 17:41:50 +01:00
|
|
|
file_changes = []
|
|
|
|
file_change = self._parse_optional_filechange()
|
2018-12-19 18:45:24 +01:00
|
|
|
had_file_changes = file_change is not None
|
2009-02-08 17:41:50 +01:00
|
|
|
while file_change:
|
2009-04-04 15:47:29 +02:00
|
|
|
if not (type(file_change) == str and file_change == 'skipped'):
|
|
|
|
file_changes.append(file_change)
|
2009-02-08 17:41:50 +01:00
|
|
|
file_change = self._parse_optional_filechange()
|
2009-03-27 02:23:13 +01:00
|
|
|
if self._currentline == '\n':
|
|
|
|
self._advance_currentline()
|
2009-02-04 01:48:01 +01:00
|
|
|
|
|
|
|
# Okay, now we can finally create the Commit object
|
|
|
|
commit = Commit(branch,
|
|
|
|
author_name, author_email, author_date,
|
|
|
|
committer_name, committer_email, committer_date,
|
2009-02-08 17:41:50 +01:00
|
|
|
commit_msg,
|
2009-02-04 01:48:01 +01:00
|
|
|
file_changes,
|
2019-04-26 15:28:16 +02:00
|
|
|
parents,
|
filter-repo: delete complex code
Over a decade ago, I added code to deal with splitting and splicing
repositories where you weren't always dealing with first parents and
linear histories, and in particular where the mainline tended to be the
second parent (because there was no integrator or special central
gatekeeper like gerrit or github; instead, everyone pushed directly to
the main repository after locally testing, and integration happened via
everyone running 'git pull'). When attempting to splice repositories
the fact that fast-export always gave changes relative to the first
parent caused some grief with my splitting and splicing efforts.
It has been over a decade, I don't know of a good testcase of this
functionality separate from the live repositories I lost access to over
six years ago, git-subtree was released in the meantime which I'm
certain handled the task better, git-fast-export since gained a
--full-tree option which might have provided a better way to attack the
problem (though with splicing repos you often want work with additive
changes rather than recreating from scratch), and I just don't
quite understand the code anymore anyway. I think it had some
fundamental limitations that I knew my usecase avoided, but I don't
remember the details (and I'm not certain if this is true).
Even though code coverage hits all but one of the lines, I'd rather
rewrite any needed functionality if the usecase arises, and in view of
what facilities exist today rather than what I was working with a decade
ago. So, just nuke this code.
Signed-off-by: Elijah Newren <newren@gmail.com>
2019-03-15 02:26:04 +01:00
|
|
|
original_id)
|
2009-02-04 01:48:01 +01:00
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
# If fast-export text had a mark for this commit, need to make sure this
|
|
|
|
# mark translates to the commit's true id.
|
|
|
|
if id_:
|
|
|
|
commit.old_id = id_
|
|
|
|
_IDS.record_rename(id_, commit.id)
|
2009-02-04 01:48:01 +01:00
|
|
|
|
filter-repo: avoid merging a commit with one of its own ancestors
Pruning of empty commits can cause an entire line of history to become
empty and be pruned, resulting in a merge commit that merges some commit
with one of its ancestors. In such a case, we should remove the
unnecessary parent(s) -- which can and will often result in the merge
commit being empty so we can remove it as well.
Currently, if the side that becomes empty is the first parent side, then
we do not detect if the commit becomes empty, due to the way that
fast-export lists changes in a merge commit relative to first parent only.
A subsequent commit will address this.
Note that the callbacks could theoretically insert additional commits or
reparent our commit on top of something else, meaning that the ancestry
graph might need post-callback updates. However, in any extreme case
where that mattered, we would more or less need full updates to the
ancestry graph to be made for all the new commits from the callback as
well, and once we expect the callback to handle any ancestry graph
updates it can handle modifying it for the current commit. However, it
is hard to come up with a case where it matters, since for the most part
we just want to know whether our filtering causes commits to become
empty and knowing the source repo we are exporting from is sufficient
information without knowing any new commits inserted or reparenting that
happens elsewhere.
Signed-off-by: Elijah Newren <newren@gmail.com>
2018-09-04 18:26:05 +02:00
|
|
|
# Record ancestry graph
|
2019-04-26 15:28:16 +02:00
|
|
|
external_parents = [p for p in commit.parents
|
2019-03-14 07:34:47 +01:00
|
|
|
if not isinstance(p, int)]
|
|
|
|
self._graph.record_external_commits(external_parents)
|
|
|
|
self._orig_graph.record_external_commits(external_parents)
|
2019-04-26 15:28:16 +02:00
|
|
|
self._graph.add_commit_and_parents(commit.id, commit.parents)
|
2019-03-01 03:12:49 +01:00
|
|
|
self._orig_graph.add_commit_and_parents(id_, orig_parents)
|
filter-repo: avoid merging a commit with one of its own ancestors
Pruning of empty commits can cause an entire line of history to become
empty and be pruned, resulting in a merge commit that merges some commit
with one of its ancestors. In such a case, we should remove the
unnecessary parent(s) -- which can and will often result in the merge
commit being empty so we can remove it as well.
Currently, if the side that becomes empty is the first parent side, then
we do not detect if the commit becomes empty, due to the way that
fast-export lists changes in a merge commit relative to first parent only.
A subsequent commit will address this.
Note that the callbacks could theoretically insert additional commits or
reparent our commit on top of something else, meaning that the ancestry
graph might need post-callback updates. However, in any extreme case
where that mattered, we would more or less need full updates to the
ancestry graph to be made for all the new commits from the callback as
well, and once we expect the callback to handle any ancestry graph
updates it can handle modifying it for the current commit. However, it
is hard to come up with a case where it matters, since for the most part
we just want to know whether our filtering causes commits to become
empty and knowing the source repo we are exporting from is sufficient
information without knowing any new commits inserted or reparenting that
happens elsewhere.
Signed-off-by: Elijah Newren <newren@gmail.com>
2018-09-04 18:26:05 +02:00
|
|
|
|
2018-12-31 16:46:05 +01:00
|
|
|
# Record the original list of file changes relative to first parent
|
|
|
|
orig_file_changes = set(commit.file_changes)
|
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
# Call any user callback to allow them to modify the commit
|
|
|
|
if self._commit_callback:
|
|
|
|
self._commit_callback(commit)
|
|
|
|
if self._everything_callback:
|
2019-03-14 17:32:02 +01:00
|
|
|
self._everything_callback(commit)
|
2009-03-27 02:23:13 +01:00
|
|
|
|
2018-12-31 16:46:05 +01:00
|
|
|
# Find out which files were modified by the callbacks. Such paths could
|
|
|
|
# lead to sebsequent commits being empty (e.g. if removed a line containing
|
|
|
|
# a password from every version of a file that had the password, and some
|
|
|
|
# later commit did nothing more than remove that line)
|
|
|
|
final_file_changes = set(commit.file_changes)
|
|
|
|
differences = orig_file_changes.symmetric_difference(final_file_changes)
|
|
|
|
self._files_tweaked.update(x.filename for x in differences)
|
|
|
|
|
2018-12-19 18:45:24 +01:00
|
|
|
# Now print the resulting commit, or if prunable skip it
|
2009-02-14 15:31:07 +01:00
|
|
|
if not commit.dumped:
|
2018-12-19 18:45:24 +01:00
|
|
|
if not self.prunable(commit, new_1st_parent, had_file_changes,
|
2019-02-02 18:59:54 +01:00
|
|
|
orig_parents):
|
2018-12-19 18:45:24 +01:00
|
|
|
self._seen_refs[commit.branch] = None # was seen, doesn't need reset
|
2009-03-27 02:23:13 +01:00
|
|
|
commit.dump(self._output)
|
2019-02-02 18:59:54 +01:00
|
|
|
self.record_remapping(commit, orig_parents)
|
2009-03-19 14:58:41 +01:00
|
|
|
else:
|
2018-12-19 18:45:24 +01:00
|
|
|
rewrite_to = new_1st_parent or commit.first_parent()
|
2018-12-19 18:05:17 +01:00
|
|
|
# We skip empty commits, but want to keep track to make sure our branch
|
|
|
|
# still gets set and/or updated appropriately.
|
2019-02-14 01:55:14 +01:00
|
|
|
if rewrite_to:
|
|
|
|
self._seen_refs[commit.branch] = rewrite_to # need reset
|
2018-12-19 18:45:24 +01:00
|
|
|
commit.skip(new_id = rewrite_to)
|
2018-09-14 03:02:41 +02:00
|
|
|
self._commit_renames[commit.original_id] = None
|
2018-12-19 18:45:24 +01:00
|
|
|
|
|
|
|
# Show progress
|
2018-10-27 00:19:48 +02:00
|
|
|
self._num_commits += 1
|
2018-11-27 18:59:49 +01:00
|
|
|
if not self._quiet:
|
2019-03-17 07:39:39 +01:00
|
|
|
self._progress_writer.show(_("Parsed %d commits") % self._num_commits)
|
2009-02-04 01:48:01 +01:00
|
|
|
|
2009-03-21 23:49:43 +01:00
|
|
|
def _parse_tag(self):
|
2009-03-27 02:23:13 +01:00
|
|
|
"""
|
|
|
|
Parse input data into a Tag object. Once the Tag has been created,
|
|
|
|
it will be handed off to the appropriate callbacks. Current-line will
|
|
|
|
be advanced until it is beyond the tag data. The Tag will be dumped
|
|
|
|
to _output once everything else is done (unless it has been skipped by
|
|
|
|
the callback).
|
|
|
|
"""
|
2009-03-21 23:49:43 +01:00
|
|
|
# Parse the Tag
|
|
|
|
tag = self._parse_ref_line('tag')
|
filter-repo: modify parse_optional_parent_ref to return original parent too
commits may not have any parents at all. As such,
parse_optional_parent_ref() is used expecting that it will sometimes
return None.
Now, when commits are skipped, we have a scheme to translate anyone that
depends on such commits to instead depend on the nearest ancestor of
such commits. If the entire ancestry of a commit was skipped along with
a comit, then that commit will be translated to None, which is
indistinguishable from there having been no parent to begin with.
Sometimes our scheme needs to distinguish between a commit that started
with no parents and one which ended up with no parents, so we need a way
to tell these apart.
Also, not knowing the original parent makes it hard for us to
determine if the original had the same weird topology that the current
commit does. For example, it is possible for a merge commit to have
one parent be the ancestor of another (particularly when --no-ff is
passed to git merge), or even for a merge commit to have the same
commit used as both parents (if you use low-level commands to create
a crazy commit). There are cases where the pruning of some commits
could cause either of these situations to arise, and it's useful to be
able to distinguish between intentionally "weird" history and history
that has been made weird due to other pruning, because the latter we
may have reason to do additional pruning on.
Signed-off-by: Elijah Newren <newren@gmail.com>
2018-12-15 18:20:40 +01:00
|
|
|
ignoreme, from_ref = self._parse_optional_parent_ref('from')
|
2018-10-31 01:55:54 +01:00
|
|
|
|
|
|
|
original_id = None
|
|
|
|
if self._currentline.startswith('original-oid'):
|
|
|
|
original_id = self._parse_original_id();
|
|
|
|
|
2019-02-02 05:52:02 +01:00
|
|
|
tagger_name, tagger_email, tagger_date = None, None, None
|
|
|
|
if self._currentline.startswith('tagger'):
|
|
|
|
(tagger_name, tagger_email, tagger_date) = self._parse_user('tagger')
|
2009-03-21 23:49:43 +01:00
|
|
|
tag_msg = self._parse_data()
|
2009-03-27 02:23:13 +01:00
|
|
|
if self._currentline == '\n':
|
|
|
|
self._advance_currentline()
|
2009-03-21 23:49:43 +01:00
|
|
|
|
|
|
|
# Create the tag
|
2018-10-31 01:55:54 +01:00
|
|
|
tag = Tag(tag, from_ref,
|
|
|
|
tagger_name, tagger_email, tagger_date, tag_msg,
|
|
|
|
original_id)
|
2009-03-21 23:49:43 +01:00
|
|
|
|
|
|
|
# Call any user callback to allow them to modify the tag
|
2009-03-27 02:23:13 +01:00
|
|
|
if self._tag_callback:
|
|
|
|
self._tag_callback(tag)
|
|
|
|
if self._everything_callback:
|
2019-03-14 17:32:02 +01:00
|
|
|
self._everything_callback(tag)
|
2009-03-21 23:49:43 +01:00
|
|
|
|
2019-03-08 07:45:59 +01:00
|
|
|
# The tag might not point at anything that still exists (self.from_ref
|
2018-11-11 01:06:42 +01:00
|
|
|
# will be None if the commit it pointed to and all its ancestors were
|
|
|
|
# pruned due to being empty)
|
2019-03-08 07:45:59 +01:00
|
|
|
if tag.from_ref:
|
|
|
|
# Print out this tag's information
|
|
|
|
if not tag.dumped:
|
|
|
|
tag.dump(self._output)
|
|
|
|
# Record the fact that this tag was seen so we don't nuke it as part
|
|
|
|
# of refs_to_nuke.
|
2018-11-11 01:06:42 +01:00
|
|
|
full_ref = 'refs/tags/{}'.format(tag.ref)
|
2019-03-08 07:45:59 +01:00
|
|
|
self._seen_refs[full_ref] = None
|
2009-03-21 23:49:43 +01:00
|
|
|
|
2009-03-22 00:03:01 +01:00
|
|
|
def _parse_progress(self):
|
2009-03-27 02:23:13 +01:00
|
|
|
"""
|
|
|
|
Parse input data into a Progress object. Once the Progress has
|
|
|
|
been created, it will be handed off to the appropriate
|
|
|
|
callbacks. Current-line will be advanced until it is beyond the
|
|
|
|
progress data. The Progress will be dumped to _output once
|
|
|
|
everything else is done (unless it has been skipped by the callback).
|
|
|
|
"""
|
2009-03-22 00:03:01 +01:00
|
|
|
# Parse the Progress
|
|
|
|
message = self._parse_ref_line('progress')
|
2009-03-27 02:23:13 +01:00
|
|
|
if self._currentline == '\n':
|
|
|
|
self._advance_currentline()
|
2009-03-22 00:03:01 +01:00
|
|
|
|
|
|
|
# Create the progress message
|
|
|
|
progress = Progress(message)
|
|
|
|
|
|
|
|
# Call any user callback to allow them to modify the progress messsage
|
2009-03-27 02:23:13 +01:00
|
|
|
if self._progress_callback:
|
|
|
|
self._progress_callback(progress)
|
|
|
|
if self._everything_callback:
|
2019-03-14 17:32:02 +01:00
|
|
|
self._everything_callback(progress)
|
2009-03-22 00:03:01 +01:00
|
|
|
|
2019-03-13 02:24:45 +01:00
|
|
|
# NOTE: By default, we do NOT print the progress message; git
|
|
|
|
# fast-import would write it to fast_import_pipes which could mess with
|
|
|
|
# our parsing of output from the 'ls' and 'get-mark' directives we send
|
|
|
|
# to fast-import. If users want these messages, they need to process
|
|
|
|
# and handle them in the appropriate callback above.
|
2009-03-22 00:03:01 +01:00
|
|
|
|
|
|
|
def _parse_checkpoint(self):
|
2009-03-27 02:23:13 +01:00
|
|
|
"""
|
|
|
|
Parse input data into a Checkpoint object. Once the Checkpoint has
|
|
|
|
been created, it will be handed off to the appropriate
|
|
|
|
callbacks. Current-line will be advanced until it is beyond the
|
|
|
|
checkpoint data. The Checkpoint will be dumped to _output once
|
|
|
|
everything else is done (unless it has been skipped by the callback).
|
|
|
|
"""
|
2009-03-22 00:03:01 +01:00
|
|
|
# Parse the Checkpoint
|
2009-03-27 02:23:13 +01:00
|
|
|
self._advance_currentline()
|
|
|
|
if self._currentline == '\n':
|
|
|
|
self._advance_currentline()
|
2009-03-22 00:03:01 +01:00
|
|
|
|
2009-03-27 02:23:13 +01:00
|
|
|
# Create the checkpoint
|
2009-03-22 00:03:01 +01:00
|
|
|
checkpoint = Checkpoint()
|
|
|
|
|
|
|
|
# Call any user callback to allow them to drop the checkpoint
|
2009-03-27 02:23:13 +01:00
|
|
|
if self._checkpoint_callback:
|
|
|
|
self._checkpoint_callback(checkpoint)
|
|
|
|
if self._everything_callback:
|
2019-03-14 17:32:02 +01:00
|
|
|
self._everything_callback(checkpoint)
|
2009-03-22 00:03:01 +01:00
|
|
|
|
2019-03-13 02:24:45 +01:00
|
|
|
# NOTE: By default, we do NOT print the checkpoint message; although it
|
|
|
|
# we would only realistically get them with --stdin, the fact that we
|
|
|
|
# are filtering makes me think the checkpointing is less likely to be
|
|
|
|
# reasonable. In fact, I don't think it's necessary in general. If
|
|
|
|
# users do want it, they should process it in the checkpoint_callback.
|
2009-03-22 00:03:01 +01:00
|
|
|
|
2018-09-03 21:10:17 +02:00
|
|
|
def _parse_literal_command(self):
|
|
|
|
"""
|
2019-03-13 06:39:58 +01:00
|
|
|
Parse literal command. Then just dump the line as is.
|
2018-09-03 21:10:17 +02:00
|
|
|
"""
|
|
|
|
# Create the literal command object
|
|
|
|
command = LiteralCommand(self._currentline)
|
|
|
|
self._advance_currentline()
|
|
|
|
|
2019-03-13 06:39:58 +01:00
|
|
|
# Now print the resulting literal command
|
2018-09-03 21:10:17 +02:00
|
|
|
if not command.dumped:
|
|
|
|
command.dump(self._output)
|
|
|
|
|
2018-09-04 04:35:25 +02:00
|
|
|
def _handle_final_commands(self):
|
|
|
|
self._finalize_handled = True
|
2019-04-01 23:36:57 +02:00
|
|
|
for ref, value in self._seen_refs.items():
|
2018-09-04 04:35:25 +02:00
|
|
|
if value is not None:
|
|
|
|
# Create a reset
|
|
|
|
reset = Reset(ref, value)
|
|
|
|
|
|
|
|
# Call any user callback to allow them to modify the reset
|
|
|
|
if self._reset_callback:
|
|
|
|
self._reset_callback(reset)
|
|
|
|
if self._everything_callback:
|
2019-03-14 17:32:02 +01:00
|
|
|
self._everything_callback(reset)
|
2018-09-04 04:35:25 +02:00
|
|
|
|
|
|
|
# Now print the resulting reset
|
|
|
|
reset.dump(self._output)
|
|
|
|
|
2018-10-17 03:07:58 +02:00
|
|
|
def record_metadata(self, metadata_dir, orig_refs, refs_nuked):
|
|
|
|
deleted_hash = '0'*40
|
2019-02-04 20:10:15 +01:00
|
|
|
self._flush_renames()
|
2018-10-17 03:07:58 +02:00
|
|
|
with open(os.path.join(metadata_dir, 'commit-map'), 'w') as f:
|
2019-03-17 07:39:39 +01:00
|
|
|
f.write("%-40s %s\n" % (_("old"), _("new")))
|
2019-04-01 23:36:57 +02:00
|
|
|
for (old,new) in self._commit_renames.items():
|
2018-10-17 03:07:58 +02:00
|
|
|
f.write('{} {}\n'.format(old, new if new != None else deleted_hash))
|
|
|
|
|
|
|
|
batch_check_process = None
|
2019-02-12 02:35:27 +01:00
|
|
|
batch_check_output_re = re.compile('^([0-9a-f]{40}) ([a-z]+) ([0-9]+)$')
|
2018-10-17 03:07:58 +02:00
|
|
|
with open(os.path.join(metadata_dir, 'ref-map'), 'w') as f:
|
2019-04-01 23:36:57 +02:00
|
|
|
for refname, old_hash in orig_refs.items():
|
2018-10-17 03:07:58 +02:00
|
|
|
if refname in refs_nuked:
|
|
|
|
new_hash = deleted_hash
|
|
|
|
elif old_hash in self._commit_renames:
|
|
|
|
new_hash = self._commit_renames[old_hash]
|
|
|
|
new_hash = new_hash if new_hash != None else deleted_hash
|
|
|
|
else: # Must be an annotated tag
|
|
|
|
if not batch_check_process:
|
|
|
|
cmd = 'git cat-file --batch-check'.split()
|
|
|
|
batch_check_process = subprocess.Popen(cmd,
|
|
|
|
stdin=subprocess.PIPE,
|
2019-01-08 20:02:53 +01:00
|
|
|
stdout=subprocess.PIPE,
|
|
|
|
cwd=self._repo_working_dir)
|
2018-10-17 03:07:58 +02:00
|
|
|
batch_check_process.stdin.write(refname+"\n")
|
|
|
|
line = batch_check_process.stdout.readline()
|
2019-02-12 02:35:27 +01:00
|
|
|
m = batch_check_output_re.match(line)
|
2018-10-17 03:07:58 +02:00
|
|
|
if not m or m.group(2) != 'tag':
|
2019-03-17 07:39:39 +01:00
|
|
|
raise SystemExit(_("Failed to find new id for %(refname)s "
|
|
|
|
"(old id was %(old_hash)s)")
|
|
|
|
% ({'refname': refname, 'old_hash': old_hash})
|
|
|
|
) # pragma: no cover
|
2018-10-17 03:07:58 +02:00
|
|
|
new_hash = m.group(1)
|
|
|
|
f.write('{} {} {}\n'.format(old_hash, new_hash, refname))
|
|
|
|
if batch_check_process:
|
|
|
|
batch_check_process.stdin.close()
|
|
|
|
batch_check_process.wait()
|
|
|
|
|
2018-10-18 03:31:26 +02:00
|
|
|
with open(os.path.join(metadata_dir, 'suboptimal-issues'), 'w') as f:
|
|
|
|
issues_found = False
|
|
|
|
if self._commits_no_longer_merges:
|
|
|
|
issues_found = True
|
|
|
|
|
2019-03-17 07:39:39 +01:00
|
|
|
f.write(textwrap.dedent(_('''
|
2018-10-18 03:31:26 +02:00
|
|
|
The following commits used to be merge commits but due to filtering
|
|
|
|
are now regular commits; they likely have suboptimal commit messages
|
|
|
|
(e.g. "Merge branch next into master"). Original commit hash on the
|
|
|
|
left, commit hash after filtering/rewriting on the right:
|
2019-03-17 07:39:39 +01:00
|
|
|
''')[1:]))
|
2018-10-18 03:31:26 +02:00
|
|
|
for oldhash, newhash in self._commits_no_longer_merges:
|
|
|
|
f.write(' {} {}\n'.format(oldhash, newhash))
|
|
|
|
f.write('\n')
|
|
|
|
|
2018-09-14 03:02:41 +02:00
|
|
|
if self._commits_referenced_but_removed:
|
|
|
|
issues_found = True
|
2019-03-17 07:39:39 +01:00
|
|
|
f.write(textwrap.dedent(_('''
|
2018-09-14 03:02:41 +02:00
|
|
|
The following commits were filtered out, but referenced in another
|
|
|
|
commit message. The reference to the now-nonexistent commit hash
|
|
|
|
(or a substring thereof) was left as-is in any commit messages:
|
2019-03-17 07:39:39 +01:00
|
|
|
''')[1:]))
|
2018-09-14 03:02:41 +02:00
|
|
|
for bad_commit_reference in self._commits_referenced_but_removed:
|
|
|
|
f.write(' {}\n'.format(bad_commit_reference))
|
|
|
|
f.write('\n')
|
|
|
|
|
2018-10-18 03:31:26 +02:00
|
|
|
if not issues_found:
|
2019-03-17 07:39:39 +01:00
|
|
|
f.write(_("No filtering problems encountered."))
|
2018-10-18 03:31:26 +02:00
|
|
|
|
2019-02-14 21:45:51 +01:00
|
|
|
with open(os.path.join(metadata_dir, 'already_ran'), 'w') as f:
|
2019-03-17 07:39:39 +01:00
|
|
|
f.write(_("This file exists to allow you to filter again without --force."))
|
2019-02-14 21:45:51 +01:00
|
|
|
|
2018-08-21 17:46:26 +02:00
|
|
|
def get_seen_refs(self):
|
2018-09-04 04:35:25 +02:00
|
|
|
return self._seen_refs.keys()
|
2018-08-21 17:46:26 +02:00
|
|
|
|
2019-01-19 03:30:42 +01:00
|
|
|
def run(self, input, output, fast_import_pipes, quiet):
|
|
|
|
"""
|
|
|
|
This method filters fast export output.
|
|
|
|
"""
|
2009-03-27 02:23:13 +01:00
|
|
|
# Set input. If no args provided, use stdin.
|
2019-01-19 03:30:42 +01:00
|
|
|
self._input = input
|
|
|
|
self._output = output
|
2019-02-02 18:59:54 +01:00
|
|
|
self._fast_import_pipes = fast_import_pipes
|
2019-01-19 03:30:42 +01:00
|
|
|
self._quiet = quiet
|
2018-10-27 00:19:48 +02:00
|
|
|
|
2009-03-12 01:09:33 +01:00
|
|
|
# Run over the input and do the filtering
|
2009-03-27 02:23:13 +01:00
|
|
|
self._advance_currentline()
|
|
|
|
while self._currentline:
|
|
|
|
if self._currentline.startswith('blob'):
|
2009-02-08 17:41:50 +01:00
|
|
|
self._parse_blob()
|
2009-03-27 02:23:13 +01:00
|
|
|
elif self._currentline.startswith('reset'):
|
2009-02-08 17:41:50 +01:00
|
|
|
self._parse_reset()
|
2009-03-27 02:23:13 +01:00
|
|
|
elif self._currentline.startswith('commit'):
|
2019-02-02 18:59:54 +01:00
|
|
|
self._parse_commit()
|
2009-03-27 02:23:13 +01:00
|
|
|
elif self._currentline.startswith('tag'):
|
2009-03-21 23:49:43 +01:00
|
|
|
self._parse_tag()
|
2009-03-27 02:23:13 +01:00
|
|
|
elif self._currentline.startswith('progress'):
|
2009-03-22 00:03:01 +01:00
|
|
|
self._parse_progress()
|
2009-03-27 02:23:13 +01:00
|
|
|
elif self._currentline.startswith('checkpoint'):
|
2009-03-22 00:03:01 +01:00
|
|
|
self._parse_checkpoint()
|
2018-09-03 21:10:17 +02:00
|
|
|
elif self._currentline.startswith('feature'):
|
|
|
|
self._parse_literal_command()
|
|
|
|
elif self._currentline.startswith('option'):
|
|
|
|
self._parse_literal_command()
|
|
|
|
elif self._currentline.startswith('done'):
|
2018-09-04 04:35:25 +02:00
|
|
|
self._handle_final_commands()
|
2018-09-03 21:10:17 +02:00
|
|
|
self._parse_literal_command()
|
|
|
|
elif self._currentline.startswith('#'):
|
|
|
|
self._parse_literal_command()
|
|
|
|
elif self._currentline.startswith('get-mark') or \
|
|
|
|
self._currentline.startswith('cat-blob') or \
|
|
|
|
self._currentline.startswith('ls'):
|
2019-03-17 07:39:39 +01:00
|
|
|
raise SystemExit(_("Unsupported command: '%s'") % self._currentline)
|
2009-02-08 17:41:50 +01:00
|
|
|
else:
|
2019-03-17 07:39:39 +01:00
|
|
|
raise SystemExit(_("Could not parse line: '%s'") % self._currentline)
|
2009-02-05 07:03:48 +01:00
|
|
|
|
2018-11-27 18:59:49 +01:00
|
|
|
if not self._quiet:
|
|
|
|
self._progress_writer.finish()
|
2018-09-04 04:35:25 +02:00
|
|
|
if not self._finalize_handled:
|
|
|
|
self._handle_final_commands()
|
|
|
|
|
2009-04-06 15:00:23 +02:00
|
|
|
def record_id_rename(old_id, new_id):
|
2009-03-27 02:23:13 +01:00
|
|
|
"""
|
|
|
|
Register a new translation
|
|
|
|
"""
|
2009-04-06 15:00:23 +02:00
|
|
|
handle_transitivity = True
|
2009-03-27 02:23:13 +01:00
|
|
|
_IDS.record_rename(old_id, new_id, handle_transitivity)
|
|
|
|
|
|
|
|
# Internal globals
|
|
|
|
_IDS = _IDs()
|
2018-12-18 02:43:15 +01:00
|
|
|
_SKIPPED_COMMITS = set()
|
2018-08-21 17:46:26 +02:00
|
|
|
|
2018-12-25 16:32:54 +01:00
|
|
|
class GitUtils(object):
|
|
|
|
@staticmethod
|
|
|
|
def get_commit_count(repo, *args):
|
|
|
|
"""
|
|
|
|
Return the number of commits that have been made on repo.
|
|
|
|
"""
|
|
|
|
if not args:
|
|
|
|
args = ['--all']
|
|
|
|
if len(args) == 1 and isinstance(args[0], list):
|
|
|
|
args = args[0]
|
|
|
|
p1 = subprocess.Popen(["git", "rev-list"] + args,
|
|
|
|
bufsize=-1,
|
|
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
|
|
cwd=repo)
|
|
|
|
p2 = subprocess.Popen(["wc", "-l"], stdin=p1.stdout, stdout=subprocess.PIPE)
|
|
|
|
count = int(p2.communicate()[0])
|
|
|
|
if p1.poll() != 0:
|
2019-03-17 07:39:39 +01:00
|
|
|
raise SystemExit(_("%s does not appear to be a valid git repository")
|
|
|
|
% repo)
|
2018-12-25 16:32:54 +01:00
|
|
|
return count
|
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
def get_total_objects(repo):
|
|
|
|
"""
|
|
|
|
Return the number of objects (both packed and unpacked)
|
|
|
|
"""
|
|
|
|
p1 = subprocess.Popen(["git", "count-objects", "-v"],
|
|
|
|
stdout=subprocess.PIPE, cwd=repo)
|
|
|
|
lines = p1.stdout.read().splitlines()
|
|
|
|
# Return unpacked objects + packed-objects
|
|
|
|
return int(lines[0].split()[1]) + int(lines[2].split()[1])
|
|
|
|
|
|
|
|
@staticmethod
|
2019-01-08 20:02:53 +01:00
|
|
|
def is_repository_bare(repo_working_dir):
|
|
|
|
out = subprocess.check_output('git rev-parse --is-bare-repository'.split(),
|
|
|
|
cwd=repo_working_dir)
|
2018-12-25 16:32:54 +01:00
|
|
|
return (out.strip() == 'true')
|
|
|
|
|
|
|
|
@staticmethod
|
2019-01-08 20:02:53 +01:00
|
|
|
def determine_git_dir(repo_working_dir):
|
|
|
|
d = subprocess.check_output('git rev-parse --git-dir'.split(),
|
|
|
|
cwd=repo_working_dir).strip()
|
|
|
|
if repo_working_dir=='.' or d.startswith('/'):
|
|
|
|
return d
|
|
|
|
return os.path.join(repo_working_dir, d)
|
2018-12-25 16:32:54 +01:00
|
|
|
|
|
|
|
@staticmethod
|
2019-01-08 20:02:53 +01:00
|
|
|
def get_refs(repo_working_dir):
|
2018-12-25 16:32:54 +01:00
|
|
|
try:
|
2019-01-08 20:02:53 +01:00
|
|
|
output = subprocess.check_output('git show-ref'.split(),
|
|
|
|
cwd=repo_working_dir)
|
2018-12-25 16:32:54 +01:00
|
|
|
except subprocess.CalledProcessError as e:
|
|
|
|
# If error code is 1, there just aren't any refs; i.e. new repo.
|
|
|
|
# If error code is other than 1, some other error (e.g. not a git repo)
|
|
|
|
if e.returncode != 1:
|
|
|
|
raise SystemExit('fatal: {}'.format(e))
|
|
|
|
output = ''
|
|
|
|
return dict(reversed(x.split()) for x in output.splitlines())
|
2018-08-21 17:46:26 +02:00
|
|
|
|
2018-12-25 07:59:38 +01:00
|
|
|
class FilteringOptions(object):
|
|
|
|
class AppendFilter(argparse.Action):
|
|
|
|
def __call__(self, parser, namespace, values, option_string=None):
|
|
|
|
suffix = option_string[len('--path-'):] or 'match'
|
|
|
|
if suffix == 'rename':
|
|
|
|
mod_type = 'rename'
|
|
|
|
match_type = 'prefix'
|
|
|
|
else:
|
|
|
|
mod_type = 'filter'
|
|
|
|
match_type = suffix
|
2019-02-12 02:35:27 +01:00
|
|
|
if match_type == 'regex':
|
|
|
|
values = re.compile(values)
|
2018-12-25 07:59:38 +01:00
|
|
|
items = getattr(namespace, self.dest, []) or []
|
|
|
|
items.append((mod_type, match_type, values))
|
|
|
|
setattr(namespace, self.dest, items)
|
|
|
|
|
|
|
|
class HelperFilter(argparse.Action):
|
|
|
|
def __call__(self, parser, namespace, values, option_string=None):
|
|
|
|
af = FilteringOptions.AppendFilter(dest='path_changes',
|
|
|
|
option_strings=None)
|
|
|
|
dirname = values if values[-1] == '/' else values+'/'
|
|
|
|
if option_string == '--subdirectory-filter':
|
|
|
|
af(parser, namespace, dirname, '--path-match')
|
|
|
|
af(parser, namespace, dirname+':', '--path-rename')
|
|
|
|
elif option_string == '--to-subdirectory-filter':
|
|
|
|
af(parser, namespace, ':'+dirname, '--path-rename')
|
|
|
|
else:
|
2019-03-17 07:39:39 +01:00
|
|
|
raise SystemExit(_("Error: HelperFilter given invalid option_string: %s")
|
|
|
|
% option_string) # pragma: no cover
|
2018-12-25 07:59:38 +01:00
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
def create_arg_parser():
|
|
|
|
# Include usage in the summary, so we can put the description first
|
2019-03-17 07:39:39 +01:00
|
|
|
summary = _('''Rewrite (or analyze) repository history
|
2018-12-25 07:59:38 +01:00
|
|
|
|
|
|
|
git-filter-repo destructively rewrites history (unless --analyze or --dry-run
|
|
|
|
are specified) according to specified rules. It refuses to do any rewriting
|
|
|
|
unless either run from a clean fresh clone, or --force was specified.
|
|
|
|
|
|
|
|
Basic Usage:
|
|
|
|
git-filter-repo --analyze
|
|
|
|
git-filter-repo [FILTER/RENAME/CONTROL OPTIONS]
|
|
|
|
|
|
|
|
See EXAMPLES section for details.
|
2019-03-17 07:39:39 +01:00
|
|
|
''').rstrip()
|
2018-12-25 07:59:38 +01:00
|
|
|
|
|
|
|
# Provide a long helpful examples section
|
2019-03-17 07:39:39 +01:00
|
|
|
example_text = _('''EXAMPLES
|
2018-12-25 07:59:38 +01:00
|
|
|
|
|
|
|
To get help:
|
|
|
|
git-filter-repo --help
|
2019-03-17 07:39:39 +01:00
|
|
|
''')
|
2018-12-25 07:59:38 +01:00
|
|
|
|
|
|
|
# Create the basic parser
|
|
|
|
parser = argparse.ArgumentParser(description=summary,
|
|
|
|
usage = argparse.SUPPRESS,
|
|
|
|
add_help = False,
|
|
|
|
epilog = example_text,
|
|
|
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
2019-02-14 19:58:42 +01:00
|
|
|
|
2019-03-17 07:39:39 +01:00
|
|
|
analyze = parser.add_argument_group(title=_("Analysis"))
|
2018-12-25 07:59:38 +01:00
|
|
|
analyze.add_argument('--analyze', action='store_true',
|
2019-03-17 07:39:39 +01:00
|
|
|
help=_("Analyze repository history and create a report that may be "
|
|
|
|
"useful in determining what to filter in a subsequent run. "
|
|
|
|
"Will not modify your repo."))
|
2018-12-25 07:59:38 +01:00
|
|
|
|
2019-03-17 07:39:39 +01:00
|
|
|
refs = parser.add_argument_group(title=_("Git References"))
|
2019-02-14 19:58:42 +01:00
|
|
|
refs.add_argument('--refs', action='store_const', const=['--all'],
|
|
|
|
default=['--all'], help=argparse.SUPPRESS)
|
2018-12-25 07:59:38 +01:00
|
|
|
|
2019-03-17 07:39:39 +01:00
|
|
|
path = parser.add_argument_group(title=_("Filtering based on paths"))
|
|
|
|
path.add_argument('--invert-paths', action='store_false', dest='inclusive',
|
|
|
|
help=_("Invert the selection of files from the specified "
|
|
|
|
"--path-{match,glob,regex} options below, i.e. only select "
|
|
|
|
"files matching none of those options."))
|
2018-12-25 07:59:38 +01:00
|
|
|
|
|
|
|
path.add_argument('--path-match', '--path', metavar='DIR_OR_FILE',
|
2019-03-17 07:39:39 +01:00
|
|
|
action=FilteringOptions.AppendFilter, dest='path_changes',
|
|
|
|
help=_("Exact paths (files or directories) to include in filtered "
|
|
|
|
"history. Multiple --path options can be specified to get "
|
|
|
|
"a union of paths."))
|
2018-12-25 07:59:38 +01:00
|
|
|
path.add_argument('--path-glob', metavar='GLOB',
|
2019-03-17 07:39:39 +01:00
|
|
|
action=FilteringOptions.AppendFilter, dest='path_changes',
|
|
|
|
help=_("Glob of paths to include in filtered history. Multiple "
|
|
|
|
"--path-glob options can be specified to get a union of "
|
|
|
|
"paths."))
|
2018-12-25 07:59:38 +01:00
|
|
|
path.add_argument('--path-regex', metavar='REGEX',
|
2019-03-17 07:39:39 +01:00
|
|
|
action=FilteringOptions.AppendFilter, dest='path_changes',
|
|
|
|
help=_("Regex of paths to include in filtered history. Multiple "
|
|
|
|
"--path-regex options can be specified to get a union of "
|
|
|
|
"paths"))
|
2018-12-25 07:59:38 +01:00
|
|
|
|
2019-03-17 07:39:39 +01:00
|
|
|
rename = parser.add_argument_group(title=_("Renaming based on paths"))
|
2018-12-25 07:59:38 +01:00
|
|
|
rename.add_argument('--path-rename', '--path-rename-prefix',
|
2019-03-17 07:39:39 +01:00
|
|
|
metavar='OLD_NAME:NEW_NAME', dest='path_changes',
|
|
|
|
action=FilteringOptions.AppendFilter,
|
|
|
|
help=_("Prefix to rename; if filename starts with OLD_NAME, "
|
|
|
|
"replace that with NEW_NAME. Multiple --path-rename "
|
|
|
|
"options can be specified."))
|
|
|
|
|
|
|
|
refrename = parser.add_argument_group(title=_("Renaming of refs"))
|
2018-12-25 07:59:38 +01:00
|
|
|
refrename.add_argument('--tag-rename', metavar='OLD:NEW',
|
2019-03-17 07:39:39 +01:00
|
|
|
help=_("Rename tags starting with OLD to start with NEW. For "
|
|
|
|
"example, --tag-rename foo:bar will rename tag foo-1.2.3 "
|
|
|
|
"to bar-1.2.3; either OLD or NEW can be empty."))
|
2018-12-25 07:59:38 +01:00
|
|
|
|
2019-03-17 07:39:39 +01:00
|
|
|
helpers = parser.add_argument_group(title=_("Shortcuts"))
|
2018-12-25 07:59:38 +01:00
|
|
|
helpers.add_argument('--subdirectory-filter', metavar='DIRECTORY',
|
2019-03-17 07:39:39 +01:00
|
|
|
action=FilteringOptions.HelperFilter,
|
|
|
|
help=_("Only look at history that touches the given subdirectory "
|
|
|
|
"and treat that directory as the project root. Equivalent "
|
|
|
|
"to using '--path DIRECTORY/ --path-rename DIRECTORY/:'"))
|
2018-12-25 07:59:38 +01:00
|
|
|
helpers.add_argument('--to-subdirectory-filter', metavar='DIRECTORY',
|
2019-03-17 07:39:39 +01:00
|
|
|
action=FilteringOptions.HelperFilter,
|
|
|
|
help=_("Treat the project root as instead being under DIRECTORY. "
|
|
|
|
"Equivalent to using '--path-rename :DIRECTORY/'"))
|
2018-12-25 07:59:38 +01:00
|
|
|
|
2019-03-17 07:39:39 +01:00
|
|
|
people = parser.add_argument_group(title=_("Filtering of names/emails"))
|
2019-01-29 02:26:49 +01:00
|
|
|
people.add_argument('--mailmap', dest='mailmap', metavar='FILENAME',
|
2019-03-17 07:39:39 +01:00
|
|
|
help=_("Use specified mailmap file (see git-shortlog(1) for "
|
|
|
|
"details on the format) when rewriting author, committer, "
|
|
|
|
"and tagger names and emails. If the specified file is "
|
|
|
|
"part of git history, historical versions of the file will "
|
|
|
|
"be ignored; only the current contents are consulted."))
|
2019-01-29 02:26:49 +01:00
|
|
|
people.add_argument('--use-mailmap', dest='mailmap',
|
2019-03-17 07:39:39 +01:00
|
|
|
action='store_const', const='.mailmap',
|
|
|
|
help=_("Same as: '--mailmap .mailmap' "))
|
2019-01-29 02:26:49 +01:00
|
|
|
|
2019-03-17 07:39:39 +01:00
|
|
|
contents = parser.add_argument_group(title=_("Content editing filters"))
|
filter-repo: add text removal (or replacement) via file of expressions
Make it easy for users to search and replace text throughout the
repository history. Instead of inventing some new syntax, reuse the
same syntax used by BFG repo filter's --replace-text option, namely,
a file with one expression per line of the form
[regex:|glob:|literal:]$MATCH_EXPR[==>$REPLACEMENT_EXPR]
Where "$MATCH_EXPR" is by default considered to be literal text, but
could be a regex or a glob if the appropriate prefix is used. Also,
$REPLACEMENT_EXPR defaults to '***REMOVED***' if not specified. If
you want a literal '==>' to be part of your $MATCH_EXPR, then you
must also manually specify a replacement expression instead of taking
the default. Some examples:
sup3rs3kr3t
(replaces 'sup3rs3kr3t' with '***REMOVED***')
HeWhoShallNotBeNamed==>Voldemort
(replaces 'HeWhoShallNotBeNamed' with 'Voldemort')
very==>
(replaces 'very' with the empty string)
regex:(\d{2})/(\d{2})/(\d{4})==>\2/\1/\3
(replaces '05/17/2012' with '17/05/2012', and vice-versa)
The format for regex is as from
re.sub(<pattern>, <repl>, <string>) from
https://docs.python.org/2/library/re.html
The <string> comes from file contents of the repo, and you specify
the <pattern> and <repl>.
glob:Copy*t==>Cartel
(replaces 'Copyright' or 'Copyleft' or 'Copy my st' with 'Cartel')
Signed-off-by: Elijah Newren <newren@gmail.com>
2019-01-30 09:56:22 +01:00
|
|
|
contents.add_argument('--replace-text', metavar='EXPRESSIONS_FILE',
|
2019-03-17 07:39:39 +01:00
|
|
|
help=_("A file with expressions that, if found, will be replaced. "
|
|
|
|
"By default, each expression is treated as literal text, "
|
|
|
|
"but 'regex:' and 'glob:' prefixes are supported. You can "
|
|
|
|
"end the line with '==>' and some replacement text to "
|
|
|
|
"choose a replacement choice other than the default of "
|
|
|
|
"'***REMOVED***'. "))
|
|
|
|
|
|
|
|
callback = parser.add_argument_group(title=_("Generic callback code snippets"))
|
2019-02-17 08:59:43 +01:00
|
|
|
callback.add_argument('--filename-callback', metavar="FUNCTION_BODY",
|
2019-03-17 07:39:39 +01:00
|
|
|
help=_("Python code body for processing filenames; see CALLBACKS "
|
|
|
|
"sections below."))
|
2019-02-17 08:59:43 +01:00
|
|
|
callback.add_argument('--message-callback', metavar="FUNCTION_BODY",
|
2019-03-17 07:39:39 +01:00
|
|
|
help=_("Python code body for processing messages (both commit "
|
|
|
|
"messages and tag messages); see CALLBACKS section below."))
|
2019-02-17 08:59:43 +01:00
|
|
|
callback.add_argument('--name-callback', metavar="FUNCTION_BODY",
|
2019-03-17 07:39:39 +01:00
|
|
|
help=_("Python code body for processing names of people; see "
|
|
|
|
"CALLBACKS section below."))
|
2019-02-17 08:59:43 +01:00
|
|
|
callback.add_argument('--email-callback', metavar="FUNCTION_BODY",
|
2019-03-17 07:39:39 +01:00
|
|
|
help=_("Python code body for processing emails addresses; see "
|
|
|
|
"CALLBACKS section below."))
|
2019-02-17 08:59:43 +01:00
|
|
|
callback.add_argument('--refname-callback', metavar="FUNCTION_BODY",
|
2019-03-17 07:39:39 +01:00
|
|
|
help=_("Python code body for processing refnames; see CALLBACKS "
|
|
|
|
"section below."))
|
2019-02-17 08:59:43 +01:00
|
|
|
|
|
|
|
callback.add_argument('--blob-callback', metavar="FUNCTION_BODY",
|
2019-03-17 07:39:39 +01:00
|
|
|
help=_("Python code body for processing blob objects; see "
|
|
|
|
"CALLBACKS section below."))
|
2019-02-17 08:59:43 +01:00
|
|
|
callback.add_argument('--commit-callback', metavar="FUNCTION_BODY",
|
2019-03-17 07:39:39 +01:00
|
|
|
help=_("Python code body for processing commit objects; see "
|
|
|
|
"CALLBACKS section below."))
|
2019-02-17 08:59:43 +01:00
|
|
|
callback.add_argument('--tag-callback', metavar="FUNCTION_BODY",
|
2019-03-17 07:39:39 +01:00
|
|
|
help=_("Python code body for processing tag objects; see CALLBACKS "
|
|
|
|
"section below."))
|
2019-02-17 08:59:43 +01:00
|
|
|
callback.add_argument('--reset-callback', metavar="FUNCTION_BODY",
|
2019-03-17 07:39:39 +01:00
|
|
|
help=_("Python code body for processing reset objects; see "
|
|
|
|
"CALLBACKS section below."))
|
2019-02-17 08:59:43 +01:00
|
|
|
|
2019-03-17 07:39:39 +01:00
|
|
|
location = parser.add_argument_group(title=_("Location to filter from/to"))
|
|
|
|
location.add_argument('--source', help=_("Git repository to read from"))
|
2019-01-08 20:02:53 +01:00
|
|
|
location.add_argument('--target',
|
2019-03-17 07:39:39 +01:00
|
|
|
help=_("Git repository to overwrite with filtered history"))
|
2019-01-08 20:02:53 +01:00
|
|
|
|
2019-03-17 07:39:39 +01:00
|
|
|
misc = parser.add_argument_group(title=_("Miscellaneous options"))
|
2018-12-25 07:59:38 +01:00
|
|
|
misc.add_argument('--help', '-h', action='store_true',
|
2019-03-17 07:39:39 +01:00
|
|
|
help=_("Show this help message and exit."))
|
2018-12-25 07:59:38 +01:00
|
|
|
misc.add_argument('--force', '-f', action='store_true',
|
2019-03-17 07:39:39 +01:00
|
|
|
help=_("Rewrite history even if the current repo does not look "
|
|
|
|
"like a fresh clone."))
|
2018-12-25 07:59:38 +01:00
|
|
|
|
|
|
|
misc.add_argument('--dry-run', action='store_true',
|
2019-03-17 07:39:39 +01:00
|
|
|
help=_("Do not change the repository. Run `git fast-export` and "
|
|
|
|
"filter its output, and save both the original and the "
|
|
|
|
"filtered version for comparison. Some filtering of empty "
|
|
|
|
"commits may not occur due to inability to query the "
|
|
|
|
"fast-import backend."))
|
2018-12-25 07:59:38 +01:00
|
|
|
misc.add_argument('--debug', action='store_true',
|
2019-03-17 07:39:39 +01:00
|
|
|
help=_("Print additional information about operations being "
|
|
|
|
"performed and commands being run. When used together "
|
|
|
|
"with --dry-run, also show extra information about what "
|
|
|
|
"would be run."))
|
2018-12-25 07:59:38 +01:00
|
|
|
misc.add_argument('--stdin', action='store_true',
|
2019-03-17 07:39:39 +01:00
|
|
|
help=_("Instead of running `git fast-export` and filtering its "
|
|
|
|
"output, filter the fast-export stream from stdin."))
|
2018-12-25 07:59:38 +01:00
|
|
|
misc.add_argument('--quiet', action='store_true',
|
2019-03-17 07:39:39 +01:00
|
|
|
help=_("Pass --quiet to other git commands called"))
|
2018-12-25 07:59:38 +01:00
|
|
|
return parser
|
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
def sanity_check_args(args):
|
|
|
|
if args.analyze and args.path_changes:
|
2019-03-17 07:39:39 +01:00
|
|
|
raise SystemExit(_("Error: --analyze is incompatible with --path* flags; "
|
|
|
|
"it's a read-only operation."))
|
2018-12-25 07:59:38 +01:00
|
|
|
if args.analyze and args.stdin:
|
2019-03-17 07:39:39 +01:00
|
|
|
raise SystemExit(_("Error: --analyze is incompatible with --stdin."))
|
2018-12-25 07:59:38 +01:00
|
|
|
# If no path_changes are found, initialize with empty list but mark as
|
|
|
|
# not inclusive so that all files match
|
|
|
|
if args.path_changes == None:
|
|
|
|
args.path_changes = []
|
2018-10-19 18:09:25 +02:00
|
|
|
args.inclusive = False
|
2018-12-25 07:59:38 +01:00
|
|
|
# Similarly, if we only have renames, all paths should match
|
|
|
|
else:
|
|
|
|
has_filter = False
|
|
|
|
for (mod_type, match_type, path_expression) in args.path_changes:
|
|
|
|
if mod_type == 'filter':
|
|
|
|
has_filter = True
|
|
|
|
if not has_filter:
|
|
|
|
args.inclusive = False
|
2019-02-02 15:35:59 +01:00
|
|
|
# Also throw in a sanity check on git version here;
|
|
|
|
# PERF: remove this check once new enough git versions are common
|
|
|
|
p = subprocess.Popen('git diff-tree -h'.split(),
|
|
|
|
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
|
|
|
p.wait()
|
|
|
|
output = p.stdout.read()
|
|
|
|
if '--combined-all-paths' not in output:
|
2019-03-17 07:39:39 +01:00
|
|
|
raise SystemExit(_("Error: need a version of git whose diff-tree command "
|
|
|
|
"has the --combined-all-paths option")) # pragma: no cover
|
2018-12-25 07:59:38 +01:00
|
|
|
|
filter-repo: add text removal (or replacement) via file of expressions
Make it easy for users to search and replace text throughout the
repository history. Instead of inventing some new syntax, reuse the
same syntax used by BFG repo filter's --replace-text option, namely,
a file with one expression per line of the form
[regex:|glob:|literal:]$MATCH_EXPR[==>$REPLACEMENT_EXPR]
Where "$MATCH_EXPR" is by default considered to be literal text, but
could be a regex or a glob if the appropriate prefix is used. Also,
$REPLACEMENT_EXPR defaults to '***REMOVED***' if not specified. If
you want a literal '==>' to be part of your $MATCH_EXPR, then you
must also manually specify a replacement expression instead of taking
the default. Some examples:
sup3rs3kr3t
(replaces 'sup3rs3kr3t' with '***REMOVED***')
HeWhoShallNotBeNamed==>Voldemort
(replaces 'HeWhoShallNotBeNamed' with 'Voldemort')
very==>
(replaces 'very' with the empty string)
regex:(\d{2})/(\d{2})/(\d{4})==>\2/\1/\3
(replaces '05/17/2012' with '17/05/2012', and vice-versa)
The format for regex is as from
re.sub(<pattern>, <repl>, <string>) from
https://docs.python.org/2/library/re.html
The <string> comes from file contents of the repo, and you specify
the <pattern> and <repl>.
glob:Copy*t==>Cartel
(replaces 'Copyright' or 'Copyleft' or 'Copy my st' with 'Cartel')
Signed-off-by: Elijah Newren <newren@gmail.com>
2019-01-30 09:56:22 +01:00
|
|
|
@staticmethod
|
|
|
|
def get_replace_text(filename):
|
|
|
|
replace_literals = []
|
|
|
|
replace_regexes = []
|
|
|
|
with open(filename) as f:
|
|
|
|
for line in f:
|
|
|
|
line = line.rstrip('\r\n')
|
|
|
|
|
|
|
|
# Determine the replacement
|
|
|
|
replacement = '***REMOVED***'
|
|
|
|
if '==>' in line:
|
|
|
|
line, replacement = line.rsplit('==>', 1)
|
|
|
|
|
|
|
|
# See if we need to match via regex
|
|
|
|
regex = None
|
|
|
|
if line.startswith('regex:'):
|
|
|
|
regex = line[6:]
|
|
|
|
elif line.startswith('glob:'):
|
|
|
|
regex = fnmatch.translate(line[5:])
|
|
|
|
if regex.endswith(r'\Z(?ms)'):
|
|
|
|
regex = regex[0:-7]
|
|
|
|
if regex:
|
|
|
|
replace_regexes.append((re.compile(regex), replacement))
|
|
|
|
else:
|
|
|
|
# Otherwise, find the literal we need to replace
|
|
|
|
if line.startswith('literal:'):
|
|
|
|
line = line[8:]
|
|
|
|
if not line:
|
|
|
|
continue
|
|
|
|
replace_literals.append((line, replacement))
|
|
|
|
return {'literals': replace_literals, 'regexes': replace_regexes}
|
|
|
|
|
2018-12-25 07:59:38 +01:00
|
|
|
@staticmethod
|
|
|
|
def default_options():
|
|
|
|
return FilteringOptions.parse_args([], error_on_empty = False)
|
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
def parse_args(input_args, error_on_empty = True):
|
|
|
|
parser = FilteringOptions.create_arg_parser()
|
|
|
|
if not input_args and error_on_empty:
|
|
|
|
parser.print_usage()
|
2019-03-17 07:39:39 +01:00
|
|
|
raise SystemExit(_("No arguments specified."))
|
2018-12-25 07:59:38 +01:00
|
|
|
args = parser.parse_args(input_args)
|
|
|
|
if args.help:
|
|
|
|
parser.print_help()
|
|
|
|
raise SystemExit()
|
|
|
|
FilteringOptions.sanity_check_args(args)
|
2019-01-29 02:26:49 +01:00
|
|
|
if args.mailmap:
|
|
|
|
args.mailmap = MailmapInfo(args.mailmap)
|
filter-repo: add text removal (or replacement) via file of expressions
Make it easy for users to search and replace text throughout the
repository history. Instead of inventing some new syntax, reuse the
same syntax used by BFG repo filter's --replace-text option, namely,
a file with one expression per line of the form
[regex:|glob:|literal:]$MATCH_EXPR[==>$REPLACEMENT_EXPR]
Where "$MATCH_EXPR" is by default considered to be literal text, but
could be a regex or a glob if the appropriate prefix is used. Also,
$REPLACEMENT_EXPR defaults to '***REMOVED***' if not specified. If
you want a literal '==>' to be part of your $MATCH_EXPR, then you
must also manually specify a replacement expression instead of taking
the default. Some examples:
sup3rs3kr3t
(replaces 'sup3rs3kr3t' with '***REMOVED***')
HeWhoShallNotBeNamed==>Voldemort
(replaces 'HeWhoShallNotBeNamed' with 'Voldemort')
very==>
(replaces 'very' with the empty string)
regex:(\d{2})/(\d{2})/(\d{4})==>\2/\1/\3
(replaces '05/17/2012' with '17/05/2012', and vice-versa)
The format for regex is as from
re.sub(<pattern>, <repl>, <string>) from
https://docs.python.org/2/library/re.html
The <string> comes from file contents of the repo, and you specify
the <pattern> and <repl>.
glob:Copy*t==>Cartel
(replaces 'Copyright' or 'Copyleft' or 'Copy my st' with 'Cartel')
Signed-off-by: Elijah Newren <newren@gmail.com>
2019-01-30 09:56:22 +01:00
|
|
|
if args.replace_text:
|
|
|
|
args.replace_text = FilteringOptions.get_replace_text(args.replace_text)
|
2018-12-25 07:59:38 +01:00
|
|
|
return args
|
2018-08-21 17:46:26 +02:00
|
|
|
|
2018-12-26 06:54:16 +01:00
|
|
|
class RepoAnalyze(object):
|
|
|
|
|
|
|
|
# First, several helper functions for analyze_commit()
|
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
def equiv_class(stats, filename):
|
2018-11-20 19:15:46 +01:00
|
|
|
return stats['equivalence'].get(filename, (filename,))
|
|
|
|
|
2018-12-26 06:54:16 +01:00
|
|
|
@staticmethod
|
2018-11-20 19:15:46 +01:00
|
|
|
def setup_equivalence_for_rename(stats, oldname, newname):
|
|
|
|
# if A is renamed to B and B is renamed to C, then the user thinks of
|
|
|
|
# A, B, and C as all being different names for the same 'file'. We record
|
|
|
|
# this as an equivalence class:
|
|
|
|
# stats['equivalence'][name] = (A,B,C)
|
|
|
|
# for name being each of A, B, and C.
|
|
|
|
old_tuple = stats['equivalence'].get(oldname, ())
|
|
|
|
if newname in old_tuple:
|
|
|
|
return
|
|
|
|
elif old_tuple:
|
|
|
|
new_tuple = tuple(list(old_tuple)+[newname])
|
|
|
|
else:
|
|
|
|
new_tuple = (oldname, newname)
|
|
|
|
for f in new_tuple:
|
|
|
|
stats['equivalence'][f] = new_tuple
|
|
|
|
|
2018-12-26 06:54:16 +01:00
|
|
|
@staticmethod
|
2018-11-20 19:15:46 +01:00
|
|
|
def setup_or_update_rename_history(stats, commit, oldname, newname):
|
|
|
|
rename_commits = stats['rename_history'].get(oldname, set())
|
|
|
|
rename_commits.add(commit)
|
|
|
|
stats['rename_history'][oldname] = rename_commits
|
|
|
|
|
2018-12-26 06:54:16 +01:00
|
|
|
@staticmethod
|
2018-11-20 19:15:46 +01:00
|
|
|
def handle_renames(stats, commit, change_types, filenames):
|
|
|
|
for index, change_type in enumerate(change_types):
|
|
|
|
if change_type == 'R':
|
|
|
|
oldname, newname = filenames[index], filenames[-1]
|
2018-12-26 06:54:16 +01:00
|
|
|
RepoAnalyze.setup_equivalence_for_rename(stats, oldname, newname)
|
|
|
|
RepoAnalyze.setup_or_update_rename_history(stats, commit,
|
|
|
|
oldname, newname)
|
2018-11-20 19:15:46 +01:00
|
|
|
|
2018-12-26 06:54:16 +01:00
|
|
|
@staticmethod
|
2018-11-20 19:15:46 +01:00
|
|
|
def handle_file(stats, graph, commit, modes, shas, filenames):
|
|
|
|
mode, sha, filename = modes[-1], shas[-1], filenames[-1]
|
|
|
|
|
|
|
|
# Figure out kind of deletions to undo for this file, and update lists
|
|
|
|
# of all-names-by-sha and all-filenames
|
|
|
|
delmode = 'tree_deletions'
|
|
|
|
if mode != '040000':
|
|
|
|
delmode = 'file_deletions'
|
|
|
|
stats['names'][sha].add(filename)
|
|
|
|
stats['allnames'].add(filename)
|
|
|
|
|
|
|
|
# If the file (or equivalence class of files) was recorded as deleted,
|
|
|
|
# clearly it isn't anymore
|
2018-12-26 06:54:16 +01:00
|
|
|
equiv = RepoAnalyze.equiv_class(stats, filename)
|
2018-11-20 19:15:46 +01:00
|
|
|
for f in equiv:
|
|
|
|
stats[delmode].pop(f, None)
|
|
|
|
|
|
|
|
# If we get a modify/add for a path that was renamed, we may need to break
|
|
|
|
# the equivalence class. However, if the modify/add was on a branch that
|
|
|
|
# doesn't have the rename in its history, we are still okay.
|
|
|
|
need_to_break_equivalence = False
|
|
|
|
if equiv[-1] != filename:
|
|
|
|
for rename_commit in stats['rename_history'][filename]:
|
|
|
|
if graph.is_ancestor(rename_commit, commit):
|
|
|
|
need_to_break_equivalence = True
|
|
|
|
|
|
|
|
if need_to_break_equivalence:
|
filter-repo: add --analyze option
This option walks through the repository history and creates a report
with basic statistics, rename related information, and sizes of objects
and when/if those have been deleted. It primarily looks at unpacked
sizes (i.e. size of object ignoring delta-ing and compression), and
sums the size of each version of the file for each path. Additionally,
it aggregates these sums by extension and by directory, and tracks
whether paths, extensions, and directories have been deleted. This can
be very useful in determining what the big things are, and whether they
might have been considered to have been mistakes to add to the
repository in the first place.
There are numerous caveats with the determination of "deleted" and
"renamed", and can give both false positives and false negatives. But
they are only meant as a helpful heuristic to give others a starting
point for an investigation, and the information provide so far is useful.
I do want to improve the equivalence classes (rename handling), but that
is for a future commit.
Signed-off-by: Elijah Newren <newren@gmail.com>
2018-10-21 16:44:33 +02:00
|
|
|
for f in equiv:
|
2018-11-20 19:15:46 +01:00
|
|
|
if f in stats['equivalence']:
|
|
|
|
del stats['equivalence'][f]
|
|
|
|
|
2018-12-26 06:54:16 +01:00
|
|
|
@staticmethod
|
|
|
|
def analyze_commit(stats, graph, commit, parents, date, file_changes):
|
|
|
|
graph.add_commit_and_parents(commit, parents)
|
|
|
|
for change in file_changes:
|
|
|
|
modes, shas, change_types, filenames = change
|
|
|
|
if len(parents) == 1 and change_types.startswith('R'):
|
|
|
|
change_types = 'R' # remove the rename score; we don't care
|
|
|
|
if modes[-1] == '160000':
|
filter-repo: add --analyze option
This option walks through the repository history and creates a report
with basic statistics, rename related information, and sizes of objects
and when/if those have been deleted. It primarily looks at unpacked
sizes (i.e. size of object ignoring delta-ing and compression), and
sums the size of each version of the file for each path. Additionally,
it aggregates these sums by extension and by directory, and tracks
whether paths, extensions, and directories have been deleted. This can
be very useful in determining what the big things are, and whether they
might have been considered to have been mistakes to add to the
repository in the first place.
There are numerous caveats with the determination of "deleted" and
"renamed", and can give both false positives and false negatives. But
they are only meant as a helpful heuristic to give others a starting
point for an investigation, and the information provide so far is useful.
I do want to improve the equivalence classes (rename handling), but that
is for a future commit.
Signed-off-by: Elijah Newren <newren@gmail.com>
2018-10-21 16:44:33 +02:00
|
|
|
continue
|
2018-12-26 06:54:16 +01:00
|
|
|
elif modes[-1] == '000000':
|
|
|
|
# Track when files/directories are deleted
|
|
|
|
for f in RepoAnalyze.equiv_class(stats, filenames[-1]):
|
|
|
|
if any(x == '040000' for x in modes[0:-1]):
|
|
|
|
stats['tree_deletions'][f] = date
|
|
|
|
else:
|
|
|
|
stats['file_deletions'][f] = date
|
|
|
|
elif change_types.strip('AMT') == '':
|
|
|
|
RepoAnalyze.handle_file(stats, graph, commit, modes, shas, filenames)
|
|
|
|
elif modes[-1] == '040000' and change_types.strip('RAM') == '':
|
|
|
|
RepoAnalyze.handle_file(stats, graph, commit, modes, shas, filenames)
|
|
|
|
elif change_types.strip('RAM') == '':
|
|
|
|
RepoAnalyze.handle_file(stats, graph, commit, modes, shas, filenames)
|
|
|
|
RepoAnalyze.handle_renames(stats, commit, change_types, filenames)
|
|
|
|
else:
|
2019-03-17 07:39:39 +01:00
|
|
|
raise SystemExit(_("Unhandled change type(s): %(change_type)s "
|
|
|
|
"(in commit %(commit)s)")
|
|
|
|
% ({'change_type': change_types, 'commit': commit})
|
|
|
|
) # pragma: no cover
|
2018-12-26 06:54:16 +01:00
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
def gather_data(args):
|
|
|
|
blob_size_progress = ProgressWriter()
|
|
|
|
num_blobs = 0
|
|
|
|
|
|
|
|
# Get sizes of blobs by sha1
|
|
|
|
cmd = '--batch-check=%(objectname) %(objecttype) ' + \
|
|
|
|
'%(objectsize) %(objectsize:disk)'
|
|
|
|
cf = subprocess.Popen(['git', 'cat-file', '--batch-all-objects', cmd],
|
|
|
|
bufsize = -1,
|
|
|
|
stdout = subprocess.PIPE)
|
|
|
|
unpacked_size = {}
|
|
|
|
packed_size = {}
|
|
|
|
for line in cf.stdout:
|
|
|
|
sha, objtype, objsize, objdisksize = line.split()
|
|
|
|
objsize, objdisksize = int(objsize), int(objdisksize)
|
|
|
|
if objtype == 'blob':
|
|
|
|
unpacked_size[sha] = objsize
|
|
|
|
packed_size[sha] = objdisksize
|
|
|
|
num_blobs += 1
|
2019-03-17 07:39:39 +01:00
|
|
|
blob_size_progress.show(_("Processed %d blob sizes") % num_blobs)
|
2018-12-26 06:54:16 +01:00
|
|
|
cf.wait()
|
|
|
|
blob_size_progress.finish()
|
|
|
|
stats = {'names': collections.defaultdict(set),
|
|
|
|
'allnames' : set(),
|
|
|
|
'file_deletions': {},
|
|
|
|
'tree_deletions': {},
|
|
|
|
'equivalence': {},
|
|
|
|
'rename_history': collections.defaultdict(set),
|
|
|
|
'unpacked_size': unpacked_size,
|
|
|
|
'packed_size': packed_size,
|
|
|
|
'num_commits': 0}
|
|
|
|
|
|
|
|
# Setup the rev-list/diff-tree process
|
|
|
|
commit_parse_progress = ProgressWriter()
|
|
|
|
num_commits = 0
|
|
|
|
cmd = ('git rev-list --topo-order --reverse {}'.format(' '.join(args.refs)) +
|
|
|
|
' | git diff-tree --stdin --always --root --format=%H%n%P%n%cd' +
|
|
|
|
' --date=short -M -t -c --raw --combined-all-paths')
|
|
|
|
dtp = subprocess.Popen(cmd, shell=True, bufsize=-1, stdout=subprocess.PIPE)
|
|
|
|
f = dtp.stdout
|
2019-02-14 20:12:29 +01:00
|
|
|
try:
|
|
|
|
line = f.next()
|
|
|
|
except StopIteration:
|
2019-03-17 07:39:39 +01:00
|
|
|
raise SystemExit(_("Nothing to analyze; repository is empty."))
|
2018-12-26 06:54:16 +01:00
|
|
|
cont = bool(line)
|
|
|
|
graph = AncestryGraph()
|
|
|
|
while cont:
|
|
|
|
commit = line.rstrip()
|
|
|
|
parents = f.next().split()
|
|
|
|
date = f.next().rstrip()
|
|
|
|
|
|
|
|
# We expect a blank line next; if we get a non-blank line then
|
|
|
|
# this commit modified no files and we need to move on to the next.
|
|
|
|
# If there is no line, we've reached end-of-input.
|
|
|
|
try:
|
|
|
|
line = f.next().rstrip()
|
|
|
|
cont = True
|
|
|
|
except StopIteration:
|
|
|
|
cont = False
|
|
|
|
|
|
|
|
# If we haven't reached end of input, and we got a blank line meaning
|
|
|
|
# a commit that has modified files, then get the file changes associated
|
|
|
|
# with this commit.
|
|
|
|
file_changes = []
|
|
|
|
if cont and not line:
|
|
|
|
cont = False
|
|
|
|
for line in f:
|
|
|
|
if not line.startswith(':'):
|
|
|
|
cont = True
|
|
|
|
break
|
|
|
|
n = 1+max(1, len(parents))
|
|
|
|
assert line.startswith(':'*(n-1))
|
|
|
|
relevant = line[n-1:-1]
|
|
|
|
splits = relevant.split(None, n)
|
|
|
|
modes = splits[0:n]
|
|
|
|
splits = splits[n].split(None, n)
|
|
|
|
shas = splits[0:n]
|
|
|
|
splits = splits[n].split('\t')
|
|
|
|
change_types = splits[0]
|
|
|
|
filenames = [PathQuoting.dequote(x) for x in splits[1:]]
|
|
|
|
file_changes.append([modes, shas, change_types, filenames])
|
|
|
|
|
|
|
|
# Analyze this commit and update progress
|
|
|
|
RepoAnalyze.analyze_commit(stats, graph, commit, parents, date,
|
|
|
|
file_changes)
|
|
|
|
num_commits += 1
|
2019-03-17 07:39:39 +01:00
|
|
|
commit_parse_progress.show(_("Processed %d commits") % num_commits)
|
2018-12-26 06:54:16 +01:00
|
|
|
|
|
|
|
# Show the final commits processed message and record the number of commits
|
|
|
|
commit_parse_progress.finish()
|
|
|
|
stats['num_commits'] = num_commits
|
|
|
|
|
|
|
|
# Close the output, ensure rev-list|diff-tree pipeline completed successfully
|
|
|
|
dtp.stdout.close()
|
|
|
|
if dtp.wait():
|
2019-03-17 07:39:39 +01:00
|
|
|
raise SystemExit(_("Error: rev-list|diff-tree pipeline failed; see above.")) # pragma: no cover
|
2018-12-26 06:54:16 +01:00
|
|
|
|
|
|
|
return stats
|
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
def write_report(reportdir, stats):
|
|
|
|
def datestr(datetimestr):
|
2019-03-17 07:39:39 +01:00
|
|
|
return datetimestr if datetimestr else _('<present>')
|
2018-12-26 06:54:16 +01:00
|
|
|
|
|
|
|
def dirnames(path):
|
|
|
|
while True:
|
|
|
|
path = os.path.dirname(path)
|
|
|
|
yield path
|
|
|
|
if path == '':
|
|
|
|
break
|
|
|
|
|
|
|
|
# Compute aggregate size information for paths, extensions, and dirs
|
|
|
|
total_size = {'packed': 0, 'unpacked': 0}
|
|
|
|
path_size = {'packed': collections.defaultdict(int),
|
|
|
|
'unpacked': collections.defaultdict(int)}
|
|
|
|
ext_size = {'packed': collections.defaultdict(int),
|
|
|
|
'unpacked': collections.defaultdict(int)}
|
|
|
|
dir_size = {'packed': collections.defaultdict(int),
|
|
|
|
'unpacked': collections.defaultdict(int)}
|
|
|
|
for sha in stats['names']:
|
|
|
|
size = {'packed': stats['packed_size'][sha],
|
|
|
|
'unpacked': stats['unpacked_size'][sha]}
|
|
|
|
for which in ('packed', 'unpacked'):
|
|
|
|
for name in stats['names'][sha]:
|
|
|
|
total_size[which] += size[which]
|
|
|
|
path_size[which][name] += size[which]
|
|
|
|
basename, ext = os.path.splitext(name)
|
|
|
|
ext_size[which][ext] += size[which]
|
|
|
|
for dirname in dirnames(name):
|
|
|
|
dir_size[which][dirname] += size[which]
|
|
|
|
|
|
|
|
# Determine if and when extensions and directories were deleted
|
|
|
|
ext_deleted_data = {}
|
|
|
|
for name in stats['allnames']:
|
|
|
|
when = stats['file_deletions'].get(name, None)
|
|
|
|
|
|
|
|
# Update the extension
|
|
|
|
basename, ext = os.path.splitext(name)
|
|
|
|
if when is None:
|
|
|
|
ext_deleted_data[ext] = None
|
|
|
|
elif ext in ext_deleted_data:
|
|
|
|
if ext_deleted_data[ext] is not None:
|
|
|
|
ext_deleted_data[ext] = max(ext_deleted_data[ext], when)
|
|
|
|
else:
|
|
|
|
ext_deleted_data[ext] = when
|
|
|
|
|
|
|
|
dir_deleted_data = {}
|
|
|
|
for name in dir_size['packed']:
|
|
|
|
dir_deleted_data[name] = stats['tree_deletions'].get(name, None)
|
|
|
|
|
|
|
|
with open(os.path.join(reportdir, "README"), 'w') as f:
|
|
|
|
# Give a basic overview of this file
|
2019-03-17 07:39:39 +01:00
|
|
|
f.write("== %s ==\n" % _("Overall Statistics"))
|
|
|
|
f.write(" %s: %d\n" % (_("Number of commits"),
|
|
|
|
stats['num_commits']))
|
|
|
|
f.write(" %s: %d\n" % (_("Number of filenames"),
|
|
|
|
len(path_size['packed'])))
|
|
|
|
f.write(" %s: %d\n" % (_("Number of directories"),
|
|
|
|
len(dir_size['packed'])))
|
|
|
|
f.write(" %s: %d\n" % (_("Number of file extensions"),
|
|
|
|
len(ext_size['packed'])))
|
2018-12-26 06:54:16 +01:00
|
|
|
f.write("\n")
|
2019-03-17 07:39:39 +01:00
|
|
|
f.write(" %s: %d\n" % (_("Total unpacked size (bytes)"),
|
|
|
|
total_size['unpacked']))
|
|
|
|
f.write(" %s: %d\n" % (_("Total packed size (bytes)"),
|
|
|
|
total_size['packed']))
|
2018-12-26 06:54:16 +01:00
|
|
|
f.write("\n")
|
|
|
|
|
|
|
|
# Mention issues with the report
|
2019-03-17 07:39:39 +01:00
|
|
|
f.write("== %s ==\n" % _("Caveats"))
|
|
|
|
f.write("=== %s ===\n" % _("Sizes"))
|
|
|
|
f.write(textwrap.dedent(_("""
|
2018-12-26 06:54:16 +01:00
|
|
|
Packed size represents what size your repository would be if no
|
|
|
|
trees, commits, tags, or other metadata were included (though it may
|
|
|
|
fail to represent de-duplication; see below). It also represents the
|
|
|
|
current packing, which may be suboptimal if you haven't gc'ed for a
|
|
|
|
while.
|
|
|
|
|
|
|
|
Unpacked size represents what size your repository would be if no if
|
|
|
|
no trees, commits, tags, or other metadata were included AND if no
|
|
|
|
files were packed; i.e., without delta-ing or compression.
|
|
|
|
|
|
|
|
Both unpacked and packed sizes can be slightly misleading. Deleting
|
|
|
|
a blob from history not save as much space as the unpacked size,
|
|
|
|
because it is obviously normally stored in packed form. Also,
|
|
|
|
deleting a blob from history may not save as much space as its packed
|
|
|
|
size either, because another blob could be stored as a delta against
|
|
|
|
that blob, so when you remove one blob another blob's packed size may
|
|
|
|
grow.
|
|
|
|
|
|
|
|
Also, the sum of the packed sizes can add up to more than the
|
|
|
|
repository size; if the same contents appeared in the repository in
|
|
|
|
multiple places, git will automatically de-dupe and store only one
|
|
|
|
copy, while the way sizes are added in this analysis adds the size
|
|
|
|
for each file path that has those contents. Further, if a file is
|
|
|
|
ever reverted to a previous version's contents, the previous
|
|
|
|
version's size will be counted multiple times in this analysis, even
|
|
|
|
though git will only store it once.
|
2019-03-17 07:39:39 +01:00
|
|
|
""")[1:]))
|
2018-12-26 06:54:16 +01:00
|
|
|
f.write("\n")
|
2019-03-17 07:39:39 +01:00
|
|
|
f.write("=== %s ===\n" % _("Deletions"))
|
|
|
|
f.write(textwrap.dedent(_("""
|
2018-12-26 06:54:16 +01:00
|
|
|
Whether a file is deleted is not a binary quality, since it can be
|
|
|
|
deleted on some branches but still exist in others. Also, it might
|
|
|
|
exist in an old tag, but have been deleted in versions newer than
|
|
|
|
that. More thorough tracking could be done, including looking at
|
|
|
|
merge commits where one side of history deleted and the other modified,
|
|
|
|
in order to give a more holistic picture of deletions. However, that
|
|
|
|
algorithm would not only be more complex to implement, it'd also be
|
|
|
|
quite difficult to present and interpret by users. Since --analyze
|
|
|
|
is just about getting a high-level rough picture of history, it instead
|
|
|
|
implements the simplistic rule that is good enough for 98% of cases:
|
|
|
|
A file is marked as deleted if the last commit in the fast-export
|
|
|
|
stream that mentions the file lists it as deleted.
|
|
|
|
This makes it dependent on topological ordering, but generally gives
|
|
|
|
the "right" answer.
|
2019-03-17 07:39:39 +01:00
|
|
|
""")[1:]))
|
2018-12-26 06:54:16 +01:00
|
|
|
f.write("\n")
|
2019-03-17 07:39:39 +01:00
|
|
|
f.write("=== %s ===\n" % _("Renames"))
|
|
|
|
f.write(textwrap.dedent(_("""
|
2018-12-26 06:54:16 +01:00
|
|
|
Renames share the same non-binary nature that deletions do, plus
|
|
|
|
additional challenges:
|
|
|
|
* If the renamed file is renamed again, instead of just two names for
|
|
|
|
a path you can have three or more.
|
|
|
|
* Rename pairs of the form (oldname, newname) that we consider to be
|
|
|
|
different names of the "same file" might only be valid over certain
|
|
|
|
commit ranges. For example, if a new commit reintroduces a file
|
|
|
|
named oldname, then new versions of oldname aren't the "same file"
|
|
|
|
anymore. We could try to portray this to the user, but it's easier
|
|
|
|
for the user to just break the pairing and only report unbroken
|
|
|
|
rename pairings to the user.
|
|
|
|
* The ability for users to rename files differently in different
|
|
|
|
branches means that our chains of renames will not necessarily be
|
|
|
|
linear but may branch out.
|
2019-03-17 07:39:39 +01:00
|
|
|
""")[1:]))
|
2018-12-26 06:54:16 +01:00
|
|
|
f.write("\n")
|
|
|
|
|
|
|
|
# Equivalence classes for names, so if folks only want to keep a
|
|
|
|
# certain set of paths, they know the old names they want to include
|
|
|
|
# too.
|
|
|
|
with open(os.path.join(reportdir, "renames.txt"), 'w') as f:
|
|
|
|
seen = set()
|
2019-04-01 23:36:57 +02:00
|
|
|
for pathname,equiv_group in sorted(stats['equivalence'].items(),
|
2019-04-01 19:58:16 +02:00
|
|
|
key=lambda x:(x[1], x[0])):
|
2018-12-26 06:54:16 +01:00
|
|
|
if equiv_group in seen:
|
|
|
|
continue
|
|
|
|
seen.add(equiv_group)
|
|
|
|
f.write("{} ->\n ".format(equiv_group[0]) +
|
|
|
|
"\n ".join(equiv_group[1:]) +
|
|
|
|
"\n")
|
|
|
|
|
|
|
|
# List directories in reverse sorted order of unpacked size
|
|
|
|
with open(os.path.join(reportdir, "directories-deleted-sizes.txt"), 'w') as f:
|
2019-03-17 07:39:39 +01:00
|
|
|
f.write("=== %s ===\n" % _("Deleted directories by reverse size"))
|
|
|
|
f.write(_("Format: unpacked size, packed size, date deleted, directory name\n"))
|
2019-04-01 23:36:57 +02:00
|
|
|
for dirname, size in sorted(dir_size['packed'].items(),
|
2019-04-01 19:58:16 +02:00
|
|
|
key=lambda x:(x[1],x[0]), reverse=True):
|
2018-12-26 06:54:16 +01:00
|
|
|
if (dir_deleted_data[dirname]):
|
|
|
|
f.write(" {:10d} {:10d} {:10s} {}\n"
|
|
|
|
.format(dir_size['unpacked'][dirname],
|
|
|
|
size,
|
|
|
|
datestr(dir_deleted_data[dirname]),
|
2019-03-17 07:39:39 +01:00
|
|
|
dirname or _('<toplevel>')))
|
2018-12-26 06:54:16 +01:00
|
|
|
|
|
|
|
with open(os.path.join(reportdir, "directories-all-sizes.txt"), 'w') as f:
|
2019-03-17 07:39:39 +01:00
|
|
|
f.write("=== %s ===\n" % _("All directories by reverse size"))
|
|
|
|
f.write(_("Format: unpacked size, packed size, date deleted, directory name\n"))
|
2019-04-01 23:36:57 +02:00
|
|
|
for dirname, size in sorted(dir_size['packed'].items(),
|
2019-04-01 19:58:16 +02:00
|
|
|
key=lambda x:(x[1],x[0]), reverse=True):
|
2018-11-15 21:04:44 +01:00
|
|
|
f.write(" {:10d} {:10d} {:10s} {}\n"
|
|
|
|
.format(dir_size['unpacked'][dirname],
|
|
|
|
size,
|
|
|
|
datestr(dir_deleted_data[dirname]),
|
2019-03-17 07:39:39 +01:00
|
|
|
dirname or _("<toplevel>")))
|
2018-11-15 17:09:39 +01:00
|
|
|
|
2018-12-26 06:54:16 +01:00
|
|
|
# List extensions in reverse sorted order of unpacked size
|
|
|
|
with open(os.path.join(reportdir, "extensions-deleted-sizes.txt"), 'w') as f:
|
2019-03-17 07:39:39 +01:00
|
|
|
f.write("=== %s ===\n" % _("Deleted extensions by reverse size"))
|
|
|
|
f.write(_("Format: unpacked size, packed size, date deleted, extension name\n"))
|
2019-04-01 23:36:57 +02:00
|
|
|
for extname, size in sorted(ext_size['packed'].items(),
|
2019-04-01 19:58:16 +02:00
|
|
|
key=lambda x:(x[1],x[0]), reverse=True):
|
2018-12-26 06:54:16 +01:00
|
|
|
if (ext_deleted_data[extname]):
|
|
|
|
f.write(" {:10d} {:10d} {:10s} {}\n"
|
|
|
|
.format(ext_size['unpacked'][extname],
|
|
|
|
size,
|
|
|
|
datestr(ext_deleted_data[extname]),
|
2019-03-17 07:39:39 +01:00
|
|
|
extname or _('<no extension>')))
|
2018-12-26 06:54:16 +01:00
|
|
|
|
|
|
|
with open(os.path.join(reportdir, "extensions-all-sizes.txt"), 'w') as f:
|
2019-03-17 07:39:39 +01:00
|
|
|
f.write("=== %s ===\n" % _("All extensions by reverse size"))
|
|
|
|
f.write(_("Format: unpacked size, packed size, date deleted, extension name\n"))
|
2019-04-01 23:36:57 +02:00
|
|
|
for extname, size in sorted(ext_size['packed'].items(),
|
2019-04-01 19:58:16 +02:00
|
|
|
key=lambda x:(x[1],x[0]), reverse=True):
|
2018-11-15 21:04:44 +01:00
|
|
|
f.write(" {:10d} {:10d} {:10s} {}\n"
|
|
|
|
.format(ext_size['unpacked'][extname],
|
|
|
|
size,
|
|
|
|
datestr(ext_deleted_data[extname]),
|
2019-03-17 07:39:39 +01:00
|
|
|
extname or _('<no extension>')))
|
2018-11-15 17:09:39 +01:00
|
|
|
|
2018-12-26 06:54:16 +01:00
|
|
|
# List files in reverse sorted order of unpacked size
|
|
|
|
with open(os.path.join(reportdir, "path-deleted-sizes.txt"), 'w') as f:
|
2019-03-17 07:39:39 +01:00
|
|
|
f.write("=== %s ===\n" % _("Deleted paths by reverse accumulated size"))
|
|
|
|
f.write(_("Format: unpacked size, packed size, date deleted, path name(s)\n"))
|
2019-04-01 23:36:57 +02:00
|
|
|
for pathname, size in sorted(path_size['packed'].items(),
|
2019-04-01 19:58:16 +02:00
|
|
|
key=lambda x:(x[1],x[0]), reverse=True):
|
2018-12-26 06:54:16 +01:00
|
|
|
when = stats['file_deletions'].get(pathname, None)
|
|
|
|
if when:
|
|
|
|
f.write(" {:10d} {:10d} {:10s} {}\n"
|
|
|
|
.format(path_size['unpacked'][pathname],
|
|
|
|
size,
|
|
|
|
datestr(when),
|
|
|
|
pathname))
|
|
|
|
|
|
|
|
with open(os.path.join(reportdir, "path-all-sizes.txt"), 'w') as f:
|
2019-03-17 07:39:39 +01:00
|
|
|
f.write("=== %s ===\n" % _("All paths by reverse accumulated size"))
|
|
|
|
f.write(_("Format: unpacked size, packed size, date deleted, pathectory name\n"))
|
2019-04-01 23:36:57 +02:00
|
|
|
for pathname, size in sorted(path_size['packed'].items(),
|
2019-04-01 19:58:16 +02:00
|
|
|
key=lambda x:(x[1],x[0]), reverse=True):
|
2018-12-26 06:54:16 +01:00
|
|
|
when = stats['file_deletions'].get(pathname, None)
|
2018-11-15 21:04:44 +01:00
|
|
|
f.write(" {:10d} {:10d} {:10s} {}\n"
|
|
|
|
.format(path_size['unpacked'][pathname],
|
|
|
|
size,
|
|
|
|
datestr(when),
|
|
|
|
pathname))
|
2018-11-15 17:09:39 +01:00
|
|
|
|
2018-12-26 06:54:16 +01:00
|
|
|
# List of filenames and sizes in descending order
|
|
|
|
with open(os.path.join(reportdir, "blob-shas-and-paths.txt"), 'w') as f:
|
2019-03-17 07:39:39 +01:00
|
|
|
f.write("=== %s ===\n" % _("Files by sha and associated pathnames in reverse size"))
|
|
|
|
f.write(_("Format: sha, unpacked size, packed size, filename(s) object stored as\n"))
|
2019-04-01 23:36:57 +02:00
|
|
|
for sha, size in sorted(stats['packed_size'].items(),
|
2019-04-01 19:58:16 +02:00
|
|
|
key=lambda x:(x[1],x[0]), reverse=True):
|
2018-12-26 06:54:16 +01:00
|
|
|
if sha not in stats['names']:
|
|
|
|
# Some objects in the repository might not be referenced, or not
|
|
|
|
# referenced by the branches/tags the user cares about; skip them.
|
|
|
|
continue
|
|
|
|
names_with_sha = stats['names'][sha]
|
|
|
|
if len(names_with_sha) == 1:
|
|
|
|
names_with_sha = names_with_sha.pop()
|
|
|
|
else:
|
|
|
|
names_with_sha = sorted(list(names_with_sha))
|
|
|
|
f.write(" {} {:10d} {:10d} {}\n".format(sha,
|
|
|
|
stats['unpacked_size'][sha],
|
|
|
|
size,
|
|
|
|
names_with_sha))
|
2018-12-14 19:30:44 +01:00
|
|
|
|
2018-12-26 06:54:16 +01:00
|
|
|
@staticmethod
|
2018-12-25 08:02:03 +01:00
|
|
|
def run(args):
|
2019-01-08 20:02:53 +01:00
|
|
|
git_dir = GitUtils.determine_git_dir('.')
|
2018-12-25 08:02:03 +01:00
|
|
|
|
2018-12-26 06:54:16 +01:00
|
|
|
# Create the report directory as necessary
|
|
|
|
results_tmp_dir = os.path.join(git_dir, 'filter-repo')
|
|
|
|
if not os.path.isdir(results_tmp_dir):
|
|
|
|
os.mkdir(results_tmp_dir)
|
|
|
|
reportdir = os.path.join(results_tmp_dir, "analysis")
|
|
|
|
if not args.force and os.path.isdir(reportdir):
|
2019-02-14 22:00:22 +01:00
|
|
|
shutil.rmtree(reportdir)
|
2018-12-26 06:54:16 +01:00
|
|
|
os.mkdir(reportdir)
|
|
|
|
|
|
|
|
# Gather the data we need
|
|
|
|
stats = RepoAnalyze.gather_data(args)
|
|
|
|
|
|
|
|
# Write the reports
|
2019-03-17 07:39:39 +01:00
|
|
|
sys.stdout.write(_("Writing reports to %s...") % reportdir)
|
2018-12-26 06:54:16 +01:00
|
|
|
sys.stdout.flush()
|
|
|
|
RepoAnalyze.write_report(reportdir, stats)
|
2019-03-17 07:39:39 +01:00
|
|
|
sys.stdout.write(_("done.\n"))
|
filter-repo: add --analyze option
This option walks through the repository history and creates a report
with basic statistics, rename related information, and sizes of objects
and when/if those have been deleted. It primarily looks at unpacked
sizes (i.e. size of object ignoring delta-ing and compression), and
sums the size of each version of the file for each path. Additionally,
it aggregates these sums by extension and by directory, and tracks
whether paths, extensions, and directories have been deleted. This can
be very useful in determining what the big things are, and whether they
might have been considered to have been mistakes to add to the
repository in the first place.
There are numerous caveats with the determination of "deleted" and
"renamed", and can give both false positives and false negatives. But
they are only meant as a helpful heuristic to give others a starting
point for an investigation, and the information provide so far is useful.
I do want to improve the equivalence classes (rename handling), but that
is for a future commit.
Signed-off-by: Elijah Newren <newren@gmail.com>
2018-10-21 16:44:33 +02:00
|
|
|
|
2018-08-29 19:26:38 +02:00
|
|
|
class InputFileBackup:
|
|
|
|
def __init__(self, input_file, output_file):
|
|
|
|
self.input_file = input_file
|
|
|
|
self.output_file = output_file
|
|
|
|
|
|
|
|
def read(self, size):
|
|
|
|
output = self.input_file.read(size)
|
|
|
|
self.output_file.write(output)
|
|
|
|
return output
|
|
|
|
|
|
|
|
def readline(self):
|
|
|
|
line = self.input_file.readline()
|
|
|
|
self.output_file.write(line)
|
|
|
|
return line
|
|
|
|
|
2018-08-29 19:27:30 +02:00
|
|
|
class DualFileWriter:
|
|
|
|
def __init__(self, file1, file2):
|
|
|
|
self.file1 = file1
|
|
|
|
self.file2 = file2
|
|
|
|
|
|
|
|
def write(self, *args):
|
|
|
|
self.file1.write(*args)
|
|
|
|
self.file2.write(*args)
|
|
|
|
|
2019-02-21 00:46:26 +01:00
|
|
|
def flush(self):
|
|
|
|
self.file1.flush()
|
|
|
|
self.file2.flush()
|
|
|
|
|
2018-08-29 19:27:30 +02:00
|
|
|
def close(self):
|
|
|
|
self.file1.close()
|
|
|
|
self.file2.close()
|
|
|
|
|
2018-12-25 08:02:03 +01:00
|
|
|
class RepoFilter(object):
|
2019-01-07 22:46:48 +01:00
|
|
|
def __init__(self,
|
|
|
|
args,
|
2019-02-16 22:25:04 +01:00
|
|
|
filename_callback = None,
|
|
|
|
message_callback = None,
|
2019-02-17 08:59:43 +01:00
|
|
|
name_callback = None,
|
2019-02-16 22:25:04 +01:00
|
|
|
email_callback = None,
|
|
|
|
refname_callback = None,
|
2019-01-07 22:46:48 +01:00
|
|
|
blob_callback = None,
|
|
|
|
commit_callback = None,
|
|
|
|
tag_callback = None,
|
|
|
|
reset_callback = None,
|
|
|
|
everything_callback = None):
|
2019-02-16 22:25:04 +01:00
|
|
|
|
2019-01-07 22:46:48 +01:00
|
|
|
self._args = args
|
2019-02-16 22:25:04 +01:00
|
|
|
|
|
|
|
# Store callbacks for acting on objects printed by FastExport
|
|
|
|
self._blob_callback = blob_callback
|
|
|
|
self._commit_callback = commit_callback
|
|
|
|
self._tag_callback = tag_callback
|
|
|
|
self._reset_callback = reset_callback
|
|
|
|
self._everything_callback = everything_callback # {blob,commit,tag,reset}
|
|
|
|
|
|
|
|
# Store callbacks for acting on slices of FastExport objects
|
2019-02-17 08:59:43 +01:00
|
|
|
self._filename_callback = filename_callback # filenames from commits
|
|
|
|
self._message_callback = message_callback # commit OR tag message
|
|
|
|
self._name_callback = name_callback # author, committer, tagger
|
|
|
|
self._email_callback = email_callback # author, committer, tagger
|
|
|
|
self._refname_callback = refname_callback # from commit/tag/reset
|
|
|
|
self._handle_arg_callbacks()
|
2019-01-07 22:46:48 +01:00
|
|
|
|
|
|
|
# Defaults for input
|
|
|
|
self._input = None
|
|
|
|
self._fep = None # Fast Export Process
|
|
|
|
self._fe_orig = None # Path to where original fast-export output stored
|
|
|
|
self._fe_filt = None # Path to where filtered fast-export output stored
|
|
|
|
|
|
|
|
# Defaults for output
|
|
|
|
self._output = None
|
|
|
|
self._fip = None # Fast Import Process
|
|
|
|
self._import_pipes = None
|
|
|
|
self._managed_output = True
|
|
|
|
|
2019-01-17 02:41:32 +01:00
|
|
|
# Other vars
|
|
|
|
self._sanity_checks_handled = False
|
|
|
|
self._orig_refs = None
|
2019-02-12 02:36:57 +01:00
|
|
|
self._newnames = {}
|
2019-01-17 02:41:32 +01:00
|
|
|
|
2019-02-17 08:59:43 +01:00
|
|
|
def _handle_arg_callbacks(self):
|
|
|
|
def make_callback(argname, str):
|
2019-05-01 08:46:05 +02:00
|
|
|
exec('def callback({}):\n'.format(argname)+
|
|
|
|
' '+'\n '.join(str.splitlines()), globals())
|
2019-02-17 08:59:43 +01:00
|
|
|
return callback #namespace['callback']
|
|
|
|
def handle(type):
|
|
|
|
callback_field = '_{}_callback'.format(type)
|
|
|
|
code_string = getattr(self._args, type+'_callback')
|
|
|
|
if code_string:
|
|
|
|
if getattr(self, callback_field):
|
2019-03-17 07:39:39 +01:00
|
|
|
raise SystemExit(_("Error: Cannot pass a %s_callback to RepoFilter "
|
|
|
|
"AND pass --%s-callback"
|
|
|
|
% (type, type)))
|
2019-02-17 08:59:43 +01:00
|
|
|
if 'return ' not in code_string and \
|
|
|
|
type not in ('blob', 'commit', 'tag', 'reset'):
|
2019-03-17 07:39:39 +01:00
|
|
|
raise SystemExit(_("Error: --%s-callback should have a return statement")
|
|
|
|
% type)
|
2019-02-17 08:59:43 +01:00
|
|
|
setattr(self, callback_field, make_callback(type, code_string))
|
|
|
|
handle('filename')
|
|
|
|
handle('message')
|
|
|
|
handle('name')
|
|
|
|
handle('email')
|
|
|
|
handle('refname')
|
|
|
|
handle('blob')
|
|
|
|
handle('commit')
|
|
|
|
handle('tag')
|
|
|
|
handle('reset')
|
|
|
|
|
2019-01-17 02:41:32 +01:00
|
|
|
def _run_sanity_checks(self):
|
|
|
|
self._sanity_checks_handled = True
|
|
|
|
if not self._managed_output:
|
|
|
|
return
|
|
|
|
|
|
|
|
if self._args.debug:
|
|
|
|
print("[DEBUG] Passed arguments:\n{}".format(self._args))
|
|
|
|
|
|
|
|
# Determine basic repository information
|
2019-01-08 20:02:53 +01:00
|
|
|
target_working_dir = self._args.target or '.'
|
|
|
|
self._orig_refs = GitUtils.get_refs(target_working_dir)
|
|
|
|
is_bare = GitUtils.is_repository_bare(target_working_dir)
|
2019-01-17 02:41:32 +01:00
|
|
|
|
|
|
|
# Do sanity checks from the correct directory
|
2019-02-14 21:45:51 +01:00
|
|
|
tmp_dir = self.results_tmp_dir(create_if_missing=False)
|
|
|
|
if not self._args.force and \
|
|
|
|
not os.path.isfile(os.path.join(tmp_dir, 'already_ran')):
|
2019-01-08 20:02:53 +01:00
|
|
|
cwd = os.getcwd()
|
|
|
|
os.chdir(target_working_dir)
|
2019-01-17 02:41:32 +01:00
|
|
|
RepoFilter.sanity_check(self._orig_refs, is_bare)
|
2019-01-08 20:02:53 +01:00
|
|
|
os.chdir(cwd)
|
2019-01-17 02:41:32 +01:00
|
|
|
|
2018-12-25 08:02:03 +01:00
|
|
|
@staticmethod
|
|
|
|
def sanity_check(refs, is_bare):
|
|
|
|
def abort(reason):
|
|
|
|
raise SystemExit(
|
2019-03-17 07:39:39 +01:00
|
|
|
_("Aborting: Refusing to overwrite repo history since this does not\n"
|
|
|
|
"look like a fresh clone.\n"
|
|
|
|
" (%s)\n"
|
|
|
|
"To override, use --force.") % reason)
|
2018-12-25 08:02:03 +01:00
|
|
|
|
|
|
|
# Make sure repo is fully packed, just like a fresh clone would be
|
|
|
|
output = subprocess.check_output('git count-objects -v'.split())
|
|
|
|
stats = dict(x.split(': ') for x in output.splitlines())
|
2019-01-16 20:39:23 +01:00
|
|
|
num_packs = int(stats['packs'])
|
|
|
|
if stats['count'] != '0' or num_packs > 1:
|
2019-03-17 07:39:39 +01:00
|
|
|
abort(_("expected freshly packed repo"))
|
2018-12-25 08:02:03 +01:00
|
|
|
|
2019-01-16 20:39:23 +01:00
|
|
|
# Make sure there is precisely one remote, named "origin"...or that this
|
|
|
|
# is a new bare repo with no packs and no remotes
|
2018-12-25 08:02:03 +01:00
|
|
|
output = subprocess.check_output('git remote'.split()).strip()
|
2019-01-16 20:39:23 +01:00
|
|
|
if not (output == "origin" or (num_packs == 0 and not output)):
|
2019-03-17 07:39:39 +01:00
|
|
|
abort(_("expected one remote, origin"))
|
2018-12-25 08:02:03 +01:00
|
|
|
|
|
|
|
# Avoid letting people running with weird setups and overwriting GIT_DIR
|
|
|
|
# elsewhere
|
2019-01-08 20:02:53 +01:00
|
|
|
git_dir = GitUtils.determine_git_dir('.')
|
2018-12-25 08:02:03 +01:00
|
|
|
if is_bare and git_dir != '.':
|
2019-03-17 07:39:39 +01:00
|
|
|
abort(_("GIT_DIR must be ."))
|
2018-12-25 08:02:03 +01:00
|
|
|
elif not is_bare and git_dir != '.git':
|
2019-03-17 07:39:39 +01:00
|
|
|
abort(_("GIT_DIR must be .git"))
|
2018-12-25 08:02:03 +01:00
|
|
|
|
|
|
|
# Make sure that all reflogs have precisely one entry
|
|
|
|
reflog_dir=os.path.join(git_dir, 'logs')
|
|
|
|
for root, dirs, files in os.walk(reflog_dir):
|
|
|
|
for filename in files:
|
|
|
|
pathname = os.path.join(root, filename)
|
|
|
|
with open(pathname) as f:
|
|
|
|
if len(f.read().splitlines()) > 1:
|
|
|
|
shortpath = pathname[len(reflog_dir)+1:]
|
2019-03-17 07:39:39 +01:00
|
|
|
abort(_("expected at most one entry in the reflog for %s") %
|
|
|
|
shortpath)
|
2018-12-25 08:02:03 +01:00
|
|
|
|
|
|
|
# Make sure there are no stashed changes
|
|
|
|
if 'refs/stash' in refs:
|
2019-03-17 07:39:39 +01:00
|
|
|
abort(_("has stashed changes"))
|
2018-12-25 08:02:03 +01:00
|
|
|
|
|
|
|
# Do extra checks in non-bare repos
|
|
|
|
if not is_bare:
|
|
|
|
# Avoid uncommitted, unstaged, or untracked changes
|
2019-03-14 01:38:03 +01:00
|
|
|
if subprocess.call('git diff --staged --quiet'.split()):
|
2019-03-17 07:39:39 +01:00
|
|
|
abort(_("you have uncommitted changes"))
|
2018-12-25 08:02:03 +01:00
|
|
|
if subprocess.call('git diff --quiet'.split()):
|
2019-03-17 07:39:39 +01:00
|
|
|
abort(_("you have unstaged changes"))
|
2018-12-25 08:02:03 +01:00
|
|
|
if len(subprocess.check_output('git ls-files -o'.split())) > 0:
|
2019-03-17 07:39:39 +01:00
|
|
|
abort(_("you have untracked changes"))
|
2018-12-25 08:02:03 +01:00
|
|
|
|
|
|
|
# Avoid unpushed changes
|
2019-04-01 23:36:57 +02:00
|
|
|
for refname, rev in refs.items():
|
2018-12-25 08:02:03 +01:00
|
|
|
if not refname.startswith('refs/heads/'):
|
|
|
|
continue
|
|
|
|
origin_ref = refname.replace('refs/heads/', 'refs/remotes/origin/')
|
|
|
|
if origin_ref not in refs:
|
2019-03-17 07:39:39 +01:00
|
|
|
abort(_('%s exists, but %s not found') % (refname, origin_ref))
|
2018-12-25 08:02:03 +01:00
|
|
|
if rev != refs[origin_ref]:
|
2019-03-17 07:39:39 +01:00
|
|
|
abort(_('%s does not match %s') % (refname, origin_ref))
|
2018-08-29 05:58:38 +02:00
|
|
|
|
filter-repo: add text removal (or replacement) via file of expressions
Make it easy for users to search and replace text throughout the
repository history. Instead of inventing some new syntax, reuse the
same syntax used by BFG repo filter's --replace-text option, namely,
a file with one expression per line of the form
[regex:|glob:|literal:]$MATCH_EXPR[==>$REPLACEMENT_EXPR]
Where "$MATCH_EXPR" is by default considered to be literal text, but
could be a regex or a glob if the appropriate prefix is used. Also,
$REPLACEMENT_EXPR defaults to '***REMOVED***' if not specified. If
you want a literal '==>' to be part of your $MATCH_EXPR, then you
must also manually specify a replacement expression instead of taking
the default. Some examples:
sup3rs3kr3t
(replaces 'sup3rs3kr3t' with '***REMOVED***')
HeWhoShallNotBeNamed==>Voldemort
(replaces 'HeWhoShallNotBeNamed' with 'Voldemort')
very==>
(replaces 'very' with the empty string)
regex:(\d{2})/(\d{2})/(\d{4})==>\2/\1/\3
(replaces '05/17/2012' with '17/05/2012', and vice-versa)
The format for regex is as from
re.sub(<pattern>, <repl>, <string>) from
https://docs.python.org/2/library/re.html
The <string> comes from file contents of the repo, and you specify
the <pattern> and <repl>.
glob:Copy*t==>Cartel
(replaces 'Copyright' or 'Copyleft' or 'Copy my st' with 'Cartel')
Signed-off-by: Elijah Newren <newren@gmail.com>
2019-01-30 09:56:22 +01:00
|
|
|
@staticmethod
|
|
|
|
def tweak_blob(args, blob):
|
|
|
|
if args.replace_text:
|
|
|
|
for literal, replacement in args.replace_text['literals']:
|
|
|
|
blob.data = blob.data.replace(literal, replacement)
|
|
|
|
for regex, replacement in args.replace_text['regexes']:
|
|
|
|
blob.data = regex.sub(replacement, blob.data)
|
|
|
|
|
2019-02-16 20:25:17 +01:00
|
|
|
def tweak_commit(self, commit):
|
2018-12-25 08:02:03 +01:00
|
|
|
def filename_matches(path_expression, pathname):
|
|
|
|
if path_expression == '':
|
|
|
|
return True
|
|
|
|
n = len(path_expression)
|
|
|
|
if (pathname.startswith(path_expression) and
|
|
|
|
(path_expression[n-1] == '/' or
|
|
|
|
len(pathname) == n or
|
|
|
|
pathname[n] == '/')):
|
|
|
|
return True
|
|
|
|
return False
|
2018-08-29 05:58:38 +02:00
|
|
|
|
2018-12-25 08:02:03 +01:00
|
|
|
def newname(path_changes, pathname, filtering_is_inclusive):
|
|
|
|
wanted = False
|
|
|
|
for (mod_type, match_type, path_exp) in path_changes:
|
|
|
|
if mod_type == 'filter' and not wanted:
|
|
|
|
assert match_type in ('match', 'glob', 'regex')
|
|
|
|
if match_type == 'match' and filename_matches(path_exp, pathname):
|
|
|
|
wanted = True
|
|
|
|
if match_type == 'glob' and fnmatch.fnmatch(pathname, path_exp):
|
|
|
|
wanted = True
|
2019-02-12 02:35:27 +01:00
|
|
|
if match_type == 'regex' and path_exp.search(pathname):
|
2018-12-25 08:02:03 +01:00
|
|
|
wanted = True
|
|
|
|
elif mod_type == 'rename':
|
|
|
|
old_exp, new_exp = path_exp.split(':')
|
|
|
|
assert match_type in ('prefix',)
|
|
|
|
if match_type == 'prefix' and pathname.startswith(old_exp):
|
|
|
|
pathname = pathname.replace(old_exp, new_exp, 1)
|
|
|
|
return pathname if (wanted == filtering_is_inclusive) else None
|
|
|
|
|
2019-02-16 22:25:04 +01:00
|
|
|
# Change the commit message according to callback
|
|
|
|
if self._message_callback:
|
|
|
|
commit.message = self._message_callback(commit.message)
|
|
|
|
|
2019-01-29 02:26:49 +01:00
|
|
|
# Change the author & committer according to mailmap rules
|
2019-02-16 20:25:17 +01:00
|
|
|
args = self._args
|
2019-01-29 02:26:49 +01:00
|
|
|
if args.mailmap:
|
|
|
|
commit.author_name, commit.author_email = \
|
|
|
|
args.mailmap.translate(commit.author_name, commit.author_email)
|
|
|
|
commit.committer_name, commit.committer_email = \
|
|
|
|
args.mailmap.translate(commit.committer_name, commit.committer_email)
|
2019-02-16 22:25:04 +01:00
|
|
|
# Change author & committer according to callbacks
|
2019-02-17 08:59:43 +01:00
|
|
|
if self._name_callback:
|
|
|
|
commit.author_name = self._name_callback(commit.author_name)
|
|
|
|
commit.committer_name = self._name_callback(commit.committer_name)
|
2019-02-16 22:25:04 +01:00
|
|
|
if self._email_callback:
|
|
|
|
commit.author_email = self._email_callback(commit.author_email)
|
|
|
|
commit.committer_email = self._email_callback(commit.committer_email)
|
2019-01-29 02:26:49 +01:00
|
|
|
|
2018-12-25 08:02:03 +01:00
|
|
|
# Sometimes the 'branch' given is a tag; if so, rename it as requested so
|
|
|
|
# we don't get any old tagnames
|
2019-02-16 20:25:17 +01:00
|
|
|
if self._args.tag_rename:
|
|
|
|
commit.branch = RepoFilter.do_tag_rename(args.tag_rename, commit.branch)
|
2019-02-16 22:25:04 +01:00
|
|
|
if self._refname_callback:
|
|
|
|
commit.branch = self._refname_callback(commit.branch)
|
2018-12-25 08:02:03 +01:00
|
|
|
|
2019-02-16 22:25:04 +01:00
|
|
|
# Filter or rename the list of file changes
|
2018-12-25 08:02:03 +01:00
|
|
|
new_file_changes = {}
|
|
|
|
for change in commit.file_changes:
|
2019-03-16 01:15:39 +01:00
|
|
|
# NEEDSWORK: _If_ we ever want to pass `--full-tree` to fast-export and
|
|
|
|
# parse that output, we'll need to modify this block; `--full-tree`
|
|
|
|
# issues a deleteall directive which has no filename, and thus this
|
|
|
|
# block would normally strip it. Of course, FileChanges() and
|
|
|
|
# _parse_optional_filechange() would need updates too.
|
2019-02-12 02:36:57 +01:00
|
|
|
if change.filename in self._newnames:
|
|
|
|
change.filename = self._newnames[change.filename]
|
|
|
|
else:
|
|
|
|
change.filename = newname(args.path_changes, change.filename,
|
|
|
|
args.inclusive)
|
2019-02-16 22:25:04 +01:00
|
|
|
if self._filename_callback:
|
|
|
|
change.filename = self._filename_callback(change.filename)
|
2019-02-12 02:36:57 +01:00
|
|
|
self._newnames[change.filename] = change.filename
|
2018-12-25 08:02:03 +01:00
|
|
|
if not change.filename:
|
|
|
|
continue # Filtering criteria excluded this file; move on to next one
|
|
|
|
if change.filename in new_file_changes:
|
|
|
|
# Getting here means that path renaming is in effect, and caused one
|
2019-03-05 20:00:40 +01:00
|
|
|
# path to collide with another. That's usually bad, but can be okay
|
|
|
|
# under two circumstances:
|
|
|
|
# 1) Sometimes people have a file named OLDFILE in old revisions of
|
|
|
|
# history, and they rename to NEWFILE, and would like to rewrite
|
|
|
|
# history so that all revisions refer to it as NEWFILE. As such,
|
|
|
|
# we can allow a collision when (at least) one of the two paths
|
|
|
|
# is a deletion. Note that if OLDFILE and NEWFILE are unrelated
|
|
|
|
# this also allows the rewrite to continue, which makes sense
|
|
|
|
# since OLDFILE is no longer in the way.
|
|
|
|
# 2) If OLDFILE and NEWFILE are exactly equal, then writing them
|
|
|
|
# both to the same location poses no problem; we only need one
|
|
|
|
# file. (This could come up if someone copied a file in some
|
|
|
|
# commit, then later either deleted the file or kept it exactly
|
|
|
|
# in sync with the original with any changes, and then decides
|
|
|
|
# they want to rewrite history to only have one of the two files)
|
|
|
|
colliding_change = new_file_changes[change.filename]
|
2018-12-25 08:02:03 +01:00
|
|
|
if change.type == 'D':
|
|
|
|
# We can just throw this one away and keep the other
|
|
|
|
continue
|
2019-03-05 20:00:40 +01:00
|
|
|
elif change.type == 'M' and (
|
|
|
|
change.mode == colliding_change.mode and
|
|
|
|
change.blob_id == colliding_change.blob_id):
|
|
|
|
# The two are identical, so we can throw this one away and keep other
|
|
|
|
continue
|
2018-12-25 08:02:03 +01:00
|
|
|
elif new_file_changes[change.filename].type != 'D':
|
2019-03-17 07:39:39 +01:00
|
|
|
raise SystemExit(_("File renaming caused colliding pathnames!\n") +
|
|
|
|
_(" Commit: {}\n").format(commit.original_id) +
|
|
|
|
_(" Filename: {}").format(change.filename))
|
2018-12-25 08:02:03 +01:00
|
|
|
new_file_changes[change.filename] = change
|
|
|
|
commit.file_changes = new_file_changes.values()
|
|
|
|
|
|
|
|
@staticmethod
|
2019-02-16 20:25:17 +01:00
|
|
|
def do_tag_rename(rename_pair, tagname):
|
|
|
|
old, new = rename_pair.split(':', 1)
|
|
|
|
old, new = 'refs/tags/'+old, 'refs/tags/'+new
|
2018-12-25 08:02:03 +01:00
|
|
|
if tagname.startswith(old):
|
|
|
|
return tagname.replace(old, new, 1)
|
|
|
|
return tagname
|
|
|
|
|
2019-02-16 20:25:17 +01:00
|
|
|
def handle_tag(self, tag):
|
2019-02-16 22:25:04 +01:00
|
|
|
# Tweak the tag message according to callbacks
|
|
|
|
if self._message_callback:
|
|
|
|
tag.message = self._message_callback(tag.message)
|
|
|
|
|
|
|
|
# Tweak the tag name according to callbacks
|
|
|
|
tag_prefix = 'refs/tags/'
|
|
|
|
fullref = tag_prefix+tag.ref
|
2019-02-16 20:25:17 +01:00
|
|
|
if self._args.tag_rename:
|
|
|
|
fullref = RepoFilter.do_tag_rename(self._args.tag_rename, fullref)
|
2019-02-16 22:25:04 +01:00
|
|
|
if self._refname_callback:
|
|
|
|
fullref = self._refname_callback(fullref)
|
|
|
|
if not fullref.startswith(tag_prefix):
|
|
|
|
msg = "Error: fast-import requires tags to be in refs/tags/ namespace."
|
|
|
|
msg += "\n {} renamed to {}".format(tag_prefix+tag.ref, fullref)
|
|
|
|
raise SystemExit(msg)
|
|
|
|
tag.ref = fullref[len(tag_prefix):]
|
|
|
|
|
|
|
|
# Tweak the tagger according to callbacks
|
2019-02-16 20:25:17 +01:00
|
|
|
if self._args.mailmap:
|
2019-01-29 02:26:49 +01:00
|
|
|
tag.tagger_name, tag.tagger_email = \
|
2019-02-16 20:25:17 +01:00
|
|
|
self._args.mailmap.translate(tag.tagger_name, tag.tagger_email)
|
2019-02-17 08:59:43 +01:00
|
|
|
if self._name_callback:
|
|
|
|
tag.tagger_name = self._name_callback(tag.tagger_name)
|
2019-02-16 22:25:04 +01:00
|
|
|
if self._email_callback:
|
|
|
|
tag.tagger_email = self._email_callback(tag.tagger_email)
|
2019-01-29 02:26:49 +01:00
|
|
|
|
2019-02-16 20:25:17 +01:00
|
|
|
def handle_reset(self, reset):
|
|
|
|
if self._args.tag_rename:
|
|
|
|
reset.ref = RepoFilter.do_tag_rename(self._args.tag_rename, reset.ref)
|
2019-02-16 22:25:04 +01:00
|
|
|
if self._refname_callback:
|
|
|
|
reset.ref = self._refname_callback(reset.ref)
|
2018-12-25 08:02:03 +01:00
|
|
|
|
2019-02-14 21:45:51 +01:00
|
|
|
def results_tmp_dir(self, create_if_missing=True):
|
2019-01-08 20:02:53 +01:00
|
|
|
working_dir = self._args.target or self._args.source or '.'
|
|
|
|
git_dir = GitUtils.determine_git_dir(working_dir)
|
2019-01-07 22:46:48 +01:00
|
|
|
d = os.path.join(git_dir, 'filter-repo')
|
2019-02-14 21:45:51 +01:00
|
|
|
if create_if_missing and not os.path.isdir(d):
|
2019-01-07 22:46:48 +01:00
|
|
|
os.mkdir(d)
|
|
|
|
return d
|
|
|
|
|
|
|
|
def importer_only(self):
|
2019-01-17 02:41:32 +01:00
|
|
|
self._run_sanity_checks()
|
2019-01-07 22:46:48 +01:00
|
|
|
self._setup_output()
|
|
|
|
|
|
|
|
def set_output(self, outputRepoFilter):
|
|
|
|
assert outputRepoFilter._output
|
|
|
|
|
|
|
|
# set_output implies this RepoFilter is doing exporting, though may not
|
|
|
|
# be the only one.
|
|
|
|
self._setup_input(use_done_feature = False)
|
|
|
|
|
|
|
|
# Set our output management up to pipe to outputRepoFilter's locations
|
|
|
|
self._managed_output = False
|
|
|
|
self._output = outputRepoFilter._output
|
|
|
|
self._import_pipes = outputRepoFilter._import_pipes
|
|
|
|
|
2019-01-17 02:41:32 +01:00
|
|
|
# Handle sanity checks, though currently none needed for export-only cases
|
|
|
|
self._run_sanity_checks()
|
|
|
|
|
2019-01-07 22:46:48 +01:00
|
|
|
def _setup_input(self, use_done_feature):
|
|
|
|
if self._args.stdin:
|
|
|
|
self._input = sys.stdin
|
|
|
|
self._fe_orig = None
|
2018-12-25 08:02:03 +01:00
|
|
|
else:
|
2019-01-08 20:02:53 +01:00
|
|
|
skip_blobs = (self._blob_callback is None and
|
|
|
|
self._everything_callback is None and
|
filter-repo: add text removal (or replacement) via file of expressions
Make it easy for users to search and replace text throughout the
repository history. Instead of inventing some new syntax, reuse the
same syntax used by BFG repo filter's --replace-text option, namely,
a file with one expression per line of the form
[regex:|glob:|literal:]$MATCH_EXPR[==>$REPLACEMENT_EXPR]
Where "$MATCH_EXPR" is by default considered to be literal text, but
could be a regex or a glob if the appropriate prefix is used. Also,
$REPLACEMENT_EXPR defaults to '***REMOVED***' if not specified. If
you want a literal '==>' to be part of your $MATCH_EXPR, then you
must also manually specify a replacement expression instead of taking
the default. Some examples:
sup3rs3kr3t
(replaces 'sup3rs3kr3t' with '***REMOVED***')
HeWhoShallNotBeNamed==>Voldemort
(replaces 'HeWhoShallNotBeNamed' with 'Voldemort')
very==>
(replaces 'very' with the empty string)
regex:(\d{2})/(\d{2})/(\d{4})==>\2/\1/\3
(replaces '05/17/2012' with '17/05/2012', and vice-versa)
The format for regex is as from
re.sub(<pattern>, <repl>, <string>) from
https://docs.python.org/2/library/re.html
The <string> comes from file contents of the repo, and you specify
the <pattern> and <repl>.
glob:Copy*t==>Cartel
(replaces 'Copyright' or 'Copyleft' or 'Copy my st' with 'Cartel')
Signed-off-by: Elijah Newren <newren@gmail.com>
2019-01-30 09:56:22 +01:00
|
|
|
self._args.replace_text is None and
|
2019-01-08 20:02:53 +01:00
|
|
|
self._args.source is None and
|
|
|
|
self._args.target is None)
|
2018-12-31 16:46:05 +01:00
|
|
|
extra_flags = ['--no-data'] if skip_blobs else []
|
2019-01-07 22:46:48 +01:00
|
|
|
done_feature = ['--use-done-feature'] if use_done_feature else []
|
2019-01-08 20:02:53 +01:00
|
|
|
location = ['-C', self._args.source] if self._args.source else []
|
|
|
|
fep_cmd = ['git'] + location + ['fast-export', '--show-original-ids',
|
2019-01-07 22:46:48 +01:00
|
|
|
'--signed-tags=strip', '--tag-of-filtered-object=rewrite'
|
|
|
|
] + done_feature + extra_flags + self._args.refs
|
|
|
|
self._fep = subprocess.Popen(fep_cmd, bufsize=-1, stdout=subprocess.PIPE)
|
|
|
|
self._input = self._fep.stdout
|
|
|
|
if self._args.dry_run or self._args.debug:
|
|
|
|
self._fe_orig = os.path.join(self.results_tmp_dir(),
|
|
|
|
'fast-export.original')
|
|
|
|
output = open(self._fe_orig, 'w')
|
|
|
|
self._input = InputFileBackup(self._input, output)
|
|
|
|
if self._args.debug:
|
2018-12-25 08:02:03 +01:00
|
|
|
print("[DEBUG] Running: {}".format(' '.join(fep_cmd)))
|
2019-01-07 22:46:48 +01:00
|
|
|
print(" (saving a copy of the output at {})".format(self._fe_orig))
|
2018-12-25 08:02:03 +01:00
|
|
|
|
2019-01-07 22:46:48 +01:00
|
|
|
def _setup_output(self):
|
|
|
|
if not self._args.dry_run:
|
2019-01-08 20:02:53 +01:00
|
|
|
location = ['-C', self._args.target] if self._args.target else []
|
|
|
|
fip_cmd = ['git'] + location + 'fast-import --force --quiet'.split()
|
2019-01-07 22:46:48 +01:00
|
|
|
self._fip = subprocess.Popen(fip_cmd,
|
|
|
|
bufsize=-1,
|
|
|
|
stdin=subprocess.PIPE,
|
|
|
|
stdout=subprocess.PIPE)
|
|
|
|
self._import_pipes = (self._fip.stdin, self._fip.stdout)
|
|
|
|
if self._args.dry_run or self._args.debug:
|
|
|
|
self._fe_filt = os.path.join(self.results_tmp_dir(),
|
|
|
|
'fast-export.filtered')
|
|
|
|
self._output = open(self._fe_filt, 'w')
|
2018-12-25 08:02:03 +01:00
|
|
|
else:
|
2019-01-07 22:46:48 +01:00
|
|
|
self._output = self._fip.stdin
|
|
|
|
if self._args.debug:
|
|
|
|
self._output = DualFileWriter(self._fip.stdin, self._output)
|
2018-12-25 08:02:03 +01:00
|
|
|
print("[DEBUG] Running: {}".format(' '.join(fip_cmd)))
|
2019-01-07 22:46:48 +01:00
|
|
|
print(" (using the following file as input: {})".format(self._fe_filt))
|
|
|
|
|
2019-02-02 16:29:43 +01:00
|
|
|
def _migrate_origin_to_heads(self):
|
|
|
|
if self._args.dry_run:
|
|
|
|
return
|
|
|
|
refs_to_migrate = set(x for x in self._orig_refs
|
|
|
|
if x.startswith('refs/remotes/origin/'))
|
|
|
|
if not refs_to_migrate:
|
|
|
|
return
|
|
|
|
if self._args.debug:
|
|
|
|
print("[DEBUG] Migrating refs/remotes/origin/* -> refs/heads/*")
|
|
|
|
target_working_dir = self._args.target or '.'
|
|
|
|
p = subprocess.Popen('git update-ref --no-deref --stdin'.split(),
|
|
|
|
stdin=subprocess.PIPE,
|
|
|
|
cwd=target_working_dir)
|
|
|
|
for ref in refs_to_migrate:
|
|
|
|
if ref == 'refs/remotes/origin/HEAD':
|
|
|
|
p.stdin.write('delete {} {}\n'.format(ref, self._orig_refs[ref]))
|
|
|
|
del self._orig_refs[ref]
|
|
|
|
continue
|
|
|
|
newref = ref.replace('refs/remotes/origin/', 'refs/heads/')
|
|
|
|
if newref not in self._orig_refs:
|
|
|
|
p.stdin.write('create {} {}\n'.format(newref, self._orig_refs[ref]))
|
|
|
|
p.stdin.write('delete {} {}\n'.format(ref, self._orig_refs[ref]))
|
|
|
|
self._orig_refs[newref] = self._orig_refs[ref]
|
|
|
|
del self._orig_refs[ref]
|
|
|
|
p.stdin.close()
|
|
|
|
if p.wait():
|
2019-03-17 07:39:39 +01:00
|
|
|
raise SystemExit(_("git update-ref failed; see above")) # pragma: no cover
|
2019-02-02 16:29:43 +01:00
|
|
|
|
|
|
|
# Now remove
|
|
|
|
if self._args.debug:
|
|
|
|
print("[DEBUG] Removing 'origin' remote (rewritten history will no ")
|
|
|
|
print(" longer be related; consider re-pushing it elsewhere.")
|
|
|
|
subprocess.call('git remote rm origin'.split(), cwd=target_working_dir)
|
|
|
|
|
2019-01-09 20:25:37 +01:00
|
|
|
def finish(self):
|
|
|
|
''' Alternative to run() when there is no input of our own to parse,
|
|
|
|
meaning that run only really needs to close the handle to fast-import
|
|
|
|
and let it finish, thus making a call to "run" feel like a misnomer. '''
|
|
|
|
assert not self._input
|
|
|
|
assert self._managed_output
|
|
|
|
self.run()
|
|
|
|
|
2019-01-07 22:46:48 +01:00
|
|
|
def run(self):
|
2019-02-04 23:03:06 +01:00
|
|
|
start = time.time()
|
2019-01-07 22:46:48 +01:00
|
|
|
if not self._input and not self._output:
|
2019-01-17 02:41:32 +01:00
|
|
|
self._run_sanity_checks()
|
2019-02-02 16:29:43 +01:00
|
|
|
self._migrate_origin_to_heads()
|
2019-01-07 22:46:48 +01:00
|
|
|
self._setup_input(use_done_feature = True)
|
|
|
|
self._setup_output()
|
2019-01-17 02:41:32 +01:00
|
|
|
assert self._sanity_checks_handled
|
2019-01-07 22:46:48 +01:00
|
|
|
|
|
|
|
if self._input:
|
|
|
|
# Set up the callbacks
|
filter-repo: add text removal (or replacement) via file of expressions
Make it easy for users to search and replace text throughout the
repository history. Instead of inventing some new syntax, reuse the
same syntax used by BFG repo filter's --replace-text option, namely,
a file with one expression per line of the form
[regex:|glob:|literal:]$MATCH_EXPR[==>$REPLACEMENT_EXPR]
Where "$MATCH_EXPR" is by default considered to be literal text, but
could be a regex or a glob if the appropriate prefix is used. Also,
$REPLACEMENT_EXPR defaults to '***REMOVED***' if not specified. If
you want a literal '==>' to be part of your $MATCH_EXPR, then you
must also manually specify a replacement expression instead of taking
the default. Some examples:
sup3rs3kr3t
(replaces 'sup3rs3kr3t' with '***REMOVED***')
HeWhoShallNotBeNamed==>Voldemort
(replaces 'HeWhoShallNotBeNamed' with 'Voldemort')
very==>
(replaces 'very' with the empty string)
regex:(\d{2})/(\d{2})/(\d{4})==>\2/\1/\3
(replaces '05/17/2012' with '17/05/2012', and vice-versa)
The format for regex is as from
re.sub(<pattern>, <repl>, <string>) from
https://docs.python.org/2/library/re.html
The <string> comes from file contents of the repo, and you specify
the <pattern> and <repl>.
glob:Copy*t==>Cartel
(replaces 'Copyright' or 'Copyleft' or 'Copy my st' with 'Cartel')
Signed-off-by: Elijah Newren <newren@gmail.com>
2019-01-30 09:56:22 +01:00
|
|
|
def combined_blob_callback(b):
|
|
|
|
RepoFilter.tweak_blob(self._args, b)
|
|
|
|
self._blob_callback and self._blob_callback(b)
|
2019-01-07 22:46:48 +01:00
|
|
|
def actual_commit_callback(c):
|
2019-02-16 20:25:17 +01:00
|
|
|
self.tweak_commit(c)
|
2019-01-07 22:46:48 +01:00
|
|
|
self._commit_callback and self._commit_callback(c)
|
|
|
|
def actual_tag_callback(t):
|
2019-02-16 20:25:17 +01:00
|
|
|
self.handle_tag(t)
|
2019-01-07 22:46:48 +01:00
|
|
|
self._tag_callback and self._tag_callback(t)
|
|
|
|
def actual_reset_callback(r):
|
2019-02-16 20:25:17 +01:00
|
|
|
self.handle_reset(r)
|
2019-01-07 22:46:48 +01:00
|
|
|
self._reset_callback and self._reset_callback(r)
|
filter-repo: add text removal (or replacement) via file of expressions
Make it easy for users to search and replace text throughout the
repository history. Instead of inventing some new syntax, reuse the
same syntax used by BFG repo filter's --replace-text option, namely,
a file with one expression per line of the form
[regex:|glob:|literal:]$MATCH_EXPR[==>$REPLACEMENT_EXPR]
Where "$MATCH_EXPR" is by default considered to be literal text, but
could be a regex or a glob if the appropriate prefix is used. Also,
$REPLACEMENT_EXPR defaults to '***REMOVED***' if not specified. If
you want a literal '==>' to be part of your $MATCH_EXPR, then you
must also manually specify a replacement expression instead of taking
the default. Some examples:
sup3rs3kr3t
(replaces 'sup3rs3kr3t' with '***REMOVED***')
HeWhoShallNotBeNamed==>Voldemort
(replaces 'HeWhoShallNotBeNamed' with 'Voldemort')
very==>
(replaces 'very' with the empty string)
regex:(\d{2})/(\d{2})/(\d{4})==>\2/\1/\3
(replaces '05/17/2012' with '17/05/2012', and vice-versa)
The format for regex is as from
re.sub(<pattern>, <repl>, <string>) from
https://docs.python.org/2/library/re.html
The <string> comes from file contents of the repo, and you specify
the <pattern> and <repl>.
glob:Copy*t==>Cartel
(replaces 'Copyright' or 'Copyleft' or 'Copy my st' with 'Cartel')
Signed-off-by: Elijah Newren <newren@gmail.com>
2019-01-30 09:56:22 +01:00
|
|
|
actual_blob_callback = self._blob_callback
|
|
|
|
if self._args.replace_text:
|
|
|
|
actual_blob_callback = combined_blob_callback
|
2019-01-07 22:46:48 +01:00
|
|
|
|
|
|
|
# Create and run the filter
|
2019-01-19 03:30:42 +01:00
|
|
|
fef = FastExportFilter(self._args.source or '.',
|
filter-repo: add text removal (or replacement) via file of expressions
Make it easy for users to search and replace text throughout the
repository history. Instead of inventing some new syntax, reuse the
same syntax used by BFG repo filter's --replace-text option, namely,
a file with one expression per line of the form
[regex:|glob:|literal:]$MATCH_EXPR[==>$REPLACEMENT_EXPR]
Where "$MATCH_EXPR" is by default considered to be literal text, but
could be a regex or a glob if the appropriate prefix is used. Also,
$REPLACEMENT_EXPR defaults to '***REMOVED***' if not specified. If
you want a literal '==>' to be part of your $MATCH_EXPR, then you
must also manually specify a replacement expression instead of taking
the default. Some examples:
sup3rs3kr3t
(replaces 'sup3rs3kr3t' with '***REMOVED***')
HeWhoShallNotBeNamed==>Voldemort
(replaces 'HeWhoShallNotBeNamed' with 'Voldemort')
very==>
(replaces 'very' with the empty string)
regex:(\d{2})/(\d{2})/(\d{4})==>\2/\1/\3
(replaces '05/17/2012' with '17/05/2012', and vice-versa)
The format for regex is as from
re.sub(<pattern>, <repl>, <string>) from
https://docs.python.org/2/library/re.html
The <string> comes from file contents of the repo, and you specify
the <pattern> and <repl>.
glob:Copy*t==>Cartel
(replaces 'Copyright' or 'Copyleft' or 'Copy my st' with 'Cartel')
Signed-off-by: Elijah Newren <newren@gmail.com>
2019-01-30 09:56:22 +01:00
|
|
|
blob_callback = actual_blob_callback,
|
2019-01-19 03:30:42 +01:00
|
|
|
commit_callback = actual_commit_callback,
|
|
|
|
tag_callback = actual_tag_callback,
|
|
|
|
reset_callback = actual_reset_callback,
|
|
|
|
everything_callback = self._everything_callback)
|
|
|
|
fef.run(self._input,
|
|
|
|
self._output,
|
|
|
|
fast_import_pipes = self._import_pipes,
|
|
|
|
quiet = self._args.quiet)
|
2019-01-07 22:46:48 +01:00
|
|
|
|
|
|
|
# Make sure fast-export completed successfully
|
|
|
|
if not self._args.stdin and self._fep.wait():
|
2019-03-17 07:39:39 +01:00
|
|
|
raise SystemExit(_("Error: fast-export failed; see above.")) # pragma: no cover
|
2019-01-07 22:46:48 +01:00
|
|
|
|
|
|
|
# If we're not the manager of self._output, we should avoid post-run cleanup
|
|
|
|
if not self._managed_output:
|
|
|
|
return
|
|
|
|
|
|
|
|
# Close the output and ensure fast-import successfully completes
|
|
|
|
self._output.close()
|
|
|
|
if not self._args.dry_run and self._fip.wait():
|
2019-03-17 07:39:39 +01:00
|
|
|
raise SystemExit(_("Error: fast-import failed; see above.")) # pragma: no cover
|
2018-12-25 08:02:03 +01:00
|
|
|
|
2019-02-04 23:03:06 +01:00
|
|
|
# Notify user how long it took, before doing a gc and such
|
|
|
|
msg = "New history written in {:.2f} seconds; now repacking/cleaning..."
|
|
|
|
print(msg.format(time.time()-start))
|
|
|
|
|
2018-12-25 08:02:03 +01:00
|
|
|
# Exit early, if requested
|
2019-01-07 22:46:48 +01:00
|
|
|
if self._args.dry_run:
|
2019-03-17 07:39:39 +01:00
|
|
|
print(_("NOTE: Not running fast-import or cleaning up; --dry-run passed."))
|
2019-01-07 22:46:48 +01:00
|
|
|
if self._fe_orig:
|
2019-03-17 07:39:39 +01:00
|
|
|
print(_(" Requested filtering can be seen by comparing:"))
|
|
|
|
print(" " + self._fe_orig)
|
2019-01-07 22:46:48 +01:00
|
|
|
else:
|
2019-03-17 07:39:39 +01:00
|
|
|
print(_(" Requested filtering can be seen at:"))
|
2019-01-07 22:46:48 +01:00
|
|
|
print(" " + self._fe_filt)
|
2019-03-15 17:48:53 +01:00
|
|
|
return
|
2018-12-25 08:02:03 +01:00
|
|
|
|
2019-01-08 20:02:53 +01:00
|
|
|
target_working_dir = self._args.target or '.'
|
2019-01-07 22:46:48 +01:00
|
|
|
if self._input:
|
|
|
|
# Remove unused refs
|
2019-01-19 03:30:42 +01:00
|
|
|
refs_to_nuke = set(self._orig_refs) - set(fef.get_seen_refs())
|
2019-01-07 22:46:48 +01:00
|
|
|
if refs_to_nuke:
|
|
|
|
if self._args.debug:
|
|
|
|
print("[DEBUG] Deleting the following refs:\n "+
|
|
|
|
"\n ".join(refs_to_nuke))
|
|
|
|
p = subprocess.Popen('git update-ref --stdin'.split(),
|
2019-01-08 20:02:53 +01:00
|
|
|
stdin=subprocess.PIPE,
|
|
|
|
cwd=target_working_dir)
|
2019-01-07 22:46:48 +01:00
|
|
|
p.stdin.write(''.join(["option no-deref\ndelete {}\n".format(x)
|
|
|
|
for x in refs_to_nuke]))
|
|
|
|
p.stdin.close()
|
|
|
|
if p.wait():
|
2019-03-17 07:39:39 +01:00
|
|
|
raise SystemExit(_("git update-ref failed; see above")) # pragma: no cover
|
2019-01-07 22:46:48 +01:00
|
|
|
|
|
|
|
# Write out data about run
|
2019-01-19 03:30:42 +01:00
|
|
|
fef.record_metadata(self.results_tmp_dir(),
|
|
|
|
self._orig_refs,
|
|
|
|
refs_to_nuke)
|
2018-12-25 08:02:03 +01:00
|
|
|
|
|
|
|
# Nuke the reflogs and repack
|
2019-01-07 22:46:48 +01:00
|
|
|
if not self._args.quiet and not self._args.debug:
|
2019-03-17 07:39:39 +01:00
|
|
|
print(_("Repacking your repo and cleaning out old unneeded objects"))
|
2019-01-07 22:46:48 +01:00
|
|
|
quiet_flags = '--quiet' if self._args.quiet else ''
|
2018-12-25 08:02:03 +01:00
|
|
|
cleanup_cmds = ['git reflog expire --expire=now --all'.split(),
|
|
|
|
'git gc {} --prune=now'.format(quiet_flags).split()]
|
2019-01-08 20:02:53 +01:00
|
|
|
if not GitUtils.is_repository_bare(target_working_dir):
|
2019-03-08 07:46:43 +01:00
|
|
|
cleanup_cmds.insert(0, 'git reset {} --hard'.format(quiet_flags).split())
|
2018-12-25 08:02:03 +01:00
|
|
|
for cmd in cleanup_cmds:
|
2019-01-07 22:46:48 +01:00
|
|
|
if self._args.debug:
|
2018-12-25 08:02:03 +01:00
|
|
|
print("[DEBUG] Running: {}".format(' '.join(cmd)))
|
2019-01-08 20:02:53 +01:00
|
|
|
subprocess.call(cmd, cwd=target_working_dir)
|
2018-08-21 17:46:26 +02:00
|
|
|
|
2019-02-04 23:03:06 +01:00
|
|
|
# Let user know how long it took
|
2019-03-17 07:39:39 +01:00
|
|
|
print(_("Completely finished after {:.2f} seconds.")
|
|
|
|
.format(time.time()-start))
|
2019-02-04 23:03:06 +01:00
|
|
|
|
2018-08-21 17:46:26 +02:00
|
|
|
if __name__ == '__main__':
|
2019-03-17 07:40:01 +01:00
|
|
|
setup_gettext()
|
2018-12-25 08:02:03 +01:00
|
|
|
args = FilteringOptions.parse_args(sys.argv[1:])
|
|
|
|
if args.analyze:
|
|
|
|
RepoAnalyze.run(args)
|
|
|
|
else:
|
2019-01-07 22:46:48 +01:00
|
|
|
filter = RepoFilter(args)
|
|
|
|
filter.run()
|