android/git-repo
1
Fork 0
mirror of https://gerrit.googlesource.com/git-repo synced 2025-02-24 02:06:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Format codebase with black and check formatting in CQ

Browse Source 
Apply rules set by https://gerrit-review.googlesource.com/c/git-repo/+/362954/ across the codebase and fix any lingering errors caught
by flake8. Also check black formatting in run_tests (and CQ).

Bug: b/267675342
Change-Id: I972d77649dac351150dcfeb1cd1ad0ea2efc1956
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/363474
Reviewed-by: Mike Frysinger <vapier@google.com>
Tested-by: Gavin Mak <gavinmak@google.com>
Commit-Queue: Gavin Mak <gavinmak@google.com>
This commit is contained in:
Gavin Mak 2023-03-11 06:46:20 +00:00 committed by LUCI
parent 1604cf255f
commit ea2e330e43
79 changed files with 19698 additions and 16679 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.flake8
View File

@ -1,5 +1,8 @@
[flake8] [flake8]
max-line-length = 80 max-line-length = 80
per-file-ignores =
# E501: line too long
tests/test_git_superproject.py: E501
extend-ignore = extend-ignore =
# E203: Whitespace before ':' # E203: Whitespace before ':'
# See https://github.com/PyCQA/pycodestyle/issues/373 # See https://github.com/PyCQA/pycodestyle/issues/373

324
color.py
View File

@ -17,196 +17,200 @@ import sys
import pager import pager
COLORS = {None: -1, COLORS = {
'normal': -1, None: -1,
'black': 0, "normal": -1,
'red': 1, "black": 0,
'green': 2, "red": 1,
'yellow': 3, "green": 2,
'blue': 4, "yellow": 3,
'magenta': 5, "blue": 4,
'cyan': 6, "magenta": 5,
'white': 7} "cyan": 6,
"white": 7,
}
ATTRS = {None: -1, ATTRS = {None: -1, "bold": 1, "dim": 2, "ul": 4, "blink": 5, "reverse": 7}
'bold': 1,
'dim': 2,
'ul': 4,
'blink': 5,
'reverse': 7}
RESET = "\033[m" RESET = "\033[m"
def is_color(s): def is_color(s):
return s in COLORS return s in COLORS
def is_attr(s): def is_attr(s):
return s in ATTRS return s in ATTRS
def _Color(fg=None, bg=None, attr=None): def _Color(fg=None, bg=None, attr=None):
fg = COLORS[fg] fg = COLORS[fg]
bg = COLORS[bg] bg = COLORS[bg]
attr = ATTRS[attr] attr = ATTRS[attr]
if attr >= 0 or fg >= 0 or bg >= 0: if attr >= 0 or fg >= 0 or bg >= 0:
need_sep = False need_sep = False
code = "\033[" code = "\033["
if attr >= 0: if attr >= 0:
code += chr(ord('0') + attr) code += chr(ord("0") + attr)
need_sep = True need_sep = True
if fg >= 0: if fg >= 0:
if need_sep: if need_sep:
code += ';' code += ";"
need_sep = True need_sep = True
if fg < 8: if fg < 8:
code += '3%c' % (ord('0') + fg) code += "3%c" % (ord("0") + fg)
else: else:
code += '38;5;%d' % fg code += "38;5;%d" % fg
if bg >= 0: if bg >= 0:
if need_sep: if need_sep:
code += ';' code += ";"
if bg < 8: if bg < 8:
code += '4%c' % (ord('0') + bg) code += "4%c" % (ord("0") + bg)
else: else:
code += '48;5;%d' % bg code += "48;5;%d" % bg
code += 'm' code += "m"
else: else:
code = '' code = ""
return code return code
DEFAULT = None DEFAULT = None
def SetDefaultColoring(state): def SetDefaultColoring(state):
"""Set coloring behavior to |state|. """Set coloring behavior to |state|.
This is useful for overriding config options via the command line. This is useful for overriding config options via the command line.
""" """
if state is None: if state is None:
# Leave it alone -- return quick! # Leave it alone -- return quick!
return return
global DEFAULT global DEFAULT
state = state.lower() state = state.lower()
if state in ('auto',): if state in ("auto",):
DEFAULT = state DEFAULT = state
elif state in ('always', 'yes', 'true', True): elif state in ("always", "yes", "true", True):
DEFAULT = 'always' DEFAULT = "always"
elif state in ('never', 'no', 'false', False): elif state in ("never", "no", "false", False):
DEFAULT = 'never' DEFAULT = "never"
class Coloring(object): class Coloring(object):
def __init__(self, config, section_type): def __init__(self, config, section_type):
self._section = 'color.%s' % section_type self._section = "color.%s" % section_type
self._config = config self._config = config
self._out = sys.stdout self._out = sys.stdout
on = DEFAULT on = DEFAULT
if on is None: if on is None:
on = self._config.GetString(self._section) on = self._config.GetString(self._section)
if on is None: if on is None:
on = self._config.GetString('color.ui') on = self._config.GetString("color.ui")
if on == 'auto': if on == "auto":
if pager.active or os.isatty(1): if pager.active or os.isatty(1):
self._on = True self._on = True
else: else:
self._on = False self._on = False
elif on in ('true', 'always'): elif on in ("true", "always"):
self._on = True self._on = True
else:
self._on = False
def redirect(self, out):
self._out = out
@property
def is_on(self):
return self._on
def write(self, fmt, *args):
self._out.write(fmt % args)
def flush(self):
self._out.flush()
def nl(self):
self._out.write('\n')
def printer(self, opt=None, fg=None, bg=None, attr=None):
s = self
c = self.colorer(opt, fg, bg, attr)
def f(fmt, *args):
s._out.write(c(fmt, *args))
return f
def nofmt_printer(self, opt=None, fg=None, bg=None, attr=None):
s = self
c = self.nofmt_colorer(opt, fg, bg, attr)
def f(fmt):
s._out.write(c(fmt))
return f
def colorer(self, opt=None, fg=None, bg=None, attr=None):
if self._on:
c = self._parse(opt, fg, bg, attr)
def f(fmt, *args):
output = fmt % args
return ''.join([c, output, RESET])
return f
else:
def f(fmt, *args):
return fmt % args
return f
def nofmt_colorer(self, opt=None, fg=None, bg=None, attr=None):
if self._on:
c = self._parse(opt, fg, bg, attr)
def f(fmt):
return ''.join([c, fmt, RESET])
return f
else:
def f(fmt):
return fmt
return f
def _parse(self, opt, fg, bg, attr):
if not opt:
return _Color(fg, bg, attr)
v = self._config.GetString('%s.%s' % (self._section, opt))
if v is None:
return _Color(fg, bg, attr)
v = v.strip().lower()
if v == "reset":
return RESET
elif v == '':
return _Color(fg, bg, attr)
have_fg = False
for a in v.split(' '):
if is_color(a):
if have_fg:
bg = a
else: else:
fg = a self._on = False
elif is_attr(a):
attr = a
return _Color(fg, bg, attr) def redirect(self, out):
self._out = out
@property
def is_on(self):
return self._on
def write(self, fmt, *args):
self._out.write(fmt % args)
def flush(self):
self._out.flush()
def nl(self):
self._out.write("\n")
def printer(self, opt=None, fg=None, bg=None, attr=None):
s = self
c = self.colorer(opt, fg, bg, attr)
def f(fmt, *args):
s._out.write(c(fmt, *args))
return f
def nofmt_printer(self, opt=None, fg=None, bg=None, attr=None):
s = self
c = self.nofmt_colorer(opt, fg, bg, attr)
def f(fmt):
s._out.write(c(fmt))
return f
def colorer(self, opt=None, fg=None, bg=None, attr=None):
if self._on:
c = self._parse(opt, fg, bg, attr)
def f(fmt, *args):
output = fmt % args
return "".join([c, output, RESET])
return f
else:
def f(fmt, *args):
return fmt % args
return f
def nofmt_colorer(self, opt=None, fg=None, bg=None, attr=None):
if self._on:
c = self._parse(opt, fg, bg, attr)
def f(fmt):
return "".join([c, fmt, RESET])
return f
else:
def f(fmt):
return fmt
return f
def _parse(self, opt, fg, bg, attr):
if not opt:
return _Color(fg, bg, attr)
v = self._config.GetString("%s.%s" % (self._section, opt))
if v is None:
return _Color(fg, bg, attr)
v = v.strip().lower()
if v == "reset":
return RESET
elif v == "":
return _Color(fg, bg, attr)
have_fg = False
for a in v.split(" "):
if is_color(a):
if have_fg:
bg = a
else:
fg = a
elif is_attr(a):
attr = a
return _Color(fg, bg, attr)

729
command.py
View File

@ -25,7 +25,7 @@ import progress
# Are we generating man-pages? # Are we generating man-pages?
GENERATE_MANPAGES = os.environ.get('_REPO_GENERATE_MANPAGES_') == ' indeed! ' GENERATE_MANPAGES = os.environ.get("_REPO_GENERATE_MANPAGES_") == " indeed! "
# Number of projects to submit to a single worker process at a time. # Number of projects to submit to a single worker process at a time.
@ -43,403 +43,470 @@ DEFAULT_LOCAL_JOBS = min(os.cpu_count(), 8)
class Command(object): class Command(object):
"""Base class for any command line action in repo. """Base class for any command line action in repo."""
"""
# Singleton for all commands to track overall repo command execution and # Singleton for all commands to track overall repo command execution and
# provide event summary to callers. Only used by sync subcommand currently. # provide event summary to callers. Only used by sync subcommand currently.
# #
# NB: This is being replaced by git trace2 events. See git_trace2_event_log. # NB: This is being replaced by git trace2 events. See git_trace2_event_log.
event_log = EventLog() event_log = EventLog()
# Whether this command is a "common" one, i.e. whether the user would commonly # Whether this command is a "common" one, i.e. whether the user would
# use it or it's a more uncommon command. This is used by the help command to # commonly use it or it's a more uncommon command. This is used by the help
# show short-vs-full summaries. # command to show short-vs-full summaries.
COMMON = False COMMON = False
# Whether this command supports running in parallel. If greater than 0, # Whether this command supports running in parallel. If greater than 0,
# it is the number of parallel jobs to default to. # it is the number of parallel jobs to default to.
PARALLEL_JOBS = None PARALLEL_JOBS = None
# Whether this command supports Multi-manifest. If False, then main.py will # Whether this command supports Multi-manifest. If False, then main.py will
# iterate over the manifests and invoke the command once per (sub)manifest. # iterate over the manifests and invoke the command once per (sub)manifest.
# This is only checked after calling ValidateOptions, so that partially # This is only checked after calling ValidateOptions, so that partially
# migrated subcommands can set it to False. # migrated subcommands can set it to False.
MULTI_MANIFEST_SUPPORT = True MULTI_MANIFEST_SUPPORT = True
def __init__(self, repodir=None, client=None, manifest=None, gitc_manifest=None, def __init__(
git_event_log=None, outer_client=None, outer_manifest=None): self,
self.repodir = repodir repodir=None,
self.client = client client=None,
self.outer_client = outer_client or client manifest=None,
self.manifest = manifest gitc_manifest=None,
self.gitc_manifest = gitc_manifest git_event_log=None,
self.git_event_log = git_event_log outer_client=None,
self.outer_manifest = outer_manifest outer_manifest=None,
):
self.repodir = repodir
self.client = client
self.outer_client = outer_client or client
self.manifest = manifest
self.gitc_manifest = gitc_manifest
self.git_event_log = git_event_log
self.outer_manifest = outer_manifest
# Cache for the OptionParser property. # Cache for the OptionParser property.
self._optparse = None self._optparse = None
def WantPager(self, _opt): def WantPager(self, _opt):
return False return False
def ReadEnvironmentOptions(self, opts): def ReadEnvironmentOptions(self, opts):
""" Set options from environment variables. """ """Set options from environment variables."""
env_options = self._RegisteredEnvironmentOptions() env_options = self._RegisteredEnvironmentOptions()
for env_key, opt_key in env_options.items(): for env_key, opt_key in env_options.items():
# Get the user-set option value if any # Get the user-set option value if any
opt_value = getattr(opts, opt_key) opt_value = getattr(opts, opt_key)
# If the value is set, it means the user has passed it as a command # If the value is set, it means the user has passed it as a command
# line option, and we should use that. Otherwise we can try to set it # line option, and we should use that. Otherwise we can try to set
# with the value from the corresponding environment variable. # it with the value from the corresponding environment variable.
if opt_value is not None: if opt_value is not None:
continue continue
env_value = os.environ.get(env_key) env_value = os.environ.get(env_key)
if env_value is not None: if env_value is not None:
setattr(opts, opt_key, env_value) setattr(opts, opt_key, env_value)
return opts return opts
@property @property
def OptionParser(self): def OptionParser(self):
if self._optparse is None: if self._optparse is None:
try: try:
me = 'repo %s' % self.NAME me = "repo %s" % self.NAME
usage = self.helpUsage.strip().replace('%prog', me) usage = self.helpUsage.strip().replace("%prog", me)
except AttributeError: except AttributeError:
usage = 'repo %s' % self.NAME usage = "repo %s" % self.NAME
epilog = 'Run `repo help %s` to view the detailed manual.' % self.NAME epilog = (
self._optparse = optparse.OptionParser(usage=usage, epilog=epilog) "Run `repo help %s` to view the detailed manual." % self.NAME
self._CommonOptions(self._optparse) )
self._Options(self._optparse) self._optparse = optparse.OptionParser(usage=usage, epilog=epilog)
return self._optparse self._CommonOptions(self._optparse)
self._Options(self._optparse)
return self._optparse
def _CommonOptions(self, p, opt_v=True): def _CommonOptions(self, p, opt_v=True):
"""Initialize the option parser with common options. """Initialize the option parser with common options.
These will show up for *all* subcommands, so use sparingly. These will show up for *all* subcommands, so use sparingly.
NB: Keep in sync with repo:InitParser(). NB: Keep in sync with repo:InitParser().
""" """
g = p.add_option_group('Logging options') g = p.add_option_group("Logging options")
opts = ['-v'] if opt_v else [] opts = ["-v"] if opt_v else []
g.add_option(*opts, '--verbose', g.add_option(
dest='output_mode', action='store_true', *opts,
help='show all output') "--verbose",
g.add_option('-q', '--quiet', dest="output_mode",
dest='output_mode', action='store_false', action="store_true",
help='only show errors') help="show all output",
)
g.add_option(
"-q",
"--quiet",
dest="output_mode",
action="store_false",
help="only show errors",
)
if self.PARALLEL_JOBS is not None: if self.PARALLEL_JOBS is not None:
default = 'based on number of CPU cores' default = "based on number of CPU cores"
if not GENERATE_MANPAGES: if not GENERATE_MANPAGES:
# Only include active cpu count if we aren't generating man pages. # Only include active cpu count if we aren't generating man
default = f'%default; {default}' # pages.
p.add_option( default = f"%default; {default}"
'-j', '--jobs', p.add_option(
type=int, default=self.PARALLEL_JOBS, "-j",
help=f'number of jobs to run in parallel (default: {default})') "--jobs",
type=int,
default=self.PARALLEL_JOBS,
help=f"number of jobs to run in parallel (default: {default})",
)
m = p.add_option_group('Multi-manifest options') m = p.add_option_group("Multi-manifest options")
m.add_option('--outer-manifest', action='store_true', default=None, m.add_option(
help='operate starting at the outermost manifest') "--outer-manifest",
m.add_option('--no-outer-manifest', dest='outer_manifest', action="store_true",
action='store_false', help='do not operate on outer manifests') default=None,
m.add_option('--this-manifest-only', action='store_true', default=None, help="operate starting at the outermost manifest",
help='only operate on this (sub)manifest') )
m.add_option('--no-this-manifest-only', '--all-manifests', m.add_option(
dest='this_manifest_only', action='store_false', "--no-outer-manifest",
help='operate on this manifest and its submanifests') dest="outer_manifest",
action="store_false",
help="do not operate on outer manifests",
)
m.add_option(
"--this-manifest-only",
action="store_true",
default=None,
help="only operate on this (sub)manifest",
)
m.add_option(
"--no-this-manifest-only",
"--all-manifests",
dest="this_manifest_only",
action="store_false",
help="operate on this manifest and its submanifests",
)
def _Options(self, p): def _Options(self, p):
"""Initialize the option parser with subcommand-specific options.""" """Initialize the option parser with subcommand-specific options."""
def _RegisteredEnvironmentOptions(self): def _RegisteredEnvironmentOptions(self):
"""Get options that can be set from environment variables. """Get options that can be set from environment variables.
Return a dictionary mapping environment variable name Return a dictionary mapping environment variable name
to option key name that it can override. to option key name that it can override.
Example: {'REPO_MY_OPTION': 'my_option'} Example: {'REPO_MY_OPTION': 'my_option'}
Will allow the option with key value 'my_option' to be set Will allow the option with key value 'my_option' to be set
from the value in the environment variable named 'REPO_MY_OPTION'. from the value in the environment variable named 'REPO_MY_OPTION'.
Note: This does not work properly for options that are explicitly Note: This does not work properly for options that are explicitly
set to None by the user, or options that are defined with a set to None by the user, or options that are defined with a
default value other than None. default value other than None.
""" """
return {} return {}
def Usage(self): def Usage(self):
"""Display usage and terminate. """Display usage and terminate."""
""" self.OptionParser.print_usage()
self.OptionParser.print_usage() sys.exit(1)
sys.exit(1)
def CommonValidateOptions(self, opt, args): def CommonValidateOptions(self, opt, args):
"""Validate common options.""" """Validate common options."""
opt.quiet = opt.output_mode is False opt.quiet = opt.output_mode is False
opt.verbose = opt.output_mode is True opt.verbose = opt.output_mode is True
if opt.outer_manifest is None: if opt.outer_manifest is None:
# By default, treat multi-manifest instances as a single manifest from # By default, treat multi-manifest instances as a single manifest
# the user's perspective. # from the user's perspective.
opt.outer_manifest = True opt.outer_manifest = True
def ValidateOptions(self, opt, args): def ValidateOptions(self, opt, args):
"""Validate the user options & arguments before executing. """Validate the user options & arguments before executing.
This is meant to help break the code up into logical steps. Some tips: This is meant to help break the code up into logical steps. Some tips:
* Use self.OptionParser.error to display CLI related errors. * Use self.OptionParser.error to display CLI related errors.
* Adjust opt member defaults as makes sense. * Adjust opt member defaults as makes sense.
* Adjust the args list, but do so inplace so the caller sees updates. * Adjust the args list, but do so inplace so the caller sees updates.
* Try to avoid updating self state. Leave that to Execute. * Try to avoid updating self state. Leave that to Execute.
""" """
def Execute(self, opt, args): def Execute(self, opt, args):
"""Perform the action, after option parsing is complete. """Perform the action, after option parsing is complete."""
""" raise NotImplementedError
raise NotImplementedError
@staticmethod @staticmethod
def ExecuteInParallel(jobs, func, inputs, callback, output=None, ordered=False): def ExecuteInParallel(
"""Helper for managing parallel execution boiler plate. jobs, func, inputs, callback, output=None, ordered=False
):
"""Helper for managing parallel execution boiler plate.
For subcommands that can easily split their work up. For subcommands that can easily split their work up.
Args: Args:
jobs: How many parallel processes to use. jobs: How many parallel processes to use.
func: The function to apply to each of the |inputs|. Usually a func: The function to apply to each of the |inputs|. Usually a
functools.partial for wrapping additional arguments. It will be run functools.partial for wrapping additional arguments. It will be
in a separate process, so it must be pickalable, so nested functions run in a separate process, so it must be pickalable, so nested
won't work. Methods on the subcommand Command class should work. functions won't work. Methods on the subcommand Command class
inputs: The list of items to process. Must be a list. should work.
callback: The function to pass the results to for processing. It will be inputs: The list of items to process. Must be a list.
executed in the main thread and process the results of |func| as they callback: The function to pass the results to for processing. It
become available. Thus it may be a local nested function. Its return will be executed in the main thread and process the results of
value is passed back directly. It takes three arguments: |func| as they become available. Thus it may be a local nested
- The processing pool (or None with one job). function. Its return value is passed back directly. It takes
- The |output| argument. three arguments:
- An iterator for the results. - The processing pool (or None with one job).
output: An output manager. May be progress.Progess or color.Coloring. - The |output| argument.
ordered: Whether the jobs should be processed in order. - An iterator for the results.
output: An output manager. May be progress.Progess or
color.Coloring.
ordered: Whether the jobs should be processed in order.
Returns: Returns:
The |callback| function's results are returned. The |callback| function's results are returned.
""" """
try:
# NB: Multiprocessing is heavy, so don't spin it up for one job.
if len(inputs) == 1 or jobs == 1:
return callback(None, output, (func(x) for x in inputs))
else:
with multiprocessing.Pool(jobs) as pool:
submit = pool.imap if ordered else pool.imap_unordered
return callback(pool, output, submit(func, inputs, chunksize=WORKER_BATCH_SIZE))
finally:
if isinstance(output, progress.Progress):
output.end()
def _ResetPathToProjectMap(self, projects):
self._by_path = dict((p.worktree, p) for p in projects)
def _UpdatePathToProjectMap(self, project):
self._by_path[project.worktree] = project
def _GetProjectByPath(self, manifest, path):
project = None
if os.path.exists(path):
oldpath = None
while (path and
path != oldpath and
path != manifest.topdir):
try: try:
project = self._by_path[path] # NB: Multiprocessing is heavy, so don't spin it up for one job.
break if len(inputs) == 1 or jobs == 1:
except KeyError: return callback(None, output, (func(x) for x in inputs))
oldpath = path else:
path = os.path.dirname(path) with multiprocessing.Pool(jobs) as pool:
if not project and path == manifest.topdir: submit = pool.imap if ordered else pool.imap_unordered
try: return callback(
project = self._by_path[path] pool,
except KeyError: output,
pass submit(func, inputs, chunksize=WORKER_BATCH_SIZE),
else: )
try: finally:
project = self._by_path[path] if isinstance(output, progress.Progress):
except KeyError: output.end()
pass
return project
def GetProjects(self, args, manifest=None, groups='', missing_ok=False, def _ResetPathToProjectMap(self, projects):
submodules_ok=False, all_manifests=False): self._by_path = dict((p.worktree, p) for p in projects)
"""A list of projects that match the arguments.
Args: def _UpdatePathToProjectMap(self, project):
args: a list of (case-insensitive) strings, projects to search for. self._by_path[project.worktree] = project
manifest: an XmlManifest, the manifest to use, or None for default.
groups: a string, the manifest groups in use.
missing_ok: a boolean, whether to allow missing projects.
submodules_ok: a boolean, whether to allow submodules.
all_manifests: a boolean, if True then all manifests and submanifests are
used. If False, then only the local (sub)manifest is used.
Returns: def _GetProjectByPath(self, manifest, path):
A list of matching Project instances. project = None
""" if os.path.exists(path):
if all_manifests: oldpath = None
if not manifest: while path and path != oldpath and path != manifest.topdir:
manifest = self.manifest.outer_client try:
all_projects_list = manifest.all_projects project = self._by_path[path]
else: break
if not manifest: except KeyError:
manifest = self.manifest oldpath = path
all_projects_list = manifest.projects path = os.path.dirname(path)
result = [] if not project and path == manifest.topdir:
try:
project = self._by_path[path]
except KeyError:
pass
else:
try:
project = self._by_path[path]
except KeyError:
pass
return project
if not groups: def GetProjects(
groups = manifest.GetGroupsStr() self,
groups = [x for x in re.split(r'[,\s]+', groups) if x] args,
manifest=None,
groups="",
missing_ok=False,
submodules_ok=False,
all_manifests=False,
):
"""A list of projects that match the arguments.
if not args: Args:
derived_projects = {} args: a list of (case-insensitive) strings, projects to search for.
for project in all_projects_list: manifest: an XmlManifest, the manifest to use, or None for default.
if submodules_ok or project.sync_s: groups: a string, the manifest groups in use.
derived_projects.update((p.name, p) missing_ok: a boolean, whether to allow missing projects.
for p in project.GetDerivedSubprojects()) submodules_ok: a boolean, whether to allow submodules.
all_projects_list.extend(derived_projects.values()) all_manifests: a boolean, if True then all manifests and
for project in all_projects_list: submanifests are used. If False, then only the local
if (missing_ok or project.Exists) and project.MatchesGroups(groups): (sub)manifest is used.
result.append(project)
else:
self._ResetPathToProjectMap(all_projects_list)
for arg in args: Returns:
# We have to filter by manifest groups in case the requested project is A list of matching Project instances.
# checked out multiple times or differently based on them. """
projects = [project if all_manifests:
if not manifest:
manifest = self.manifest.outer_client
all_projects_list = manifest.all_projects
else:
if not manifest:
manifest = self.manifest
all_projects_list = manifest.projects
result = []
if not groups:
groups = manifest.GetGroupsStr()
groups = [x for x in re.split(r"[,\s]+", groups) if x]
if not args:
derived_projects = {}
for project in all_projects_list:
if submodules_ok or project.sync_s:
derived_projects.update(
(p.name, p) for p in project.GetDerivedSubprojects()
)
all_projects_list.extend(derived_projects.values())
for project in all_projects_list:
if (missing_ok or project.Exists) and project.MatchesGroups(
groups
):
result.append(project)
else:
self._ResetPathToProjectMap(all_projects_list)
for arg in args:
# We have to filter by manifest groups in case the requested
# project is checked out multiple times or differently based on
# them.
projects = [
project
for project in manifest.GetProjectsWithName( for project in manifest.GetProjectsWithName(
arg, all_manifests=all_manifests) arg, all_manifests=all_manifests
if project.MatchesGroups(groups)] )
if project.MatchesGroups(groups)
]
if not projects: if not projects:
path = os.path.abspath(arg).replace('\\', '/') path = os.path.abspath(arg).replace("\\", "/")
tree = manifest tree = manifest
if all_manifests: if all_manifests:
# Look for the deepest matching submanifest. # Look for the deepest matching submanifest.
for tree in reversed(list(manifest.all_manifests)): for tree in reversed(list(manifest.all_manifests)):
if path.startswith(tree.topdir): if path.startswith(tree.topdir):
break break
project = self._GetProjectByPath(tree, path) project = self._GetProjectByPath(tree, path)
# If it's not a derived project, update path->project mapping and # If it's not a derived project, update path->project
# search again, as arg might actually point to a derived subproject. # mapping and search again, as arg might actually point to
if (project and not project.Derived and (submodules_ok or # a derived subproject.
project.sync_s)): if (
search_again = False project
for subproject in project.GetDerivedSubprojects(): and not project.Derived
self._UpdatePathToProjectMap(subproject) and (submodules_ok or project.sync_s)
search_again = True ):
if search_again: search_again = False
project = self._GetProjectByPath(manifest, path) or project for subproject in project.GetDerivedSubprojects():
self._UpdatePathToProjectMap(subproject)
search_again = True
if search_again:
project = (
self._GetProjectByPath(manifest, path)
or project
)
if project: if project:
projects = [project] projects = [project]
if not projects: if not projects:
raise NoSuchProjectError(arg) raise NoSuchProjectError(arg)
for project in projects: for project in projects:
if not missing_ok and not project.Exists: if not missing_ok and not project.Exists:
raise NoSuchProjectError('%s (%s)' % ( raise NoSuchProjectError(
arg, project.RelPath(local=not all_manifests))) "%s (%s)"
if not project.MatchesGroups(groups): % (arg, project.RelPath(local=not all_manifests))
raise InvalidProjectGroupsError(arg) )
if not project.MatchesGroups(groups):
raise InvalidProjectGroupsError(arg)
result.extend(projects) result.extend(projects)
def _getpath(x): def _getpath(x):
return x.relpath return x.relpath
result.sort(key=_getpath)
return result
def FindProjects(self, args, inverse=False, all_manifests=False): result.sort(key=_getpath)
"""Find projects from command line arguments. return result
Args: def FindProjects(self, args, inverse=False, all_manifests=False):
args: a list of (case-insensitive) strings, projects to search for. """Find projects from command line arguments.
inverse: a boolean, if True, then projects not matching any |args| are
returned.
all_manifests: a boolean, if True then all manifests and submanifests are
used. If False, then only the local (sub)manifest is used.
"""
result = []
patterns = [re.compile(r'%s' % a, re.IGNORECASE) for a in args]
for project in self.GetProjects('', all_manifests=all_manifests):
paths = [project.name, project.RelPath(local=not all_manifests)]
for pattern in patterns:
match = any(pattern.search(x) for x in paths)
if not inverse and match:
result.append(project)
break
if inverse and match:
break
else:
if inverse:
result.append(project)
result.sort(key=lambda project: (project.manifest.path_prefix,
project.relpath))
return result
def ManifestList(self, opt): Args:
"""Yields all of the manifests to traverse. args: a list of (case-insensitive) strings, projects to search for.
inverse: a boolean, if True, then projects not matching any |args|
are returned.
all_manifests: a boolean, if True then all manifests and
submanifests are used. If False, then only the local
(sub)manifest is used.
"""
result = []
patterns = [re.compile(r"%s" % a, re.IGNORECASE) for a in args]
for project in self.GetProjects("", all_manifests=all_manifests):
paths = [project.name, project.RelPath(local=not all_manifests)]
for pattern in patterns:
match = any(pattern.search(x) for x in paths)
if not inverse and match:
result.append(project)
break
if inverse and match:
break
else:
if inverse:
result.append(project)
result.sort(
key=lambda project: (project.manifest.path_prefix, project.relpath)
)
return result
Args: def ManifestList(self, opt):
opt: The command options. """Yields all of the manifests to traverse.
"""
top = self.outer_manifest Args:
if not opt.outer_manifest or opt.this_manifest_only: opt: The command options.
top = self.manifest """
yield top top = self.outer_manifest
if not opt.this_manifest_only: if not opt.outer_manifest or opt.this_manifest_only:
for child in top.all_children: top = self.manifest
yield child yield top
if not opt.this_manifest_only:
for child in top.all_children:
yield child
class InteractiveCommand(Command): class InteractiveCommand(Command):
"""Command which requires user interaction on the tty and """Command which requires user interaction on the tty and must not run
must not run within a pager, even if the user asks to. within a pager, even if the user asks to.
""" """
def WantPager(self, _opt): def WantPager(self, _opt):
return False return False
class PagedCommand(Command): class PagedCommand(Command):
"""Command which defaults to output in a pager, as its """Command which defaults to output in a pager, as its display tends to be
display tends to be larger than one screen full. larger than one screen full.
""" """
def WantPager(self, _opt): def WantPager(self, _opt):
return True return True
class MirrorSafeCommand(object): class MirrorSafeCommand(object):
"""Command permits itself to run within a mirror, """Command permits itself to run within a mirror, and does not require a
and does not require a working directory. working directory.
""" """
class GitcAvailableCommand(object): class GitcAvailableCommand(object):
"""Command that requires GITC to be available, but does """Command that requires GITC to be available, but does not require the
not require the local client to be a GITC client. local client to be a GITC client.
""" """
class GitcClientCommand(object): class GitcClientCommand(object):
"""Command that requires the local client to be a GITC """Command that requires the local client to be a GITC client."""
client.
"""

152
editor.py
View File

@ -23,93 +23,99 @@ import platform_utils
class Editor(object): class Editor(object):
"""Manages the user's preferred text editor.""" """Manages the user's preferred text editor."""
_editor = None _editor = None
globalConfig = None globalConfig = None
@classmethod @classmethod
def _GetEditor(cls): def _GetEditor(cls):
if cls._editor is None: if cls._editor is None:
cls._editor = cls._SelectEditor() cls._editor = cls._SelectEditor()
return cls._editor return cls._editor
@classmethod @classmethod
def _SelectEditor(cls): def _SelectEditor(cls):
e = os.getenv('GIT_EDITOR') e = os.getenv("GIT_EDITOR")
if e: if e:
return e return e
if cls.globalConfig: if cls.globalConfig:
e = cls.globalConfig.GetString('core.editor') e = cls.globalConfig.GetString("core.editor")
if e: if e:
return e return e
e = os.getenv('VISUAL') e = os.getenv("VISUAL")
if e: if e:
return e return e
e = os.getenv('EDITOR') e = os.getenv("EDITOR")
if e: if e:
return e return e
if os.getenv('TERM') == 'dumb': if os.getenv("TERM") == "dumb":
print( print(
"""No editor specified in GIT_EDITOR, core.editor, VISUAL or EDITOR. """No editor specified in GIT_EDITOR, core.editor, VISUAL or EDITOR.
Tried to fall back to vi but terminal is dumb. Please configure at Tried to fall back to vi but terminal is dumb. Please configure at
least one of these before using this command.""", file=sys.stderr) least one of these before using this command.""", # noqa: E501
sys.exit(1) file=sys.stderr,
)
sys.exit(1)
return 'vi' return "vi"
@classmethod @classmethod
def EditString(cls, data): def EditString(cls, data):
"""Opens an editor to edit the given content. """Opens an editor to edit the given content.
Args: Args:
data: The text to edit. data: The text to edit.
Returns: Returns:
New value of edited text. New value of edited text.
Raises: Raises:
EditorError: The editor failed to run. EditorError: The editor failed to run.
""" """
editor = cls._GetEditor() editor = cls._GetEditor()
if editor == ':': if editor == ":":
return data return data
fd, path = tempfile.mkstemp() fd, path = tempfile.mkstemp()
try: try:
os.write(fd, data.encode('utf-8')) os.write(fd, data.encode("utf-8"))
os.close(fd) os.close(fd)
fd = None fd = None
if platform_utils.isWindows(): if platform_utils.isWindows():
# Split on spaces, respecting quoted strings # Split on spaces, respecting quoted strings
import shlex import shlex
args = shlex.split(editor)
shell = False
elif re.compile("^.*[$ \t'].*$").match(editor):
args = [editor + ' "$@"', 'sh']
shell = True
else:
args = [editor]
shell = False
args.append(path)
try: args = shlex.split(editor)
rc = subprocess.Popen(args, shell=shell).wait() shell = False
except OSError as e: elif re.compile("^.*[$ \t'].*$").match(editor):
raise EditorError('editor failed, %s: %s %s' args = [editor + ' "$@"', "sh"]
% (str(e), editor, path)) shell = True
if rc != 0: else:
raise EditorError('editor failed with exit status %d: %s %s' args = [editor]
% (rc, editor, path)) shell = False
args.append(path)
with open(path, mode='rb') as fd2: try:
return fd2.read().decode('utf-8') rc = subprocess.Popen(args, shell=shell).wait()
finally: except OSError as e:
if fd: raise EditorError(
os.close(fd) "editor failed, %s: %s %s" % (str(e), editor, path)
platform_utils.remove(path) )
if rc != 0:
raise EditorError(
"editor failed with exit status %d: %s %s"
% (rc, editor, path)
)
with open(path, mode="rb") as fd2:
return fd2.read().decode("utf-8")
finally:
if fd:
os.close(fd)
platform_utils.remove(path)

130
error.py
View File

@ -14,122 +14,112 @@
class ManifestParseError(Exception): class ManifestParseError(Exception):
"""Failed to parse the manifest file. """Failed to parse the manifest file."""
"""
class ManifestInvalidRevisionError(ManifestParseError): class ManifestInvalidRevisionError(ManifestParseError):
"""The revision value in a project is incorrect. """The revision value in a project is incorrect."""
"""
class ManifestInvalidPathError(ManifestParseError): class ManifestInvalidPathError(ManifestParseError):
"""A path used in <copyfile> or <linkfile> is incorrect. """A path used in <copyfile> or <linkfile> is incorrect."""
"""
class NoManifestException(Exception): class NoManifestException(Exception):
"""The required manifest does not exist. """The required manifest does not exist."""
"""
def __init__(self, path, reason): def __init__(self, path, reason):
super().__init__(path, reason) super().__init__(path, reason)
self.path = path self.path = path
self.reason = reason self.reason = reason
def __str__(self): def __str__(self):
return self.reason return self.reason
class EditorError(Exception): class EditorError(Exception):
"""Unspecified error from the user's text editor. """Unspecified error from the user's text editor."""
"""
def __init__(self, reason): def __init__(self, reason):
super().__init__(reason) super().__init__(reason)
self.reason = reason self.reason = reason
def __str__(self): def __str__(self):
return self.reason return self.reason
class GitError(Exception): class GitError(Exception):
"""Unspecified internal error from git. """Unspecified internal error from git."""
"""
def __init__(self, command): def __init__(self, command):
super().__init__(command) super().__init__(command)
self.command = command self.command = command
def __str__(self): def __str__(self):
return self.command return self.command
class UploadError(Exception): class UploadError(Exception):
"""A bundle upload to Gerrit did not succeed. """A bundle upload to Gerrit did not succeed."""
"""
def __init__(self, reason): def __init__(self, reason):
super().__init__(reason) super().__init__(reason)
self.reason = reason self.reason = reason
def __str__(self): def __str__(self):
return self.reason return self.reason
class DownloadError(Exception): class DownloadError(Exception):
"""Cannot download a repository. """Cannot download a repository."""
"""
def __init__(self, reason): def __init__(self, reason):
super().__init__(reason) super().__init__(reason)
self.reason = reason self.reason = reason
def __str__(self): def __str__(self):
return self.reason return self.reason
class NoSuchProjectError(Exception): class NoSuchProjectError(Exception):
"""A specified project does not exist in the work tree. """A specified project does not exist in the work tree."""
"""
def __init__(self, name=None): def __init__(self, name=None):
super().__init__(name) super().__init__(name)
self.name = name self.name = name
def __str__(self): def __str__(self):
if self.name is None: if self.name is None:
return 'in current directory' return "in current directory"
return self.name return self.name
class InvalidProjectGroupsError(Exception): class InvalidProjectGroupsError(Exception):
"""A specified project is not suitable for the specified groups """A specified project is not suitable for the specified groups"""
"""
def __init__(self, name=None): def __init__(self, name=None):
super().__init__(name) super().__init__(name)
self.name = name self.name = name
def __str__(self): def __str__(self):
if self.name is None: if self.name is None:
return 'in current directory' return "in current directory"
return self.name return self.name
class RepoChangedException(Exception): class RepoChangedException(Exception):
"""Thrown if 'repo sync' results in repo updating its internal """Thrown if 'repo sync' results in repo updating its internal
repo or manifest repositories. In this special case we must repo or manifest repositories. In this special case we must
use exec to re-execute repo with the new code and manifest. use exec to re-execute repo with the new code and manifest.
""" """
def __init__(self, extra_args=None): def __init__(self, extra_args=None):
super().__init__(extra_args) super().__init__(extra_args)
self.extra_args = extra_args or [] self.extra_args = extra_args or []
class HookError(Exception): class HookError(Exception):
"""Thrown if a 'repo-hook' could not be run. """Thrown if a 'repo-hook' could not be run.
The common case is that the file wasn't present when we tried to run it. The common case is that the file wasn't present when we tried to run it.
""" """

262
event_log.py
View File

@ -15,161 +15,169 @@
import json import json
import multiprocessing import multiprocessing
TASK_COMMAND = 'command' TASK_COMMAND = "command"
TASK_SYNC_NETWORK = 'sync-network' TASK_SYNC_NETWORK = "sync-network"
TASK_SYNC_LOCAL = 'sync-local' TASK_SYNC_LOCAL = "sync-local"
class EventLog(object): class EventLog(object):
"""Event log that records events that occurred during a repo invocation. """Event log that records events that occurred during a repo invocation.
Events are written to the log as a consecutive JSON entries, one per line. Events are written to the log as a consecutive JSON entries, one per line.
Each entry contains the following keys: Each entry contains the following keys:
- id: A ('RepoOp', ID) tuple, suitable for storing in a datastore. - id: A ('RepoOp', ID) tuple, suitable for storing in a datastore.
The ID is only unique for the invocation of the repo command. The ID is only unique for the invocation of the repo command.
- name: Name of the object being operated upon. - name: Name of the object being operated upon.
- task_name: The task that was performed. - task_name: The task that was performed.
- start: Timestamp of when the operation started. - start: Timestamp of when the operation started.
- finish: Timestamp of when the operation finished. - finish: Timestamp of when the operation finished.
- success: Boolean indicating if the operation was successful. - success: Boolean indicating if the operation was successful.
- try_count: A counter indicating the try count of this task. - try_count: A counter indicating the try count of this task.
Optionally: Optionally:
- parent: A ('RepoOp', ID) tuple indicating the parent event for nested - parent: A ('RepoOp', ID) tuple indicating the parent event for nested
events. events.
Valid task_names include: Valid task_names include:
- command: The invocation of a subcommand. - command: The invocation of a subcommand.
- sync-network: The network component of a sync command. - sync-network: The network component of a sync command.
- sync-local: The local component of a sync command. - sync-local: The local component of a sync command.
Specific tasks may include additional informational properties. Specific tasks may include additional informational properties.
"""
def __init__(self):
"""Initializes the event log."""
self._log = []
self._parent = None
def Add(self, name, task_name, start, finish=None, success=None,
try_count=1, kind='RepoOp'):
"""Add an event to the log.
Args:
name: Name of the object being operated upon.
task_name: A sub-task that was performed for name.
start: Timestamp of when the operation started.
finish: Timestamp of when the operation finished.
success: Boolean indicating if the operation was successful.
try_count: A counter indicating the try count of this task.
kind: The kind of the object for the unique identifier.
Returns:
A dictionary of the event added to the log.
""" """
event = {
'id': (kind, _NextEventId()),
'name': name,
'task_name': task_name,
'start_time': start,
'try': try_count,
}
if self._parent: def __init__(self):
event['parent'] = self._parent['id'] """Initializes the event log."""
self._log = []
self._parent = None
if success is not None or finish is not None: def Add(
self.FinishEvent(event, finish, success) self,
name,
task_name,
start,
finish=None,
success=None,
try_count=1,
kind="RepoOp",
):
"""Add an event to the log.
self._log.append(event) Args:
return event name: Name of the object being operated upon.
task_name: A sub-task that was performed for name.
start: Timestamp of when the operation started.
finish: Timestamp of when the operation finished.
success: Boolean indicating if the operation was successful.
try_count: A counter indicating the try count of this task.
kind: The kind of the object for the unique identifier.
def AddSync(self, project, task_name, start, finish, success): Returns:
"""Add a event to the log for a sync command. A dictionary of the event added to the log.
"""
event = {
"id": (kind, _NextEventId()),
"name": name,
"task_name": task_name,
"start_time": start,
"try": try_count,
}
Args: if self._parent:
project: Project being synced. event["parent"] = self._parent["id"]
task_name: A sub-task that was performed for name.
One of (TASK_SYNC_NETWORK, TASK_SYNC_LOCAL)
start: Timestamp of when the operation started.
finish: Timestamp of when the operation finished.
success: Boolean indicating if the operation was successful.
Returns: if success is not None or finish is not None:
A dictionary of the event added to the log. self.FinishEvent(event, finish, success)
"""
event = self.Add(project.relpath, task_name, start, finish, success)
if event is not None:
event['project'] = project.name
if project.revisionExpr:
event['revision'] = project.revisionExpr
if project.remote.url:
event['project_url'] = project.remote.url
if project.remote.fetchUrl:
event['remote_url'] = project.remote.fetchUrl
try:
event['git_hash'] = project.GetCommitRevisionId()
except Exception:
pass
return event
def GetStatusString(self, success): self._log.append(event)
"""Converst a boolean success to a status string. return event
Args: def AddSync(self, project, task_name, start, finish, success):
success: Boolean indicating if the operation was successful. """Add a event to the log for a sync command.
Returns: Args:
status string. project: Project being synced.
""" task_name: A sub-task that was performed for name.
return 'pass' if success else 'fail' One of (TASK_SYNC_NETWORK, TASK_SYNC_LOCAL)
start: Timestamp of when the operation started.
finish: Timestamp of when the operation finished.
success: Boolean indicating if the operation was successful.
def FinishEvent(self, event, finish, success): Returns:
"""Finishes an incomplete event. A dictionary of the event added to the log.
"""
event = self.Add(project.relpath, task_name, start, finish, success)
if event is not None:
event["project"] = project.name
if project.revisionExpr:
event["revision"] = project.revisionExpr
if project.remote.url:
event["project_url"] = project.remote.url
if project.remote.fetchUrl:
event["remote_url"] = project.remote.fetchUrl
try:
event["git_hash"] = project.GetCommitRevisionId()
except Exception:
pass
return event
Args: def GetStatusString(self, success):
event: An event that has been added to the log. """Converst a boolean success to a status string.
finish: Timestamp of when the operation finished.
success: Boolean indicating if the operation was successful.
Returns: Args:
A dictionary of the event added to the log. success: Boolean indicating if the operation was successful.
"""
event['status'] = self.GetStatusString(success)
event['finish_time'] = finish
return event
def SetParent(self, event): Returns:
"""Set a parent event for all new entities. status string.
"""
return "pass" if success else "fail"
Args: def FinishEvent(self, event, finish, success):
event: The event to use as a parent. """Finishes an incomplete event.
"""
self._parent = event
def Write(self, filename): Args:
"""Writes the log out to a file. event: An event that has been added to the log.
finish: Timestamp of when the operation finished.
success: Boolean indicating if the operation was successful.
Args: Returns:
filename: The file to write the log to. A dictionary of the event added to the log.
""" """
with open(filename, 'w+') as f: event["status"] = self.GetStatusString(success)
for e in self._log: event["finish_time"] = finish
json.dump(e, f, sort_keys=True) return event
f.write('\n')
def SetParent(self, event):
"""Set a parent event for all new entities.
Args:
event: The event to use as a parent.
"""
self._parent = event
def Write(self, filename):
"""Writes the log out to a file.
Args:
filename: The file to write the log to.
"""
with open(filename, "w+") as f:
for e in self._log:
json.dump(e, f, sort_keys=True)
f.write("\n")
# An integer id that is unique across this invocation of the program. # An integer id that is unique across this invocation of the program.
_EVENT_ID = multiprocessing.Value('i', 1) _EVENT_ID = multiprocessing.Value("i", 1)
def _NextEventId(): def _NextEventId():
"""Helper function for grabbing the next unique id. """Helper function for grabbing the next unique id.
Returns: Returns:
A unique, to this invocation of the program, integer id. A unique, to this invocation of the program, integer id.
""" """
with _EVENT_ID.get_lock(): with _EVENT_ID.get_lock():
val = _EVENT_ID.value val = _EVENT_ID.value
_EVENT_ID.value += 1 _EVENT_ID.value += 1
return val return val

46
fetch.py
View File

@ -21,25 +21,29 @@ from urllib.request import urlopen
def fetch_file(url, verbose=False): def fetch_file(url, verbose=False):
"""Fetch a file from the specified source using the appropriate protocol. """Fetch a file from the specified source using the appropriate protocol.
Returns: Returns:
The contents of the file as bytes. The contents of the file as bytes.
""" """
scheme = urlparse(url).scheme scheme = urlparse(url).scheme
if scheme == 'gs': if scheme == "gs":
cmd = ['gsutil', 'cat', url] cmd = ["gsutil", "cat", url]
try: try:
result = subprocess.run( result = subprocess.run(
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True
check=True) )
if result.stderr and verbose: if result.stderr and verbose:
print('warning: non-fatal error running "gsutil": %s' % result.stderr, print(
file=sys.stderr) 'warning: non-fatal error running "gsutil": %s'
return result.stdout % result.stderr,
except subprocess.CalledProcessError as e: file=sys.stderr,
print('fatal: error running "gsutil": %s' % e.stderr, )
file=sys.stderr) return result.stdout
sys.exit(1) except subprocess.CalledProcessError as e:
with urlopen(url) as f: print(
return f.read() 'fatal: error running "gsutil": %s' % e.stderr, file=sys.stderr
)
sys.exit(1)
with urlopen(url) as f:
return f.read()

497
git_command.py
View File

@ -24,7 +24,7 @@ import platform_utils
from repo_trace import REPO_TRACE, IsTrace, Trace from repo_trace import REPO_TRACE, IsTrace, Trace
from wrapper import Wrapper from wrapper import Wrapper
GIT = 'git' GIT = "git"
# NB: These do not need to be kept in sync with the repo launcher script. # NB: These do not need to be kept in sync with the repo launcher script.
# These may be much newer as it allows the repo launcher to roll between # These may be much newer as it allows the repo launcher to roll between
# different repo releases while source versions might require a newer git. # different repo releases while source versions might require a newer git.
@ -36,126 +36,138 @@ GIT = 'git'
# git-1.7 is in (EOL) Ubuntu Precise. git-1.9 is in Ubuntu Trusty. # git-1.7 is in (EOL) Ubuntu Precise. git-1.9 is in Ubuntu Trusty.
MIN_GIT_VERSION_SOFT = (1, 9, 1) MIN_GIT_VERSION_SOFT = (1, 9, 1)
MIN_GIT_VERSION_HARD = (1, 7, 2) MIN_GIT_VERSION_HARD = (1, 7, 2)
GIT_DIR = 'GIT_DIR' GIT_DIR = "GIT_DIR"
LAST_GITDIR = None LAST_GITDIR = None
LAST_CWD = None LAST_CWD = None
class _GitCall(object): class _GitCall(object):
@functools.lru_cache(maxsize=None) @functools.lru_cache(maxsize=None)
def version_tuple(self): def version_tuple(self):
ret = Wrapper().ParseGitVersion() ret = Wrapper().ParseGitVersion()
if ret is None: if ret is None:
print('fatal: unable to detect git version', file=sys.stderr) print("fatal: unable to detect git version", file=sys.stderr)
sys.exit(1) sys.exit(1)
return ret return ret
def __getattr__(self, name): def __getattr__(self, name):
name = name.replace('_', '-') name = name.replace("_", "-")
def fun(*cmdv): def fun(*cmdv):
command = [name] command = [name]
command.extend(cmdv) command.extend(cmdv)
return GitCommand(None, command).Wait() == 0 return GitCommand(None, command).Wait() == 0
return fun
return fun
git = _GitCall() git = _GitCall()
def RepoSourceVersion(): def RepoSourceVersion():
"""Return the version of the repo.git tree.""" """Return the version of the repo.git tree."""
ver = getattr(RepoSourceVersion, 'version', None) ver = getattr(RepoSourceVersion, "version", None)
# We avoid GitCommand so we don't run into circular deps -- GitCommand needs # We avoid GitCommand so we don't run into circular deps -- GitCommand needs
# to initialize version info we provide. # to initialize version info we provide.
if ver is None: if ver is None:
env = GitCommand._GetBasicEnv() env = GitCommand._GetBasicEnv()
proj = os.path.dirname(os.path.abspath(__file__)) proj = os.path.dirname(os.path.abspath(__file__))
env[GIT_DIR] = os.path.join(proj, '.git') env[GIT_DIR] = os.path.join(proj, ".git")
result = subprocess.run([GIT, 'describe', HEAD], stdout=subprocess.PIPE, result = subprocess.run(
stderr=subprocess.DEVNULL, encoding='utf-8', [GIT, "describe", HEAD],
env=env, check=False) stdout=subprocess.PIPE,
if result.returncode == 0: stderr=subprocess.DEVNULL,
ver = result.stdout.strip() encoding="utf-8",
if ver.startswith('v'): env=env,
ver = ver[1:] check=False,
else: )
ver = 'unknown' if result.returncode == 0:
setattr(RepoSourceVersion, 'version', ver) ver = result.stdout.strip()
if ver.startswith("v"):
ver = ver[1:]
else:
ver = "unknown"
setattr(RepoSourceVersion, "version", ver)
return ver return ver
class UserAgent(object): class UserAgent(object):
"""Mange User-Agent settings when talking to external services """Mange User-Agent settings when talking to external services
We follow the style as documented here: We follow the style as documented here:
https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent
""" """
_os = None _os = None
_repo_ua = None _repo_ua = None
_git_ua = None _git_ua = None
@property @property
def os(self): def os(self):
"""The operating system name.""" """The operating system name."""
if self._os is None: if self._os is None:
os_name = sys.platform os_name = sys.platform
if os_name.lower().startswith('linux'): if os_name.lower().startswith("linux"):
os_name = 'Linux' os_name = "Linux"
elif os_name == 'win32': elif os_name == "win32":
os_name = 'Win32' os_name = "Win32"
elif os_name == 'cygwin': elif os_name == "cygwin":
os_name = 'Cygwin' os_name = "Cygwin"
elif os_name == 'darwin': elif os_name == "darwin":
os_name = 'Darwin' os_name = "Darwin"
self._os = os_name self._os = os_name
return self._os return self._os
@property @property
def repo(self): def repo(self):
"""The UA when connecting directly from repo.""" """The UA when connecting directly from repo."""
if self._repo_ua is None: if self._repo_ua is None:
py_version = sys.version_info py_version = sys.version_info
self._repo_ua = 'git-repo/%s (%s) git/%s Python/%d.%d.%d' % ( self._repo_ua = "git-repo/%s (%s) git/%s Python/%d.%d.%d" % (
RepoSourceVersion(), RepoSourceVersion(),
self.os, self.os,
git.version_tuple().full, git.version_tuple().full,
py_version.major, py_version.minor, py_version.micro) py_version.major,
py_version.minor,
py_version.micro,
)
return self._repo_ua return self._repo_ua
@property @property
def git(self): def git(self):
"""The UA when running git.""" """The UA when running git."""
if self._git_ua is None: if self._git_ua is None:
self._git_ua = 'git/%s (%s) git-repo/%s' % ( self._git_ua = "git/%s (%s) git-repo/%s" % (
git.version_tuple().full, git.version_tuple().full,
self.os, self.os,
RepoSourceVersion()) RepoSourceVersion(),
)
return self._git_ua return self._git_ua
user_agent = UserAgent() user_agent = UserAgent()
def git_require(min_version, fail=False, msg=''): def git_require(min_version, fail=False, msg=""):
git_version = git.version_tuple() git_version = git.version_tuple()
if min_version <= git_version: if min_version <= git_version:
return True return True
if fail: if fail:
need = '.'.join(map(str, min_version)) need = ".".join(map(str, min_version))
if msg: if msg:
msg = ' for ' + msg msg = " for " + msg
print('fatal: git %s or later required%s' % (need, msg), file=sys.stderr) print(
sys.exit(1) "fatal: git %s or later required%s" % (need, msg), file=sys.stderr
return False )
sys.exit(1)
return False
def _build_env( def _build_env(
@ -164,175 +176,194 @@ def _build_env(
disable_editor: Optional[bool] = False, disable_editor: Optional[bool] = False,
ssh_proxy: Optional[Any] = None, ssh_proxy: Optional[Any] = None,
gitdir: Optional[str] = None, gitdir: Optional[str] = None,
objdir: Optional[str] = None objdir: Optional[str] = None,
): ):
"""Constucts an env dict for command execution.""" """Constucts an env dict for command execution."""
assert _kwargs_only == (), '_build_env only accepts keyword arguments.' assert _kwargs_only == (), "_build_env only accepts keyword arguments."
env = GitCommand._GetBasicEnv() env = GitCommand._GetBasicEnv()
if disable_editor: if disable_editor:
env['GIT_EDITOR'] = ':' env["GIT_EDITOR"] = ":"
if ssh_proxy: if ssh_proxy:
env['REPO_SSH_SOCK'] = ssh_proxy.sock() env["REPO_SSH_SOCK"] = ssh_proxy.sock()
env['GIT_SSH'] = ssh_proxy.proxy env["GIT_SSH"] = ssh_proxy.proxy
env['GIT_SSH_VARIANT'] = 'ssh' env["GIT_SSH_VARIANT"] = "ssh"
if 'http_proxy' in env and 'darwin' == sys.platform: if "http_proxy" in env and "darwin" == sys.platform:
s = "'http.proxy=%s'" % (env['http_proxy'],) s = "'http.proxy=%s'" % (env["http_proxy"],)
p = env.get('GIT_CONFIG_PARAMETERS') p = env.get("GIT_CONFIG_PARAMETERS")
if p is not None: if p is not None:
s = p + ' ' + s s = p + " " + s
env['GIT_CONFIG_PARAMETERS'] = s env["GIT_CONFIG_PARAMETERS"] = s
if 'GIT_ALLOW_PROTOCOL' not in env: if "GIT_ALLOW_PROTOCOL" not in env:
env['GIT_ALLOW_PROTOCOL'] = ( env[
'file:git:http:https:ssh:persistent-http:persistent-https:sso:rpc') "GIT_ALLOW_PROTOCOL"
env['GIT_HTTP_USER_AGENT'] = user_agent.git ] = "file:git:http:https:ssh:persistent-http:persistent-https:sso:rpc"
env["GIT_HTTP_USER_AGENT"] = user_agent.git
if objdir: if objdir:
# Set to the place we want to save the objects. # Set to the place we want to save the objects.
env['GIT_OBJECT_DIRECTORY'] = objdir env["GIT_OBJECT_DIRECTORY"] = objdir
alt_objects = os.path.join(gitdir, 'objects') if gitdir else None alt_objects = os.path.join(gitdir, "objects") if gitdir else None
if alt_objects and os.path.realpath(alt_objects) != os.path.realpath(objdir): if alt_objects and os.path.realpath(alt_objects) != os.path.realpath(
# Allow git to search the original place in case of local or unique refs objdir
# that git will attempt to resolve even if we aren't fetching them. ):
env['GIT_ALTERNATE_OBJECT_DIRECTORIES'] = alt_objects # Allow git to search the original place in case of local or unique
if bare and gitdir is not None: # refs that git will attempt to resolve even if we aren't fetching
env[GIT_DIR] = gitdir # them.
env["GIT_ALTERNATE_OBJECT_DIRECTORIES"] = alt_objects
if bare and gitdir is not None:
env[GIT_DIR] = gitdir
return env return env
class GitCommand(object): class GitCommand(object):
"""Wrapper around a single git invocation.""" """Wrapper around a single git invocation."""
def __init__(self, def __init__(
project, self,
cmdv, project,
bare=False, cmdv,
input=None, bare=False,
capture_stdout=False, input=None,
capture_stderr=False, capture_stdout=False,
merge_output=False, capture_stderr=False,
disable_editor=False, merge_output=False,
ssh_proxy=None, disable_editor=False,
cwd=None, ssh_proxy=None,
gitdir=None, cwd=None,
objdir=None): gitdir=None,
objdir=None,
):
if project:
if not cwd:
cwd = project.worktree
if not gitdir:
gitdir = project.gitdir
if project: # Git on Windows wants its paths only using / for reliability.
if not cwd: if platform_utils.isWindows():
cwd = project.worktree if objdir:
if not gitdir: objdir = objdir.replace("\\", "/")
gitdir = project.gitdir if gitdir:
gitdir = gitdir.replace("\\", "/")
# Git on Windows wants its paths only using / for reliability. env = _build_env(
if platform_utils.isWindows(): disable_editor=disable_editor,
if objdir: ssh_proxy=ssh_proxy,
objdir = objdir.replace('\\', '/') objdir=objdir,
if gitdir: gitdir=gitdir,
gitdir = gitdir.replace('\\', '/') bare=bare,
)
env = _build_env( command = [GIT]
disable_editor=disable_editor, if bare:
ssh_proxy=ssh_proxy, cwd = None
objdir=objdir, command.append(cmdv[0])
gitdir=gitdir, # Need to use the --progress flag for fetch/clone so output will be
bare=bare, # displayed as by default git only does progress output if stderr is a
) # TTY.
if sys.stderr.isatty() and cmdv[0] in ("fetch", "clone"):
if "--progress" not in cmdv and "--quiet" not in cmdv:
command.append("--progress")
command.extend(cmdv[1:])
command = [GIT] stdin = subprocess.PIPE if input else None
if bare: stdout = subprocess.PIPE if capture_stdout else None
cwd = None stderr = (
command.append(cmdv[0]) subprocess.STDOUT
# Need to use the --progress flag for fetch/clone so output will be if merge_output
# displayed as by default git only does progress output if stderr is a TTY. else (subprocess.PIPE if capture_stderr else None)
if sys.stderr.isatty() and cmdv[0] in ('fetch', 'clone'): )
if '--progress' not in cmdv and '--quiet' not in cmdv:
command.append('--progress')
command.extend(cmdv[1:])
stdin = subprocess.PIPE if input else None dbg = ""
stdout = subprocess.PIPE if capture_stdout else None if IsTrace():
stderr = (subprocess.STDOUT if merge_output else global LAST_CWD
(subprocess.PIPE if capture_stderr else None)) global LAST_GITDIR
dbg = '' if cwd and LAST_CWD != cwd:
if IsTrace(): if LAST_GITDIR or LAST_CWD:
global LAST_CWD dbg += "\n"
global LAST_GITDIR dbg += ": cd %s\n" % cwd
LAST_CWD = cwd
if cwd and LAST_CWD != cwd: if GIT_DIR in env and LAST_GITDIR != env[GIT_DIR]:
if LAST_GITDIR or LAST_CWD: if LAST_GITDIR or LAST_CWD:
dbg += '\n' dbg += "\n"
dbg += ': cd %s\n' % cwd dbg += ": export GIT_DIR=%s\n" % env[GIT_DIR]
LAST_CWD = cwd LAST_GITDIR = env[GIT_DIR]
if GIT_DIR in env and LAST_GITDIR != env[GIT_DIR]: if "GIT_OBJECT_DIRECTORY" in env:
if LAST_GITDIR or LAST_CWD: dbg += (
dbg += '\n' ": export GIT_OBJECT_DIRECTORY=%s\n"
dbg += ': export GIT_DIR=%s\n' % env[GIT_DIR] % env["GIT_OBJECT_DIRECTORY"]
LAST_GITDIR = env[GIT_DIR] )
if "GIT_ALTERNATE_OBJECT_DIRECTORIES" in env:
dbg += ": export GIT_ALTERNATE_OBJECT_DIRECTORIES=%s\n" % (
env["GIT_ALTERNATE_OBJECT_DIRECTORIES"]
)
if 'GIT_OBJECT_DIRECTORY' in env: dbg += ": "
dbg += ': export GIT_OBJECT_DIRECTORY=%s\n' % env['GIT_OBJECT_DIRECTORY'] dbg += " ".join(command)
if 'GIT_ALTERNATE_OBJECT_DIRECTORIES' in env: if stdin == subprocess.PIPE:
dbg += ': export GIT_ALTERNATE_OBJECT_DIRECTORIES=%s\n' % ( dbg += " 0<|"
env['GIT_ALTERNATE_OBJECT_DIRECTORIES']) if stdout == subprocess.PIPE:
dbg += " 1>|"
if stderr == subprocess.PIPE:
dbg += " 2>|"
elif stderr == subprocess.STDOUT:
dbg += " 2>&1"
dbg += ': ' with Trace(
dbg += ' '.join(command) "git command %s %s with debug: %s", LAST_GITDIR, command, dbg
if stdin == subprocess.PIPE: ):
dbg += ' 0<|' try:
if stdout == subprocess.PIPE: p = subprocess.Popen(
dbg += ' 1>|' command,
if stderr == subprocess.PIPE: cwd=cwd,
dbg += ' 2>|' env=env,
elif stderr == subprocess.STDOUT: encoding="utf-8",
dbg += ' 2>&1' errors="backslashreplace",
stdin=stdin,
stdout=stdout,
stderr=stderr,
)
except Exception as e:
raise GitError("%s: %s" % (command[1], e))
with Trace('git command %s %s with debug: %s', LAST_GITDIR, command, dbg): if ssh_proxy:
try: ssh_proxy.add_client(p)
p = subprocess.Popen(command,
cwd=cwd,
env=env,
encoding='utf-8',
errors='backslashreplace',
stdin=stdin,
stdout=stdout,
stderr=stderr)
except Exception as e:
raise GitError('%s: %s' % (command[1], e))
if ssh_proxy: self.process = p
ssh_proxy.add_client(p)
self.process = p try:
self.stdout, self.stderr = p.communicate(input=input)
finally:
if ssh_proxy:
ssh_proxy.remove_client(p)
self.rc = p.wait()
try: @staticmethod
self.stdout, self.stderr = p.communicate(input=input) def _GetBasicEnv():
finally: """Return a basic env for running git under.
if ssh_proxy:
ssh_proxy.remove_client(p)
self.rc = p.wait()
@staticmethod This is guaranteed to be side-effect free.
def _GetBasicEnv(): """
"""Return a basic env for running git under. env = os.environ.copy()
for key in (
REPO_TRACE,
GIT_DIR,
"GIT_ALTERNATE_OBJECT_DIRECTORIES",
"GIT_OBJECT_DIRECTORY",
"GIT_WORK_TREE",
"GIT_GRAFT_FILE",
"GIT_INDEX_FILE",
):
env.pop(key, None)
return env
This is guaranteed to be side-effect free. def Wait(self):
""" return self.rc
env = os.environ.copy()
for key in (REPO_TRACE,
GIT_DIR,
'GIT_ALTERNATE_OBJECT_DIRECTORIES',
'GIT_OBJECT_DIRECTORY',
'GIT_WORK_TREE',
'GIT_GRAFT_FILE',
'GIT_INDEX_FILE'):
env.pop(key, None)
return env
def Wait(self):
return self.rc

1342
git_config.py
View File

File diff suppressed because it is too large Load Diff

267
git_refs.py
View File

@ -16,149 +16,150 @@ import os
from repo_trace import Trace from repo_trace import Trace
import platform_utils import platform_utils
HEAD = 'HEAD' HEAD = "HEAD"
R_CHANGES = 'refs/changes/' R_CHANGES = "refs/changes/"
R_HEADS = 'refs/heads/' R_HEADS = "refs/heads/"
R_TAGS = 'refs/tags/' R_TAGS = "refs/tags/"
R_PUB = 'refs/published/' R_PUB = "refs/published/"
R_WORKTREE = 'refs/worktree/' R_WORKTREE = "refs/worktree/"
R_WORKTREE_M = R_WORKTREE + 'm/' R_WORKTREE_M = R_WORKTREE + "m/"
R_M = 'refs/remotes/m/' R_M = "refs/remotes/m/"
class GitRefs(object): class GitRefs(object):
def __init__(self, gitdir): def __init__(self, gitdir):
self._gitdir = gitdir self._gitdir = gitdir
self._phyref = None self._phyref = None
self._symref = None self._symref = None
self._mtime = {} self._mtime = {}
@property @property
def all(self): def all(self):
self._EnsureLoaded() self._EnsureLoaded()
return self._phyref return self._phyref
def get(self, name): def get(self, name):
try:
return self.all[name]
except KeyError:
return ''
def deleted(self, name):
if self._phyref is not None:
if name in self._phyref:
del self._phyref[name]
if name in self._symref:
del self._symref[name]
if name in self._mtime:
del self._mtime[name]
def symref(self, name):
try:
self._EnsureLoaded()
return self._symref[name]
except KeyError:
return ''
def _EnsureLoaded(self):
if self._phyref is None or self._NeedUpdate():
self._LoadAll()
def _NeedUpdate(self):
with Trace(': scan refs %s', self._gitdir):
for name, mtime in self._mtime.items():
try: try:
if mtime != os.path.getmtime(os.path.join(self._gitdir, name)): return self.all[name]
return True except KeyError:
return ""
def deleted(self, name):
if self._phyref is not None:
if name in self._phyref:
del self._phyref[name]
if name in self._symref:
del self._symref[name]
if name in self._mtime:
del self._mtime[name]
def symref(self, name):
try:
self._EnsureLoaded()
return self._symref[name]
except KeyError:
return ""
def _EnsureLoaded(self):
if self._phyref is None or self._NeedUpdate():
self._LoadAll()
def _NeedUpdate(self):
with Trace(": scan refs %s", self._gitdir):
for name, mtime in self._mtime.items():
try:
if mtime != os.path.getmtime(
os.path.join(self._gitdir, name)
):
return True
except OSError:
return True
return False
def _LoadAll(self):
with Trace(": load refs %s", self._gitdir):
self._phyref = {}
self._symref = {}
self._mtime = {}
self._ReadPackedRefs()
self._ReadLoose("refs/")
self._ReadLoose1(os.path.join(self._gitdir, HEAD), HEAD)
scan = self._symref
attempts = 0
while scan and attempts < 5:
scan_next = {}
for name, dest in scan.items():
if dest in self._phyref:
self._phyref[name] = self._phyref[dest]
else:
scan_next[name] = dest
scan = scan_next
attempts += 1
def _ReadPackedRefs(self):
path = os.path.join(self._gitdir, "packed-refs")
try:
fd = open(path, "r")
mtime = os.path.getmtime(path)
except IOError:
return
except OSError: except OSError:
return True return
return False try:
for line in fd:
line = str(line)
if line[0] == "#":
continue
if line[0] == "^":
continue
def _LoadAll(self): line = line[:-1]
with Trace(': load refs %s', self._gitdir): p = line.split(" ")
ref_id = p[0]
name = p[1]
self._phyref = {} self._phyref[name] = ref_id
self._symref = {} finally:
self._mtime = {} fd.close()
self._mtime["packed-refs"] = mtime
self._ReadPackedRefs() def _ReadLoose(self, prefix):
self._ReadLoose('refs/') base = os.path.join(self._gitdir, prefix)
self._ReadLoose1(os.path.join(self._gitdir, HEAD), HEAD) for name in platform_utils.listdir(base):
p = os.path.join(base, name)
# We don't implement the full ref validation algorithm, just the
# simple rules that would show up in local filesystems.
# https://git-scm.com/docs/git-check-ref-format
if name.startswith(".") or name.endswith(".lock"):
pass
elif platform_utils.isdir(p):
self._mtime[prefix] = os.path.getmtime(base)
self._ReadLoose(prefix + name + "/")
else:
self._ReadLoose1(p, prefix + name)
scan = self._symref def _ReadLoose1(self, path, name):
attempts = 0 try:
while scan and attempts < 5: with open(path) as fd:
scan_next = {} mtime = os.path.getmtime(path)
for name, dest in scan.items(): ref_id = fd.readline()
if dest in self._phyref: except (OSError, UnicodeError):
self._phyref[name] = self._phyref[dest] return
else:
scan_next[name] = dest
scan = scan_next
attempts += 1
def _ReadPackedRefs(self): try:
path = os.path.join(self._gitdir, 'packed-refs') ref_id = ref_id.decode()
try: except AttributeError:
fd = open(path, 'r') pass
mtime = os.path.getmtime(path) if not ref_id:
except IOError: return
return ref_id = ref_id[:-1]
except OSError:
return
try:
for line in fd:
line = str(line)
if line[0] == '#':
continue
if line[0] == '^':
continue
line = line[:-1] if ref_id.startswith("ref: "):
p = line.split(' ') self._symref[name] = ref_id[5:]
ref_id = p[0] else:
name = p[1] self._phyref[name] = ref_id
self._mtime[name] = mtime
self._phyref[name] = ref_id
finally:
fd.close()
self._mtime['packed-refs'] = mtime
def _ReadLoose(self, prefix):
base = os.path.join(self._gitdir, prefix)
for name in platform_utils.listdir(base):
p = os.path.join(base, name)
# We don't implement the full ref validation algorithm, just the simple
# rules that would show up in local filesystems.
# https://git-scm.com/docs/git-check-ref-format
if name.startswith('.') or name.endswith('.lock'):
pass
elif platform_utils.isdir(p):
self._mtime[prefix] = os.path.getmtime(base)
self._ReadLoose(prefix + name + '/')
else:
self._ReadLoose1(p, prefix + name)
def _ReadLoose1(self, path, name):
try:
with open(path) as fd:
mtime = os.path.getmtime(path)
ref_id = fd.readline()
except (OSError, UnicodeError):
return
try:
ref_id = ref_id.decode()
except AttributeError:
pass
if not ref_id:
return
ref_id = ref_id[:-1]
if ref_id.startswith('ref: '):
self._symref[name] = ref_id[5:]
else:
self._phyref[name] = ref_id
self._mtime[name] = mtime

790
git_superproject.py
View File

@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Provide functionality to get all projects and their commit ids from Superproject. """Provide functionality to get projects and their commit ids from Superproject.
For more information on superproject, check out: For more information on superproject, check out:
https://en.wikibooks.org/wiki/Git/Submodules_and_Superprojects https://en.wikibooks.org/wiki/Git/Submodules_and_Superprojects
@ -33,434 +33,524 @@ from git_command import git_require, GitCommand
from git_config import RepoConfig from git_config import RepoConfig
from git_refs import GitRefs from git_refs import GitRefs
_SUPERPROJECT_GIT_NAME = 'superproject.git' _SUPERPROJECT_GIT_NAME = "superproject.git"
_SUPERPROJECT_MANIFEST_NAME = 'superproject_override.xml' _SUPERPROJECT_MANIFEST_NAME = "superproject_override.xml"
class SyncResult(NamedTuple): class SyncResult(NamedTuple):
"""Return the status of sync and whether caller should exit.""" """Return the status of sync and whether caller should exit."""
# Whether the superproject sync was successful. # Whether the superproject sync was successful.
success: bool success: bool
# Whether the caller should exit. # Whether the caller should exit.
fatal: bool fatal: bool
class CommitIdsResult(NamedTuple): class CommitIdsResult(NamedTuple):
"""Return the commit ids and whether caller should exit.""" """Return the commit ids and whether caller should exit."""
# A dictionary with the projects/commit ids on success, otherwise None. # A dictionary with the projects/commit ids on success, otherwise None.
commit_ids: dict commit_ids: dict
# Whether the caller should exit. # Whether the caller should exit.
fatal: bool fatal: bool
class UpdateProjectsResult(NamedTuple): class UpdateProjectsResult(NamedTuple):
"""Return the overriding manifest file and whether caller should exit.""" """Return the overriding manifest file and whether caller should exit."""
# Path name of the overriding manifest file if successful, otherwise None. # Path name of the overriding manifest file if successful, otherwise None.
manifest_path: str manifest_path: str
# Whether the caller should exit. # Whether the caller should exit.
fatal: bool fatal: bool
class Superproject(object): class Superproject(object):
"""Get commit ids from superproject. """Get commit ids from superproject.
Initializes a local copy of a superproject for the manifest. This allows Initializes a local copy of a superproject for the manifest. This allows
lookup of commit ids for all projects. It contains _project_commit_ids which lookup of commit ids for all projects. It contains _project_commit_ids which
is a dictionary with project/commit id entries. is a dictionary with project/commit id entries.
"""
def __init__(self, manifest, name, remote, revision,
superproject_dir='exp-superproject'):
"""Initializes superproject.
Args:
manifest: A Manifest object that is to be written to a file.
name: The unique name of the superproject
remote: The RemoteSpec for the remote.
revision: The name of the git branch to track.
superproject_dir: Relative path under |manifest.subdir| to checkout
superproject.
""" """
self._project_commit_ids = None
self._manifest = manifest
self.name = name
self.remote = remote
self.revision = self._branch = revision
self._repodir = manifest.repodir
self._superproject_dir = superproject_dir
self._superproject_path = manifest.SubmanifestInfoDir(manifest.path_prefix,
superproject_dir)
self._manifest_path = os.path.join(self._superproject_path,
_SUPERPROJECT_MANIFEST_NAME)
git_name = hashlib.md5(remote.name.encode('utf8')).hexdigest() + '-'
self._remote_url = remote.url
self._work_git_name = git_name + _SUPERPROJECT_GIT_NAME
self._work_git = os.path.join(self._superproject_path, self._work_git_name)
# The following are command arguemnts, rather than superproject attributes, def __init__(
# and were included here originally. They should eventually become self,
# arguments that are passed down from the public methods, instead of being manifest,
# treated as attributes. name,
self._git_event_log = None remote,
self._quiet = False revision,
self._print_messages = False superproject_dir="exp-superproject",
):
"""Initializes superproject.
def SetQuiet(self, value): Args:
"""Set the _quiet attribute.""" manifest: A Manifest object that is to be written to a file.
self._quiet = value name: The unique name of the superproject
remote: The RemoteSpec for the remote.
revision: The name of the git branch to track.
superproject_dir: Relative path under |manifest.subdir| to checkout
superproject.
"""
self._project_commit_ids = None
self._manifest = manifest
self.name = name
self.remote = remote
self.revision = self._branch = revision
self._repodir = manifest.repodir
self._superproject_dir = superproject_dir
self._superproject_path = manifest.SubmanifestInfoDir(
manifest.path_prefix, superproject_dir
)
self._manifest_path = os.path.join(
self._superproject_path, _SUPERPROJECT_MANIFEST_NAME
)
git_name = hashlib.md5(remote.name.encode("utf8")).hexdigest() + "-"
self._remote_url = remote.url
self._work_git_name = git_name + _SUPERPROJECT_GIT_NAME
self._work_git = os.path.join(
self._superproject_path, self._work_git_name
)
def SetPrintMessages(self, value): # The following are command arguemnts, rather than superproject
"""Set the _print_messages attribute.""" # attributes, and were included here originally. They should eventually
self._print_messages = value # become arguments that are passed down from the public methods, instead
# of being treated as attributes.
self._git_event_log = None
self._quiet = False
self._print_messages = False
@property def SetQuiet(self, value):
def project_commit_ids(self): """Set the _quiet attribute."""
"""Returns a dictionary of projects and their commit ids.""" self._quiet = value
return self._project_commit_ids
@property def SetPrintMessages(self, value):
def manifest_path(self): """Set the _print_messages attribute."""
"""Returns the manifest path if the path exists or None.""" self._print_messages = value
return self._manifest_path if os.path.exists(self._manifest_path) else None
def _LogMessage(self, fmt, *inputs): @property
"""Logs message to stderr and _git_event_log.""" def project_commit_ids(self):
message = f'{self._LogMessagePrefix()} {fmt.format(*inputs)}' """Returns a dictionary of projects and their commit ids."""
if self._print_messages: return self._project_commit_ids
print(message, file=sys.stderr)
self._git_event_log.ErrorEvent(message, fmt)
def _LogMessagePrefix(self): @property
"""Returns the prefix string to be logged in each log message""" def manifest_path(self):
return f'repo superproject branch: {self._branch} url: {self._remote_url}' """Returns the manifest path if the path exists or None."""
return (
self._manifest_path if os.path.exists(self._manifest_path) else None
)
def _LogError(self, fmt, *inputs): def _LogMessage(self, fmt, *inputs):
"""Logs error message to stderr and _git_event_log.""" """Logs message to stderr and _git_event_log."""
self._LogMessage(f'error: {fmt}', *inputs) message = f"{self._LogMessagePrefix()} {fmt.format(*inputs)}"
if self._print_messages:
print(message, file=sys.stderr)
self._git_event_log.ErrorEvent(message, fmt)
def _LogWarning(self, fmt, *inputs): def _LogMessagePrefix(self):
"""Logs warning message to stderr and _git_event_log.""" """Returns the prefix string to be logged in each log message"""
self._LogMessage(f'warning: {fmt}', *inputs) return (
f"repo superproject branch: {self._branch} url: {self._remote_url}"
)
def _Init(self): def _LogError(self, fmt, *inputs):
"""Sets up a local Git repository to get a copy of a superproject. """Logs error message to stderr and _git_event_log."""
self._LogMessage(f"error: {fmt}", *inputs)
Returns: def _LogWarning(self, fmt, *inputs):
True if initialization is successful, or False. """Logs warning message to stderr and _git_event_log."""
""" self._LogMessage(f"warning: {fmt}", *inputs)
if not os.path.exists(self._superproject_path):
os.mkdir(self._superproject_path)
if not self._quiet and not os.path.exists(self._work_git):
print('%s: Performing initial setup for superproject; this might take '
'several minutes.' % self._work_git)
cmd = ['init', '--bare', self._work_git_name]
p = GitCommand(None,
cmd,
cwd=self._superproject_path,
capture_stdout=True,
capture_stderr=True)
retval = p.Wait()
if retval:
self._LogWarning('git init call failed, command: git {}, '
'return code: {}, stderr: {}', cmd, retval, p.stderr)
return False
return True
def _Fetch(self): def _Init(self):
"""Fetches a local copy of a superproject for the manifest based on |_remote_url|. """Sets up a local Git repository to get a copy of a superproject.
Returns: Returns:
True if fetch is successful, or False. True if initialization is successful, or False.
""" """
if not os.path.exists(self._work_git): if not os.path.exists(self._superproject_path):
self._LogWarning('git fetch missing directory: {}', self._work_git) os.mkdir(self._superproject_path)
return False if not self._quiet and not os.path.exists(self._work_git):
if not git_require((2, 28, 0)): print(
self._LogWarning('superproject requires a git version 2.28 or later') "%s: Performing initial setup for superproject; this might "
return False "take several minutes." % self._work_git
cmd = ['fetch', self._remote_url, '--depth', '1', '--force', '--no-tags', )
'--filter', 'blob:none'] cmd = ["init", "--bare", self._work_git_name]
p = GitCommand(
None,
cmd,
cwd=self._superproject_path,
capture_stdout=True,
capture_stderr=True,
)
retval = p.Wait()
if retval:
self._LogWarning(
"git init call failed, command: git {}, "
"return code: {}, stderr: {}",
cmd,
retval,
p.stderr,
)
return False
return True
# Check if there is a local ref that we can pass to --negotiation-tip. def _Fetch(self):
# If this is the first fetch, it does not exist yet. """Fetches a superproject for the manifest based on |_remote_url|.
# We use --negotiation-tip to speed up the fetch. Superproject branches do
# not share commits. So this lets git know it only needs to send commits
# reachable from the specified local refs.
rev_commit = GitRefs(self._work_git).get(f'refs/heads/{self.revision}')
if rev_commit:
cmd.extend(['--negotiation-tip', rev_commit])
if self._branch: This runs git fetch which stores a local copy the superproject.
cmd += [self._branch + ':' + self._branch]
p = GitCommand(None,
cmd,
cwd=self._work_git,
capture_stdout=True,
capture_stderr=True)
retval = p.Wait()
if retval:
self._LogWarning('git fetch call failed, command: git {}, '
'return code: {}, stderr: {}', cmd, retval, p.stderr)
return False
return True
def _LsTree(self): Returns:
"""Gets the commit ids for all projects. True if fetch is successful, or False.
"""
if not os.path.exists(self._work_git):
self._LogWarning("git fetch missing directory: {}", self._work_git)
return False
if not git_require((2, 28, 0)):
self._LogWarning(
"superproject requires a git version 2.28 or later"
)
return False
cmd = [
"fetch",
self._remote_url,
"--depth",
"1",
"--force",
"--no-tags",
"--filter",
"blob:none",
]
Works only in git repositories. # Check if there is a local ref that we can pass to --negotiation-tip.
# If this is the first fetch, it does not exist yet.
# We use --negotiation-tip to speed up the fetch. Superproject branches
# do not share commits. So this lets git know it only needs to send
# commits reachable from the specified local refs.
rev_commit = GitRefs(self._work_git).get(f"refs/heads/{self.revision}")
if rev_commit:
cmd.extend(["--negotiation-tip", rev_commit])
Returns: if self._branch:
data: data returned from 'git ls-tree ...' instead of None. cmd += [self._branch + ":" + self._branch]
""" p = GitCommand(
if not os.path.exists(self._work_git): None,
self._LogWarning('git ls-tree missing directory: {}', self._work_git) cmd,
return None cwd=self._work_git,
data = None capture_stdout=True,
branch = 'HEAD' if not self._branch else self._branch capture_stderr=True,
cmd = ['ls-tree', '-z', '-r', branch] )
retval = p.Wait()
if retval:
self._LogWarning(
"git fetch call failed, command: git {}, "
"return code: {}, stderr: {}",
cmd,
retval,
p.stderr,
)
return False
return True
p = GitCommand(None, def _LsTree(self):
cmd, """Gets the commit ids for all projects.
cwd=self._work_git,
capture_stdout=True,
capture_stderr=True)
retval = p.Wait()
if retval == 0:
data = p.stdout
else:
self._LogWarning('git ls-tree call failed, command: git {}, '
'return code: {}, stderr: {}', cmd, retval, p.stderr)
return data
def Sync(self, git_event_log): Works only in git repositories.
"""Gets a local copy of a superproject for the manifest.
Args: Returns:
git_event_log: an EventLog, for git tracing. data: data returned from 'git ls-tree ...' instead of None.
"""
if not os.path.exists(self._work_git):
self._LogWarning(
"git ls-tree missing directory: {}", self._work_git
)
return None
data = None
branch = "HEAD" if not self._branch else self._branch
cmd = ["ls-tree", "-z", "-r", branch]
Returns: p = GitCommand(
SyncResult None,
""" cmd,
self._git_event_log = git_event_log cwd=self._work_git,
if not self._manifest.superproject: capture_stdout=True,
self._LogWarning('superproject tag is not defined in manifest: {}', capture_stderr=True,
self._manifest.manifestFile) )
return SyncResult(False, False) retval = p.Wait()
if retval == 0:
data = p.stdout
else:
self._LogWarning(
"git ls-tree call failed, command: git {}, "
"return code: {}, stderr: {}",
cmd,
retval,
p.stderr,
)
return data
_PrintBetaNotice() def Sync(self, git_event_log):
"""Gets a local copy of a superproject for the manifest.
should_exit = True Args:
if not self._remote_url: git_event_log: an EventLog, for git tracing.
self._LogWarning('superproject URL is not defined in manifest: {}',
self._manifest.manifestFile)
return SyncResult(False, should_exit)
if not self._Init(): Returns:
return SyncResult(False, should_exit) SyncResult
if not self._Fetch(): """
return SyncResult(False, should_exit) self._git_event_log = git_event_log
if not self._quiet: if not self._manifest.superproject:
print('%s: Initial setup for superproject completed.' % self._work_git) self._LogWarning(
return SyncResult(True, False) "superproject tag is not defined in manifest: {}",
self._manifest.manifestFile,
)
return SyncResult(False, False)
def _GetAllProjectsCommitIds(self): _PrintBetaNotice()
"""Get commit ids for all projects from superproject and save them in _project_commit_ids.
Returns: should_exit = True
CommitIdsResult if not self._remote_url:
""" self._LogWarning(
sync_result = self.Sync(self._git_event_log) "superproject URL is not defined in manifest: {}",
if not sync_result.success: self._manifest.manifestFile,
return CommitIdsResult(None, sync_result.fatal) )
return SyncResult(False, should_exit)
data = self._LsTree() if not self._Init():
if not data: return SyncResult(False, should_exit)
self._LogWarning('git ls-tree failed to return data for manifest: {}', if not self._Fetch():
self._manifest.manifestFile) return SyncResult(False, should_exit)
return CommitIdsResult(None, True) if not self._quiet:
print(
"%s: Initial setup for superproject completed." % self._work_git
)
return SyncResult(True, False)
# Parse lines like the following to select lines starting with '160000' and def _GetAllProjectsCommitIds(self):
# build a dictionary with project path (last element) and its commit id (3rd element). """Get commit ids for all projects from superproject and save them.
#
# 160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00
# 120000 blob acc2cbdf438f9d2141f0ae424cec1d8fc4b5d97f\tbootstrap.bash\x00
commit_ids = {}
for line in data.split('\x00'):
ls_data = line.split(None, 3)
if not ls_data:
break
if ls_data[0] == '160000':
commit_ids[ls_data[3]] = ls_data[2]
self._project_commit_ids = commit_ids Commit ids are saved in _project_commit_ids.
return CommitIdsResult(commit_ids, False)
def _WriteManifestFile(self): Returns:
"""Writes manifest to a file. CommitIdsResult
"""
sync_result = self.Sync(self._git_event_log)
if not sync_result.success:
return CommitIdsResult(None, sync_result.fatal)
Returns: data = self._LsTree()
manifest_path: Path name of the file into which manifest is written instead of None. if not data:
""" self._LogWarning(
if not os.path.exists(self._superproject_path): "git ls-tree failed to return data for manifest: {}",
self._LogWarning('missing superproject directory: {}', self._superproject_path) self._manifest.manifestFile,
return None )
manifest_str = self._manifest.ToXml(groups=self._manifest.GetGroupsStr(), return CommitIdsResult(None, True)
omit_local=True).toxml()
manifest_path = self._manifest_path
try:
with open(manifest_path, 'w', encoding='utf-8') as fp:
fp.write(manifest_str)
except IOError as e:
self._LogError('cannot write manifest to : {} {}',
manifest_path, e)
return None
return manifest_path
def _SkipUpdatingProjectRevisionId(self, project): # Parse lines like the following to select lines starting with '160000'
"""Checks if a project's revision id needs to be updated or not. # and build a dictionary with project path (last element) and its commit
# id (3rd element).
#
# 160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00
# 120000 blob acc2cbdf438f9d2141f0ae424cec1d8fc4b5d97f\tbootstrap.bash\x00 # noqa: E501
commit_ids = {}
for line in data.split("\x00"):
ls_data = line.split(None, 3)
if not ls_data:
break
if ls_data[0] == "160000":
commit_ids[ls_data[3]] = ls_data[2]
Revision id for projects from local manifest will not be updated. self._project_commit_ids = commit_ids
return CommitIdsResult(commit_ids, False)
Args: def _WriteManifestFile(self):
project: project whose revision id is being updated. """Writes manifest to a file.
Returns: Returns:
True if a project's revision id should not be updated, or False, manifest_path: Path name of the file into which manifest is written
""" instead of None.
path = project.relpath """
if not path: if not os.path.exists(self._superproject_path):
return True self._LogWarning(
# Skip the project with revisionId. "missing superproject directory: {}", self._superproject_path
if project.revisionId: )
return True return None
# Skip the project if it comes from the local manifest. manifest_str = self._manifest.ToXml(
return project.manifest.IsFromLocalManifest(project) groups=self._manifest.GetGroupsStr(), omit_local=True
).toxml()
manifest_path = self._manifest_path
try:
with open(manifest_path, "w", encoding="utf-8") as fp:
fp.write(manifest_str)
except IOError as e:
self._LogError("cannot write manifest to : {} {}", manifest_path, e)
return None
return manifest_path
def UpdateProjectsRevisionId(self, projects, git_event_log): def _SkipUpdatingProjectRevisionId(self, project):
"""Update revisionId of every project in projects with the commit id. """Checks if a project's revision id needs to be updated or not.
Args: Revision id for projects from local manifest will not be updated.
projects: a list of projects whose revisionId needs to be updated.
git_event_log: an EventLog, for git tracing.
Returns: Args:
UpdateProjectsResult project: project whose revision id is being updated.
"""
self._git_event_log = git_event_log
commit_ids_result = self._GetAllProjectsCommitIds()
commit_ids = commit_ids_result.commit_ids
if not commit_ids:
return UpdateProjectsResult(None, commit_ids_result.fatal)
projects_missing_commit_ids = [] Returns:
for project in projects: True if a project's revision id should not be updated, or False,
if self._SkipUpdatingProjectRevisionId(project): """
continue path = project.relpath
path = project.relpath if not path:
commit_id = commit_ids.get(path) return True
if not commit_id: # Skip the project with revisionId.
projects_missing_commit_ids.append(path) if project.revisionId:
return True
# Skip the project if it comes from the local manifest.
return project.manifest.IsFromLocalManifest(project)
# If superproject doesn't have a commit id for a project, then report an def UpdateProjectsRevisionId(self, projects, git_event_log):
# error event and continue as if do not use superproject is specified. """Update revisionId of every project in projects with the commit id.
if projects_missing_commit_ids:
self._LogWarning('please file a bug using {} to report missing '
'commit_ids for: {}', self._manifest.contactinfo.bugurl,
projects_missing_commit_ids)
return UpdateProjectsResult(None, False)
for project in projects: Args:
if not self._SkipUpdatingProjectRevisionId(project): projects: a list of projects whose revisionId needs to be updated.
project.SetRevisionId(commit_ids.get(project.relpath)) git_event_log: an EventLog, for git tracing.
manifest_path = self._WriteManifestFile() Returns:
return UpdateProjectsResult(manifest_path, False) UpdateProjectsResult
"""
self._git_event_log = git_event_log
commit_ids_result = self._GetAllProjectsCommitIds()
commit_ids = commit_ids_result.commit_ids
if not commit_ids:
return UpdateProjectsResult(None, commit_ids_result.fatal)
projects_missing_commit_ids = []
for project in projects:
if self._SkipUpdatingProjectRevisionId(project):
continue
path = project.relpath
commit_id = commit_ids.get(path)
if not commit_id:
projects_missing_commit_ids.append(path)
# If superproject doesn't have a commit id for a project, then report an
# error event and continue as if do not use superproject is specified.
if projects_missing_commit_ids:
self._LogWarning(
"please file a bug using {} to report missing "
"commit_ids for: {}",
self._manifest.contactinfo.bugurl,
projects_missing_commit_ids,
)
return UpdateProjectsResult(None, False)
for project in projects:
if not self._SkipUpdatingProjectRevisionId(project):
project.SetRevisionId(commit_ids.get(project.relpath))
manifest_path = self._WriteManifestFile()
return UpdateProjectsResult(manifest_path, False)
@functools.lru_cache(maxsize=10) @functools.lru_cache(maxsize=10)
def _PrintBetaNotice(): def _PrintBetaNotice():
"""Print the notice of beta status.""" """Print the notice of beta status."""
print('NOTICE: --use-superproject is in beta; report any issues to the ' print(
'address described in `repo version`', file=sys.stderr) "NOTICE: --use-superproject is in beta; report any issues to the "
"address described in `repo version`",
file=sys.stderr,
)
@functools.lru_cache(maxsize=None) @functools.lru_cache(maxsize=None)
def _UseSuperprojectFromConfiguration(): def _UseSuperprojectFromConfiguration():
"""Returns the user choice of whether to use superproject.""" """Returns the user choice of whether to use superproject."""
user_cfg = RepoConfig.ForUser() user_cfg = RepoConfig.ForUser()
time_now = int(time.time()) time_now = int(time.time())
user_value = user_cfg.GetBoolean('repo.superprojectChoice') user_value = user_cfg.GetBoolean("repo.superprojectChoice")
if user_value is not None: if user_value is not None:
user_expiration = user_cfg.GetInt('repo.superprojectChoiceExpire') user_expiration = user_cfg.GetInt("repo.superprojectChoiceExpire")
if user_expiration is None or user_expiration <= 0 or user_expiration >= time_now: if (
# TODO(b/190688390) - Remove prompt when we are comfortable with the new user_expiration is None
# default value. or user_expiration <= 0
if user_value: or user_expiration >= time_now
print(('You are currently enrolled in Git submodules experiment ' ):
'(go/android-submodules-quickstart). Use --no-use-superproject ' # TODO(b/190688390) - Remove prompt when we are comfortable with the
'to override.\n'), file=sys.stderr) # new default value.
else: if user_value:
print(('You are not currently enrolled in Git submodules experiment ' print(
'(go/android-submodules-quickstart). Use --use-superproject ' (
'to override.\n'), file=sys.stderr) "You are currently enrolled in Git submodules "
return user_value "experiment (go/android-submodules-quickstart). Use "
"--no-use-superproject to override.\n"
),
file=sys.stderr,
)
else:
print(
(
"You are not currently enrolled in Git submodules "
"experiment (go/android-submodules-quickstart). Use "
"--use-superproject to override.\n"
),
file=sys.stderr,
)
return user_value
# We don't have an unexpired choice, ask for one. # We don't have an unexpired choice, ask for one.
system_cfg = RepoConfig.ForSystem() system_cfg = RepoConfig.ForSystem()
system_value = system_cfg.GetBoolean('repo.superprojectChoice') system_value = system_cfg.GetBoolean("repo.superprojectChoice")
if system_value: if system_value:
# The system configuration is proposing that we should enable the # The system configuration is proposing that we should enable the
# use of superproject. Treat the user as enrolled for two weeks. # use of superproject. Treat the user as enrolled for two weeks.
# #
# TODO(b/190688390) - Remove prompt when we are comfortable with the new # TODO(b/190688390) - Remove prompt when we are comfortable with the new
# default value. # default value.
userchoice = True userchoice = True
time_choiceexpire = time_now + (86400 * 14) time_choiceexpire = time_now + (86400 * 14)
user_cfg.SetString('repo.superprojectChoiceExpire', str(time_choiceexpire)) user_cfg.SetString(
user_cfg.SetBoolean('repo.superprojectChoice', userchoice) "repo.superprojectChoiceExpire", str(time_choiceexpire)
print('You are automatically enrolled in Git submodules experiment ' )
'(go/android-submodules-quickstart) for another two weeks.\n', user_cfg.SetBoolean("repo.superprojectChoice", userchoice)
file=sys.stderr) print(
return True "You are automatically enrolled in Git submodules experiment "
"(go/android-submodules-quickstart) for another two weeks.\n",
file=sys.stderr,
)
return True
# For all other cases, we would not use superproject by default. # For all other cases, we would not use superproject by default.
return False return False
def PrintMessages(use_superproject, manifest): def PrintMessages(use_superproject, manifest):
"""Returns a boolean if error/warning messages are to be printed. """Returns a boolean if error/warning messages are to be printed.
Args: Args:
use_superproject: option value from optparse. use_superproject: option value from optparse.
manifest: manifest to use. manifest: manifest to use.
""" """
return use_superproject is not None or bool(manifest.superproject) return use_superproject is not None or bool(manifest.superproject)
def UseSuperproject(use_superproject, manifest): def UseSuperproject(use_superproject, manifest):
"""Returns a boolean if use-superproject option is enabled. """Returns a boolean if use-superproject option is enabled.
Args: Args:
use_superproject: option value from optparse. use_superproject: option value from optparse.
manifest: manifest to use. manifest: manifest to use.
Returns: Returns:
Whether the superproject should be used. Whether the superproject should be used.
""" """
if not manifest.superproject: if not manifest.superproject:
# This (sub) manifest does not have a superproject definition. # This (sub) manifest does not have a superproject definition.
return False return False
elif use_superproject is not None: elif use_superproject is not None:
return use_superproject return use_superproject
else:
client_value = manifest.manifestProject.use_superproject
if client_value is not None:
return client_value
elif manifest.superproject:
return _UseSuperprojectFromConfiguration()
else: else:
return False client_value = manifest.manifestProject.use_superproject
if client_value is not None:
return client_value
elif manifest.superproject:
return _UseSuperprojectFromConfiguration()
else:
return False

593
git_trace2_event_log.py
View File

@ -41,291 +41,330 @@ from git_command import GitCommand, RepoSourceVersion
class EventLog(object): class EventLog(object):
"""Event log that records events that occurred during a repo invocation. """Event log that records events that occurred during a repo invocation.
Events are written to the log as a consecutive JSON entries, one per line. Events are written to the log as a consecutive JSON entries, one per line.
Entries follow the git trace2 EVENT format. Entries follow the git trace2 EVENT format.
Each entry contains the following common keys: Each entry contains the following common keys:
- event: The event name - event: The event name
- sid: session-id - Unique string to allow process instance to be identified. - sid: session-id - Unique string to allow process instance to be
- thread: The thread name. identified.
- time: is the UTC time of the event. - thread: The thread name.
- time: is the UTC time of the event.
Valid 'event' names and event specific fields are documented here: Valid 'event' names and event specific fields are documented here:
https://git-scm.com/docs/api-trace2#_event_format https://git-scm.com/docs/api-trace2#_event_format
"""
def __init__(self, env=None):
"""Initializes the event log."""
self._log = []
# Try to get session-id (sid) from environment (setup in repo launcher).
KEY = 'GIT_TRACE2_PARENT_SID'
if env is None:
env = os.environ
now = datetime.datetime.utcnow()
# Save both our sid component and the complete sid.
# We use our sid component (self._sid) as the unique filename prefix and
# the full sid (self._full_sid) in the log itself.
self._sid = 'repo-%s-P%08x' % (now.strftime('%Y%m%dT%H%M%SZ'), os.getpid())
parent_sid = env.get(KEY)
# Append our sid component to the parent sid (if it exists).
if parent_sid is not None:
self._full_sid = parent_sid + '/' + self._sid
else:
self._full_sid = self._sid
# Set/update the environment variable.
# Environment handling across systems is messy.
try:
env[KEY] = self._full_sid
except UnicodeEncodeError:
env[KEY] = self._full_sid.encode()
# Add a version event to front of the log.
self._AddVersionEvent()
@property
def full_sid(self):
return self._full_sid
def _AddVersionEvent(self):
"""Adds a 'version' event at the beginning of current log."""
version_event = self._CreateEventDict('version')
version_event['evt'] = "2"
version_event['exe'] = RepoSourceVersion()
self._log.insert(0, version_event)
def _CreateEventDict(self, event_name):
"""Returns a dictionary with the common keys/values for git trace2 events.
Args:
event_name: The event name.
Returns:
Dictionary with the common event fields populated.
"""
return {
'event': event_name,
'sid': self._full_sid,
'thread': threading.current_thread().name,
'time': datetime.datetime.utcnow().isoformat() + 'Z',
}
def StartEvent(self):
"""Append a 'start' event to the current log."""
start_event = self._CreateEventDict('start')
start_event['argv'] = sys.argv
self._log.append(start_event)
def ExitEvent(self, result):
"""Append an 'exit' event to the current log.
Args:
result: Exit code of the event
"""
exit_event = self._CreateEventDict('exit')
# Consider 'None' success (consistent with event_log result handling).
if result is None:
result = 0
exit_event['code'] = result
self._log.append(exit_event)
def CommandEvent(self, name, subcommands):
"""Append a 'command' event to the current log.
Args:
name: Name of the primary command (ex: repo, git)
subcommands: List of the sub-commands (ex: version, init, sync)
"""
command_event = self._CreateEventDict('command')
command_event['name'] = name
command_event['subcommands'] = subcommands
self._log.append(command_event)
def LogConfigEvents(self, config, event_dict_name):
"""Append a |event_dict_name| event for each config key in |config|.
Args:
config: Configuration dictionary.
event_dict_name: Name of the event dictionary for items to be logged under.
"""
for param, value in config.items():
event = self._CreateEventDict(event_dict_name)
event['param'] = param
event['value'] = value
self._log.append(event)
def DefParamRepoEvents(self, config):
"""Append a 'def_param' event for each repo.* config key to the current log.
Args:
config: Repo configuration dictionary
"""
# Only output the repo.* config parameters.
repo_config = {k: v for k, v in config.items() if k.startswith('repo.')}
self.LogConfigEvents(repo_config, 'def_param')
def GetDataEventName(self, value):
"""Returns 'data-json' if the value is an array else returns 'data'."""
return 'data-json' if value[0] == '[' and value[-1] == ']' else 'data'
def LogDataConfigEvents(self, config, prefix):
"""Append a 'data' event for each config key/value in |config| to the current log.
For each keyX and valueX of the config, "key" field of the event is '|prefix|/keyX'
and the "value" of the "key" field is valueX.
Args:
config: Configuration dictionary.
prefix: Prefix for each key that is logged.
"""
for key, value in config.items():
event = self._CreateEventDict(self.GetDataEventName(value))
event['key'] = f'{prefix}/{key}'
event['value'] = value
self._log.append(event)
def ErrorEvent(self, msg, fmt):
"""Append a 'error' event to the current log."""
error_event = self._CreateEventDict('error')
error_event['msg'] = msg
error_event['fmt'] = fmt
self._log.append(error_event)
def _GetEventTargetPath(self):
"""Get the 'trace2.eventtarget' path from git configuration.
Returns:
path: git config's 'trace2.eventtarget' path if it exists, or None
"""
path = None
cmd = ['config', '--get', 'trace2.eventtarget']
# TODO(https://crbug.com/gerrit/13706): Use GitConfig when it supports
# system git config variables.
p = GitCommand(None, cmd, capture_stdout=True, capture_stderr=True,
bare=True)
retval = p.Wait()
if retval == 0:
# Strip trailing carriage-return in path.
path = p.stdout.rstrip('\n')
elif retval != 1:
# `git config --get` is documented to produce an exit status of `1` if
# the requested variable is not present in the configuration. Report any
# other return value as an error.
print("repo: error: 'git config --get' call failed with return code: %r, stderr: %r" % (
retval, p.stderr), file=sys.stderr)
return path
def _WriteLog(self, write_fn):
"""Writes the log out using a provided writer function.
Generate compact JSON output for each item in the log, and write it using
write_fn.
Args:
write_fn: A function that accepts byts and writes them to a destination.
""" """
for e in self._log: def __init__(self, env=None):
# Dump in compact encoding mode. """Initializes the event log."""
# See 'Compact encoding' in Python docs: self._log = []
# https://docs.python.org/3/library/json.html#module-json # Try to get session-id (sid) from environment (setup in repo launcher).
write_fn(json.dumps(e, indent=None, separators=(',', ':')).encode('utf-8') + b'\n') KEY = "GIT_TRACE2_PARENT_SID"
if env is None:
env = os.environ
def Write(self, path=None): now = datetime.datetime.utcnow()
"""Writes the log out to a file or socket.
Log is only written if 'path' or 'git config --get trace2.eventtarget' # Save both our sid component and the complete sid.
provide a valid path (or socket) to write logs to. # We use our sid component (self._sid) as the unique filename prefix and
# the full sid (self._full_sid) in the log itself.
self._sid = "repo-%s-P%08x" % (
now.strftime("%Y%m%dT%H%M%SZ"),
os.getpid(),
)
parent_sid = env.get(KEY)
# Append our sid component to the parent sid (if it exists).
if parent_sid is not None:
self._full_sid = parent_sid + "/" + self._sid
else:
self._full_sid = self._sid
Logging filename format follows the git trace2 style of being a unique # Set/update the environment variable.
(exclusive writable) file. # Environment handling across systems is messy.
Args:
path: Path to where logs should be written. The path may have a prefix of
the form "af_unix:[{stream|dgram}:]", in which case the path is
treated as a Unix domain socket. See
https://git-scm.com/docs/api-trace2#_enabling_a_target for details.
Returns:
log_path: Path to the log file or socket if log is written, otherwise None
"""
log_path = None
# If no logging path is specified, get the path from 'trace2.eventtarget'.
if path is None:
path = self._GetEventTargetPath()
# If no logging path is specified, exit.
if path is None:
return None
path_is_socket = False
socket_type = None
if isinstance(path, str):
parts = path.split(':', 1)
if parts[0] == 'af_unix' and len(parts) == 2:
path_is_socket = True
path = parts[1]
parts = path.split(':', 1)
if parts[0] == 'stream' and len(parts) == 2:
socket_type = socket.SOCK_STREAM
path = parts[1]
elif parts[0] == 'dgram' and len(parts) == 2:
socket_type = socket.SOCK_DGRAM
path = parts[1]
else:
# Get absolute path.
path = os.path.abspath(os.path.expanduser(path))
else:
raise TypeError('path: str required but got %s.' % type(path))
# Git trace2 requires a directory to write log to.
# TODO(https://crbug.com/gerrit/13706): Support file (append) mode also.
if not (path_is_socket or os.path.isdir(path)):
return None
if path_is_socket:
if socket_type == socket.SOCK_STREAM or socket_type is None:
try: try:
with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock: env[KEY] = self._full_sid
sock.connect(path) except UnicodeEncodeError:
self._WriteLog(sock.sendall) env[KEY] = self._full_sid.encode()
return f'af_unix:stream:{path}'
except OSError as err: # Add a version event to front of the log.
# If we tried to connect to a DGRAM socket using STREAM, ignore the self._AddVersionEvent()
# attempt and continue to DGRAM below. Otherwise, issue a warning.
if err.errno != errno.EPROTOTYPE: @property
print(f'repo: warning: git trace2 logging failed: {err}', file=sys.stderr) def full_sid(self):
return self._full_sid
def _AddVersionEvent(self):
"""Adds a 'version' event at the beginning of current log."""
version_event = self._CreateEventDict("version")
version_event["evt"] = "2"
version_event["exe"] = RepoSourceVersion()
self._log.insert(0, version_event)
def _CreateEventDict(self, event_name):
"""Returns a dictionary with common keys/values for git trace2 events.
Args:
event_name: The event name.
Returns:
Dictionary with the common event fields populated.
"""
return {
"event": event_name,
"sid": self._full_sid,
"thread": threading.current_thread().name,
"time": datetime.datetime.utcnow().isoformat() + "Z",
}
def StartEvent(self):
"""Append a 'start' event to the current log."""
start_event = self._CreateEventDict("start")
start_event["argv"] = sys.argv
self._log.append(start_event)
def ExitEvent(self, result):
"""Append an 'exit' event to the current log.
Args:
result: Exit code of the event
"""
exit_event = self._CreateEventDict("exit")
# Consider 'None' success (consistent with event_log result handling).
if result is None:
result = 0
exit_event["code"] = result
self._log.append(exit_event)
def CommandEvent(self, name, subcommands):
"""Append a 'command' event to the current log.
Args:
name: Name of the primary command (ex: repo, git)
subcommands: List of the sub-commands (ex: version, init, sync)
"""
command_event = self._CreateEventDict("command")
command_event["name"] = name
command_event["subcommands"] = subcommands
self._log.append(command_event)
def LogConfigEvents(self, config, event_dict_name):
"""Append a |event_dict_name| event for each config key in |config|.
Args:
config: Configuration dictionary.
event_dict_name: Name of the event dictionary for items to be logged
under.
"""
for param, value in config.items():
event = self._CreateEventDict(event_dict_name)
event["param"] = param
event["value"] = value
self._log.append(event)
def DefParamRepoEvents(self, config):
"""Append 'def_param' events for repo config keys to the current log.
This appends one event for each repo.* config key.
Args:
config: Repo configuration dictionary
"""
# Only output the repo.* config parameters.
repo_config = {k: v for k, v in config.items() if k.startswith("repo.")}
self.LogConfigEvents(repo_config, "def_param")
def GetDataEventName(self, value):
"""Returns 'data-json' if the value is an array else returns 'data'."""
return "data-json" if value[0] == "[" and value[-1] == "]" else "data"
def LogDataConfigEvents(self, config, prefix):
"""Append a 'data' event for each entry in |config| to the current log.
For each keyX and valueX of the config, "key" field of the event is
'|prefix|/keyX' and the "value" of the "key" field is valueX.
Args:
config: Configuration dictionary.
prefix: Prefix for each key that is logged.
"""
for key, value in config.items():
event = self._CreateEventDict(self.GetDataEventName(value))
event["key"] = f"{prefix}/{key}"
event["value"] = value
self._log.append(event)
def ErrorEvent(self, msg, fmt):
"""Append a 'error' event to the current log."""
error_event = self._CreateEventDict("error")
error_event["msg"] = msg
error_event["fmt"] = fmt
self._log.append(error_event)
def _GetEventTargetPath(self):
"""Get the 'trace2.eventtarget' path from git configuration.
Returns:
path: git config's 'trace2.eventtarget' path if it exists, or None
"""
path = None
cmd = ["config", "--get", "trace2.eventtarget"]
# TODO(https://crbug.com/gerrit/13706): Use GitConfig when it supports
# system git config variables.
p = GitCommand(
None, cmd, capture_stdout=True, capture_stderr=True, bare=True
)
retval = p.Wait()
if retval == 0:
# Strip trailing carriage-return in path.
path = p.stdout.rstrip("\n")
elif retval != 1:
# `git config --get` is documented to produce an exit status of `1`
# if the requested variable is not present in the configuration.
# Report any other return value as an error.
print(
"repo: error: 'git config --get' call failed with return code: "
"%r, stderr: %r" % (retval, p.stderr),
file=sys.stderr,
)
return path
def _WriteLog(self, write_fn):
"""Writes the log out using a provided writer function.
Generate compact JSON output for each item in the log, and write it
using write_fn.
Args:
write_fn: A function that accepts byts and writes them to a
destination.
"""
for e in self._log:
# Dump in compact encoding mode.
# See 'Compact encoding' in Python docs:
# https://docs.python.org/3/library/json.html#module-json
write_fn(
json.dumps(e, indent=None, separators=(",", ":")).encode(
"utf-8"
)
+ b"\n"
)
def Write(self, path=None):
"""Writes the log out to a file or socket.
Log is only written if 'path' or 'git config --get trace2.eventtarget'
provide a valid path (or socket) to write logs to.
Logging filename format follows the git trace2 style of being a unique
(exclusive writable) file.
Args:
path: Path to where logs should be written. The path may have a
prefix of the form "af_unix:[{stream|dgram}:]", in which case
the path is treated as a Unix domain socket. See
https://git-scm.com/docs/api-trace2#_enabling_a_target for
details.
Returns:
log_path: Path to the log file or socket if log is written,
otherwise None
"""
log_path = None
# If no logging path is specified, get the path from
# 'trace2.eventtarget'.
if path is None:
path = self._GetEventTargetPath()
# If no logging path is specified, exit.
if path is None:
return None return None
if socket_type == socket.SOCK_DGRAM or socket_type is None:
try:
with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock:
self._WriteLog(lambda bs: sock.sendto(bs, path))
return f'af_unix:dgram:{path}'
except OSError as err:
print(f'repo: warning: git trace2 logging failed: {err}', file=sys.stderr)
return None
# Tried to open a socket but couldn't connect (SOCK_STREAM) or write
# (SOCK_DGRAM).
print('repo: warning: git trace2 logging failed: could not write to socket', file=sys.stderr)
return None
# Path is an absolute path path_is_socket = False
# Use NamedTemporaryFile to generate a unique filename as required by git trace2. socket_type = None
try: if isinstance(path, str):
with tempfile.NamedTemporaryFile(mode='xb', prefix=self._sid, dir=path, parts = path.split(":", 1)
delete=False) as f: if parts[0] == "af_unix" and len(parts) == 2:
# TODO(https://crbug.com/gerrit/13706): Support writing events as they path_is_socket = True
# occur. path = parts[1]
self._WriteLog(f.write) parts = path.split(":", 1)
log_path = f.name if parts[0] == "stream" and len(parts) == 2:
except FileExistsError as err: socket_type = socket.SOCK_STREAM
print('repo: warning: git trace2 logging failed: %r' % err, path = parts[1]
file=sys.stderr) elif parts[0] == "dgram" and len(parts) == 2:
return None socket_type = socket.SOCK_DGRAM
return log_path path = parts[1]
else:
# Get absolute path.
path = os.path.abspath(os.path.expanduser(path))
else:
raise TypeError("path: str required but got %s." % type(path))
# Git trace2 requires a directory to write log to.
# TODO(https://crbug.com/gerrit/13706): Support file (append) mode also.
if not (path_is_socket or os.path.isdir(path)):
return None
if path_is_socket:
if socket_type == socket.SOCK_STREAM or socket_type is None:
try:
with socket.socket(
socket.AF_UNIX, socket.SOCK_STREAM
) as sock:
sock.connect(path)
self._WriteLog(sock.sendall)
return f"af_unix:stream:{path}"
except OSError as err:
# If we tried to connect to a DGRAM socket using STREAM,
# ignore the attempt and continue to DGRAM below. Otherwise,
# issue a warning.
if err.errno != errno.EPROTOTYPE:
print(
f"repo: warning: git trace2 logging failed: {err}",
file=sys.stderr,
)
return None
if socket_type == socket.SOCK_DGRAM or socket_type is None:
try:
with socket.socket(
socket.AF_UNIX, socket.SOCK_DGRAM
) as sock:
self._WriteLog(lambda bs: sock.sendto(bs, path))
return f"af_unix:dgram:{path}"
except OSError as err:
print(
f"repo: warning: git trace2 logging failed: {err}",
file=sys.stderr,
)
return None
# Tried to open a socket but couldn't connect (SOCK_STREAM) or write
# (SOCK_DGRAM).
print(
"repo: warning: git trace2 logging failed: could not write to "
"socket",
file=sys.stderr,
)
return None
# Path is an absolute path
# Use NamedTemporaryFile to generate a unique filename as required by
# git trace2.
try:
with tempfile.NamedTemporaryFile(
mode="xb", prefix=self._sid, dir=path, delete=False
) as f:
# TODO(https://crbug.com/gerrit/13706): Support writing events
# as they occur.
self._WriteLog(f.write)
log_path = f.name
except FileExistsError as err:
print(
"repo: warning: git trace2 logging failed: %r" % err,
file=sys.stderr,
)
return None
return log_path

203
gitc_utils.py
View File

@ -28,128 +28,139 @@ NUM_BATCH_RETRIEVE_REVISIONID = 32
def get_gitc_manifest_dir(): def get_gitc_manifest_dir():
return wrapper.Wrapper().get_gitc_manifest_dir() return wrapper.Wrapper().get_gitc_manifest_dir()
def parse_clientdir(gitc_fs_path): def parse_clientdir(gitc_fs_path):
return wrapper.Wrapper().gitc_parse_clientdir(gitc_fs_path) return wrapper.Wrapper().gitc_parse_clientdir(gitc_fs_path)
def _get_project_revision(args): def _get_project_revision(args):
"""Worker for _set_project_revisions to lookup one project remote.""" """Worker for _set_project_revisions to lookup one project remote."""
(i, url, expr) = args (i, url, expr) = args
gitcmd = git_command.GitCommand( gitcmd = git_command.GitCommand(
None, ['ls-remote', url, expr], capture_stdout=True, cwd='/tmp') None, ["ls-remote", url, expr], capture_stdout=True, cwd="/tmp"
rc = gitcmd.Wait() )
return (i, rc, gitcmd.stdout.split('\t', 1)[0]) rc = gitcmd.Wait()
return (i, rc, gitcmd.stdout.split("\t", 1)[0])
def _set_project_revisions(projects): def _set_project_revisions(projects):
"""Sets the revisionExpr for a list of projects. """Sets the revisionExpr for a list of projects.
Because of the limit of open file descriptors allowed, length of projects Because of the limit of open file descriptors allowed, length of projects
should not be overly large. Recommend calling this function multiple times should not be overly large. Recommend calling this function multiple times
with each call not exceeding NUM_BATCH_RETRIEVE_REVISIONID projects. with each call not exceeding NUM_BATCH_RETRIEVE_REVISIONID projects.
Args: Args:
projects: List of project objects to set the revionExpr for. projects: List of project objects to set the revionExpr for.
""" """
# Retrieve the commit id for each project based off of it's current # Retrieve the commit id for each project based off of its current
# revisionExpr and it is not already a commit id. # revisionExpr and it is not already a commit id.
with multiprocessing.Pool(NUM_BATCH_RETRIEVE_REVISIONID) as pool: with multiprocessing.Pool(NUM_BATCH_RETRIEVE_REVISIONID) as pool:
results_iter = pool.imap_unordered( results_iter = pool.imap_unordered(
_get_project_revision, _get_project_revision,
((i, project.remote.url, project.revisionExpr) (
for i, project in enumerate(projects) (i, project.remote.url, project.revisionExpr)
if not git_config.IsId(project.revisionExpr)), for i, project in enumerate(projects)
chunksize=8) if not git_config.IsId(project.revisionExpr)
for (i, rc, revisionExpr) in results_iter: ),
project = projects[i] chunksize=8,
if rc: )
print('FATAL: Failed to retrieve revisionExpr for %s' % project.name) for i, rc, revisionExpr in results_iter:
pool.terminate() project = projects[i]
sys.exit(1) if rc:
if not revisionExpr: print(
pool.terminate() "FATAL: Failed to retrieve revisionExpr for %s"
raise ManifestParseError('Invalid SHA-1 revision project %s (%s)' % % project.name
(project.remote.url, project.revisionExpr)) )
project.revisionExpr = revisionExpr pool.terminate()
sys.exit(1)
if not revisionExpr:
pool.terminate()
raise ManifestParseError(
"Invalid SHA-1 revision project %s (%s)"
% (project.remote.url, project.revisionExpr)
)
project.revisionExpr = revisionExpr
def generate_gitc_manifest(gitc_manifest, manifest, paths=None): def generate_gitc_manifest(gitc_manifest, manifest, paths=None):
"""Generate a manifest for shafsd to use for this GITC client. """Generate a manifest for shafsd to use for this GITC client.
Args: Args:
gitc_manifest: Current gitc manifest, or None if there isn't one yet. gitc_manifest: Current gitc manifest, or None if there isn't one yet.
manifest: A GitcManifest object loaded with the current repo manifest. manifest: A GitcManifest object loaded with the current repo manifest.
paths: List of project paths we want to update. paths: List of project paths we want to update.
""" """
print('Generating GITC Manifest by fetching revision SHAs for each ' print(
'project.') "Generating GITC Manifest by fetching revision SHAs for each "
if paths is None: "project."
paths = list(manifest.paths.keys()) )
if paths is None:
paths = list(manifest.paths.keys())
groups = [x for x in re.split(r'[,\s]+', manifest.GetGroupsStr()) if x] groups = [x for x in re.split(r"[,\s]+", manifest.GetGroupsStr()) if x]
# Convert the paths to projects, and filter them to the matched groups. # Convert the paths to projects, and filter them to the matched groups.
projects = [manifest.paths[p] for p in paths] projects = [manifest.paths[p] for p in paths]
projects = [p for p in projects if p.MatchesGroups(groups)] projects = [p for p in projects if p.MatchesGroups(groups)]
if gitc_manifest is not None: if gitc_manifest is not None:
for path, proj in manifest.paths.items(): for path, proj in manifest.paths.items():
if not proj.MatchesGroups(groups): if not proj.MatchesGroups(groups):
continue continue
if not proj.upstream and not git_config.IsId(proj.revisionExpr): if not proj.upstream and not git_config.IsId(proj.revisionExpr):
proj.upstream = proj.revisionExpr proj.upstream = proj.revisionExpr
if path not in gitc_manifest.paths: if path not in gitc_manifest.paths:
# Any new projects need their first revision, even if we weren't asked # Any new projects need their first revision, even if we weren't
# for them. # asked for them.
projects.append(proj) projects.append(proj)
elif path not in paths: elif path not in paths:
# And copy revisions from the previous manifest if we're not updating # And copy revisions from the previous manifest if we're not
# them now. # updating them now.
gitc_proj = gitc_manifest.paths[path] gitc_proj = gitc_manifest.paths[path]
if gitc_proj.old_revision: if gitc_proj.old_revision:
proj.revisionExpr = None proj.revisionExpr = None
proj.old_revision = gitc_proj.old_revision proj.old_revision = gitc_proj.old_revision
else: else:
proj.revisionExpr = gitc_proj.revisionExpr proj.revisionExpr = gitc_proj.revisionExpr
_set_project_revisions(projects) _set_project_revisions(projects)
if gitc_manifest is not None: if gitc_manifest is not None:
for path, proj in gitc_manifest.paths.items(): for path, proj in gitc_manifest.paths.items():
if proj.old_revision and path in paths: if proj.old_revision and path in paths:
# If we updated a project that has been started, keep the old-revision # If we updated a project that has been started, keep the
# updated. # old-revision updated.
repo_proj = manifest.paths[path] repo_proj = manifest.paths[path]
repo_proj.old_revision = repo_proj.revisionExpr repo_proj.old_revision = repo_proj.revisionExpr
repo_proj.revisionExpr = None repo_proj.revisionExpr = None
# Convert URLs from relative to absolute. # Convert URLs from relative to absolute.
for _name, remote in manifest.remotes.items(): for _name, remote in manifest.remotes.items():
remote.fetchUrl = remote.resolvedFetchUrl remote.fetchUrl = remote.resolvedFetchUrl
# Save the manifest. # Save the manifest.
save_manifest(manifest) save_manifest(manifest)
def save_manifest(manifest, client_dir=None): def save_manifest(manifest, client_dir=None):
"""Save the manifest file in the client_dir. """Save the manifest file in the client_dir.
Args: Args:
manifest: Manifest object to save. manifest: Manifest object to save.
client_dir: Client directory to save the manifest in. client_dir: Client directory to save the manifest in.
""" """
if not client_dir: if not client_dir:
manifest_file = manifest.manifestFile manifest_file = manifest.manifestFile
else: else:
manifest_file = os.path.join(client_dir, '.manifest') manifest_file = os.path.join(client_dir, ".manifest")
with open(manifest_file, 'w') as f: with open(manifest_file, "w") as f:
manifest.Save(f, groups=manifest.GetGroupsStr()) manifest.Save(f, groups=manifest.GetGroupsStr())
# TODO(sbasi/jorg): Come up with a solution to remove the sleep below. # TODO(sbasi/jorg): Come up with a solution to remove the sleep below.
# Give the GITC filesystem time to register the manifest changes. # Give the GITC filesystem time to register the manifest changes.
time.sleep(3) time.sleep(3)

870
hooks.py
View File

@ -26,271 +26,293 @@ from git_refs import HEAD
class RepoHook(object): class RepoHook(object):
"""A RepoHook contains information about a script to run as a hook. """A RepoHook contains information about a script to run as a hook.
Hooks are used to run a python script before running an upload (for instance, Hooks are used to run a python script before running an upload (for
to run presubmit checks). Eventually, we may have hooks for other actions. instance, to run presubmit checks). Eventually, we may have hooks for other
actions.
This shouldn't be confused with files in the 'repo/hooks' directory. Those This shouldn't be confused with files in the 'repo/hooks' directory. Those
files are copied into each '.git/hooks' folder for each project. Repo-level files are copied into each '.git/hooks' folder for each project. Repo-level
hooks are associated instead with repo actions. hooks are associated instead with repo actions.
Hooks are always python. When a hook is run, we will load the hook into the Hooks are always python. When a hook is run, we will load the hook into the
interpreter and execute its main() function. interpreter and execute its main() function.
Combinations of hook option flags: Combinations of hook option flags:
- no-verify=False, verify=False (DEFAULT): - no-verify=False, verify=False (DEFAULT):
If stdout is a tty, can prompt about running hooks if needed. If stdout is a tty, can prompt about running hooks if needed.
If user denies running hooks, the action is cancelled. If stdout is If user denies running hooks, the action is cancelled. If stdout is
not a tty and we would need to prompt about hooks, action is not a tty and we would need to prompt about hooks, action is
cancelled. cancelled.
- no-verify=False, verify=True: - no-verify=False, verify=True:
Always run hooks with no prompt. Always run hooks with no prompt.
- no-verify=True, verify=False: - no-verify=True, verify=False:
Never run hooks, but run action anyway (AKA bypass hooks). Never run hooks, but run action anyway (AKA bypass hooks).
- no-verify=True, verify=True: - no-verify=True, verify=True:
Invalid Invalid
"""
def __init__(self,
hook_type,
hooks_project,
repo_topdir,
manifest_url,
bypass_hooks=False,
allow_all_hooks=False,
ignore_hooks=False,
abort_if_user_denies=False):
"""RepoHook constructor.
Params:
hook_type: A string representing the type of hook. This is also used
to figure out the name of the file containing the hook. For
example: 'pre-upload'.
hooks_project: The project containing the repo hooks.
If you have a manifest, this is manifest.repo_hooks_project.
OK if this is None, which will make the hook a no-op.
repo_topdir: The top directory of the repo client checkout.
This is the one containing the .repo directory. Scripts will
run with CWD as this directory.
If you have a manifest, this is manifest.topdir.
manifest_url: The URL to the manifest git repo.
bypass_hooks: If True, then 'Do not run the hook'.
allow_all_hooks: If True, then 'Run the hook without prompting'.
ignore_hooks: If True, then 'Do not abort action if hooks fail'.
abort_if_user_denies: If True, we'll abort running the hook if the user
doesn't allow us to run the hook.
""" """
self._hook_type = hook_type
self._hooks_project = hooks_project
self._repo_topdir = repo_topdir
self._manifest_url = manifest_url
self._bypass_hooks = bypass_hooks
self._allow_all_hooks = allow_all_hooks
self._ignore_hooks = ignore_hooks
self._abort_if_user_denies = abort_if_user_denies
# Store the full path to the script for convenience. def __init__(
if self._hooks_project: self,
self._script_fullpath = os.path.join(self._hooks_project.worktree, hook_type,
self._hook_type + '.py') hooks_project,
else: repo_topdir,
self._script_fullpath = None manifest_url,
bypass_hooks=False,
allow_all_hooks=False,
ignore_hooks=False,
abort_if_user_denies=False,
):
"""RepoHook constructor.
def _GetHash(self): Params:
"""Return a hash of the contents of the hooks directory. hook_type: A string representing the type of hook. This is also used
to figure out the name of the file containing the hook. For
example: 'pre-upload'.
hooks_project: The project containing the repo hooks.
If you have a manifest, this is manifest.repo_hooks_project.
OK if this is None, which will make the hook a no-op.
repo_topdir: The top directory of the repo client checkout.
This is the one containing the .repo directory. Scripts will
run with CWD as this directory.
If you have a manifest, this is manifest.topdir.
manifest_url: The URL to the manifest git repo.
bypass_hooks: If True, then 'Do not run the hook'.
allow_all_hooks: If True, then 'Run the hook without prompting'.
ignore_hooks: If True, then 'Do not abort action if hooks fail'.
abort_if_user_denies: If True, we'll abort running the hook if the
user doesn't allow us to run the hook.
"""
self._hook_type = hook_type
self._hooks_project = hooks_project
self._repo_topdir = repo_topdir
self._manifest_url = manifest_url
self._bypass_hooks = bypass_hooks
self._allow_all_hooks = allow_all_hooks
self._ignore_hooks = ignore_hooks
self._abort_if_user_denies = abort_if_user_denies
We'll just use git to do this. This hash has the property that if anything # Store the full path to the script for convenience.
changes in the directory we will return a different has. if self._hooks_project:
self._script_fullpath = os.path.join(
self._hooks_project.worktree, self._hook_type + ".py"
)
else:
self._script_fullpath = None
SECURITY CONSIDERATION: def _GetHash(self):
This hash only represents the contents of files in the hook directory, not """Return a hash of the contents of the hooks directory.
any other files imported or called by hooks. Changes to imported files
can change the script behavior without affecting the hash.
Returns: We'll just use git to do this. This hash has the property that if
A string representing the hash. This will always be ASCII so that it can anything changes in the directory we will return a different has.
be printed to the user easily.
"""
assert self._hooks_project, "Must have hooks to calculate their hash."
# We will use the work_git object rather than just calling GetRevisionId(). SECURITY CONSIDERATION:
# That gives us a hash of the latest checked in version of the files that This hash only represents the contents of files in the hook
# the user will actually be executing. Specifically, GetRevisionId() directory, not any other files imported or called by hooks. Changes
# doesn't appear to change even if a user checks out a different version to imported files can change the script behavior without affecting
# of the hooks repo (via git checkout) nor if a user commits their own revs. the hash.
#
# NOTE: Local (non-committed) changes will not be factored into this hash.
# I think this is OK, since we're really only worried about warning the user
# about upstream changes.
return self._hooks_project.work_git.rev_parse(HEAD)
def _GetMustVerb(self): Returns:
"""Return 'must' if the hook is required; 'should' if not.""" A string representing the hash. This will always be ASCII so that
if self._abort_if_user_denies: it can be printed to the user easily.
return 'must' """
else: assert self._hooks_project, "Must have hooks to calculate their hash."
return 'should'
def _CheckForHookApproval(self): # We will use the work_git object rather than just calling
"""Check to see whether this hook has been approved. # GetRevisionId(). That gives us a hash of the latest checked in version
# of the files that the user will actually be executing. Specifically,
# GetRevisionId() doesn't appear to change even if a user checks out a
# different version of the hooks repo (via git checkout) nor if a user
# commits their own revs.
#
# NOTE: Local (non-committed) changes will not be factored into this
# hash. I think this is OK, since we're really only worried about
# warning the user about upstream changes.
return self._hooks_project.work_git.rev_parse(HEAD)
We'll accept approval of manifest URLs if they're using secure transports. def _GetMustVerb(self):
This way the user can say they trust the manifest hoster. For insecure """Return 'must' if the hook is required; 'should' if not."""
hosts, we fall back to checking the hash of the hooks repo. if self._abort_if_user_denies:
return "must"
else:
return "should"
Note that we ask permission for each individual hook even though we use def _CheckForHookApproval(self):
the hash of all hooks when detecting changes. We'd like the user to be """Check to see whether this hook has been approved.
able to approve / deny each hook individually. We only use the hash of all
hooks because there is no other easy way to detect changes to local imports.
Returns: We'll accept approval of manifest URLs if they're using secure
True if this hook is approved to run; False otherwise. transports. This way the user can say they trust the manifest hoster.
For insecure hosts, we fall back to checking the hash of the hooks repo.
Raises: Note that we ask permission for each individual hook even though we use
HookError: Raised if the user doesn't approve and abort_if_user_denies the hash of all hooks when detecting changes. We'd like the user to be
was passed to the consturctor. able to approve / deny each hook individually. We only use the hash of
""" all hooks because there is no other easy way to detect changes to local
if self._ManifestUrlHasSecureScheme(): imports.
return self._CheckForHookApprovalManifest()
else:
return self._CheckForHookApprovalHash()
def _CheckForHookApprovalHelper(self, subkey, new_val, main_prompt, Returns:
changed_prompt): True if this hook is approved to run; False otherwise.
"""Check for approval for a particular attribute and hook.
Args: Raises:
subkey: The git config key under [repo.hooks.<hook_type>] to store the HookError: Raised if the user doesn't approve and
last approved string. abort_if_user_denies was passed to the consturctor.
new_val: The new value to compare against the last approved one. """
main_prompt: Message to display to the user to ask for approval. if self._ManifestUrlHasSecureScheme():
changed_prompt: Message explaining why we're re-asking for approval. return self._CheckForHookApprovalManifest()
else:
return self._CheckForHookApprovalHash()
Returns: def _CheckForHookApprovalHelper(
True if this hook is approved to run; False otherwise. self, subkey, new_val, main_prompt, changed_prompt
):
"""Check for approval for a particular attribute and hook.
Raises: Args:
HookError: Raised if the user doesn't approve and abort_if_user_denies subkey: The git config key under [repo.hooks.<hook_type>] to store
was passed to the consturctor. the last approved string.
""" new_val: The new value to compare against the last approved one.
hooks_config = self._hooks_project.config main_prompt: Message to display to the user to ask for approval.
git_approval_key = 'repo.hooks.%s.%s' % (self._hook_type, subkey) changed_prompt: Message explaining why we're re-asking for approval.
# Get the last value that the user approved for this hook; may be None. Returns:
old_val = hooks_config.GetString(git_approval_key) True if this hook is approved to run; False otherwise.
if old_val is not None: Raises:
# User previously approved hook and asked not to be prompted again. HookError: Raised if the user doesn't approve and
if new_val == old_val: abort_if_user_denies was passed to the consturctor.
# Approval matched. We're done. """
return True hooks_config = self._hooks_project.config
else: git_approval_key = "repo.hooks.%s.%s" % (self._hook_type, subkey)
# Give the user a reason why we're prompting, since they last told
# us to "never ask again".
prompt = 'WARNING: %s\n\n' % (changed_prompt,)
else:
prompt = ''
# Prompt the user if we're not on a tty; on a tty we'll assume "no". # Get the last value that the user approved for this hook; may be None.
if sys.stdout.isatty(): old_val = hooks_config.GetString(git_approval_key)
prompt += main_prompt + ' (yes/always/NO)? '
response = input(prompt).lower()
print()
# User is doing a one-time approval. if old_val is not None:
if response in ('y', 'yes'): # User previously approved hook and asked not to be prompted again.
return True if new_val == old_val:
elif response == 'always': # Approval matched. We're done.
hooks_config.SetString(git_approval_key, new_val) return True
return True else:
# Give the user a reason why we're prompting, since they last
# told us to "never ask again".
prompt = "WARNING: %s\n\n" % (changed_prompt,)
else:
prompt = ""
# For anything else, we'll assume no approval. # Prompt the user if we're not on a tty; on a tty we'll assume "no".
if self._abort_if_user_denies: if sys.stdout.isatty():
raise HookError('You must allow the %s hook or use --no-verify.' % prompt += main_prompt + " (yes/always/NO)? "
self._hook_type) response = input(prompt).lower()
print()
return False # User is doing a one-time approval.
if response in ("y", "yes"):
return True
elif response == "always":
hooks_config.SetString(git_approval_key, new_val)
return True
def _ManifestUrlHasSecureScheme(self): # For anything else, we'll assume no approval.
"""Check if the URI for the manifest is a secure transport.""" if self._abort_if_user_denies:
secure_schemes = ('file', 'https', 'ssh', 'persistent-https', 'sso', 'rpc') raise HookError(
parse_results = urllib.parse.urlparse(self._manifest_url) "You must allow the %s hook or use --no-verify."
return parse_results.scheme in secure_schemes % self._hook_type
)
def _CheckForHookApprovalManifest(self): return False
"""Check whether the user has approved this manifest host.
Returns: def _ManifestUrlHasSecureScheme(self):
True if this hook is approved to run; False otherwise. """Check if the URI for the manifest is a secure transport."""
""" secure_schemes = (
return self._CheckForHookApprovalHelper( "file",
'approvedmanifest', "https",
self._manifest_url, "ssh",
'Run hook scripts from %s' % (self._manifest_url,), "persistent-https",
'Manifest URL has changed since %s was allowed.' % (self._hook_type,)) "sso",
"rpc",
)
parse_results = urllib.parse.urlparse(self._manifest_url)
return parse_results.scheme in secure_schemes
def _CheckForHookApprovalHash(self): def _CheckForHookApprovalManifest(self):
"""Check whether the user has approved the hooks repo. """Check whether the user has approved this manifest host.
Returns: Returns:
True if this hook is approved to run; False otherwise. True if this hook is approved to run; False otherwise.
""" """
prompt = ('Repo %s run the script:\n' return self._CheckForHookApprovalHelper(
' %s\n' "approvedmanifest",
'\n' self._manifest_url,
'Do you want to allow this script to run') "Run hook scripts from %s" % (self._manifest_url,),
return self._CheckForHookApprovalHelper( "Manifest URL has changed since %s was allowed."
'approvedhash', % (self._hook_type,),
self._GetHash(), )
prompt % (self._GetMustVerb(), self._script_fullpath),
'Scripts have changed since %s was allowed.' % (self._hook_type,))
@staticmethod def _CheckForHookApprovalHash(self):
def _ExtractInterpFromShebang(data): """Check whether the user has approved the hooks repo.
"""Extract the interpreter used in the shebang.
Try to locate the interpreter the script is using (ignoring `env`). Returns:
True if this hook is approved to run; False otherwise.
"""
prompt = (
"Repo %s run the script:\n"
" %s\n"
"\n"
"Do you want to allow this script to run"
)
return self._CheckForHookApprovalHelper(
"approvedhash",
self._GetHash(),
prompt % (self._GetMustVerb(), self._script_fullpath),
"Scripts have changed since %s was allowed." % (self._hook_type,),
)
Args: @staticmethod
data: The file content of the script. def _ExtractInterpFromShebang(data):
"""Extract the interpreter used in the shebang.
Returns: Try to locate the interpreter the script is using (ignoring `env`).
The basename of the main script interpreter, or None if a shebang is not
used or could not be parsed out.
"""
firstline = data.splitlines()[:1]
if not firstline:
return None
# The format here can be tricky. Args:
shebang = firstline[0].strip() data: The file content of the script.
m = re.match(r'^#!\s*([^\s]+)(?:\s+([^\s]+))?', shebang)
if not m:
return None
# If the using `env`, find the target program. Returns:
interp = m.group(1) The basename of the main script interpreter, or None if a shebang is
if os.path.basename(interp) == 'env': not used or could not be parsed out.
interp = m.group(2) """
firstline = data.splitlines()[:1]
if not firstline:
return None
return interp # The format here can be tricky.
shebang = firstline[0].strip()
m = re.match(r"^#!\s*([^\s]+)(?:\s+([^\s]+))?", shebang)
if not m:
return None
def _ExecuteHookViaReexec(self, interp, context, **kwargs): # If the using `env`, find the target program.
"""Execute the hook script through |interp|. interp = m.group(1)
if os.path.basename(interp) == "env":
interp = m.group(2)
Note: Support for this feature should be dropped ~Jun 2021. return interp
Args: def _ExecuteHookViaReexec(self, interp, context, **kwargs):
interp: The Python program to run. """Execute the hook script through |interp|.
context: Basic Python context to execute the hook inside.
kwargs: Arbitrary arguments to pass to the hook script.
Raises: Note: Support for this feature should be dropped ~Jun 2021.
HookError: When the hooks failed for any reason.
""" Args:
# This logic needs to be kept in sync with _ExecuteHookViaImport below. interp: The Python program to run.
script = """ context: Basic Python context to execute the hook inside.
kwargs: Arbitrary arguments to pass to the hook script.
Raises:
HookError: When the hooks failed for any reason.
"""
# This logic needs to be kept in sync with _ExecuteHookViaImport below.
script = """
import json, os, sys import json, os, sys
path = '''%(path)s''' path = '''%(path)s'''
kwargs = json.loads('''%(kwargs)s''') kwargs = json.loads('''%(kwargs)s''')
@ -300,210 +322,240 @@ data = open(path).read()
exec(compile(data, path, 'exec'), context) exec(compile(data, path, 'exec'), context)
context['main'](**kwargs) context['main'](**kwargs)
""" % { """ % {
'path': self._script_fullpath, "path": self._script_fullpath,
'kwargs': json.dumps(kwargs), "kwargs": json.dumps(kwargs),
'context': json.dumps(context), "context": json.dumps(context),
} }
# We pass the script via stdin to avoid OS argv limits. It also makes # We pass the script via stdin to avoid OS argv limits. It also makes
# unhandled exception tracebacks less verbose/confusing for users. # unhandled exception tracebacks less verbose/confusing for users.
cmd = [interp, '-c', 'import sys; exec(sys.stdin.read())'] cmd = [interp, "-c", "import sys; exec(sys.stdin.read())"]
proc = subprocess.Popen(cmd, stdin=subprocess.PIPE) proc = subprocess.Popen(cmd, stdin=subprocess.PIPE)
proc.communicate(input=script.encode('utf-8')) proc.communicate(input=script.encode("utf-8"))
if proc.returncode: if proc.returncode:
raise HookError('Failed to run %s hook.' % (self._hook_type,)) raise HookError("Failed to run %s hook." % (self._hook_type,))
def _ExecuteHookViaImport(self, data, context, **kwargs): def _ExecuteHookViaImport(self, data, context, **kwargs):
"""Execute the hook code in |data| directly. """Execute the hook code in |data| directly.
Args: Args:
data: The code of the hook to execute. data: The code of the hook to execute.
context: Basic Python context to execute the hook inside. context: Basic Python context to execute the hook inside.
kwargs: Arbitrary arguments to pass to the hook script. kwargs: Arbitrary arguments to pass to the hook script.
Raises: Raises:
HookError: When the hooks failed for any reason. HookError: When the hooks failed for any reason.
""" """
# Exec, storing global context in the context dict. We catch exceptions # Exec, storing global context in the context dict. We catch exceptions
# and convert to a HookError w/ just the failing traceback. # and convert to a HookError w/ just the failing traceback.
try:
exec(compile(data, self._script_fullpath, 'exec'), context)
except Exception:
raise HookError('%s\nFailed to import %s hook; see traceback above.' %
(traceback.format_exc(), self._hook_type))
# Running the script should have defined a main() function.
if 'main' not in context:
raise HookError('Missing main() in: "%s"' % self._script_fullpath)
# Call the main function in the hook. If the hook should cause the
# build to fail, it will raise an Exception. We'll catch that convert
# to a HookError w/ just the failing traceback.
try:
context['main'](**kwargs)
except Exception:
raise HookError('%s\nFailed to run main() for %s hook; see traceback '
'above.' % (traceback.format_exc(), self._hook_type))
def _ExecuteHook(self, **kwargs):
"""Actually execute the given hook.
This will run the hook's 'main' function in our python interpreter.
Args:
kwargs: Keyword arguments to pass to the hook. These are often specific
to the hook type. For instance, pre-upload hooks will contain
a project_list.
"""
# Keep sys.path and CWD stashed away so that we can always restore them
# upon function exit.
orig_path = os.getcwd()
orig_syspath = sys.path
try:
# Always run hooks with CWD as topdir.
os.chdir(self._repo_topdir)
# Put the hook dir as the first item of sys.path so hooks can do
# relative imports. We want to replace the repo dir as [0] so
# hooks can't import repo files.
sys.path = [os.path.dirname(self._script_fullpath)] + sys.path[1:]
# Initial global context for the hook to run within.
context = {'__file__': self._script_fullpath}
# Add 'hook_should_take_kwargs' to the arguments to be passed to main.
# We don't actually want hooks to define their main with this argument--
# it's there to remind them that their hook should always take **kwargs.
# For instance, a pre-upload hook should be defined like:
# def main(project_list, **kwargs):
#
# This allows us to later expand the API without breaking old hooks.
kwargs = kwargs.copy()
kwargs['hook_should_take_kwargs'] = True
# See what version of python the hook has been written against.
data = open(self._script_fullpath).read()
interp = self._ExtractInterpFromShebang(data)
reexec = False
if interp:
prog = os.path.basename(interp)
if prog.startswith('python2') and sys.version_info.major != 2:
reexec = True
elif prog.startswith('python3') and sys.version_info.major == 2:
reexec = True
# Attempt to execute the hooks through the requested version of Python.
if reexec:
try: try:
self._ExecuteHookViaReexec(interp, context, **kwargs) exec(compile(data, self._script_fullpath, "exec"), context)
except OSError as e: except Exception:
if e.errno == errno.ENOENT: raise HookError(
# We couldn't find the interpreter, so fallback to importing. "%s\nFailed to import %s hook; see traceback above."
% (traceback.format_exc(), self._hook_type)
)
# Running the script should have defined a main() function.
if "main" not in context:
raise HookError('Missing main() in: "%s"' % self._script_fullpath)
# Call the main function in the hook. If the hook should cause the
# build to fail, it will raise an Exception. We'll catch that convert
# to a HookError w/ just the failing traceback.
try:
context["main"](**kwargs)
except Exception:
raise HookError(
"%s\nFailed to run main() for %s hook; see traceback "
"above." % (traceback.format_exc(), self._hook_type)
)
def _ExecuteHook(self, **kwargs):
"""Actually execute the given hook.
This will run the hook's 'main' function in our python interpreter.
Args:
kwargs: Keyword arguments to pass to the hook. These are often
specific to the hook type. For instance, pre-upload hooks will
contain a project_list.
"""
# Keep sys.path and CWD stashed away so that we can always restore them
# upon function exit.
orig_path = os.getcwd()
orig_syspath = sys.path
try:
# Always run hooks with CWD as topdir.
os.chdir(self._repo_topdir)
# Put the hook dir as the first item of sys.path so hooks can do
# relative imports. We want to replace the repo dir as [0] so
# hooks can't import repo files.
sys.path = [os.path.dirname(self._script_fullpath)] + sys.path[1:]
# Initial global context for the hook to run within.
context = {"__file__": self._script_fullpath}
# Add 'hook_should_take_kwargs' to the arguments to be passed to
# main. We don't actually want hooks to define their main with this
# argument--it's there to remind them that their hook should always
# take **kwargs.
# For instance, a pre-upload hook should be defined like:
# def main(project_list, **kwargs):
#
# This allows us to later expand the API without breaking old hooks.
kwargs = kwargs.copy()
kwargs["hook_should_take_kwargs"] = True
# See what version of python the hook has been written against.
data = open(self._script_fullpath).read()
interp = self._ExtractInterpFromShebang(data)
reexec = False reexec = False
else: if interp:
raise prog = os.path.basename(interp)
if prog.startswith("python2") and sys.version_info.major != 2:
reexec = True
elif prog.startswith("python3") and sys.version_info.major == 2:
reexec = True
# Run the hook by importing directly. # Attempt to execute the hooks through the requested version of
if not reexec: # Python.
self._ExecuteHookViaImport(data, context, **kwargs) if reexec:
finally: try:
# Restore sys.path and CWD. self._ExecuteHookViaReexec(interp, context, **kwargs)
sys.path = orig_syspath except OSError as e:
os.chdir(orig_path) if e.errno == errno.ENOENT:
# We couldn't find the interpreter, so fallback to
# importing.
reexec = False
else:
raise
def _CheckHook(self): # Run the hook by importing directly.
# Bail with a nice error if we can't find the hook. if not reexec:
if not os.path.isfile(self._script_fullpath): self._ExecuteHookViaImport(data, context, **kwargs)
raise HookError('Couldn\'t find repo hook: %s' % self._script_fullpath) finally:
# Restore sys.path and CWD.
sys.path = orig_syspath
os.chdir(orig_path)
def Run(self, **kwargs): def _CheckHook(self):
"""Run the hook. # Bail with a nice error if we can't find the hook.
if not os.path.isfile(self._script_fullpath):
raise HookError(
"Couldn't find repo hook: %s" % self._script_fullpath
)
If the hook doesn't exist (because there is no hooks project or because def Run(self, **kwargs):
this particular hook is not enabled), this is a no-op. """Run the hook.
Args: If the hook doesn't exist (because there is no hooks project or because
user_allows_all_hooks: If True, we will never prompt about running the this particular hook is not enabled), this is a no-op.
hook--we'll just assume it's OK to run it.
kwargs: Keyword arguments to pass to the hook. These are often specific
to the hook type. For instance, pre-upload hooks will contain
a project_list.
Returns: Args:
True: On success or ignore hooks by user-request user_allows_all_hooks: If True, we will never prompt about running
False: The hook failed. The caller should respond with aborting the action. the hook--we'll just assume it's OK to run it.
Some examples in which False is returned: kwargs: Keyword arguments to pass to the hook. These are often
* Finding the hook failed while it was enabled, or specific to the hook type. For instance, pre-upload hooks will
* the user declined to run a required hook (from _CheckForHookApproval) contain a project_list.
In all these cases the user did not pass the proper arguments to
ignore the result through the option combinations as listed in
AddHookOptionGroup().
"""
# Do not do anything in case bypass_hooks is set, or
# no-op if there is no hooks project or if hook is disabled.
if (self._bypass_hooks or
not self._hooks_project or
self._hook_type not in self._hooks_project.enabled_repo_hooks):
return True
passed = True Returns:
try: True: On success or ignore hooks by user-request
self._CheckHook() False: The hook failed. The caller should respond with aborting the
action. Some examples in which False is returned:
* Finding the hook failed while it was enabled, or
* the user declined to run a required hook (from
_CheckForHookApproval)
In all these cases the user did not pass the proper arguments to
ignore the result through the option combinations as listed in
AddHookOptionGroup().
"""
# Do not do anything in case bypass_hooks is set, or
# no-op if there is no hooks project or if hook is disabled.
if (
self._bypass_hooks
or not self._hooks_project
or self._hook_type not in self._hooks_project.enabled_repo_hooks
):
return True
# Make sure the user is OK with running the hook. passed = True
if self._allow_all_hooks or self._CheckForHookApproval(): try:
# Run the hook with the same version of python we're using. self._CheckHook()
self._ExecuteHook(**kwargs)
except SystemExit as e:
passed = False
print('ERROR: %s hooks exited with exit code: %s' % (self._hook_type, str(e)),
file=sys.stderr)
except HookError as e:
passed = False
print('ERROR: %s' % str(e), file=sys.stderr)
if not passed and self._ignore_hooks: # Make sure the user is OK with running the hook.
print('\nWARNING: %s hooks failed, but continuing anyways.' % self._hook_type, if self._allow_all_hooks or self._CheckForHookApproval():
file=sys.stderr) # Run the hook with the same version of python we're using.
passed = True self._ExecuteHook(**kwargs)
except SystemExit as e:
passed = False
print(
"ERROR: %s hooks exited with exit code: %s"
% (self._hook_type, str(e)),
file=sys.stderr,
)
except HookError as e:
passed = False
print("ERROR: %s" % str(e), file=sys.stderr)
return passed if not passed and self._ignore_hooks:
print(
"\nWARNING: %s hooks failed, but continuing anyways."
% self._hook_type,
file=sys.stderr,
)
passed = True
@classmethod return passed
def FromSubcmd(cls, manifest, opt, *args, **kwargs):
"""Method to construct the repo hook class
Args: @classmethod
manifest: The current active manifest for this command from which we def FromSubcmd(cls, manifest, opt, *args, **kwargs):
extract a couple of fields. """Method to construct the repo hook class
opt: Contains the commandline options for the action of this hook.
It should contain the options added by AddHookOptionGroup() in which
we are interested in RepoHook execution.
"""
for key in ('bypass_hooks', 'allow_all_hooks', 'ignore_hooks'):
kwargs.setdefault(key, getattr(opt, key))
kwargs.update({
'hooks_project': manifest.repo_hooks_project,
'repo_topdir': manifest.topdir,
'manifest_url': manifest.manifestProject.GetRemote('origin').url,
})
return cls(*args, **kwargs)
@staticmethod Args:
def AddOptionGroup(parser, name): manifest: The current active manifest for this command from which we
"""Help options relating to the various hooks.""" extract a couple of fields.
opt: Contains the commandline options for the action of this hook.
It should contain the options added by AddHookOptionGroup() in
which we are interested in RepoHook execution.
"""
for key in ("bypass_hooks", "allow_all_hooks", "ignore_hooks"):
kwargs.setdefault(key, getattr(opt, key))
kwargs.update(
{
"hooks_project": manifest.repo_hooks_project,
"repo_topdir": manifest.topdir,
"manifest_url": manifest.manifestProject.GetRemote(
"origin"
).url,
}
)
return cls(*args, **kwargs)
# Note that verify and no-verify are NOT opposites of each other, which @staticmethod
# is why they store to different locations. We are using them to match def AddOptionGroup(parser, name):
# 'git commit' syntax. """Help options relating to the various hooks."""
group = parser.add_option_group(name + ' hooks')
group.add_option('--no-verify', # Note that verify and no-verify are NOT opposites of each other, which
dest='bypass_hooks', action='store_true', # is why they store to different locations. We are using them to match
help='Do not run the %s hook.' % name) # 'git commit' syntax.
group.add_option('--verify', group = parser.add_option_group(name + " hooks")
dest='allow_all_hooks', action='store_true', group.add_option(
help='Run the %s hook without prompting.' % name) "--no-verify",
group.add_option('--ignore-hooks', dest="bypass_hooks",
action='store_true', action="store_true",
help='Do not abort if %s hooks fail.' % name) help="Do not run the %s hook." % name,
)
group.add_option(
"--verify",
dest="allow_all_hooks",
action="store_true",
help="Run the %s hook without prompting." % name,
)
group.add_option(
"--ignore-hooks",
action="store_true",
help="Do not abort if %s hooks fail." % name,
)

1218
main.py
View File

File diff suppressed because it is too large Load Diff

3967
manifest_xml.py
View File

File diff suppressed because it is too large Load Diff

151
pager.py
View File

@ -26,102 +26,101 @@ old_stderr = None
def RunPager(globalConfig): def RunPager(globalConfig):
if not os.isatty(0) or not os.isatty(1): if not os.isatty(0) or not os.isatty(1):
return return
pager = _SelectPager(globalConfig) pager = _SelectPager(globalConfig)
if pager == '' or pager == 'cat': if pager == "" or pager == "cat":
return return
if platform_utils.isWindows(): if platform_utils.isWindows():
_PipePager(pager) _PipePager(pager)
else: else:
_ForkPager(pager) _ForkPager(pager)
def TerminatePager(): def TerminatePager():
global pager_process, old_stdout, old_stderr global pager_process, old_stdout, old_stderr
if pager_process: if pager_process:
sys.stdout.flush() sys.stdout.flush()
sys.stderr.flush() sys.stderr.flush()
pager_process.stdin.close() pager_process.stdin.close()
pager_process.wait() pager_process.wait()
pager_process = None pager_process = None
# Restore initial stdout/err in case there is more output in this process # Restore initial stdout/err in case there is more output in this
# after shutting down the pager process # process after shutting down the pager process.
sys.stdout = old_stdout sys.stdout = old_stdout
sys.stderr = old_stderr sys.stderr = old_stderr
def _PipePager(pager): def _PipePager(pager):
global pager_process, old_stdout, old_stderr global pager_process, old_stdout, old_stderr
assert pager_process is None, "Only one active pager process at a time" assert pager_process is None, "Only one active pager process at a time"
# Create pager process, piping stdout/err into its stdin # Create pager process, piping stdout/err into its stdin.
try: try:
pager_process = subprocess.Popen([pager], stdin=subprocess.PIPE, stdout=sys.stdout, pager_process = subprocess.Popen(
stderr=sys.stderr) [pager], stdin=subprocess.PIPE, stdout=sys.stdout, stderr=sys.stderr
except FileNotFoundError: )
sys.exit(f'fatal: cannot start pager "{pager}"') except FileNotFoundError:
old_stdout = sys.stdout sys.exit(f'fatal: cannot start pager "{pager}"')
old_stderr = sys.stderr old_stdout = sys.stdout
sys.stdout = pager_process.stdin old_stderr = sys.stderr
sys.stderr = pager_process.stdin sys.stdout = pager_process.stdin
sys.stderr = pager_process.stdin
def _ForkPager(pager): def _ForkPager(pager):
global active global active
# This process turns into the pager; a child it forks will # This process turns into the pager; a child it forks will
# do the real processing and output back to the pager. This # do the real processing and output back to the pager. This
# is necessary to keep the pager in control of the tty. # is necessary to keep the pager in control of the tty.
# try:
try: r, w = os.pipe()
r, w = os.pipe() pid = os.fork()
pid = os.fork() if not pid:
if not pid: os.dup2(w, 1)
os.dup2(w, 1) os.dup2(w, 2)
os.dup2(w, 2) os.close(r)
os.close(r) os.close(w)
os.close(w) active = True
active = True return
return
os.dup2(r, 0) os.dup2(r, 0)
os.close(r) os.close(r)
os.close(w) os.close(w)
_BecomePager(pager) _BecomePager(pager)
except Exception: except Exception:
print("fatal: cannot start pager '%s'" % pager, file=sys.stderr) print("fatal: cannot start pager '%s'" % pager, file=sys.stderr)
sys.exit(255) sys.exit(255)
def _SelectPager(globalConfig): def _SelectPager(globalConfig):
try: try:
return os.environ['GIT_PAGER'] return os.environ["GIT_PAGER"]
except KeyError: except KeyError:
pass pass
pager = globalConfig.GetString('core.pager') pager = globalConfig.GetString("core.pager")
if pager: if pager:
return pager return pager
try: try:
return os.environ['PAGER'] return os.environ["PAGER"]
except KeyError: except KeyError:
pass pass
return 'less' return "less"
def _BecomePager(pager): def _BecomePager(pager):
# Delaying execution of the pager until we have output # Delaying execution of the pager until we have output
# ready works around a long-standing bug in popularly # ready works around a long-standing bug in popularly
# available versions of 'less', a better 'more'. # available versions of 'less', a better 'more'.
# _a, _b, _c = select.select([0], [], [0])
_a, _b, _c = select.select([0], [], [0])
os.environ['LESS'] = 'FRSX' os.environ["LESS"] = "FRSX"
try: try:
os.execvp(pager, [pager]) os.execvp(pager, [pager])
except OSError: except OSError:
os.execv('/bin/sh', ['sh', '-c', pager]) os.execv("/bin/sh", ["sh", "-c", pager])

378
platform_utils.py
View File

@ -20,246 +20,264 @@ import stat
def isWindows(): def isWindows():
""" Returns True when running with the native port of Python for Windows, """Returns True when running with the native port of Python for Windows,
False when running on any other platform (including the Cygwin port of False when running on any other platform (including the Cygwin port of
Python). Python).
""" """
# Note: The cygwin port of Python returns "CYGWIN_NT_xxx" # Note: The cygwin port of Python returns "CYGWIN_NT_xxx"
return platform.system() == "Windows" return platform.system() == "Windows"
def symlink(source, link_name): def symlink(source, link_name):
"""Creates a symbolic link pointing to source named link_name. """Creates a symbolic link pointing to source named link_name.
Note: On Windows, source must exist on disk, as the implementation needs
to know whether to create a "File" or a "Directory" symbolic link. Note: On Windows, source must exist on disk, as the implementation needs
""" to know whether to create a "File" or a "Directory" symbolic link.
if isWindows(): """
import platform_utils_win32 if isWindows():
source = _validate_winpath(source) import platform_utils_win32
link_name = _validate_winpath(link_name)
target = os.path.join(os.path.dirname(link_name), source) source = _validate_winpath(source)
if isdir(target): link_name = _validate_winpath(link_name)
platform_utils_win32.create_dirsymlink(_makelongpath(source), link_name) target = os.path.join(os.path.dirname(link_name), source)
if isdir(target):
platform_utils_win32.create_dirsymlink(
_makelongpath(source), link_name
)
else:
platform_utils_win32.create_filesymlink(
_makelongpath(source), link_name
)
else: else:
platform_utils_win32.create_filesymlink(_makelongpath(source), link_name) return os.symlink(source, link_name)
else:
return os.symlink(source, link_name)
def _validate_winpath(path): def _validate_winpath(path):
path = os.path.normpath(path) path = os.path.normpath(path)
if _winpath_is_valid(path): if _winpath_is_valid(path):
return path return path
raise ValueError("Path \"%s\" must be a relative path or an absolute " raise ValueError(
"path starting with a drive letter".format(path)) 'Path "{}" must be a relative path or an absolute '
"path starting with a drive letter".format(path)
)
def _winpath_is_valid(path): def _winpath_is_valid(path):
"""Windows only: returns True if path is relative (e.g. ".\\foo") or is """Windows only: returns True if path is relative (e.g. ".\\foo") or is
absolute including a drive letter (e.g. "c:\\foo"). Returns False if path absolute including a drive letter (e.g. "c:\\foo"). Returns False if path
is ambiguous (e.g. "x:foo" or "\\foo"). is ambiguous (e.g. "x:foo" or "\\foo").
""" """
assert isWindows() assert isWindows()
path = os.path.normpath(path) path = os.path.normpath(path)
drive, tail = os.path.splitdrive(path) drive, tail = os.path.splitdrive(path)
if tail: if tail:
if not drive: if not drive:
return tail[0] != os.sep # "\\foo" is invalid return tail[0] != os.sep # "\\foo" is invalid
else:
return tail[0] == os.sep # "x:foo" is invalid
else: else:
return tail[0] == os.sep # "x:foo" is invalid return not drive # "x:" is invalid
else:
return not drive # "x:" is invalid
def _makelongpath(path): def _makelongpath(path):
"""Return the input path normalized to support the Windows long path syntax """Return the input path normalized to support the Windows long path syntax
("\\\\?\\" prefix) if needed, i.e. if the input path is longer than the ("\\\\?\\" prefix) if needed, i.e. if the input path is longer than the
MAX_PATH limit. MAX_PATH limit.
""" """
if isWindows(): if isWindows():
# Note: MAX_PATH is 260, but, for directories, the maximum value is actually 246. # Note: MAX_PATH is 260, but, for directories, the maximum value is
if len(path) < 246: # actually 246.
return path if len(path) < 246:
if path.startswith(u"\\\\?\\"): return path
return path if path.startswith("\\\\?\\"):
if not os.path.isabs(path): return path
return path if not os.path.isabs(path):
# Append prefix and ensure unicode so that the special longpath syntax return path
# is supported by underlying Win32 API calls # Append prefix and ensure unicode so that the special longpath syntax
return u"\\\\?\\" + os.path.normpath(path) # is supported by underlying Win32 API calls
else: return "\\\\?\\" + os.path.normpath(path)
return path else:
return path
def rmtree(path, ignore_errors=False): def rmtree(path, ignore_errors=False):
"""shutil.rmtree(path) wrapper with support for long paths on Windows. """shutil.rmtree(path) wrapper with support for long paths on Windows.
Availability: Unix, Windows.""" Availability: Unix, Windows.
onerror = None """
if isWindows(): onerror = None
path = _makelongpath(path) if isWindows():
onerror = handle_rmtree_error path = _makelongpath(path)
shutil.rmtree(path, ignore_errors=ignore_errors, onerror=onerror) onerror = handle_rmtree_error
shutil.rmtree(path, ignore_errors=ignore_errors, onerror=onerror)
def handle_rmtree_error(function, path, excinfo): def handle_rmtree_error(function, path, excinfo):
# Allow deleting read-only files # Allow deleting read-only files.
os.chmod(path, stat.S_IWRITE) os.chmod(path, stat.S_IWRITE)
function(path) function(path)
def rename(src, dst): def rename(src, dst):
"""os.rename(src, dst) wrapper with support for long paths on Windows. """os.rename(src, dst) wrapper with support for long paths on Windows.
Availability: Unix, Windows.""" Availability: Unix, Windows.
if isWindows(): """
# On Windows, rename fails if destination exists, see if isWindows():
# https://docs.python.org/2/library/os.html#os.rename # On Windows, rename fails if destination exists, see
try: # https://docs.python.org/2/library/os.html#os.rename
os.rename(_makelongpath(src), _makelongpath(dst)) try:
except OSError as e: os.rename(_makelongpath(src), _makelongpath(dst))
if e.errno == errno.EEXIST: except OSError as e:
os.remove(_makelongpath(dst)) if e.errno == errno.EEXIST:
os.rename(_makelongpath(src), _makelongpath(dst)) os.remove(_makelongpath(dst))
else: os.rename(_makelongpath(src), _makelongpath(dst))
raise else:
else: raise
shutil.move(src, dst) else:
shutil.move(src, dst)
def remove(path, missing_ok=False): def remove(path, missing_ok=False):
"""Remove (delete) the file path. This is a replacement for os.remove that """Remove (delete) the file path. This is a replacement for os.remove that
allows deleting read-only files on Windows, with support for long paths and allows deleting read-only files on Windows, with support for long paths and
for deleting directory symbolic links. for deleting directory symbolic links.
Availability: Unix, Windows.""" Availability: Unix, Windows.
longpath = _makelongpath(path) if isWindows() else path """
try: longpath = _makelongpath(path) if isWindows() else path
os.remove(longpath) try:
except OSError as e:
if e.errno == errno.EACCES:
os.chmod(longpath, stat.S_IWRITE)
# Directory symbolic links must be deleted with 'rmdir'.
if islink(longpath) and isdir(longpath):
os.rmdir(longpath)
else:
os.remove(longpath) os.remove(longpath)
elif missing_ok and e.errno == errno.ENOENT: except OSError as e:
pass if e.errno == errno.EACCES:
else: os.chmod(longpath, stat.S_IWRITE)
raise # Directory symbolic links must be deleted with 'rmdir'.
if islink(longpath) and isdir(longpath):
os.rmdir(longpath)
else:
os.remove(longpath)
elif missing_ok and e.errno == errno.ENOENT:
pass
else:
raise
def walk(top, topdown=True, onerror=None, followlinks=False): def walk(top, topdown=True, onerror=None, followlinks=False):
"""os.walk(path) wrapper with support for long paths on Windows. """os.walk(path) wrapper with support for long paths on Windows.
Availability: Windows, Unix. Availability: Windows, Unix.
""" """
if isWindows(): if isWindows():
return _walk_windows_impl(top, topdown, onerror, followlinks) return _walk_windows_impl(top, topdown, onerror, followlinks)
else: else:
return os.walk(top, topdown, onerror, followlinks) return os.walk(top, topdown, onerror, followlinks)
def _walk_windows_impl(top, topdown, onerror, followlinks): def _walk_windows_impl(top, topdown, onerror, followlinks):
try: try:
names = listdir(top) names = listdir(top)
except Exception as err: except Exception as err:
if onerror is not None: if onerror is not None:
onerror(err) onerror(err)
return return
dirs, nondirs = [], [] dirs, nondirs = [], []
for name in names: for name in names:
if isdir(os.path.join(top, name)): if isdir(os.path.join(top, name)):
dirs.append(name) dirs.append(name)
else: else:
nondirs.append(name) nondirs.append(name)
if topdown: if topdown:
yield top, dirs, nondirs yield top, dirs, nondirs
for name in dirs: for name in dirs:
new_path = os.path.join(top, name) new_path = os.path.join(top, name)
if followlinks or not islink(new_path): if followlinks or not islink(new_path):
for x in _walk_windows_impl(new_path, topdown, onerror, followlinks): for x in _walk_windows_impl(
yield x new_path, topdown, onerror, followlinks
if not topdown: ):
yield top, dirs, nondirs yield x
if not topdown:
yield top, dirs, nondirs
def listdir(path): def listdir(path):
"""os.listdir(path) wrapper with support for long paths on Windows. """os.listdir(path) wrapper with support for long paths on Windows.
Availability: Windows, Unix. Availability: Windows, Unix.
""" """
return os.listdir(_makelongpath(path)) return os.listdir(_makelongpath(path))
def rmdir(path): def rmdir(path):
"""os.rmdir(path) wrapper with support for long paths on Windows. """os.rmdir(path) wrapper with support for long paths on Windows.
Availability: Windows, Unix. Availability: Windows, Unix.
""" """
os.rmdir(_makelongpath(path)) os.rmdir(_makelongpath(path))
def isdir(path): def isdir(path):
"""os.path.isdir(path) wrapper with support for long paths on Windows. """os.path.isdir(path) wrapper with support for long paths on Windows.
Availability: Windows, Unix. Availability: Windows, Unix.
""" """
return os.path.isdir(_makelongpath(path)) return os.path.isdir(_makelongpath(path))
def islink(path): def islink(path):
"""os.path.islink(path) wrapper with support for long paths on Windows. """os.path.islink(path) wrapper with support for long paths on Windows.
Availability: Windows, Unix. Availability: Windows, Unix.
""" """
if isWindows(): if isWindows():
import platform_utils_win32 import platform_utils_win32
return platform_utils_win32.islink(_makelongpath(path))
else: return platform_utils_win32.islink(_makelongpath(path))
return os.path.islink(path) else:
return os.path.islink(path)
def readlink(path): def readlink(path):
"""Return a string representing the path to which the symbolic link """Return a string representing the path to which the symbolic link
points. The result may be either an absolute or relative pathname; points. The result may be either an absolute or relative pathname;
if it is relative, it may be converted to an absolute pathname using if it is relative, it may be converted to an absolute pathname using
os.path.join(os.path.dirname(path), result). os.path.join(os.path.dirname(path), result).
Availability: Windows, Unix. Availability: Windows, Unix.
""" """
if isWindows(): if isWindows():
import platform_utils_win32 import platform_utils_win32
return platform_utils_win32.readlink(_makelongpath(path))
else: return platform_utils_win32.readlink(_makelongpath(path))
return os.readlink(path) else:
return os.readlink(path)
def realpath(path): def realpath(path):
"""Return the canonical path of the specified filename, eliminating """Return the canonical path of the specified filename, eliminating
any symbolic links encountered in the path. any symbolic links encountered in the path.
Availability: Windows, Unix. Availability: Windows, Unix.
""" """
if isWindows(): if isWindows():
current_path = os.path.abspath(path) current_path = os.path.abspath(path)
path_tail = [] path_tail = []
for c in range(0, 100): # Avoid cycles for c in range(0, 100): # Avoid cycles
if islink(current_path): if islink(current_path):
target = readlink(current_path) target = readlink(current_path)
current_path = os.path.join(os.path.dirname(current_path), target) current_path = os.path.join(
else: os.path.dirname(current_path), target
basename = os.path.basename(current_path) )
if basename == '': else:
path_tail.append(current_path) basename = os.path.basename(current_path)
break if basename == "":
path_tail.append(basename) path_tail.append(current_path)
current_path = os.path.dirname(current_path) break
path_tail.reverse() path_tail.append(basename)
result = os.path.normpath(os.path.join(*path_tail)) current_path = os.path.dirname(current_path)
return result path_tail.reverse()
else: result = os.path.normpath(os.path.join(*path_tail))
return os.path.realpath(path) return result
else:
return os.path.realpath(path)

243
platform_utils_win32.py
View File

@ -19,7 +19,7 @@ from ctypes import c_buffer, c_ubyte, Structure, Union, byref
from ctypes.wintypes import BOOL, BOOLEAN, LPCWSTR, DWORD, HANDLE from ctypes.wintypes import BOOL, BOOLEAN, LPCWSTR, DWORD, HANDLE
from ctypes.wintypes import WCHAR, USHORT, LPVOID, ULONG, LPDWORD from ctypes.wintypes import WCHAR, USHORT, LPVOID, ULONG, LPDWORD
kernel32 = WinDLL('kernel32', use_last_error=True) kernel32 = WinDLL("kernel32", use_last_error=True)
UCHAR = c_ubyte UCHAR = c_ubyte
@ -31,14 +31,17 @@ ERROR_PRIVILEGE_NOT_HELD = 1314
# Win32 API entry points # Win32 API entry points
CreateSymbolicLinkW = kernel32.CreateSymbolicLinkW CreateSymbolicLinkW = kernel32.CreateSymbolicLinkW
CreateSymbolicLinkW.restype = BOOLEAN CreateSymbolicLinkW.restype = BOOLEAN
CreateSymbolicLinkW.argtypes = (LPCWSTR, # lpSymlinkFileName In CreateSymbolicLinkW.argtypes = (
LPCWSTR, # lpTargetFileName In LPCWSTR, # lpSymlinkFileName In
DWORD) # dwFlags In LPCWSTR, # lpTargetFileName In
DWORD, # dwFlags In
)
# Symbolic link creation flags # Symbolic link creation flags
SYMBOLIC_LINK_FLAG_FILE = 0x00 SYMBOLIC_LINK_FLAG_FILE = 0x00
SYMBOLIC_LINK_FLAG_DIRECTORY = 0x01 SYMBOLIC_LINK_FLAG_DIRECTORY = 0x01
# symlink support for CreateSymbolicLink() starting with Windows 10 (1703, v10.0.14972) # symlink support for CreateSymbolicLink() starting with Windows 10 (1703,
# v10.0.14972)
SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE = 0x02 SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE = 0x02
GetFileAttributesW = kernel32.GetFileAttributesW GetFileAttributesW = kernel32.GetFileAttributesW
@ -50,13 +53,15 @@ FILE_ATTRIBUTE_REPARSE_POINT = 0x00400
CreateFileW = kernel32.CreateFileW CreateFileW = kernel32.CreateFileW
CreateFileW.restype = HANDLE CreateFileW.restype = HANDLE
CreateFileW.argtypes = (LPCWSTR, # lpFileName In CreateFileW.argtypes = (
DWORD, # dwDesiredAccess In LPCWSTR, # lpFileName In
DWORD, # dwShareMode In DWORD, # dwDesiredAccess In
LPVOID, # lpSecurityAttributes In_opt DWORD, # dwShareMode In
DWORD, # dwCreationDisposition In LPVOID, # lpSecurityAttributes In_opt
DWORD, # dwFlagsAndAttributes In DWORD, # dwCreationDisposition In
HANDLE) # hTemplateFile In_opt DWORD, # dwFlagsAndAttributes In
HANDLE, # hTemplateFile In_opt
)
CloseHandle = kernel32.CloseHandle CloseHandle = kernel32.CloseHandle
CloseHandle.restype = BOOL CloseHandle.restype = BOOL
@ -69,14 +74,16 @@ FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000
DeviceIoControl = kernel32.DeviceIoControl DeviceIoControl = kernel32.DeviceIoControl
DeviceIoControl.restype = BOOL DeviceIoControl.restype = BOOL
DeviceIoControl.argtypes = (HANDLE, # hDevice In DeviceIoControl.argtypes = (
DWORD, # dwIoControlCode In HANDLE, # hDevice In
LPVOID, # lpInBuffer In_opt DWORD, # dwIoControlCode In
DWORD, # nInBufferSize In LPVOID, # lpInBuffer In_opt
LPVOID, # lpOutBuffer Out_opt DWORD, # nInBufferSize In
DWORD, # nOutBufferSize In LPVOID, # lpOutBuffer Out_opt
LPDWORD, # lpBytesReturned Out_opt DWORD, # nOutBufferSize In
LPVOID) # lpOverlapped Inout_opt LPDWORD, # lpBytesReturned Out_opt
LPVOID, # lpOverlapped Inout_opt
)
# Device I/O control flags and options # Device I/O control flags and options
FSCTL_GET_REPARSE_POINT = 0x000900A8 FSCTL_GET_REPARSE_POINT = 0x000900A8
@ -86,124 +93,138 @@ MAXIMUM_REPARSE_DATA_BUFFER_SIZE = 0x4000
class GENERIC_REPARSE_BUFFER(Structure): class GENERIC_REPARSE_BUFFER(Structure):
_fields_ = (('DataBuffer', UCHAR * 1),) _fields_ = (("DataBuffer", UCHAR * 1),)
class SYMBOLIC_LINK_REPARSE_BUFFER(Structure): class SYMBOLIC_LINK_REPARSE_BUFFER(Structure):
_fields_ = (('SubstituteNameOffset', USHORT), _fields_ = (
('SubstituteNameLength', USHORT), ("SubstituteNameOffset", USHORT),
('PrintNameOffset', USHORT), ("SubstituteNameLength", USHORT),
('PrintNameLength', USHORT), ("PrintNameOffset", USHORT),
('Flags', ULONG), ("PrintNameLength", USHORT),
('PathBuffer', WCHAR * 1)) ("Flags", ULONG),
("PathBuffer", WCHAR * 1),
)
@property @property
def PrintName(self): def PrintName(self):
arrayt = WCHAR * (self.PrintNameLength // 2) arrayt = WCHAR * (self.PrintNameLength // 2)
offset = type(self).PathBuffer.offset + self.PrintNameOffset offset = type(self).PathBuffer.offset + self.PrintNameOffset
return arrayt.from_address(addressof(self) + offset).value return arrayt.from_address(addressof(self) + offset).value
class MOUNT_POINT_REPARSE_BUFFER(Structure): class MOUNT_POINT_REPARSE_BUFFER(Structure):
_fields_ = (('SubstituteNameOffset', USHORT), _fields_ = (
('SubstituteNameLength', USHORT), ("SubstituteNameOffset", USHORT),
('PrintNameOffset', USHORT), ("SubstituteNameLength", USHORT),
('PrintNameLength', USHORT), ("PrintNameOffset", USHORT),
('PathBuffer', WCHAR * 1)) ("PrintNameLength", USHORT),
("PathBuffer", WCHAR * 1),
)
@property @property
def PrintName(self): def PrintName(self):
arrayt = WCHAR * (self.PrintNameLength // 2) arrayt = WCHAR * (self.PrintNameLength // 2)
offset = type(self).PathBuffer.offset + self.PrintNameOffset offset = type(self).PathBuffer.offset + self.PrintNameOffset
return arrayt.from_address(addressof(self) + offset).value return arrayt.from_address(addressof(self) + offset).value
class REPARSE_DATA_BUFFER(Structure): class REPARSE_DATA_BUFFER(Structure):
class REPARSE_BUFFER(Union): class REPARSE_BUFFER(Union):
_fields_ = (('SymbolicLinkReparseBuffer', SYMBOLIC_LINK_REPARSE_BUFFER), _fields_ = (
('MountPointReparseBuffer', MOUNT_POINT_REPARSE_BUFFER), ("SymbolicLinkReparseBuffer", SYMBOLIC_LINK_REPARSE_BUFFER),
('GenericReparseBuffer', GENERIC_REPARSE_BUFFER)) ("MountPointReparseBuffer", MOUNT_POINT_REPARSE_BUFFER),
_fields_ = (('ReparseTag', ULONG), ("GenericReparseBuffer", GENERIC_REPARSE_BUFFER),
('ReparseDataLength', USHORT), )
('Reserved', USHORT),
('ReparseBuffer', REPARSE_BUFFER)) _fields_ = (
_anonymous_ = ('ReparseBuffer',) ("ReparseTag", ULONG),
("ReparseDataLength", USHORT),
("Reserved", USHORT),
("ReparseBuffer", REPARSE_BUFFER),
)
_anonymous_ = ("ReparseBuffer",)
def create_filesymlink(source, link_name): def create_filesymlink(source, link_name):
"""Creates a Windows file symbolic link source pointing to link_name.""" """Creates a Windows file symbolic link source pointing to link_name."""
_create_symlink(source, link_name, SYMBOLIC_LINK_FLAG_FILE) _create_symlink(source, link_name, SYMBOLIC_LINK_FLAG_FILE)
def create_dirsymlink(source, link_name): def create_dirsymlink(source, link_name):
"""Creates a Windows directory symbolic link source pointing to link_name. """Creates a Windows directory symbolic link source pointing to link_name.""" # noqa: E501
""" _create_symlink(source, link_name, SYMBOLIC_LINK_FLAG_DIRECTORY)
_create_symlink(source, link_name, SYMBOLIC_LINK_FLAG_DIRECTORY)
def _create_symlink(source, link_name, dwFlags): def _create_symlink(source, link_name, dwFlags):
if not CreateSymbolicLinkW(link_name, source, if not CreateSymbolicLinkW(
dwFlags | SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE): link_name,
# See https://github.com/golang/go/pull/24307/files#diff-b87bc12e4da2497308f9ef746086e4f0 source,
# "the unprivileged create flag is unsupported below Windows 10 (1703, v10.0.14972). dwFlags | SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE,
# retry without it." ):
if not CreateSymbolicLinkW(link_name, source, dwFlags): # See https://github.com/golang/go/pull/24307/files#diff-b87bc12e4da2497308f9ef746086e4f0 # noqa: E501
code = get_last_error() # "the unprivileged create flag is unsupported below Windows 10 (1703,
error_desc = FormatError(code).strip() # v10.0.14972). retry without it."
if code == ERROR_PRIVILEGE_NOT_HELD: if not CreateSymbolicLinkW(link_name, source, dwFlags):
raise OSError(errno.EPERM, error_desc, link_name) code = get_last_error()
_raise_winerror( error_desc = FormatError(code).strip()
code, if code == ERROR_PRIVILEGE_NOT_HELD:
'Error creating symbolic link \"%s\"'.format(link_name)) raise OSError(errno.EPERM, error_desc, link_name)
_raise_winerror(
code, 'Error creating symbolic link "{}"'.format(link_name)
)
def islink(path): def islink(path):
result = GetFileAttributesW(path) result = GetFileAttributesW(path)
if result == INVALID_FILE_ATTRIBUTES: if result == INVALID_FILE_ATTRIBUTES:
return False return False
return bool(result & FILE_ATTRIBUTE_REPARSE_POINT) return bool(result & FILE_ATTRIBUTE_REPARSE_POINT)
def readlink(path): def readlink(path):
reparse_point_handle = CreateFileW(path, reparse_point_handle = CreateFileW(
0, path,
0, 0,
None, 0,
OPEN_EXISTING, None,
FILE_FLAG_OPEN_REPARSE_POINT | OPEN_EXISTING,
FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS,
None) None,
if reparse_point_handle == INVALID_HANDLE_VALUE: )
if reparse_point_handle == INVALID_HANDLE_VALUE:
_raise_winerror(
get_last_error(), 'Error opening symbolic link "{}"'.format(path)
)
target_buffer = c_buffer(MAXIMUM_REPARSE_DATA_BUFFER_SIZE)
n_bytes_returned = DWORD()
io_result = DeviceIoControl(
reparse_point_handle,
FSCTL_GET_REPARSE_POINT,
None,
0,
target_buffer,
len(target_buffer),
byref(n_bytes_returned),
None,
)
CloseHandle(reparse_point_handle)
if not io_result:
_raise_winerror(
get_last_error(), 'Error reading symbolic link "{}"'.format(path)
)
rdb = REPARSE_DATA_BUFFER.from_buffer(target_buffer)
if rdb.ReparseTag == IO_REPARSE_TAG_SYMLINK:
return rdb.SymbolicLinkReparseBuffer.PrintName
elif rdb.ReparseTag == IO_REPARSE_TAG_MOUNT_POINT:
return rdb.MountPointReparseBuffer.PrintName
# Unsupported reparse point type.
_raise_winerror( _raise_winerror(
get_last_error(), ERROR_NOT_SUPPORTED, 'Error reading symbolic link "{}"'.format(path)
'Error opening symbolic link \"%s\"'.format(path)) )
target_buffer = c_buffer(MAXIMUM_REPARSE_DATA_BUFFER_SIZE)
n_bytes_returned = DWORD()
io_result = DeviceIoControl(reparse_point_handle,
FSCTL_GET_REPARSE_POINT,
None,
0,
target_buffer,
len(target_buffer),
byref(n_bytes_returned),
None)
CloseHandle(reparse_point_handle)
if not io_result:
_raise_winerror(
get_last_error(),
'Error reading symbolic link \"%s\"'.format(path))
rdb = REPARSE_DATA_BUFFER.from_buffer(target_buffer)
if rdb.ReparseTag == IO_REPARSE_TAG_SYMLINK:
return rdb.SymbolicLinkReparseBuffer.PrintName
elif rdb.ReparseTag == IO_REPARSE_TAG_MOUNT_POINT:
return rdb.MountPointReparseBuffer.PrintName
# Unsupported reparse point type
_raise_winerror(
ERROR_NOT_SUPPORTED,
'Error reading symbolic link \"%s\"'.format(path))
def _raise_winerror(code, error_desc): def _raise_winerror(code, error_desc):
win_error_desc = FormatError(code).strip() win_error_desc = FormatError(code).strip()
error_desc = "%s: %s".format(error_desc, win_error_desc) error_desc = "{0}: {1}".format(error_desc, win_error_desc)
raise WinError(code, error_desc) raise WinError(code, error_desc)

205
progress.py
View File

@ -22,115 +22,136 @@ _NOT_TTY = not os.isatty(2)
# This will erase all content in the current line (wherever the cursor is). # This will erase all content in the current line (wherever the cursor is).
# It does not move the cursor, so this is usually followed by \r to move to # It does not move the cursor, so this is usually followed by \r to move to
# column 0. # column 0.
CSI_ERASE_LINE = '\x1b[2K' CSI_ERASE_LINE = "\x1b[2K"
# This will erase all content in the current line after the cursor. This is # This will erase all content in the current line after the cursor. This is
# useful for partial updates & progress messages as the terminal can display # useful for partial updates & progress messages as the terminal can display
# it better. # it better.
CSI_ERASE_LINE_AFTER = '\x1b[K' CSI_ERASE_LINE_AFTER = "\x1b[K"
def duration_str(total): def duration_str(total):
"""A less noisy timedelta.__str__. """A less noisy timedelta.__str__.
The default timedelta stringification contains a lot of leading zeros and The default timedelta stringification contains a lot of leading zeros and
uses microsecond resolution. This makes for noisy output. uses microsecond resolution. This makes for noisy output.
""" """
hours, rem = divmod(total, 3600) hours, rem = divmod(total, 3600)
mins, secs = divmod(rem, 60) mins, secs = divmod(rem, 60)
ret = '%.3fs' % (secs,) ret = "%.3fs" % (secs,)
if mins: if mins:
ret = '%im%s' % (mins, ret) ret = "%im%s" % (mins, ret)
if hours: if hours:
ret = '%ih%s' % (hours, ret) ret = "%ih%s" % (hours, ret)
return ret return ret
class Progress(object): class Progress(object):
def __init__(self, title, total=0, units='', print_newline=False, delay=True, def __init__(
quiet=False): self,
self._title = title title,
self._total = total total=0,
self._done = 0 units="",
self._start = time() print_newline=False,
self._show = not delay delay=True,
self._units = units quiet=False,
self._print_newline = print_newline ):
# Only show the active jobs section if we run more than one in parallel. self._title = title
self._show_jobs = False self._total = total
self._active = 0 self._done = 0
self._start = time()
self._show = not delay
self._units = units
self._print_newline = print_newline
# Only show the active jobs section if we run more than one in parallel.
self._show_jobs = False
self._active = 0
# When quiet, never show any output. It's a bit hacky, but reusing the # When quiet, never show any output. It's a bit hacky, but reusing the
# existing logic that delays initial output keeps the rest of the class # existing logic that delays initial output keeps the rest of the class
# clean. Basically we set the start time to years in the future. # clean. Basically we set the start time to years in the future.
if quiet: if quiet:
self._show = False self._show = False
self._start += 2**32 self._start += 2**32
def start(self, name): def start(self, name):
self._active += 1 self._active += 1
if not self._show_jobs: if not self._show_jobs:
self._show_jobs = self._active > 1 self._show_jobs = self._active > 1
self.update(inc=0, msg='started ' + name) self.update(inc=0, msg="started " + name)
def finish(self, name): def finish(self, name):
self.update(msg='finished ' + name) self.update(msg="finished " + name)
self._active -= 1 self._active -= 1
def update(self, inc=1, msg=''): def update(self, inc=1, msg=""):
self._done += inc self._done += inc
if _NOT_TTY or IsTraceToStderr(): if _NOT_TTY or IsTraceToStderr():
return return
if not self._show: if not self._show:
if 0.5 <= time() - self._start: if 0.5 <= time() - self._start:
self._show = True self._show = True
else: else:
return return
if self._total <= 0: if self._total <= 0:
sys.stderr.write('\r%s: %d,%s' % ( sys.stderr.write(
self._title, "\r%s: %d,%s" % (self._title, self._done, CSI_ERASE_LINE_AFTER)
self._done, )
CSI_ERASE_LINE_AFTER)) sys.stderr.flush()
sys.stderr.flush() else:
else: p = (100 * self._done) / self._total
p = (100 * self._done) / self._total if self._show_jobs:
if self._show_jobs: jobs = "[%d job%s] " % (
jobs = '[%d job%s] ' % (self._active, 's' if self._active > 1 else '') self._active,
else: "s" if self._active > 1 else "",
jobs = '' )
sys.stderr.write('\r%s: %2d%% %s(%d%s/%d%s)%s%s%s%s' % ( else:
self._title, jobs = ""
p, sys.stderr.write(
jobs, "\r%s: %2d%% %s(%d%s/%d%s)%s%s%s%s"
self._done, self._units, % (
self._total, self._units, self._title,
' ' if msg else '', msg, p,
CSI_ERASE_LINE_AFTER, jobs,
'\n' if self._print_newline else '')) self._done,
sys.stderr.flush() self._units,
self._total,
self._units,
" " if msg else "",
msg,
CSI_ERASE_LINE_AFTER,
"\n" if self._print_newline else "",
)
)
sys.stderr.flush()
def end(self): def end(self):
if _NOT_TTY or IsTraceToStderr() or not self._show: if _NOT_TTY or IsTraceToStderr() or not self._show:
return return
duration = duration_str(time() - self._start) duration = duration_str(time() - self._start)
if self._total <= 0: if self._total <= 0:
sys.stderr.write('\r%s: %d, done in %s%s\n' % ( sys.stderr.write(
self._title, "\r%s: %d, done in %s%s\n"
self._done, % (self._title, self._done, duration, CSI_ERASE_LINE_AFTER)
duration, )
CSI_ERASE_LINE_AFTER)) sys.stderr.flush()
sys.stderr.flush() else:
else: p = (100 * self._done) / self._total
p = (100 * self._done) / self._total sys.stderr.write(
sys.stderr.write('\r%s: %3d%% (%d%s/%d%s), done in %s%s\n' % ( "\r%s: %3d%% (%d%s/%d%s), done in %s%s\n"
self._title, % (
p, self._title,
self._done, self._units, p,
self._total, self._units, self._done,
duration, self._units,
CSI_ERASE_LINE_AFTER)) self._total,
sys.stderr.flush() self._units,
duration,
CSI_ERASE_LINE_AFTER,
)
)
sys.stderr.flush()

7732
project.py
View File

File diff suppressed because it is too large Load Diff

162
release/sign-launcher.py
View File

@ -28,43 +28,56 @@ import util
def sign(opts): def sign(opts):
"""Sign the launcher!""" """Sign the launcher!"""
output = '' output = ""
for key in opts.keys: for key in opts.keys:
# We use ! at the end of the key so that gpg uses this specific key. # We use ! at the end of the key so that gpg uses this specific key.
# Otherwise it uses the key as a lookup into the overall key and uses the # Otherwise it uses the key as a lookup into the overall key and uses
# default signing key. i.e. It will see that KEYID_RSA is a subkey of # the default signing key. i.e. It will see that KEYID_RSA is a subkey
# another key, and use the primary key to sign instead of the subkey. # of another key, and use the primary key to sign instead of the subkey.
cmd = ['gpg', '--homedir', opts.gpgdir, '-u', f'{key}!', '--batch', '--yes', cmd = [
'--armor', '--detach-sign', '--output', '-', opts.launcher] "gpg",
ret = util.run(opts, cmd, encoding='utf-8', stdout=subprocess.PIPE) "--homedir",
output += ret.stdout opts.gpgdir,
"-u",
f"{key}!",
"--batch",
"--yes",
"--armor",
"--detach-sign",
"--output",
"-",
opts.launcher,
]
ret = util.run(opts, cmd, encoding="utf-8", stdout=subprocess.PIPE)
output += ret.stdout
# Save the combined signatures into one file. # Save the combined signatures into one file.
with open(f'{opts.launcher}.asc', 'w', encoding='utf-8') as fp: with open(f"{opts.launcher}.asc", "w", encoding="utf-8") as fp:
fp.write(output) fp.write(output)
def check(opts): def check(opts):
"""Check the signature.""" """Check the signature."""
util.run(opts, ['gpg', '--verify', f'{opts.launcher}.asc']) util.run(opts, ["gpg", "--verify", f"{opts.launcher}.asc"])
def get_version(opts): def get_version(opts):
"""Get the version from |launcher|.""" """Get the version from |launcher|."""
# Make sure we don't search $PATH when signing the "repo" file in the cwd. # Make sure we don't search $PATH when signing the "repo" file in the cwd.
launcher = os.path.join('.', opts.launcher) launcher = os.path.join(".", opts.launcher)
cmd = [launcher, '--version'] cmd = [launcher, "--version"]
ret = util.run(opts, cmd, encoding='utf-8', stdout=subprocess.PIPE) ret = util.run(opts, cmd, encoding="utf-8", stdout=subprocess.PIPE)
m = re.search(r'repo launcher version ([0-9.]+)', ret.stdout) m = re.search(r"repo launcher version ([0-9.]+)", ret.stdout)
if not m: if not m:
sys.exit(f'{opts.launcher}: unable to detect repo version') sys.exit(f"{opts.launcher}: unable to detect repo version")
return m.group(1) return m.group(1)
def postmsg(opts, version): def postmsg(opts, version):
"""Helpful info to show at the end for release manager.""" """Helpful info to show at the end for release manager."""
print(f""" print(
f"""
Repo launcher bucket: Repo launcher bucket:
gs://git-repo-downloads/ gs://git-repo-downloads/
@ -81,55 +94,72 @@ NB: If a rollback is necessary, the GS bucket archives old versions, and may be
gsutil ls -la gs://git-repo-downloads/repo gs://git-repo-downloads/repo.asc gsutil ls -la gs://git-repo-downloads/repo gs://git-repo-downloads/repo.asc
gsutil cp -a public-read gs://git-repo-downloads/repo#<unique id> gs://git-repo-downloads/repo gsutil cp -a public-read gs://git-repo-downloads/repo#<unique id> gs://git-repo-downloads/repo
gsutil cp -a public-read gs://git-repo-downloads/repo.asc#<unique id> gs://git-repo-downloads/repo.asc gsutil cp -a public-read gs://git-repo-downloads/repo.asc#<unique id> gs://git-repo-downloads/repo.asc
""") """ # noqa: E501
)
def get_parser(): def get_parser():
"""Get a CLI parser.""" """Get a CLI parser."""
parser = argparse.ArgumentParser(description=__doc__) parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('-n', '--dry-run', parser.add_argument(
dest='dryrun', action='store_true', "-n",
help='show everything that would be done') "--dry-run",
parser.add_argument('--gpgdir', dest="dryrun",
default=os.path.join(util.HOMEDIR, '.gnupg', 'repo'), action="store_true",
help='path to dedicated gpg dir with release keys ' help="show everything that would be done",
'(default: ~/.gnupg/repo/)') )
parser.add_argument('--keyid', dest='keys', default=[], action='append', parser.add_argument(
help='alternative signing keys to use') "--gpgdir",
parser.add_argument('launcher', default=os.path.join(util.HOMEDIR, ".gnupg", "repo"),
default=os.path.join(util.TOPDIR, 'repo'), nargs='?', help="path to dedicated gpg dir with release keys "
help='the launcher script to sign') "(default: ~/.gnupg/repo/)",
return parser )
parser.add_argument(
"--keyid",
dest="keys",
default=[],
action="append",
help="alternative signing keys to use",
)
parser.add_argument(
"launcher",
default=os.path.join(util.TOPDIR, "repo"),
nargs="?",
help="the launcher script to sign",
)
return parser
def main(argv): def main(argv):
"""The main func!""" """The main func!"""
parser = get_parser() parser = get_parser()
opts = parser.parse_args(argv) opts = parser.parse_args(argv)
if not os.path.exists(opts.gpgdir): if not os.path.exists(opts.gpgdir):
parser.error(f'--gpgdir does not exist: {opts.gpgdir}') parser.error(f"--gpgdir does not exist: {opts.gpgdir}")
if not os.path.exists(opts.launcher): if not os.path.exists(opts.launcher):
parser.error(f'launcher does not exist: {opts.launcher}') parser.error(f"launcher does not exist: {opts.launcher}")
opts.launcher = os.path.relpath(opts.launcher) opts.launcher = os.path.relpath(opts.launcher)
print(f'Signing "{opts.launcher}" launcher script and saving to ' print(
f'"{opts.launcher}.asc"') f'Signing "{opts.launcher}" launcher script and saving to '
f'"{opts.launcher}.asc"'
)
if opts.keys: if opts.keys:
print(f'Using custom keys to sign: {" ".join(opts.keys)}') print(f'Using custom keys to sign: {" ".join(opts.keys)}')
else: else:
print('Using official Repo release keys to sign') print("Using official Repo release keys to sign")
opts.keys = [util.KEYID_DSA, util.KEYID_RSA, util.KEYID_ECC] opts.keys = [util.KEYID_DSA, util.KEYID_RSA, util.KEYID_ECC]
util.import_release_key(opts) util.import_release_key(opts)
version = get_version(opts) version = get_version(opts)
sign(opts) sign(opts)
check(opts) check(opts)
postmsg(opts, version) postmsg(opts, version)
return 0 return 0
if __name__ == '__main__': if __name__ == "__main__":
sys.exit(main(sys.argv[1:])) sys.exit(main(sys.argv[1:]))

165
release/sign-tag.py
View File

@ -35,46 +35,61 @@ import util
KEYID = util.KEYID_DSA KEYID = util.KEYID_DSA
# Regular expression to validate tag names. # Regular expression to validate tag names.
RE_VALID_TAG = r'^v([0-9]+[.])+[0-9]+$' RE_VALID_TAG = r"^v([0-9]+[.])+[0-9]+$"
def sign(opts): def sign(opts):
"""Tag the commit & sign it!""" """Tag the commit & sign it!"""
# We use ! at the end of the key so that gpg uses this specific key. # We use ! at the end of the key so that gpg uses this specific key.
# Otherwise it uses the key as a lookup into the overall key and uses the # Otherwise it uses the key as a lookup into the overall key and uses the
# default signing key. i.e. It will see that KEYID_RSA is a subkey of # default signing key. i.e. It will see that KEYID_RSA is a subkey of
# another key, and use the primary key to sign instead of the subkey. # another key, and use the primary key to sign instead of the subkey.
cmd = ['git', 'tag', '-s', opts.tag, '-u', f'{opts.key}!', cmd = [
'-m', f'repo {opts.tag}', opts.commit] "git",
"tag",
"-s",
opts.tag,
"-u",
f"{opts.key}!",
"-m",
f"repo {opts.tag}",
opts.commit,
]
key = 'GNUPGHOME' key = "GNUPGHOME"
print('+', f'export {key}="{opts.gpgdir}"') print("+", f'export {key}="{opts.gpgdir}"')
oldvalue = os.getenv(key) oldvalue = os.getenv(key)
os.putenv(key, opts.gpgdir) os.putenv(key, opts.gpgdir)
util.run(opts, cmd) util.run(opts, cmd)
if oldvalue is None: if oldvalue is None:
os.unsetenv(key) os.unsetenv(key)
else: else:
os.putenv(key, oldvalue) os.putenv(key, oldvalue)
def check(opts): def check(opts):
"""Check the signature.""" """Check the signature."""
util.run(opts, ['git', 'tag', '--verify', opts.tag]) util.run(opts, ["git", "tag", "--verify", opts.tag])
def postmsg(opts): def postmsg(opts):
"""Helpful info to show at the end for release manager.""" """Helpful info to show at the end for release manager."""
cmd = ['git', 'rev-parse', 'remotes/origin/stable'] cmd = ["git", "rev-parse", "remotes/origin/stable"]
ret = util.run(opts, cmd, encoding='utf-8', stdout=subprocess.PIPE) ret = util.run(opts, cmd, encoding="utf-8", stdout=subprocess.PIPE)
current_release = ret.stdout.strip() current_release = ret.stdout.strip()
cmd = ['git', 'log', '--format=%h (%aN) %s', '--no-merges', cmd = [
f'remotes/origin/stable..{opts.tag}'] "git",
ret = util.run(opts, cmd, encoding='utf-8', stdout=subprocess.PIPE) "log",
shortlog = ret.stdout.strip() "--format=%h (%aN) %s",
"--no-merges",
f"remotes/origin/stable..{opts.tag}",
]
ret = util.run(opts, cmd, encoding="utf-8", stdout=subprocess.PIPE)
shortlog = ret.stdout.strip()
print(f""" print(
f"""
Here's the short log since the last release. Here's the short log since the last release.
{shortlog} {shortlog}
@ -84,57 +99,69 @@ NB: People will start upgrading to this version immediately.
To roll back a release: To roll back a release:
git push origin --force {current_release}:stable -n git push origin --force {current_release}:stable -n
""") """
)
def get_parser(): def get_parser():
"""Get a CLI parser.""" """Get a CLI parser."""
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description=__doc__, description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter) formatter_class=argparse.RawDescriptionHelpFormatter,
parser.add_argument('-n', '--dry-run', )
dest='dryrun', action='store_true', parser.add_argument(
help='show everything that would be done') "-n",
parser.add_argument('--gpgdir', "--dry-run",
default=os.path.join(util.HOMEDIR, '.gnupg', 'repo'), dest="dryrun",
help='path to dedicated gpg dir with release keys ' action="store_true",
'(default: ~/.gnupg/repo/)') help="show everything that would be done",
parser.add_argument('-f', '--force', action='store_true', )
help='force signing of any tag') parser.add_argument(
parser.add_argument('--keyid', dest='key', "--gpgdir",
help='alternative signing key to use') default=os.path.join(util.HOMEDIR, ".gnupg", "repo"),
parser.add_argument('tag', help="path to dedicated gpg dir with release keys "
help='the tag to create (e.g. "v2.0")') "(default: ~/.gnupg/repo/)",
parser.add_argument('commit', default='HEAD', nargs='?', )
help='the commit to tag') parser.add_argument(
return parser "-f", "--force", action="store_true", help="force signing of any tag"
)
parser.add_argument(
"--keyid", dest="key", help="alternative signing key to use"
)
parser.add_argument("tag", help='the tag to create (e.g. "v2.0")')
parser.add_argument(
"commit", default="HEAD", nargs="?", help="the commit to tag"
)
return parser
def main(argv): def main(argv):
"""The main func!""" """The main func!"""
parser = get_parser() parser = get_parser()
opts = parser.parse_args(argv) opts = parser.parse_args(argv)
if not os.path.exists(opts.gpgdir): if not os.path.exists(opts.gpgdir):
parser.error(f'--gpgdir does not exist: {opts.gpgdir}') parser.error(f"--gpgdir does not exist: {opts.gpgdir}")
if not opts.force and not re.match(RE_VALID_TAG, opts.tag): if not opts.force and not re.match(RE_VALID_TAG, opts.tag):
parser.error(f'tag "{opts.tag}" does not match regex "{RE_VALID_TAG}"; ' parser.error(
'use --force to sign anyways') f'tag "{opts.tag}" does not match regex "{RE_VALID_TAG}"; '
"use --force to sign anyways"
)
if opts.key: if opts.key:
print(f'Using custom key to sign: {opts.key}') print(f"Using custom key to sign: {opts.key}")
else: else:
print('Using official Repo release key to sign') print("Using official Repo release key to sign")
opts.key = KEYID opts.key = KEYID
util.import_release_key(opts) util.import_release_key(opts)
sign(opts) sign(opts)
check(opts) check(opts)
postmsg(opts) postmsg(opts)
return 0 return 0
if __name__ == '__main__': if __name__ == "__main__":
sys.exit(main(sys.argv[1:])) sys.exit(main(sys.argv[1:]))

160
release/update_manpages.py
View File

@ -29,91 +29,125 @@ import sys
import tempfile import tempfile
TOPDIR = Path(__file__).resolve().parent.parent TOPDIR = Path(__file__).resolve().parent.parent
MANDIR = TOPDIR.joinpath('man') MANDIR = TOPDIR.joinpath("man")
# Load repo local modules. # Load repo local modules.
sys.path.insert(0, str(TOPDIR)) sys.path.insert(0, str(TOPDIR))
from git_command import RepoSourceVersion from git_command import RepoSourceVersion
import subcmds import subcmds
def worker(cmd, **kwargs): def worker(cmd, **kwargs):
subprocess.run(cmd, **kwargs) subprocess.run(cmd, **kwargs)
def main(argv): def main(argv):
parser = argparse.ArgumentParser(description=__doc__) parser = argparse.ArgumentParser(description=__doc__)
opts = parser.parse_args(argv) parser.parse_args(argv)
if not shutil.which('help2man'): if not shutil.which("help2man"):
sys.exit('Please install help2man to continue.') sys.exit("Please install help2man to continue.")
# Let repo know we're generating man pages so it can avoid some dynamic # Let repo know we're generating man pages so it can avoid some dynamic
# behavior (like probing active number of CPUs). We use a weird name & # behavior (like probing active number of CPUs). We use a weird name &
# value to make it less likely for users to set this var themselves. # value to make it less likely for users to set this var themselves.
os.environ['_REPO_GENERATE_MANPAGES_'] = ' indeed! ' os.environ["_REPO_GENERATE_MANPAGES_"] = " indeed! "
# "repo branch" is an alias for "repo branches". # "repo branch" is an alias for "repo branches".
del subcmds.all_commands['branch'] del subcmds.all_commands["branch"]
(MANDIR / 'repo-branch.1').write_text('.so man1/repo-branches.1') (MANDIR / "repo-branch.1").write_text(".so man1/repo-branches.1")
version = RepoSourceVersion() version = RepoSourceVersion()
cmdlist = [['help2man', '-N', '-n', f'repo {cmd} - manual page for repo {cmd}', cmdlist = [
'-S', f'repo {cmd}', '-m', 'Repo Manual', f'--version-string={version}', [
'-o', MANDIR.joinpath(f'repo-{cmd}.1.tmp'), './repo', "help2man",
'-h', f'help {cmd}'] for cmd in subcmds.all_commands] "-N",
cmdlist.append(['help2man', '-N', '-n', 'repository management tool built on top of git', "-n",
'-S', 'repo', '-m', 'Repo Manual', f'--version-string={version}', f"repo {cmd} - manual page for repo {cmd}",
'-o', MANDIR.joinpath('repo.1.tmp'), './repo', "-S",
'-h', '--help-all']) f"repo {cmd}",
"-m",
"Repo Manual",
f"--version-string={version}",
"-o",
MANDIR.joinpath(f"repo-{cmd}.1.tmp"),
"./repo",
"-h",
f"help {cmd}",
]
for cmd in subcmds.all_commands
]
cmdlist.append(
[
"help2man",
"-N",
"-n",
"repository management tool built on top of git",
"-S",
"repo",
"-m",
"Repo Manual",
f"--version-string={version}",
"-o",
MANDIR.joinpath("repo.1.tmp"),
"./repo",
"-h",
"--help-all",
]
)
with tempfile.TemporaryDirectory() as tempdir: with tempfile.TemporaryDirectory() as tempdir:
tempdir = Path(tempdir) tempdir = Path(tempdir)
repo_dir = tempdir / '.repo' repo_dir = tempdir / ".repo"
repo_dir.mkdir() repo_dir.mkdir()
(repo_dir / 'repo').symlink_to(TOPDIR) (repo_dir / "repo").symlink_to(TOPDIR)
# Create a repo wrapper using the active Python executable. We can't pass # Create a repo wrapper using the active Python executable. We can't
# this directly to help2man as it's too simple, so insert it via shebang. # pass this directly to help2man as it's too simple, so insert it via
data = (TOPDIR / 'repo').read_text(encoding='utf-8') # shebang.
tempbin = tempdir / 'repo' data = (TOPDIR / "repo").read_text(encoding="utf-8")
tempbin.write_text(f'#!{sys.executable}\n' + data, encoding='utf-8') tempbin = tempdir / "repo"
tempbin.chmod(0o755) tempbin.write_text(f"#!{sys.executable}\n" + data, encoding="utf-8")
tempbin.chmod(0o755)
# Run all cmd in parallel, and wait for them to finish. # Run all cmd in parallel, and wait for them to finish.
with multiprocessing.Pool() as pool: with multiprocessing.Pool() as pool:
pool.map(partial(worker, cwd=tempdir, check=True), cmdlist) pool.map(partial(worker, cwd=tempdir, check=True), cmdlist)
for tmp_path in MANDIR.glob('*.1.tmp'): for tmp_path in MANDIR.glob("*.1.tmp"):
path = tmp_path.parent / tmp_path.stem path = tmp_path.parent / tmp_path.stem
old_data = path.read_text() if path.exists() else '' old_data = path.read_text() if path.exists() else ""
data = tmp_path.read_text() data = tmp_path.read_text()
tmp_path.unlink() tmp_path.unlink()
data = replace_regex(data) data = replace_regex(data)
# If the only thing that changed was the date, don't refresh. This avoids # If the only thing that changed was the date, don't refresh. This
# a lot of noise when only one file actually updates. # avoids a lot of noise when only one file actually updates.
old_data = re.sub(r'^(\.TH REPO "1" ")([^"]+)', r'\1', old_data, flags=re.M) old_data = re.sub(
new_data = re.sub(r'^(\.TH REPO "1" ")([^"]+)', r'\1', data, flags=re.M) r'^(\.TH REPO "1" ")([^"]+)', r"\1", old_data, flags=re.M
if old_data != new_data: )
path.write_text(data) new_data = re.sub(r'^(\.TH REPO "1" ")([^"]+)', r"\1", data, flags=re.M)
if old_data != new_data:
path.write_text(data)
def replace_regex(data): def replace_regex(data):
"""Replace semantically null regexes in the data. """Replace semantically null regexes in the data.
Args: Args:
data: manpage text. data: manpage text.
Returns: Returns:
Updated manpage text. Updated manpage text.
""" """
regex = ( regex = (
(r'(It was generated by help2man) [0-9.]+', r'\g<1>.'), (r"(It was generated by help2man) [0-9.]+", r"\g<1>."),
(r'^\033\[[0-9;]*m([^\033]*)\033\[m', r'\g<1>'), (r"^\033\[[0-9;]*m([^\033]*)\033\[m", r"\g<1>"),
(r'^\.IP\n(.*:)\n', r'.SS \g<1>\n'), (r"^\.IP\n(.*:)\n", r".SS \g<1>\n"),
(r'^\.PP\nDescription', r'.SH DETAILS'), (r"^\.PP\nDescription", r".SH DETAILS"),
) )
for pattern, replacement in regex: for pattern, replacement in regex:
data = re.sub(pattern, replacement, data, flags=re.M) data = re.sub(pattern, replacement, data, flags=re.M)
return data return data

74
release/util.py
View File

@ -20,54 +20,60 @@ import subprocess
import sys import sys
assert sys.version_info >= (3, 6), 'This module requires Python 3.6+' assert sys.version_info >= (3, 6), "This module requires Python 3.6+"
TOPDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) TOPDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
HOMEDIR = os.path.expanduser('~') HOMEDIR = os.path.expanduser("~")
# These are the release keys we sign with. # These are the release keys we sign with.
KEYID_DSA = '8BB9AD793E8E6153AF0F9A4416530D5E920F5C65' KEYID_DSA = "8BB9AD793E8E6153AF0F9A4416530D5E920F5C65"
KEYID_RSA = 'A34A13BE8E76BFF46A0C022DA2E75A824AAB9624' KEYID_RSA = "A34A13BE8E76BFF46A0C022DA2E75A824AAB9624"
KEYID_ECC = 'E1F9040D7A3F6DAFAC897CD3D3B95DA243E48A39' KEYID_ECC = "E1F9040D7A3F6DAFAC897CD3D3B95DA243E48A39"
def cmdstr(cmd): def cmdstr(cmd):
"""Get a nicely quoted shell command.""" """Get a nicely quoted shell command."""
ret = [] ret = []
for arg in cmd: for arg in cmd:
if not re.match(r'^[a-zA-Z0-9/_.=-]+$', arg): if not re.match(r"^[a-zA-Z0-9/_.=-]+$", arg):
arg = f'"{arg}"' arg = f'"{arg}"'
ret.append(arg) ret.append(arg)
return ' '.join(ret) return " ".join(ret)
def run(opts, cmd, check=True, **kwargs): def run(opts, cmd, check=True, **kwargs):
"""Helper around subprocess.run to include logging.""" """Helper around subprocess.run to include logging."""
print('+', cmdstr(cmd)) print("+", cmdstr(cmd))
if opts.dryrun: if opts.dryrun:
cmd = ['true', '--'] + cmd cmd = ["true", "--"] + cmd
try: try:
return subprocess.run(cmd, check=check, **kwargs) return subprocess.run(cmd, check=check, **kwargs)
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
print(f'aborting: {e}', file=sys.stderr) print(f"aborting: {e}", file=sys.stderr)
sys.exit(1) sys.exit(1)
def import_release_key(opts): def import_release_key(opts):
"""Import the public key of the official release repo signing key.""" """Import the public key of the official release repo signing key."""
# Extract the key from our repo launcher. # Extract the key from our repo launcher.
launcher = getattr(opts, 'launcher', os.path.join(TOPDIR, 'repo')) launcher = getattr(opts, "launcher", os.path.join(TOPDIR, "repo"))
print(f'Importing keys from "{launcher}" launcher script') print(f'Importing keys from "{launcher}" launcher script')
with open(launcher, encoding='utf-8') as fp: with open(launcher, encoding="utf-8") as fp:
data = fp.read() data = fp.read()
keys = re.findall( keys = re.findall(
r'\n-----BEGIN PGP PUBLIC KEY BLOCK-----\n[^-]*' r"\n-----BEGIN PGP PUBLIC KEY BLOCK-----\n[^-]*"
r'\n-----END PGP PUBLIC KEY BLOCK-----\n', data, flags=re.M) r"\n-----END PGP PUBLIC KEY BLOCK-----\n",
run(opts, ['gpg', '--import'], input='\n'.join(keys).encode('utf-8')) data,
flags=re.M,
)
run(opts, ["gpg", "--import"], input="\n".join(keys).encode("utf-8"))
print('Marking keys as fully trusted') print("Marking keys as fully trusted")
run(opts, ['gpg', '--import-ownertrust'], run(
input=f'{KEYID_DSA}:6:\n'.encode('utf-8')) opts,
["gpg", "--import-ownertrust"],
input=f"{KEYID_DSA}:6:\n".encode("utf-8"),
)

18
repo
View File

@ -506,10 +506,10 @@ def gitc_parse_clientdir(gitc_fs_path):
"""Parse a path in the GITC FS and return its client name. """Parse a path in the GITC FS and return its client name.
Args: Args:
gitc_fs_path: A subdirectory path within the GITC_FS_ROOT_DIR. gitc_fs_path: A subdirectory path within the GITC_FS_ROOT_DIR.
Returns: Returns:
The GITC client name. The GITC client name.
""" """
if gitc_fs_path == GITC_FS_ROOT_DIR: if gitc_fs_path == GITC_FS_ROOT_DIR:
return None return None
@ -942,14 +942,14 @@ def resolve_repo_rev(cwd, committish):
* xxx: Branch or tag or commit. * xxx: Branch or tag or commit.
Args: Args:
cwd: The git checkout to run in. cwd: The git checkout to run in.
committish: The REPO_REV argument to resolve. committish: The REPO_REV argument to resolve.
Returns: Returns:
A tuple of (remote ref, commit) as makes sense for the committish. A tuple of (remote ref, commit) as makes sense for the committish.
For branches, this will look like ('refs/heads/stable', <revision>). For branches, this will look like ('refs/heads/stable', <revision>).
For tags, this will look like ('refs/tags/v1.0', <revision>). For tags, this will look like ('refs/tags/v1.0', <revision>).
For commits, this will be (<revision>, <revision>). For commits, this will be (<revision>, <revision>).
""" """
def resolve(committish): def resolve(committish):
ret = run_git('rev-parse', '--verify', '%s^{commit}' % (committish,), ret = run_git('rev-parse', '--verify', '%s^{commit}' % (committish,),
@ -1104,7 +1104,7 @@ class Requirements(object):
"""Initialize. """Initialize.
Args: Args:
requirements: A dictionary of settings. requirements: A dictionary of settings.
""" """
self.requirements = requirements self.requirements = requirements

180
repo_trace.py
View File

@ -29,138 +29,142 @@ from contextlib import ContextDecorator
import platform_utils import platform_utils
# Env var to implicitly turn on tracing. # Env var to implicitly turn on tracing.
REPO_TRACE = 'REPO_TRACE' REPO_TRACE = "REPO_TRACE"
# Temporarily set tracing to always on unless user expicitly sets to 0. # Temporarily set tracing to always on unless user expicitly sets to 0.
_TRACE = os.environ.get(REPO_TRACE) != '0' _TRACE = os.environ.get(REPO_TRACE) != "0"
_TRACE_TO_STDERR = False _TRACE_TO_STDERR = False
_TRACE_FILE = None _TRACE_FILE = None
_TRACE_FILE_NAME = 'TRACE_FILE' _TRACE_FILE_NAME = "TRACE_FILE"
_MAX_SIZE = 70 # in MiB _MAX_SIZE = 70 # in MiB
_NEW_COMMAND_SEP = '+++++++++++++++NEW COMMAND+++++++++++++++++++' _NEW_COMMAND_SEP = "+++++++++++++++NEW COMMAND+++++++++++++++++++"
def IsTraceToStderr(): def IsTraceToStderr():
"""Whether traces are written to stderr.""" """Whether traces are written to stderr."""
return _TRACE_TO_STDERR return _TRACE_TO_STDERR
def IsTrace(): def IsTrace():
"""Whether tracing is enabled.""" """Whether tracing is enabled."""
return _TRACE return _TRACE
def SetTraceToStderr(): def SetTraceToStderr():
"""Enables tracing logging to stderr.""" """Enables tracing logging to stderr."""
global _TRACE_TO_STDERR global _TRACE_TO_STDERR
_TRACE_TO_STDERR = True _TRACE_TO_STDERR = True
def SetTrace(): def SetTrace():
"""Enables tracing.""" """Enables tracing."""
global _TRACE global _TRACE
_TRACE = True _TRACE = True
def _SetTraceFile(quiet): def _SetTraceFile(quiet):
"""Sets the trace file location.""" """Sets the trace file location."""
global _TRACE_FILE global _TRACE_FILE
_TRACE_FILE = _GetTraceFile(quiet) _TRACE_FILE = _GetTraceFile(quiet)
class Trace(ContextDecorator): class Trace(ContextDecorator):
"""Used to capture and save git traces.""" """Used to capture and save git traces."""
def _time(self): def _time(self):
"""Generate nanoseconds of time in a py3.6 safe way""" """Generate nanoseconds of time in a py3.6 safe way"""
return int(time.time() * 1e+9) return int(time.time() * 1e9)
def __init__(self, fmt, *args, first_trace=False, quiet=True): def __init__(self, fmt, *args, first_trace=False, quiet=True):
"""Initialize the object. """Initialize the object.
Args: Args:
fmt: The format string for the trace. fmt: The format string for the trace.
*args: Arguments to pass to formatting. *args: Arguments to pass to formatting.
first_trace: Whether this is the first trace of a `repo` invocation. first_trace: Whether this is the first trace of a `repo` invocation.
quiet: Whether to suppress notification of trace file location. quiet: Whether to suppress notification of trace file location.
""" """
if not IsTrace(): if not IsTrace():
return return
self._trace_msg = fmt % args self._trace_msg = fmt % args
if not _TRACE_FILE: if not _TRACE_FILE:
_SetTraceFile(quiet) _SetTraceFile(quiet)
if first_trace: if first_trace:
_ClearOldTraces() _ClearOldTraces()
self._trace_msg = f'{_NEW_COMMAND_SEP} {self._trace_msg}' self._trace_msg = f"{_NEW_COMMAND_SEP} {self._trace_msg}"
def __enter__(self): def __enter__(self):
if not IsTrace(): if not IsTrace():
return self return self
print_msg = f'PID: {os.getpid()} START: {self._time()} :{self._trace_msg}\n' print_msg = (
f"PID: {os.getpid()} START: {self._time()} :{self._trace_msg}\n"
)
with open(_TRACE_FILE, 'a') as f: with open(_TRACE_FILE, "a") as f:
print(print_msg, file=f) print(print_msg, file=f)
if _TRACE_TO_STDERR: if _TRACE_TO_STDERR:
print(print_msg, file=sys.stderr) print(print_msg, file=sys.stderr)
return self return self
def __exit__(self, *exc): def __exit__(self, *exc):
if not IsTrace(): if not IsTrace():
return False return False
print_msg = f'PID: {os.getpid()} END: {self._time()} :{self._trace_msg}\n' print_msg = (
f"PID: {os.getpid()} END: {self._time()} :{self._trace_msg}\n"
)
with open(_TRACE_FILE, 'a') as f: with open(_TRACE_FILE, "a") as f:
print(print_msg, file=f) print(print_msg, file=f)
if _TRACE_TO_STDERR: if _TRACE_TO_STDERR:
print(print_msg, file=sys.stderr) print(print_msg, file=sys.stderr)
return False return False
def _GetTraceFile(quiet): def _GetTraceFile(quiet):
"""Get the trace file or create one.""" """Get the trace file or create one."""
# TODO: refactor to pass repodir to Trace. # TODO: refactor to pass repodir to Trace.
repo_dir = os.path.dirname(os.path.dirname(__file__)) repo_dir = os.path.dirname(os.path.dirname(__file__))
trace_file = os.path.join(repo_dir, _TRACE_FILE_NAME) trace_file = os.path.join(repo_dir, _TRACE_FILE_NAME)
if not quiet: if not quiet:
print(f'Trace outputs in {trace_file}', file=sys.stderr) print(f"Trace outputs in {trace_file}", file=sys.stderr)
return trace_file return trace_file
def _ClearOldTraces(): def _ClearOldTraces():
"""Clear the oldest commands if trace file is too big.""" """Clear the oldest commands if trace file is too big."""
try: try:
with open(_TRACE_FILE, 'r', errors='ignore') as f: with open(_TRACE_FILE, "r", errors="ignore") as f:
if os.path.getsize(f.name) / (1024 * 1024) <= _MAX_SIZE: if os.path.getsize(f.name) / (1024 * 1024) <= _MAX_SIZE:
return
trace_lines = f.readlines()
except FileNotFoundError:
return return
trace_lines = f.readlines()
except FileNotFoundError:
return
while sum(len(x) for x in trace_lines) / (1024 * 1024) > _MAX_SIZE: while sum(len(x) for x in trace_lines) / (1024 * 1024) > _MAX_SIZE:
for i, line in enumerate(trace_lines): for i, line in enumerate(trace_lines):
if 'END:' in line and _NEW_COMMAND_SEP in line: if "END:" in line and _NEW_COMMAND_SEP in line:
trace_lines = trace_lines[i + 1:] trace_lines = trace_lines[i + 1 :]
break break
else: else:
# The last chunk is bigger than _MAX_SIZE, so just throw everything away. # The last chunk is bigger than _MAX_SIZE, so just throw everything
trace_lines = [] # away.
trace_lines = []
while trace_lines and trace_lines[-1] == '\n': while trace_lines and trace_lines[-1] == "\n":
trace_lines = trace_lines[:-1] trace_lines = trace_lines[:-1]
# Write to a temporary file with a unique name in the same filesystem # Write to a temporary file with a unique name in the same filesystem
# before replacing the original trace file. # before replacing the original trace file.
temp_dir, temp_prefix = os.path.split(_TRACE_FILE) temp_dir, temp_prefix = os.path.split(_TRACE_FILE)
with tempfile.NamedTemporaryFile('w', with tempfile.NamedTemporaryFile(
dir=temp_dir, "w", dir=temp_dir, prefix=temp_prefix, delete=False
prefix=temp_prefix, ) as f:
delete=False) as f: f.writelines(trace_lines)
f.writelines(trace_lines) platform_utils.rename(f.name, _TRACE_FILE)
platform_utils.rename(f.name, _TRACE_FILE)

24
run_tests
View File

@ -13,10 +13,28 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Wrapper to run pytest with the right settings.""" """Wrapper to run black and pytest with the right settings."""
import os
import subprocess
import sys import sys
import pytest import pytest
if __name__ == '__main__':
sys.exit(pytest.main(sys.argv[1:])) def run_black():
"""Returns the exit code of running `black --check`."""
dirpath = os.path.dirname(os.path.realpath(__file__))
return subprocess.run(
[sys.executable, "-m", "black", "--check", dirpath], check=False
).returncode
def main(argv):
"""The main entry."""
black_ret = 0 if argv else run_black()
pytest_ret = pytest.main(argv)
return 0 if not black_ret and not pytest_ret else 1
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))

45
run_tests.vpython3
View File

@ -26,8 +26,8 @@ wheel: <
# Required by pytest==6.2.2 # Required by pytest==6.2.2
wheel: < wheel: <
name: "infra/python/wheels/packaging-py2_py3" name: "infra/python/wheels/packaging-py3"
version: "version:16.8" version: "version:23.0"
> >
# Required by pytest==6.2.2 # Required by pytest==6.2.2
@ -59,3 +59,44 @@ wheel: <
name: "infra/python/wheels/six-py2_py3" name: "infra/python/wheels/six-py2_py3"
version: "version:1.16.0" version: "version:1.16.0"
> >
wheel: <
name: "infra/python/wheels/black-py3"
version: "version:23.1.0"
>
# Required by black==23.1.0
wheel: <
name: "infra/python/wheels/mypy-extensions-py3"
version: "version:0.4.3"
>
# Required by black==23.1.0
wheel: <
name: "infra/python/wheels/tomli-py3"
version: "version:2.0.1"
>
# Required by black==23.1.0
wheel: <
name: "infra/python/wheels/platformdirs-py3"
version: "version:2.5.2"
>
# Required by black==23.1.0
wheel: <
name: "infra/python/wheels/pathspec-py3"
version: "version:0.9.0"
>
# Required by black==23.1.0
wheel: <
name: "infra/python/wheels/typing-extensions-py3"
version: "version:4.3.0"
>
# Required by black==23.1.0
wheel: <
name: "infra/python/wheels/click-py3"
version: "version:8.0.3"
>

48
setup.py
View File

@ -23,39 +23,39 @@ TOPDIR = os.path.dirname(os.path.abspath(__file__))
# Rip out the first intro paragraph. # Rip out the first intro paragraph.
with open(os.path.join(TOPDIR, 'README.md')) as fp: with open(os.path.join(TOPDIR, "README.md")) as fp:
lines = fp.read().splitlines()[2:] lines = fp.read().splitlines()[2:]
end = lines.index('') end = lines.index("")
long_description = ' '.join(lines[0:end]) long_description = " ".join(lines[0:end])
# https://packaging.python.org/tutorials/packaging-projects/ # https://packaging.python.org/tutorials/packaging-projects/
setuptools.setup( setuptools.setup(
name='repo', name="repo",
version='2', version="2",
maintainer='Various', maintainer="Various",
maintainer_email='repo-discuss@googlegroups.com', maintainer_email="repo-discuss@googlegroups.com",
description='Repo helps manage many Git repositories', description="Repo helps manage many Git repositories",
long_description=long_description, long_description=long_description,
long_description_content_type='text/plain', long_description_content_type="text/plain",
url='https://gerrit.googlesource.com/git-repo/', url="https://gerrit.googlesource.com/git-repo/",
project_urls={ project_urls={
'Bug Tracker': 'https://bugs.chromium.org/p/gerrit/issues/list?q=component:Applications%3Erepo', "Bug Tracker": "https://bugs.chromium.org/p/gerrit/issues/list?q=component:Applications%3Erepo", # noqa: E501
}, },
# https://pypi.org/classifiers/ # https://pypi.org/classifiers/
classifiers=[ classifiers=[
'Development Status :: 6 - Mature', "Development Status :: 6 - Mature",
'Environment :: Console', "Environment :: Console",
'Intended Audience :: Developers', "Intended Audience :: Developers",
'License :: OSI Approved :: Apache Software License', "License :: OSI Approved :: Apache Software License",
'Natural Language :: English', "Natural Language :: English",
'Operating System :: MacOS :: MacOS X', "Operating System :: MacOS :: MacOS X",
'Operating System :: Microsoft :: Windows :: Windows 10', "Operating System :: Microsoft :: Windows :: Windows 10",
'Operating System :: POSIX :: Linux', "Operating System :: POSIX :: Linux",
'Programming Language :: Python :: 3', "Programming Language :: Python :: 3",
'Programming Language :: Python :: 3 :: Only', "Programming Language :: Python :: 3 :: Only",
'Topic :: Software Development :: Version Control :: Git', "Topic :: Software Development :: Version Control :: Git",
], ],
python_requires='>=3.6', python_requires=">=3.6",
packages=['subcmds'], packages=["subcmds"],
) )

434
ssh.py
View File

@ -28,254 +28,264 @@ import platform_utils
from repo_trace import Trace from repo_trace import Trace
PROXY_PATH = os.path.join(os.path.dirname(__file__), 'git_ssh') PROXY_PATH = os.path.join(os.path.dirname(__file__), "git_ssh")
def _run_ssh_version(): def _run_ssh_version():
"""run ssh -V to display the version number""" """run ssh -V to display the version number"""
return subprocess.check_output(['ssh', '-V'], stderr=subprocess.STDOUT).decode() return subprocess.check_output(
["ssh", "-V"], stderr=subprocess.STDOUT
).decode()
def _parse_ssh_version(ver_str=None): def _parse_ssh_version(ver_str=None):
"""parse a ssh version string into a tuple""" """parse a ssh version string into a tuple"""
if ver_str is None: if ver_str is None:
ver_str = _run_ssh_version() ver_str = _run_ssh_version()
m = re.match(r'^OpenSSH_([0-9.]+)(p[0-9]+)?\s', ver_str) m = re.match(r"^OpenSSH_([0-9.]+)(p[0-9]+)?\s", ver_str)
if m: if m:
return tuple(int(x) for x in m.group(1).split('.')) return tuple(int(x) for x in m.group(1).split("."))
else: else:
return () return ()
@functools.lru_cache(maxsize=None) @functools.lru_cache(maxsize=None)
def version(): def version():
"""return ssh version as a tuple""" """return ssh version as a tuple"""
try: try:
return _parse_ssh_version() return _parse_ssh_version()
except FileNotFoundError: except FileNotFoundError:
print('fatal: ssh not installed', file=sys.stderr) print("fatal: ssh not installed", file=sys.stderr)
sys.exit(1) sys.exit(1)
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
print('fatal: unable to detect ssh version', file=sys.stderr) print("fatal: unable to detect ssh version", file=sys.stderr)
sys.exit(1) sys.exit(1)
URI_SCP = re.compile(r'^([^@:]*@?[^:/]{1,}):') URI_SCP = re.compile(r"^([^@:]*@?[^:/]{1,}):")
URI_ALL = re.compile(r'^([a-z][a-z+-]*)://([^@/]*@?[^/]*)/') URI_ALL = re.compile(r"^([a-z][a-z+-]*)://([^@/]*@?[^/]*)/")
class ProxyManager: class ProxyManager:
"""Manage various ssh clients & masters that we spawn. """Manage various ssh clients & masters that we spawn.
This will take care of sharing state between multiprocessing children, and This will take care of sharing state between multiprocessing children, and
make sure that if we crash, we don't leak any of the ssh sessions. make sure that if we crash, we don't leak any of the ssh sessions.
The code should work with a single-process scenario too, and not add too much The code should work with a single-process scenario too, and not add too
overhead due to the manager. much overhead due to the manager.
"""
# Path to the ssh program to run which will pass our master settings along.
# Set here more as a convenience API.
proxy = PROXY_PATH
def __init__(self, manager):
# Protect access to the list of active masters.
self._lock = multiprocessing.Lock()
# List of active masters (pid). These will be spawned on demand, and we are
# responsible for shutting them all down at the end.
self._masters = manager.list()
# Set of active masters indexed by "host:port" information.
# The value isn't used, but multiprocessing doesn't provide a set class.
self._master_keys = manager.dict()
# Whether ssh masters are known to be broken, so we give up entirely.
self._master_broken = manager.Value('b', False)
# List of active ssh sesssions. Clients will be added & removed as
# connections finish, so this list is just for safety & cleanup if we crash.
self._clients = manager.list()
# Path to directory for holding master sockets.
self._sock_path = None
def __enter__(self):
"""Enter a new context."""
return self
def __exit__(self, exc_type, exc_value, traceback):
"""Exit a context & clean up all resources."""
self.close()
def add_client(self, proc):
"""Track a new ssh session."""
self._clients.append(proc.pid)
def remove_client(self, proc):
"""Remove a completed ssh session."""
try:
self._clients.remove(proc.pid)
except ValueError:
pass
def add_master(self, proc):
"""Track a new master connection."""
self._masters.append(proc.pid)
def _terminate(self, procs):
"""Kill all |procs|."""
for pid in procs:
try:
os.kill(pid, signal.SIGTERM)
os.waitpid(pid, 0)
except OSError:
pass
# The multiprocessing.list() API doesn't provide many standard list()
# methods, so we have to manually clear the list.
while True:
try:
procs.pop(0)
except:
break
def close(self):
"""Close this active ssh session.
Kill all ssh clients & masters we created, and nuke the socket dir.
""" """
self._terminate(self._clients)
self._terminate(self._masters)
d = self.sock(create=False) # Path to the ssh program to run which will pass our master settings along.
if d: # Set here more as a convenience API.
try: proxy = PROXY_PATH
platform_utils.rmdir(os.path.dirname(d))
except OSError:
pass
def _open_unlocked(self, host, port=None): def __init__(self, manager):
"""Make sure a ssh master session exists for |host| & |port|. # Protect access to the list of active masters.
self._lock = multiprocessing.Lock()
# List of active masters (pid). These will be spawned on demand, and we
# are responsible for shutting them all down at the end.
self._masters = manager.list()
# Set of active masters indexed by "host:port" information.
# The value isn't used, but multiprocessing doesn't provide a set class.
self._master_keys = manager.dict()
# Whether ssh masters are known to be broken, so we give up entirely.
self._master_broken = manager.Value("b", False)
# List of active ssh sesssions. Clients will be added & removed as
# connections finish, so this list is just for safety & cleanup if we
# crash.
self._clients = manager.list()
# Path to directory for holding master sockets.
self._sock_path = None
If one doesn't exist already, we'll create it. def __enter__(self):
"""Enter a new context."""
return self
We won't grab any locks, so the caller has to do that. This helps keep the def __exit__(self, exc_type, exc_value, traceback):
business logic of actually creating the master separate from grabbing locks. """Exit a context & clean up all resources."""
""" self.close()
# Check to see whether we already think that the master is running; if we
# think it's already running, return right away.
if port is not None:
key = '%s:%s' % (host, port)
else:
key = host
if key in self._master_keys: def add_client(self, proc):
return True """Track a new ssh session."""
self._clients.append(proc.pid)
if self._master_broken.value or 'GIT_SSH' in os.environ: def remove_client(self, proc):
# Failed earlier, so don't retry. """Remove a completed ssh session."""
return False try:
self._clients.remove(proc.pid)
except ValueError:
pass
# We will make two calls to ssh; this is the common part of both calls. def add_master(self, proc):
command_base = ['ssh', '-o', 'ControlPath %s' % self.sock(), host] """Track a new master connection."""
if port is not None: self._masters.append(proc.pid)
command_base[1:1] = ['-p', str(port)]
# Since the key wasn't in _master_keys, we think that master isn't running. def _terminate(self, procs):
# ...but before actually starting a master, we'll double-check. This can """Kill all |procs|."""
# be important because we can't tell that that 'git@myhost.com' is the same for pid in procs:
# as 'myhost.com' where "User git" is setup in the user's ~/.ssh/config file. try:
check_command = command_base + ['-O', 'check'] os.kill(pid, signal.SIGTERM)
with Trace('Call to ssh (check call): %s', ' '.join(check_command)): os.waitpid(pid, 0)
try: except OSError:
check_process = subprocess.Popen(check_command, pass
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
check_process.communicate() # read output, but ignore it...
isnt_running = check_process.wait()
if not isnt_running: # The multiprocessing.list() API doesn't provide many standard list()
# Our double-check found that the master _was_ infact running. Add to # methods, so we have to manually clear the list.
# the list of keys. while True:
self._master_keys[key] = True try:
return True procs.pop(0)
except Exception: except: # noqa: E722
# Ignore excpetions. We we will fall back to the normal command and break
# print to the log there.
pass
command = command_base[:1] + ['-M', '-N'] + command_base[1:] def close(self):
p = None """Close this active ssh session.
try:
with Trace('Call to ssh: %s', ' '.join(command)):
p = subprocess.Popen(command)
except Exception as e:
self._master_broken.value = True
print('\nwarn: cannot enable ssh control master for %s:%s\n%s'
% (host, port, str(e)), file=sys.stderr)
return False
time.sleep(1) Kill all ssh clients & masters we created, and nuke the socket dir.
ssh_died = (p.poll() is not None) """
if ssh_died: self._terminate(self._clients)
return False self._terminate(self._masters)
self.add_master(p) d = self.sock(create=False)
self._master_keys[key] = True if d:
return True try:
platform_utils.rmdir(os.path.dirname(d))
except OSError:
pass
def _open(self, host, port=None): def _open_unlocked(self, host, port=None):
"""Make sure a ssh master session exists for |host| & |port|. """Make sure a ssh master session exists for |host| & |port|.
If one doesn't exist already, we'll create it. If one doesn't exist already, we'll create it.
This will obtain any necessary locks to avoid inter-process races. We won't grab any locks, so the caller has to do that. This helps keep
""" the business logic of actually creating the master separate from
# Bail before grabbing the lock if we already know that we aren't going to grabbing locks.
# try creating new masters below. """
if sys.platform in ('win32', 'cygwin'): # Check to see whether we already think that the master is running; if
return False # we think it's already running, return right away.
if port is not None:
key = "%s:%s" % (host, port)
else:
key = host
# Acquire the lock. This is needed to prevent opening multiple masters for if key in self._master_keys:
# the same host when we're running "repo sync -jN" (for N > 1) _and_ the return True
# manifest <remote fetch="ssh://xyz"> specifies a different host from the
# one that was passed to repo init.
with self._lock:
return self._open_unlocked(host, port)
def preconnect(self, url): if self._master_broken.value or "GIT_SSH" in os.environ:
"""If |uri| will create a ssh connection, setup the ssh master for it.""" # Failed earlier, so don't retry.
m = URI_ALL.match(url) return False
if m:
scheme = m.group(1)
host = m.group(2)
if ':' in host:
host, port = host.split(':')
else:
port = None
if scheme in ('ssh', 'git+ssh', 'ssh+git'):
return self._open(host, port)
return False
m = URI_SCP.match(url) # We will make two calls to ssh; this is the common part of both calls.
if m: command_base = ["ssh", "-o", "ControlPath %s" % self.sock(), host]
host = m.group(1) if port is not None:
return self._open(host) command_base[1:1] = ["-p", str(port)]
return False # Since the key wasn't in _master_keys, we think that master isn't
# running... but before actually starting a master, we'll double-check.
# This can be important because we can't tell that that 'git@myhost.com'
# is the same as 'myhost.com' where "User git" is setup in the user's
# ~/.ssh/config file.
check_command = command_base + ["-O", "check"]
with Trace("Call to ssh (check call): %s", " ".join(check_command)):
try:
check_process = subprocess.Popen(
check_command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
check_process.communicate() # read output, but ignore it...
isnt_running = check_process.wait()
def sock(self, create=True): if not isnt_running:
"""Return the path to the ssh socket dir. # Our double-check found that the master _was_ infact
# running. Add to the list of keys.
self._master_keys[key] = True
return True
except Exception:
# Ignore excpetions. We we will fall back to the normal command
# and print to the log there.
pass
This has all the master sockets so clients can talk to them. command = command_base[:1] + ["-M", "-N"] + command_base[1:]
""" p = None
if self._sock_path is None: try:
if not create: with Trace("Call to ssh: %s", " ".join(command)):
return None p = subprocess.Popen(command)
tmp_dir = '/tmp' except Exception as e:
if not os.path.exists(tmp_dir): self._master_broken.value = True
tmp_dir = tempfile.gettempdir() print(
if version() < (6, 7): "\nwarn: cannot enable ssh control master for %s:%s\n%s"
tokens = '%r@%h:%p' % (host, port, str(e)),
else: file=sys.stderr,
tokens = '%C' # hash of %l%h%p%r )
self._sock_path = os.path.join( return False
tempfile.mkdtemp('', 'ssh-', tmp_dir),
'master-' + tokens) time.sleep(1)
return self._sock_path ssh_died = p.poll() is not None
if ssh_died:
return False
self.add_master(p)
self._master_keys[key] = True
return True
def _open(self, host, port=None):
"""Make sure a ssh master session exists for |host| & |port|.
If one doesn't exist already, we'll create it.
This will obtain any necessary locks to avoid inter-process races.
"""
# Bail before grabbing the lock if we already know that we aren't going
# to try creating new masters below.
if sys.platform in ("win32", "cygwin"):
return False
# Acquire the lock. This is needed to prevent opening multiple masters
# for the same host when we're running "repo sync -jN" (for N > 1) _and_
# the manifest <remote fetch="ssh://xyz"> specifies a different host
# from the one that was passed to repo init.
with self._lock:
return self._open_unlocked(host, port)
def preconnect(self, url):
"""If |uri| will create a ssh connection, setup the ssh master for it.""" # noqa: E501
m = URI_ALL.match(url)
if m:
scheme = m.group(1)
host = m.group(2)
if ":" in host:
host, port = host.split(":")
else:
port = None
if scheme in ("ssh", "git+ssh", "ssh+git"):
return self._open(host, port)
return False
m = URI_SCP.match(url)
if m:
host = m.group(1)
return self._open(host)
return False
def sock(self, create=True):
"""Return the path to the ssh socket dir.
This has all the master sockets so clients can talk to them.
"""
if self._sock_path is None:
if not create:
return None
tmp_dir = "/tmp"
if not os.path.exists(tmp_dir):
tmp_dir = tempfile.gettempdir()
if version() < (6, 7):
tokens = "%r@%h:%p"
else:
tokens = "%C" # hash of %l%h%p%r
self._sock_path = os.path.join(
tempfile.mkdtemp("", "ssh-", tmp_dir), "master-" + tokens
)
return self._sock_path

42
subcmds/__init__.py
View File

@ -19,31 +19,29 @@ all_commands = {}
my_dir = os.path.dirname(__file__) my_dir = os.path.dirname(__file__)
for py in os.listdir(my_dir): for py in os.listdir(my_dir):
if py == '__init__.py': if py == "__init__.py":
continue continue
if py.endswith('.py'): if py.endswith(".py"):
name = py[:-3] name = py[:-3]
clsn = name.capitalize() clsn = name.capitalize()
while clsn.find('_') > 0: while clsn.find("_") > 0:
h = clsn.index('_') h = clsn.index("_")
clsn = clsn[0:h] + clsn[h + 1:].capitalize() clsn = clsn[0:h] + clsn[h + 1 :].capitalize()
mod = __import__(__name__, mod = __import__(__name__, globals(), locals(), ["%s" % name])
globals(), mod = getattr(mod, name)
locals(), try:
['%s' % name]) cmd = getattr(mod, clsn)
mod = getattr(mod, name) except AttributeError:
try: raise SyntaxError(
cmd = getattr(mod, clsn) "%s/%s does not define class %s" % (__name__, py, clsn)
except AttributeError: )
raise SyntaxError('%s/%s does not define class %s' % (
__name__, py, clsn))
name = name.replace('_', '-') name = name.replace("_", "-")
cmd.NAME = name cmd.NAME = name
all_commands[name] = cmd all_commands[name] = cmd
# Add 'branch' as an alias for 'branches'. # Add 'branch' as an alias for 'branches'.
all_commands['branch'] = all_commands['branches'] all_commands["branch"] = all_commands["branches"]

179
subcmds/abandon.py
View File

@ -23,9 +23,9 @@ from progress import Progress
class Abandon(Command): class Abandon(Command):
COMMON = True COMMON = True
helpSummary = "Permanently abandon a development branch" helpSummary = "Permanently abandon a development branch"
helpUsage = """ helpUsage = """
%prog [--all | <branchname>] [<project>...] %prog [--all | <branchname>] [<project>...]
This subcommand permanently abandons a development branch by This subcommand permanently abandons a development branch by
@ -33,83 +33,104 @@ deleting it (and all its history) from your local repository.
It is equivalent to "git branch -D <branchname>". It is equivalent to "git branch -D <branchname>".
""" """
PARALLEL_JOBS = DEFAULT_LOCAL_JOBS PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
def _Options(self, p): def _Options(self, p):
p.add_option('--all', p.add_option(
dest='all', action='store_true', "--all",
help='delete all branches in all projects') dest="all",
action="store_true",
help="delete all branches in all projects",
)
def ValidateOptions(self, opt, args): def ValidateOptions(self, opt, args):
if not opt.all and not args: if not opt.all and not args:
self.Usage() self.Usage()
if not opt.all: if not opt.all:
nb = args[0] nb = args[0]
if not git.check_ref_format('heads/%s' % nb): if not git.check_ref_format("heads/%s" % nb):
self.OptionParser.error("'%s' is not a valid branch name" % nb) self.OptionParser.error("'%s' is not a valid branch name" % nb)
else:
args.insert(0, "'All local branches'")
def _ExecuteOne(self, all_branches, nb, project):
"""Abandon one project."""
if all_branches:
branches = project.GetBranches()
else:
branches = [nb]
ret = {}
for name in branches:
status = project.AbandonBranch(name)
if status is not None:
ret[name] = status
return (ret, project)
def Execute(self, opt, args):
nb = args[0]
err = defaultdict(list)
success = defaultdict(list)
all_projects = self.GetProjects(args[1:], all_manifests=not opt.this_manifest_only)
_RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
def _ProcessResults(_pool, pm, states):
for (results, project) in states:
for branch, status in results.items():
if status:
success[branch].append(project)
else:
err[branch].append(project)
pm.update()
self.ExecuteInParallel(
opt.jobs,
functools.partial(self._ExecuteOne, opt.all, nb),
all_projects,
callback=_ProcessResults,
output=Progress('Abandon %s' % (nb,), len(all_projects), quiet=opt.quiet))
width = max(itertools.chain(
[25], (len(x) for x in itertools.chain(success, err))))
if err:
for br in err.keys():
err_msg = "error: cannot abandon %s" % br
print(err_msg, file=sys.stderr)
for proj in err[br]:
print(' ' * len(err_msg) + " | %s" % _RelPath(proj), file=sys.stderr)
sys.exit(1)
elif not success:
print('error: no project has local branch(es) : %s' % nb,
file=sys.stderr)
sys.exit(1)
else:
# Everything below here is displaying status.
if opt.quiet:
return
print('Abandoned branches:')
for br in success.keys():
if len(all_projects) > 1 and len(all_projects) == len(success[br]):
result = "all project"
else: else:
result = "%s" % ( args.insert(0, "'All local branches'")
('\n' + ' ' * width + '| ').join(_RelPath(p) for p in success[br]))
print("%s%s| %s\n" % (br, ' ' * (width - len(br)), result)) def _ExecuteOne(self, all_branches, nb, project):
"""Abandon one project."""
if all_branches:
branches = project.GetBranches()
else:
branches = [nb]
ret = {}
for name in branches:
status = project.AbandonBranch(name)
if status is not None:
ret[name] = status
return (ret, project)
def Execute(self, opt, args):
nb = args[0]
err = defaultdict(list)
success = defaultdict(list)
all_projects = self.GetProjects(
args[1:], all_manifests=not opt.this_manifest_only
)
_RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
def _ProcessResults(_pool, pm, states):
for results, project in states:
for branch, status in results.items():
if status:
success[branch].append(project)
else:
err[branch].append(project)
pm.update()
self.ExecuteInParallel(
opt.jobs,
functools.partial(self._ExecuteOne, opt.all, nb),
all_projects,
callback=_ProcessResults,
output=Progress(
"Abandon %s" % (nb,), len(all_projects), quiet=opt.quiet
),
)
width = max(
itertools.chain(
[25], (len(x) for x in itertools.chain(success, err))
)
)
if err:
for br in err.keys():
err_msg = "error: cannot abandon %s" % br
print(err_msg, file=sys.stderr)
for proj in err[br]:
print(
" " * len(err_msg) + " | %s" % _RelPath(proj),
file=sys.stderr,
)
sys.exit(1)
elif not success:
print(
"error: no project has local branch(es) : %s" % nb,
file=sys.stderr,
)
sys.exit(1)
else:
# Everything below here is displaying status.
if opt.quiet:
return
print("Abandoned branches:")
for br in success.keys():
if len(all_projects) > 1 and len(all_projects) == len(
success[br]
):
result = "all project"
else:
result = "%s" % (
("\n" + " " * width + "| ").join(
_RelPath(p) for p in success[br]
)
)
print("%s%s| %s\n" % (br, " " * (width - len(br)), result))

255
subcmds/branches.py
View File

@ -20,51 +20,51 @@ from command import Command, DEFAULT_LOCAL_JOBS
class BranchColoring(Coloring): class BranchColoring(Coloring):
def __init__(self, config): def __init__(self, config):
Coloring.__init__(self, config, 'branch') Coloring.__init__(self, config, "branch")
self.current = self.printer('current', fg='green') self.current = self.printer("current", fg="green")
self.local = self.printer('local') self.local = self.printer("local")
self.notinproject = self.printer('notinproject', fg='red') self.notinproject = self.printer("notinproject", fg="red")
class BranchInfo(object): class BranchInfo(object):
def __init__(self, name): def __init__(self, name):
self.name = name self.name = name
self.current = 0 self.current = 0
self.published = 0 self.published = 0
self.published_equal = 0 self.published_equal = 0
self.projects = [] self.projects = []
def add(self, b): def add(self, b):
if b.current: if b.current:
self.current += 1 self.current += 1
if b.published: if b.published:
self.published += 1 self.published += 1
if b.revision == b.published: if b.revision == b.published:
self.published_equal += 1 self.published_equal += 1
self.projects.append(b) self.projects.append(b)
@property @property
def IsCurrent(self): def IsCurrent(self):
return self.current > 0 return self.current > 0
@property @property
def IsSplitCurrent(self): def IsSplitCurrent(self):
return self.current != 0 and self.current != len(self.projects) return self.current != 0 and self.current != len(self.projects)
@property @property
def IsPublished(self): def IsPublished(self):
return self.published > 0 return self.published > 0
@property @property
def IsPublishedEqual(self): def IsPublishedEqual(self):
return self.published_equal == len(self.projects) return self.published_equal == len(self.projects)
class Branches(Command): class Branches(Command):
COMMON = True COMMON = True
helpSummary = "View current topic branches" helpSummary = "View current topic branches"
helpUsage = """ helpUsage = """
%prog [<project>...] %prog [<project>...]
Summarizes the currently available topic branches. Summarizes the currently available topic branches.
@ -95,111 +95,114 @@ the branch appears in, or does not appear in. If no project list
is shown, then the branch appears in all projects. is shown, then the branch appears in all projects.
""" """
PARALLEL_JOBS = DEFAULT_LOCAL_JOBS PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
def Execute(self, opt, args): def Execute(self, opt, args):
projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only) projects = self.GetProjects(
out = BranchColoring(self.manifest.manifestProject.config) args, all_manifests=not opt.this_manifest_only
all_branches = {} )
project_cnt = len(projects) out = BranchColoring(self.manifest.manifestProject.config)
all_branches = {}
project_cnt = len(projects)
def _ProcessResults(_pool, _output, results): def _ProcessResults(_pool, _output, results):
for name, b in itertools.chain.from_iterable(results): for name, b in itertools.chain.from_iterable(results):
if name not in all_branches: if name not in all_branches:
all_branches[name] = BranchInfo(name) all_branches[name] = BranchInfo(name)
all_branches[name].add(b) all_branches[name].add(b)
self.ExecuteInParallel( self.ExecuteInParallel(
opt.jobs, opt.jobs,
expand_project_to_branches, expand_project_to_branches,
projects, projects,
callback=_ProcessResults) callback=_ProcessResults,
)
names = sorted(all_branches) names = sorted(all_branches)
if not names: if not names:
print(' (no branches)', file=sys.stderr) print(" (no branches)", file=sys.stderr)
return return
width = 25 width = 25
for name in names: for name in names:
if width < len(name): if width < len(name):
width = len(name) width = len(name)
for name in names: for name in names:
i = all_branches[name] i = all_branches[name]
in_cnt = len(i.projects) in_cnt = len(i.projects)
if i.IsCurrent: if i.IsCurrent:
current = '*' current = "*"
hdr = out.current hdr = out.current
else:
current = ' '
hdr = out.local
if i.IsPublishedEqual:
published = 'P'
elif i.IsPublished:
published = 'p'
else:
published = ' '
hdr('%c%c %-*s' % (current, published, width, name))
out.write(' |')
_RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
if in_cnt < project_cnt:
fmt = out.write
paths = []
non_cur_paths = []
if i.IsSplitCurrent or (in_cnt <= project_cnt - in_cnt):
in_type = 'in'
for b in i.projects:
relpath = _RelPath(b.project)
if not i.IsSplitCurrent or b.current:
paths.append(relpath)
else: else:
non_cur_paths.append(relpath) current = " "
else: hdr = out.local
fmt = out.notinproject
in_type = 'not in'
have = set()
for b in i.projects:
have.add(_RelPath(b.project))
for p in projects:
if _RelPath(p) not in have:
paths.append(_RelPath(p))
s = ' %s %s' % (in_type, ', '.join(paths)) if i.IsPublishedEqual:
if not i.IsSplitCurrent and (width + 7 + len(s) < 80): published = "P"
fmt = out.current if i.IsCurrent else fmt elif i.IsPublished:
fmt(s) published = "p"
else: else:
fmt(' %s:' % in_type) published = " "
fmt = out.current if i.IsCurrent else out.write
for p in paths: hdr("%c%c %-*s" % (current, published, width, name))
out.write(" |")
_RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
if in_cnt < project_cnt:
fmt = out.write
paths = []
non_cur_paths = []
if i.IsSplitCurrent or (in_cnt <= project_cnt - in_cnt):
in_type = "in"
for b in i.projects:
relpath = _RelPath(b.project)
if not i.IsSplitCurrent or b.current:
paths.append(relpath)
else:
non_cur_paths.append(relpath)
else:
fmt = out.notinproject
in_type = "not in"
have = set()
for b in i.projects:
have.add(_RelPath(b.project))
for p in projects:
if _RelPath(p) not in have:
paths.append(_RelPath(p))
s = " %s %s" % (in_type, ", ".join(paths))
if not i.IsSplitCurrent and (width + 7 + len(s) < 80):
fmt = out.current if i.IsCurrent else fmt
fmt(s)
else:
fmt(" %s:" % in_type)
fmt = out.current if i.IsCurrent else out.write
for p in paths:
out.nl()
fmt(width * " " + " %s" % p)
fmt = out.write
for p in non_cur_paths:
out.nl()
fmt(width * " " + " %s" % p)
else:
out.write(" in all projects")
out.nl() out.nl()
fmt(width * ' ' + ' %s' % p)
fmt = out.write
for p in non_cur_paths:
out.nl()
fmt(width * ' ' + ' %s' % p)
else:
out.write(' in all projects')
out.nl()
def expand_project_to_branches(project): def expand_project_to_branches(project):
"""Expands a project into a list of branch names & associated information. """Expands a project into a list of branch names & associated information.
Args: Args:
project: project.Project project: project.Project
Returns: Returns:
List[Tuple[str, git_config.Branch]] List[Tuple[str, git_config.Branch]]
""" """
branches = [] branches = []
for name, b in project.GetBranches().items(): for name, b in project.GetBranches().items():
b.project = project b.project = project
branches.append((name, b)) branches.append((name, b))
return branches return branches

83
subcmds/checkout.py
View File

@ -20,12 +20,12 @@ from progress import Progress
class Checkout(Command): class Checkout(Command):
COMMON = True COMMON = True
helpSummary = "Checkout a branch for development" helpSummary = "Checkout a branch for development"
helpUsage = """ helpUsage = """
%prog <branchname> [<project>...] %prog <branchname> [<project>...]
""" """
helpDescription = """ helpDescription = """
The '%prog' command checks out an existing branch that was previously The '%prog' command checks out an existing branch that was previously
created by 'repo start'. created by 'repo start'.
@ -33,43 +33,50 @@ The command is equivalent to:
repo forall [<project>...] -c git checkout <branchname> repo forall [<project>...] -c git checkout <branchname>
""" """
PARALLEL_JOBS = DEFAULT_LOCAL_JOBS PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
def ValidateOptions(self, opt, args): def ValidateOptions(self, opt, args):
if not args: if not args:
self.Usage() self.Usage()
def _ExecuteOne(self, nb, project): def _ExecuteOne(self, nb, project):
"""Checkout one project.""" """Checkout one project."""
return (project.CheckoutBranch(nb), project) return (project.CheckoutBranch(nb), project)
def Execute(self, opt, args): def Execute(self, opt, args):
nb = args[0] nb = args[0]
err = [] err = []
success = [] success = []
all_projects = self.GetProjects(args[1:], all_manifests=not opt.this_manifest_only) all_projects = self.GetProjects(
args[1:], all_manifests=not opt.this_manifest_only
)
def _ProcessResults(_pool, pm, results): def _ProcessResults(_pool, pm, results):
for status, project in results: for status, project in results:
if status is not None: if status is not None:
if status: if status:
success.append(project) success.append(project)
else: else:
err.append(project) err.append(project)
pm.update() pm.update()
self.ExecuteInParallel( self.ExecuteInParallel(
opt.jobs, opt.jobs,
functools.partial(self._ExecuteOne, nb), functools.partial(self._ExecuteOne, nb),
all_projects, all_projects,
callback=_ProcessResults, callback=_ProcessResults,
output=Progress('Checkout %s' % (nb,), len(all_projects), quiet=opt.quiet)) output=Progress(
"Checkout %s" % (nb,), len(all_projects), quiet=opt.quiet
),
)
if err: if err:
for p in err: for p in err:
print("error: %s/: cannot checkout %s" % (p.relpath, nb), print(
file=sys.stderr) "error: %s/: cannot checkout %s" % (p.relpath, nb),
sys.exit(1) file=sys.stderr,
elif not success: )
print('error: no project has branch %s' % nb, file=sys.stderr) sys.exit(1)
sys.exit(1) elif not success:
print("error: no project has branch %s" % nb, file=sys.stderr)
sys.exit(1)

147
subcmds/cherry_pick.py
View File

@ -17,96 +17,107 @@ import sys
from command import Command from command import Command
from git_command import GitCommand from git_command import GitCommand
CHANGE_ID_RE = re.compile(r'^\s*Change-Id: I([0-9a-f]{40})\s*$') CHANGE_ID_RE = re.compile(r"^\s*Change-Id: I([0-9a-f]{40})\s*$")
class CherryPick(Command): class CherryPick(Command):
COMMON = True COMMON = True
helpSummary = "Cherry-pick a change." helpSummary = "Cherry-pick a change."
helpUsage = """ helpUsage = """
%prog <sha1> %prog <sha1>
""" """
helpDescription = """ helpDescription = """
'%prog' cherry-picks a change from one branch to another. '%prog' cherry-picks a change from one branch to another.
The change id will be updated, and a reference to the old The change id will be updated, and a reference to the old
change id will be added. change id will be added.
""" """
def ValidateOptions(self, opt, args): def ValidateOptions(self, opt, args):
if len(args) != 1: if len(args) != 1:
self.Usage() self.Usage()
def Execute(self, opt, args): def Execute(self, opt, args):
reference = args[0] reference = args[0]
p = GitCommand(None, p = GitCommand(
['rev-parse', '--verify', reference], None,
capture_stdout=True, ["rev-parse", "--verify", reference],
capture_stderr=True) capture_stdout=True,
if p.Wait() != 0: capture_stderr=True,
print(p.stderr, file=sys.stderr) )
sys.exit(1) if p.Wait() != 0:
sha1 = p.stdout.strip() print(p.stderr, file=sys.stderr)
sys.exit(1)
sha1 = p.stdout.strip()
p = GitCommand(None, ['cat-file', 'commit', sha1], capture_stdout=True) p = GitCommand(None, ["cat-file", "commit", sha1], capture_stdout=True)
if p.Wait() != 0: if p.Wait() != 0:
print("error: Failed to retrieve old commit message", file=sys.stderr) print(
sys.exit(1) "error: Failed to retrieve old commit message", file=sys.stderr
old_msg = self._StripHeader(p.stdout) )
sys.exit(1)
old_msg = self._StripHeader(p.stdout)
p = GitCommand(None, p = GitCommand(
['cherry-pick', sha1], None,
capture_stdout=True, ["cherry-pick", sha1],
capture_stderr=True) capture_stdout=True,
status = p.Wait() capture_stderr=True,
)
status = p.Wait()
if p.stdout: if p.stdout:
print(p.stdout.strip(), file=sys.stdout) print(p.stdout.strip(), file=sys.stdout)
if p.stderr: if p.stderr:
print(p.stderr.strip(), file=sys.stderr) print(p.stderr.strip(), file=sys.stderr)
if status == 0: if status == 0:
# The cherry-pick was applied correctly. We just need to edit the # The cherry-pick was applied correctly. We just need to edit the
# commit message. # commit message.
new_msg = self._Reformat(old_msg, sha1) new_msg = self._Reformat(old_msg, sha1)
p = GitCommand(None, ['commit', '--amend', '-F', '-'], p = GitCommand(
input=new_msg, None,
capture_stdout=True, ["commit", "--amend", "-F", "-"],
capture_stderr=True) input=new_msg,
if p.Wait() != 0: capture_stdout=True,
print("error: Failed to update commit message", file=sys.stderr) capture_stderr=True,
sys.exit(1) )
if p.Wait() != 0:
print("error: Failed to update commit message", file=sys.stderr)
sys.exit(1)
else: else:
print('NOTE: When committing (please see above) and editing the commit ' print(
'message, please remove the old Change-Id-line and add:') "NOTE: When committing (please see above) and editing the "
print(self._GetReference(sha1), file=sys.stderr) "commit message, please remove the old Change-Id-line and add:"
print(file=sys.stderr) )
print(self._GetReference(sha1), file=sys.stderr)
print(file=sys.stderr)
def _IsChangeId(self, line): def _IsChangeId(self, line):
return CHANGE_ID_RE.match(line) return CHANGE_ID_RE.match(line)
def _GetReference(self, sha1): def _GetReference(self, sha1):
return "(cherry picked from commit %s)" % sha1 return "(cherry picked from commit %s)" % sha1
def _StripHeader(self, commit_msg): def _StripHeader(self, commit_msg):
lines = commit_msg.splitlines() lines = commit_msg.splitlines()
return "\n".join(lines[lines.index("") + 1:]) return "\n".join(lines[lines.index("") + 1 :])
def _Reformat(self, old_msg, sha1): def _Reformat(self, old_msg, sha1):
new_msg = [] new_msg = []
for line in old_msg.splitlines(): for line in old_msg.splitlines():
if not self._IsChangeId(line): if not self._IsChangeId(line):
new_msg.append(line) new_msg.append(line)
# Add a blank line between the message and the change id/reference # Add a blank line between the message and the change id/reference.
try: try:
if new_msg[-1].strip() != "": if new_msg[-1].strip() != "":
new_msg.append("") new_msg.append("")
except IndexError: except IndexError:
pass pass
new_msg.append(self._GetReference(sha1)) new_msg.append(self._GetReference(sha1))
return "\n".join(new_msg) return "\n".join(new_msg)

85
subcmds/diff.py
View File

@ -19,54 +19,63 @@ from command import DEFAULT_LOCAL_JOBS, PagedCommand
class Diff(PagedCommand): class Diff(PagedCommand):
COMMON = True COMMON = True
helpSummary = "Show changes between commit and working tree" helpSummary = "Show changes between commit and working tree"
helpUsage = """ helpUsage = """
%prog [<project>...] %prog [<project>...]
The -u option causes '%prog' to generate diff output with file paths The -u option causes '%prog' to generate diff output with file paths
relative to the repository root, so the output can be applied relative to the repository root, so the output can be applied
to the Unix 'patch' command. to the Unix 'patch' command.
""" """
PARALLEL_JOBS = DEFAULT_LOCAL_JOBS PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
def _Options(self, p): def _Options(self, p):
p.add_option('-u', '--absolute', p.add_option(
dest='absolute', action='store_true', "-u",
help='paths are relative to the repository root') "--absolute",
dest="absolute",
action="store_true",
help="paths are relative to the repository root",
)
def _ExecuteOne(self, absolute, local, project): def _ExecuteOne(self, absolute, local, project):
"""Obtains the diff for a specific project. """Obtains the diff for a specific project.
Args: Args:
absolute: Paths are relative to the root. absolute: Paths are relative to the root.
local: a boolean, if True, the path is relative to the local local: a boolean, if True, the path is relative to the local
(sub)manifest. If false, the path is relative to the (sub)manifest. If false, the path is relative to the outermost
outermost manifest. manifest.
project: Project to get status of. project: Project to get status of.
Returns: Returns:
The status of the project. The status of the project.
""" """
buf = io.StringIO() buf = io.StringIO()
ret = project.PrintWorkTreeDiff(absolute, output_redir=buf, local=local) ret = project.PrintWorkTreeDiff(absolute, output_redir=buf, local=local)
return (ret, buf.getvalue()) return (ret, buf.getvalue())
def Execute(self, opt, args): def Execute(self, opt, args):
all_projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only) all_projects = self.GetProjects(
args, all_manifests=not opt.this_manifest_only
)
def _ProcessResults(_pool, _output, results): def _ProcessResults(_pool, _output, results):
ret = 0 ret = 0
for (state, output) in results: for state, output in results:
if output: if output:
print(output, end='') print(output, end="")
if not state: if not state:
ret = 1 ret = 1
return ret return ret
return self.ExecuteInParallel( return self.ExecuteInParallel(
opt.jobs, opt.jobs,
functools.partial(self._ExecuteOne, opt.absolute, opt.this_manifest_only), functools.partial(
all_projects, self._ExecuteOne, opt.absolute, opt.this_manifest_only
callback=_ProcessResults, ),
ordered=True) all_projects,
callback=_ProcessResults,
ordered=True,
)

370
subcmds/diffmanifests.py
View File

@ -18,24 +18,24 @@ from manifest_xml import RepoClient
class _Coloring(Coloring): class _Coloring(Coloring):
def __init__(self, config): def __init__(self, config):
Coloring.__init__(self, config, "status") Coloring.__init__(self, config, "status")
class Diffmanifests(PagedCommand): class Diffmanifests(PagedCommand):
""" A command to see logs in projects represented by manifests """A command to see logs in projects represented by manifests
This is used to see deeper differences between manifests. Where a simple This is used to see deeper differences between manifests. Where a simple
diff would only show a diff of sha1s for example, this command will display diff would only show a diff of sha1s for example, this command will display
the logs of the project between both sha1s, allowing user to see diff at a the logs of the project between both sha1s, allowing user to see diff at a
deeper level. deeper level.
""" """
COMMON = True COMMON = True
helpSummary = "Manifest diff utility" helpSummary = "Manifest diff utility"
helpUsage = """%prog manifest1.xml [manifest2.xml] [options]""" helpUsage = """%prog manifest1.xml [manifest2.xml] [options]"""
helpDescription = """ helpDescription = """
The %prog command shows differences between project revisions of manifest1 and The %prog command shows differences between project revisions of manifest1 and
manifest2. if manifest2 is not specified, current manifest.xml will be used manifest2. if manifest2 is not specified, current manifest.xml will be used
instead. Both absolute and relative paths may be used for manifests. Relative instead. Both absolute and relative paths may be used for manifests. Relative
@ -65,159 +65,209 @@ synced and their revisions won't be found.
""" """
def _Options(self, p): def _Options(self, p):
p.add_option('--raw', p.add_option(
dest='raw', action='store_true', "--raw", dest="raw", action="store_true", help="display raw diff"
help='display raw diff') )
p.add_option('--no-color', p.add_option(
dest='color', action='store_false', default=True, "--no-color",
help='does not display the diff in color') dest="color",
p.add_option('--pretty-format', action="store_false",
dest='pretty_format', action='store', default=True,
metavar='<FORMAT>', help="does not display the diff in color",
help='print the log using a custom git pretty format string') )
p.add_option(
"--pretty-format",
dest="pretty_format",
action="store",
metavar="<FORMAT>",
help="print the log using a custom git pretty format string",
)
def _printRawDiff(self, diff, pretty_format=None, local=False): def _printRawDiff(self, diff, pretty_format=None, local=False):
_RelPath = lambda p: p.RelPath(local=local) _RelPath = lambda p: p.RelPath(local=local)
for project in diff['added']: for project in diff["added"]:
self.printText("A %s %s" % (_RelPath(project), project.revisionExpr)) self.printText(
self.out.nl() "A %s %s" % (_RelPath(project), project.revisionExpr)
)
for project in diff['removed']:
self.printText("R %s %s" % (_RelPath(project), project.revisionExpr))
self.out.nl()
for project, otherProject in diff['changed']:
self.printText("C %s %s %s" % (_RelPath(project), project.revisionExpr,
otherProject.revisionExpr))
self.out.nl()
self._printLogs(project, otherProject, raw=True, color=False, pretty_format=pretty_format)
for project, otherProject in diff['unreachable']:
self.printText("U %s %s %s" % (_RelPath(project), project.revisionExpr,
otherProject.revisionExpr))
self.out.nl()
def _printDiff(self, diff, color=True, pretty_format=None, local=False):
_RelPath = lambda p: p.RelPath(local=local)
if diff['added']:
self.out.nl()
self.printText('added projects : \n')
self.out.nl()
for project in diff['added']:
self.printProject('\t%s' % (_RelPath(project)))
self.printText(' at revision ')
self.printRevision(project.revisionExpr)
self.out.nl()
if diff['removed']:
self.out.nl()
self.printText('removed projects : \n')
self.out.nl()
for project in diff['removed']:
self.printProject('\t%s' % (_RelPath(project)))
self.printText(' at revision ')
self.printRevision(project.revisionExpr)
self.out.nl()
if diff['missing']:
self.out.nl()
self.printText('missing projects : \n')
self.out.nl()
for project in diff['missing']:
self.printProject('\t%s' % (_RelPath(project)))
self.printText(' at revision ')
self.printRevision(project.revisionExpr)
self.out.nl()
if diff['changed']:
self.out.nl()
self.printText('changed projects : \n')
self.out.nl()
for project, otherProject in diff['changed']:
self.printProject('\t%s' % (_RelPath(project)))
self.printText(' changed from ')
self.printRevision(project.revisionExpr)
self.printText(' to ')
self.printRevision(otherProject.revisionExpr)
self.out.nl()
self._printLogs(project, otherProject, raw=False, color=color,
pretty_format=pretty_format)
self.out.nl()
if diff['unreachable']:
self.out.nl()
self.printText('projects with unreachable revisions : \n')
self.out.nl()
for project, otherProject in diff['unreachable']:
self.printProject('\t%s ' % (_RelPath(project)))
self.printRevision(project.revisionExpr)
self.printText(' or ')
self.printRevision(otherProject.revisionExpr)
self.printText(' not found')
self.out.nl()
def _printLogs(self, project, otherProject, raw=False, color=True,
pretty_format=None):
logs = project.getAddedAndRemovedLogs(otherProject,
oneline=(pretty_format is None),
color=color,
pretty_format=pretty_format)
if logs['removed']:
removedLogs = logs['removed'].split('\n')
for log in removedLogs:
if log.strip():
if raw:
self.printText(' R ' + log)
self.out.nl()
else:
self.printRemoved('\t\t[-] ')
self.printText(log)
self.out.nl() self.out.nl()
if logs['added']: for project in diff["removed"]:
addedLogs = logs['added'].split('\n') self.printText(
for log in addedLogs: "R %s %s" % (_RelPath(project), project.revisionExpr)
if log.strip(): )
if raw:
self.printText(' A ' + log)
self.out.nl()
else:
self.printAdded('\t\t[+] ')
self.printText(log)
self.out.nl() self.out.nl()
def ValidateOptions(self, opt, args): for project, otherProject in diff["changed"]:
if not args or len(args) > 2: self.printText(
self.OptionParser.error('missing manifests to diff') "C %s %s %s"
if opt.this_manifest_only is False: % (
raise self.OptionParser.error( _RelPath(project),
'`diffmanifest` only supports the current tree') project.revisionExpr,
otherProject.revisionExpr,
)
)
self.out.nl()
self._printLogs(
project,
otherProject,
raw=True,
color=False,
pretty_format=pretty_format,
)
def Execute(self, opt, args): for project, otherProject in diff["unreachable"]:
self.out = _Coloring(self.client.globalConfig) self.printText(
self.printText = self.out.nofmt_printer('text') "U %s %s %s"
if opt.color: % (
self.printProject = self.out.nofmt_printer('project', attr='bold') _RelPath(project),
self.printAdded = self.out.nofmt_printer('green', fg='green', attr='bold') project.revisionExpr,
self.printRemoved = self.out.nofmt_printer('red', fg='red', attr='bold') otherProject.revisionExpr,
self.printRevision = self.out.nofmt_printer('revision', fg='yellow') )
else: )
self.printProject = self.printAdded = self.printRemoved = self.printRevision = self.printText self.out.nl()
manifest1 = RepoClient(self.repodir) def _printDiff(self, diff, color=True, pretty_format=None, local=False):
manifest1.Override(args[0], load_local_manifests=False) _RelPath = lambda p: p.RelPath(local=local)
if len(args) == 1: if diff["added"]:
manifest2 = self.manifest self.out.nl()
else: self.printText("added projects : \n")
manifest2 = RepoClient(self.repodir) self.out.nl()
manifest2.Override(args[1], load_local_manifests=False) for project in diff["added"]:
self.printProject("\t%s" % (_RelPath(project)))
self.printText(" at revision ")
self.printRevision(project.revisionExpr)
self.out.nl()
diff = manifest1.projectsDiff(manifest2) if diff["removed"]:
if opt.raw: self.out.nl()
self._printRawDiff(diff, pretty_format=opt.pretty_format, self.printText("removed projects : \n")
local=opt.this_manifest_only) self.out.nl()
else: for project in diff["removed"]:
self._printDiff(diff, color=opt.color, pretty_format=opt.pretty_format, self.printProject("\t%s" % (_RelPath(project)))
local=opt.this_manifest_only) self.printText(" at revision ")
self.printRevision(project.revisionExpr)
self.out.nl()
if diff["missing"]:
self.out.nl()
self.printText("missing projects : \n")
self.out.nl()
for project in diff["missing"]:
self.printProject("\t%s" % (_RelPath(project)))
self.printText(" at revision ")
self.printRevision(project.revisionExpr)
self.out.nl()
if diff["changed"]:
self.out.nl()
self.printText("changed projects : \n")
self.out.nl()
for project, otherProject in diff["changed"]:
self.printProject("\t%s" % (_RelPath(project)))
self.printText(" changed from ")
self.printRevision(project.revisionExpr)
self.printText(" to ")
self.printRevision(otherProject.revisionExpr)
self.out.nl()
self._printLogs(
project,
otherProject,
raw=False,
color=color,
pretty_format=pretty_format,
)
self.out.nl()
if diff["unreachable"]:
self.out.nl()
self.printText("projects with unreachable revisions : \n")
self.out.nl()
for project, otherProject in diff["unreachable"]:
self.printProject("\t%s " % (_RelPath(project)))
self.printRevision(project.revisionExpr)
self.printText(" or ")
self.printRevision(otherProject.revisionExpr)
self.printText(" not found")
self.out.nl()
def _printLogs(
self, project, otherProject, raw=False, color=True, pretty_format=None
):
logs = project.getAddedAndRemovedLogs(
otherProject,
oneline=(pretty_format is None),
color=color,
pretty_format=pretty_format,
)
if logs["removed"]:
removedLogs = logs["removed"].split("\n")
for log in removedLogs:
if log.strip():
if raw:
self.printText(" R " + log)
self.out.nl()
else:
self.printRemoved("\t\t[-] ")
self.printText(log)
self.out.nl()
if logs["added"]:
addedLogs = logs["added"].split("\n")
for log in addedLogs:
if log.strip():
if raw:
self.printText(" A " + log)
self.out.nl()
else:
self.printAdded("\t\t[+] ")
self.printText(log)
self.out.nl()
def ValidateOptions(self, opt, args):
if not args or len(args) > 2:
self.OptionParser.error("missing manifests to diff")
if opt.this_manifest_only is False:
raise self.OptionParser.error(
"`diffmanifest` only supports the current tree"
)
def Execute(self, opt, args):
self.out = _Coloring(self.client.globalConfig)
self.printText = self.out.nofmt_printer("text")
if opt.color:
self.printProject = self.out.nofmt_printer("project", attr="bold")
self.printAdded = self.out.nofmt_printer(
"green", fg="green", attr="bold"
)
self.printRemoved = self.out.nofmt_printer(
"red", fg="red", attr="bold"
)
self.printRevision = self.out.nofmt_printer("revision", fg="yellow")
else:
self.printProject = (
self.printAdded
) = self.printRemoved = self.printRevision = self.printText
manifest1 = RepoClient(self.repodir)
manifest1.Override(args[0], load_local_manifests=False)
if len(args) == 1:
manifest2 = self.manifest
else:
manifest2 = RepoClient(self.repodir)
manifest2.Override(args[1], load_local_manifests=False)
diff = manifest1.projectsDiff(manifest2)
if opt.raw:
self._printRawDiff(
diff,
pretty_format=opt.pretty_format,
local=opt.this_manifest_only,
)
else:
self._printDiff(
diff,
color=opt.color,
pretty_format=opt.pretty_format,
local=opt.this_manifest_only,
)

280
subcmds/download.py
View File

@ -18,143 +18,187 @@ import sys
from command import Command from command import Command
from error import GitError, NoSuchProjectError from error import GitError, NoSuchProjectError
CHANGE_RE = re.compile(r'^([1-9][0-9]*)(?:[/\.-]([1-9][0-9]*))?$') CHANGE_RE = re.compile(r"^([1-9][0-9]*)(?:[/\.-]([1-9][0-9]*))?$")
class Download(Command): class Download(Command):
COMMON = True COMMON = True
helpSummary = "Download and checkout a change" helpSummary = "Download and checkout a change"
helpUsage = """ helpUsage = """
%prog {[project] change[/patchset]}... %prog {[project] change[/patchset]}...
""" """
helpDescription = """ helpDescription = """
The '%prog' command downloads a change from the review system and The '%prog' command downloads a change from the review system and
makes it available in your project's local working directory. makes it available in your project's local working directory.
If no project is specified try to use current directory as a project. If no project is specified try to use current directory as a project.
""" """
def _Options(self, p): def _Options(self, p):
p.add_option('-b', '--branch', p.add_option("-b", "--branch", help="create a new branch first")
help='create a new branch first') p.add_option(
p.add_option('-c', '--cherry-pick', "-c",
dest='cherrypick', action='store_true', "--cherry-pick",
help="cherry-pick instead of checkout") dest="cherrypick",
p.add_option('-x', '--record-origin', action='store_true', action="store_true",
help='pass -x when cherry-picking') help="cherry-pick instead of checkout",
p.add_option('-r', '--revert', )
dest='revert', action='store_true', p.add_option(
help="revert instead of checkout") "-x",
p.add_option('-f', '--ff-only', "--record-origin",
dest='ffonly', action='store_true', action="store_true",
help="force fast-forward merge") help="pass -x when cherry-picking",
)
p.add_option(
"-r",
"--revert",
dest="revert",
action="store_true",
help="revert instead of checkout",
)
p.add_option(
"-f",
"--ff-only",
dest="ffonly",
action="store_true",
help="force fast-forward merge",
)
def _ParseChangeIds(self, opt, args): def _ParseChangeIds(self, opt, args):
if not args: if not args:
self.Usage() self.Usage()
to_get = [] to_get = []
project = None project = None
for a in args: for a in args:
m = CHANGE_RE.match(a) m = CHANGE_RE.match(a)
if m: if m:
if not project: if not project:
project = self.GetProjects(".")[0] project = self.GetProjects(".")[0]
print('Defaulting to cwd project', project.name) print("Defaulting to cwd project", project.name)
chg_id = int(m.group(1)) chg_id = int(m.group(1))
if m.group(2): if m.group(2):
ps_id = int(m.group(2)) ps_id = int(m.group(2))
else: else:
ps_id = 1 ps_id = 1
refs = 'refs/changes/%2.2d/%d/' % (chg_id % 100, chg_id) refs = "refs/changes/%2.2d/%d/" % (chg_id % 100, chg_id)
output = project._LsRemote(refs + '*') output = project._LsRemote(refs + "*")
if output: if output:
regex = refs + r'(\d+)' regex = refs + r"(\d+)"
rcomp = re.compile(regex, re.I) rcomp = re.compile(regex, re.I)
for line in output.splitlines(): for line in output.splitlines():
match = rcomp.search(line) match = rcomp.search(line)
if match: if match:
ps_id = max(int(match.group(1)), ps_id) ps_id = max(int(match.group(1)), ps_id)
to_get.append((project, chg_id, ps_id)) to_get.append((project, chg_id, ps_id))
else: else:
projects = self.GetProjects([a], all_manifests=not opt.this_manifest_only) projects = self.GetProjects(
if len(projects) > 1: [a], all_manifests=not opt.this_manifest_only
# If the cwd is one of the projects, assume they want that. )
try: if len(projects) > 1:
project = self.GetProjects('.')[0] # If the cwd is one of the projects, assume they want that.
except NoSuchProjectError: try:
project = None project = self.GetProjects(".")[0]
if project not in projects: except NoSuchProjectError:
print('error: %s matches too many projects; please re-run inside ' project = None
'the project checkout.' % (a,), file=sys.stderr) if project not in projects:
for project in projects: print(
print(' %s/ @ %s' % (project.RelPath(local=opt.this_manifest_only), "error: %s matches too many projects; please "
project.revisionExpr), file=sys.stderr) "re-run inside the project checkout." % (a,),
sys.exit(1) file=sys.stderr,
else: )
project = projects[0] for project in projects:
print('Defaulting to cwd project', project.name) print(
return to_get " %s/ @ %s"
% (
project.RelPath(
local=opt.this_manifest_only
),
project.revisionExpr,
),
file=sys.stderr,
)
sys.exit(1)
else:
project = projects[0]
print("Defaulting to cwd project", project.name)
return to_get
def ValidateOptions(self, opt, args): def ValidateOptions(self, opt, args):
if opt.record_origin: if opt.record_origin:
if not opt.cherrypick: if not opt.cherrypick:
self.OptionParser.error('-x only makes sense with --cherry-pick') self.OptionParser.error(
"-x only makes sense with --cherry-pick"
)
if opt.ffonly: if opt.ffonly:
self.OptionParser.error('-x and --ff are mutually exclusive options') self.OptionParser.error(
"-x and --ff are mutually exclusive options"
)
def Execute(self, opt, args): def Execute(self, opt, args):
for project, change_id, ps_id in self._ParseChangeIds(opt, args): for project, change_id, ps_id in self._ParseChangeIds(opt, args):
dl = project.DownloadPatchSet(change_id, ps_id) dl = project.DownloadPatchSet(change_id, ps_id)
if not dl: if not dl:
print('[%s] change %d/%d not found' print(
% (project.name, change_id, ps_id), "[%s] change %d/%d not found"
file=sys.stderr) % (project.name, change_id, ps_id),
sys.exit(1) file=sys.stderr,
)
sys.exit(1)
if not opt.revert and not dl.commits: if not opt.revert and not dl.commits:
print('[%s] change %d/%d has already been merged' print(
% (project.name, change_id, ps_id), "[%s] change %d/%d has already been merged"
file=sys.stderr) % (project.name, change_id, ps_id),
continue file=sys.stderr,
)
continue
if len(dl.commits) > 1: if len(dl.commits) > 1:
print('[%s] %d/%d depends on %d unmerged changes:' print(
% (project.name, change_id, ps_id, len(dl.commits)), "[%s] %d/%d depends on %d unmerged changes:"
file=sys.stderr) % (project.name, change_id, ps_id, len(dl.commits)),
for c in dl.commits: file=sys.stderr,
print(' %s' % (c), file=sys.stderr) )
for c in dl.commits:
print(" %s" % (c), file=sys.stderr)
if opt.cherrypick: if opt.cherrypick:
mode = 'cherry-pick' mode = "cherry-pick"
elif opt.revert: elif opt.revert:
mode = 'revert' mode = "revert"
elif opt.ffonly: elif opt.ffonly:
mode = 'fast-forward merge' mode = "fast-forward merge"
else: else:
mode = 'checkout' mode = "checkout"
# We'll combine the branch+checkout operation, but all the rest need a # We'll combine the branch+checkout operation, but all the rest need
# dedicated branch start. # a dedicated branch start.
if opt.branch and mode != 'checkout': if opt.branch and mode != "checkout":
project.StartBranch(opt.branch) project.StartBranch(opt.branch)
try: try:
if opt.cherrypick: if opt.cherrypick:
project._CherryPick(dl.commit, ffonly=opt.ffonly, project._CherryPick(
record_origin=opt.record_origin) dl.commit,
elif opt.revert: ffonly=opt.ffonly,
project._Revert(dl.commit) record_origin=opt.record_origin,
elif opt.ffonly: )
project._FastForward(dl.commit, ffonly=True) elif opt.revert:
else: project._Revert(dl.commit)
if opt.branch: elif opt.ffonly:
project.StartBranch(opt.branch, revision=dl.commit) project._FastForward(dl.commit, ffonly=True)
else: else:
project._Checkout(dl.commit) if opt.branch:
project.StartBranch(opt.branch, revision=dl.commit)
else:
project._Checkout(dl.commit)
except GitError: except GitError:
print('[%s] Could not complete the %s of %s' print(
% (project.name, mode, dl.commit), file=sys.stderr) "[%s] Could not complete the %s of %s"
sys.exit(1) % (project.name, mode, dl.commit),
file=sys.stderr,
)
sys.exit(1)

466
subcmds/forall.py
View File

@ -23,31 +23,36 @@ import sys
import subprocess import subprocess
from color import Coloring from color import Coloring
from command import DEFAULT_LOCAL_JOBS, Command, MirrorSafeCommand, WORKER_BATCH_SIZE from command import (
DEFAULT_LOCAL_JOBS,
Command,
MirrorSafeCommand,
WORKER_BATCH_SIZE,
)
from error import ManifestInvalidRevisionError from error import ManifestInvalidRevisionError
_CAN_COLOR = [ _CAN_COLOR = [
'branch', "branch",
'diff', "diff",
'grep', "grep",
'log', "log",
] ]
class ForallColoring(Coloring): class ForallColoring(Coloring):
def __init__(self, config): def __init__(self, config):
Coloring.__init__(self, config, 'forall') Coloring.__init__(self, config, "forall")
self.project = self.printer('project', attr='bold') self.project = self.printer("project", attr="bold")
class Forall(Command, MirrorSafeCommand): class Forall(Command, MirrorSafeCommand):
COMMON = False COMMON = False
helpSummary = "Run a shell command in each project" helpSummary = "Run a shell command in each project"
helpUsage = """ helpUsage = """
%prog [<project>...] -c <command> [<arg>...] %prog [<project>...] -c <command> [<arg>...]
%prog -r str1 [str2] ... -c <command> [<arg>...] %prog -r str1 [str2] ... -c <command> [<arg>...]
""" """
helpDescription = """ helpDescription = """
Executes the same shell command in each project. Executes the same shell command in each project.
The -r option allows running the command only on projects matching The -r option allows running the command only on projects matching
@ -125,236 +130,285 @@ terminal and are not redirected.
If -e is used, when a command exits unsuccessfully, '%prog' will abort If -e is used, when a command exits unsuccessfully, '%prog' will abort
without iterating through the remaining projects. without iterating through the remaining projects.
""" """
PARALLEL_JOBS = DEFAULT_LOCAL_JOBS PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
@staticmethod @staticmethod
def _cmd_option(option, _opt_str, _value, parser): def _cmd_option(option, _opt_str, _value, parser):
setattr(parser.values, option.dest, list(parser.rargs)) setattr(parser.values, option.dest, list(parser.rargs))
while parser.rargs: while parser.rargs:
del parser.rargs[0] del parser.rargs[0]
def _Options(self, p): def _Options(self, p):
p.add_option('-r', '--regex', p.add_option(
dest='regex', action='store_true', "-r",
help='execute the command only on projects matching regex or wildcard expression') "--regex",
p.add_option('-i', '--inverse-regex', dest="regex",
dest='inverse_regex', action='store_true', action="store_true",
help='execute the command only on projects not matching regex or ' help="execute the command only on projects matching regex or "
'wildcard expression') "wildcard expression",
p.add_option('-g', '--groups', )
dest='groups', p.add_option(
help='execute the command only on projects matching the specified groups') "-i",
p.add_option('-c', '--command', "--inverse-regex",
help='command (and arguments) to execute', dest="inverse_regex",
dest='command', action="store_true",
action='callback', help="execute the command only on projects not matching regex or "
callback=self._cmd_option) "wildcard expression",
p.add_option('-e', '--abort-on-errors', )
dest='abort_on_errors', action='store_true', p.add_option(
help='abort if a command exits unsuccessfully') "-g",
p.add_option('--ignore-missing', action='store_true', "--groups",
help='silently skip & do not exit non-zero due missing ' dest="groups",
'checkouts') help="execute the command only on projects matching the specified "
"groups",
)
p.add_option(
"-c",
"--command",
help="command (and arguments) to execute",
dest="command",
action="callback",
callback=self._cmd_option,
)
p.add_option(
"-e",
"--abort-on-errors",
dest="abort_on_errors",
action="store_true",
help="abort if a command exits unsuccessfully",
)
p.add_option(
"--ignore-missing",
action="store_true",
help="silently skip & do not exit non-zero due missing "
"checkouts",
)
g = p.get_option_group('--quiet') g = p.get_option_group("--quiet")
g.add_option('-p', g.add_option(
dest='project_header', action='store_true', "-p",
help='show project headers before output') dest="project_header",
p.add_option('--interactive', action="store_true",
action='store_true', help="show project headers before output",
help='force interactive usage') )
p.add_option(
"--interactive", action="store_true", help="force interactive usage"
)
def WantPager(self, opt): def WantPager(self, opt):
return opt.project_header and opt.jobs == 1 return opt.project_header and opt.jobs == 1
def ValidateOptions(self, opt, args): def ValidateOptions(self, opt, args):
if not opt.command: if not opt.command:
self.Usage() self.Usage()
def Execute(self, opt, args): def Execute(self, opt, args):
cmd = [opt.command[0]] cmd = [opt.command[0]]
all_trees = not opt.this_manifest_only all_trees = not opt.this_manifest_only
shell = True shell = True
if re.compile(r'^[a-z0-9A-Z_/\.-]+$').match(cmd[0]): if re.compile(r"^[a-z0-9A-Z_/\.-]+$").match(cmd[0]):
shell = False shell = False
if shell: if shell:
cmd.append(cmd[0]) cmd.append(cmd[0])
cmd.extend(opt.command[1:]) cmd.extend(opt.command[1:])
# Historically, forall operated interactively, and in serial. If the user # Historically, forall operated interactively, and in serial. If the
# has selected 1 job, then default to interacive mode. # user has selected 1 job, then default to interacive mode.
if opt.jobs == 1: if opt.jobs == 1:
opt.interactive = True opt.interactive = True
if opt.project_header \ if opt.project_header and not shell and cmd[0] == "git":
and not shell \ # If this is a direct git command that can enable colorized
and cmd[0] == 'git': # output and the user prefers coloring, add --color into the
# If this is a direct git command that can enable colorized # command line because we are going to wrap the command into
# output and the user prefers coloring, add --color into the # a pipe and git won't know coloring should activate.
# command line because we are going to wrap the command into #
# a pipe and git won't know coloring should activate. for cn in cmd[1:]:
# if not cn.startswith("-"):
for cn in cmd[1:]: break
if not cn.startswith('-'): else:
break cn = None
else: if cn and cn in _CAN_COLOR:
cn = None
if cn and cn in _CAN_COLOR:
class ColorCmd(Coloring):
def __init__(self, config, cmd):
Coloring.__init__(self, config, cmd)
if ColorCmd(self.manifest.manifestProject.config, cn).is_on:
cmd.insert(cmd.index(cn) + 1, '--color')
mirror = self.manifest.IsMirror class ColorCmd(Coloring):
rc = 0 def __init__(self, config, cmd):
Coloring.__init__(self, config, cmd)
smart_sync_manifest_name = "smart_sync_override.xml" if ColorCmd(self.manifest.manifestProject.config, cn).is_on:
smart_sync_manifest_path = os.path.join( cmd.insert(cmd.index(cn) + 1, "--color")
self.manifest.manifestProject.worktree, smart_sync_manifest_name)
if os.path.isfile(smart_sync_manifest_path): mirror = self.manifest.IsMirror
self.manifest.Override(smart_sync_manifest_path) rc = 0
if opt.regex: smart_sync_manifest_name = "smart_sync_override.xml"
projects = self.FindProjects(args, all_manifests=all_trees) smart_sync_manifest_path = os.path.join(
elif opt.inverse_regex: self.manifest.manifestProject.worktree, smart_sync_manifest_name
projects = self.FindProjects(args, inverse=True, all_manifests=all_trees) )
else:
projects = self.GetProjects(args, groups=opt.groups, all_manifests=all_trees)
os.environ['REPO_COUNT'] = str(len(projects)) if os.path.isfile(smart_sync_manifest_path):
self.manifest.Override(smart_sync_manifest_path)
try: if opt.regex:
config = self.manifest.manifestProject.config projects = self.FindProjects(args, all_manifests=all_trees)
with multiprocessing.Pool(opt.jobs, InitWorker) as pool: elif opt.inverse_regex:
results_it = pool.imap( projects = self.FindProjects(
functools.partial(DoWorkWrapper, mirror, opt, cmd, shell, config), args, inverse=True, all_manifests=all_trees
enumerate(projects), )
chunksize=WORKER_BATCH_SIZE) else:
first = True projects = self.GetProjects(
for (r, output) in results_it: args, groups=opt.groups, all_manifests=all_trees
if output: )
if first:
first = False os.environ["REPO_COUNT"] = str(len(projects))
elif opt.project_header:
print() try:
# To simplify the DoWorkWrapper, take care of automatic newlines. config = self.manifest.manifestProject.config
end = '\n' with multiprocessing.Pool(opt.jobs, InitWorker) as pool:
if output[-1] == '\n': results_it = pool.imap(
end = '' functools.partial(
print(output, end=end) DoWorkWrapper, mirror, opt, cmd, shell, config
rc = rc or r ),
if r != 0 and opt.abort_on_errors: enumerate(projects),
raise Exception('Aborting due to previous error') chunksize=WORKER_BATCH_SIZE,
except (KeyboardInterrupt, WorkerKeyboardInterrupt): )
# Catch KeyboardInterrupt raised inside and outside of workers first = True
rc = rc or errno.EINTR for r, output in results_it:
except Exception as e: if output:
# Catch any other exceptions raised if first:
print('forall: unhandled error, terminating the pool: %s: %s' % first = False
(type(e).__name__, e), elif opt.project_header:
file=sys.stderr) print()
rc = rc or getattr(e, 'errno', 1) # To simplify the DoWorkWrapper, take care of automatic
if rc != 0: # newlines.
sys.exit(rc) end = "\n"
if output[-1] == "\n":
end = ""
print(output, end=end)
rc = rc or r
if r != 0 and opt.abort_on_errors:
raise Exception("Aborting due to previous error")
except (KeyboardInterrupt, WorkerKeyboardInterrupt):
# Catch KeyboardInterrupt raised inside and outside of workers
rc = rc or errno.EINTR
except Exception as e:
# Catch any other exceptions raised
print(
"forall: unhandled error, terminating the pool: %s: %s"
% (type(e).__name__, e),
file=sys.stderr,
)
rc = rc or getattr(e, "errno", 1)
if rc != 0:
sys.exit(rc)
class WorkerKeyboardInterrupt(Exception): class WorkerKeyboardInterrupt(Exception):
""" Keyboard interrupt exception for worker processes. """ """Keyboard interrupt exception for worker processes."""
def InitWorker(): def InitWorker():
signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGINT, signal.SIG_IGN)
def DoWorkWrapper(mirror, opt, cmd, shell, config, args): def DoWorkWrapper(mirror, opt, cmd, shell, config, args):
""" A wrapper around the DoWork() method. """A wrapper around the DoWork() method.
Catch the KeyboardInterrupt exceptions here and re-raise them as a different, Catch the KeyboardInterrupt exceptions here and re-raise them as a
``Exception``-based exception to stop it flooding the console with stacktraces different, ``Exception``-based exception to stop it flooding the console
and making the parent hang indefinitely. with stacktraces and making the parent hang indefinitely.
""" """
cnt, project = args cnt, project = args
try: try:
return DoWork(project, mirror, opt, cmd, shell, cnt, config) return DoWork(project, mirror, opt, cmd, shell, cnt, config)
except KeyboardInterrupt: except KeyboardInterrupt:
print('%s: Worker interrupted' % project.name) print("%s: Worker interrupted" % project.name)
raise WorkerKeyboardInterrupt() raise WorkerKeyboardInterrupt()
def DoWork(project, mirror, opt, cmd, shell, cnt, config): def DoWork(project, mirror, opt, cmd, shell, cnt, config):
env = os.environ.copy() env = os.environ.copy()
def setenv(name, val): def setenv(name, val):
if val is None: if val is None:
val = '' val = ""
env[name] = val env[name] = val
setenv('REPO_PROJECT', project.name) setenv("REPO_PROJECT", project.name)
setenv('REPO_OUTERPATH', project.manifest.path_prefix) setenv("REPO_OUTERPATH", project.manifest.path_prefix)
setenv('REPO_INNERPATH', project.relpath) setenv("REPO_INNERPATH", project.relpath)
setenv('REPO_PATH', project.RelPath(local=opt.this_manifest_only)) setenv("REPO_PATH", project.RelPath(local=opt.this_manifest_only))
setenv('REPO_REMOTE', project.remote.name) setenv("REPO_REMOTE", project.remote.name)
try: try:
# If we aren't in a fully synced state and we don't have the ref the manifest # If we aren't in a fully synced state and we don't have the ref the
# wants, then this will fail. Ignore it for the purposes of this code. # manifest wants, then this will fail. Ignore it for the purposes of
lrev = '' if mirror else project.GetRevisionId() # this code.
except ManifestInvalidRevisionError: lrev = "" if mirror else project.GetRevisionId()
lrev = '' except ManifestInvalidRevisionError:
setenv('REPO_LREV', lrev) lrev = ""
setenv('REPO_RREV', project.revisionExpr) setenv("REPO_LREV", lrev)
setenv('REPO_UPSTREAM', project.upstream) setenv("REPO_RREV", project.revisionExpr)
setenv('REPO_DEST_BRANCH', project.dest_branch) setenv("REPO_UPSTREAM", project.upstream)
setenv('REPO_I', str(cnt + 1)) setenv("REPO_DEST_BRANCH", project.dest_branch)
for annotation in project.annotations: setenv("REPO_I", str(cnt + 1))
setenv("REPO__%s" % (annotation.name), annotation.value) for annotation in project.annotations:
setenv("REPO__%s" % (annotation.name), annotation.value)
if mirror: if mirror:
setenv('GIT_DIR', project.gitdir) setenv("GIT_DIR", project.gitdir)
cwd = project.gitdir cwd = project.gitdir
else: else:
cwd = project.worktree cwd = project.worktree
if not os.path.exists(cwd): if not os.path.exists(cwd):
# Allow the user to silently ignore missing checkouts so they can run on # Allow the user to silently ignore missing checkouts so they can run on
# partial checkouts (good for infra recovery tools). # partial checkouts (good for infra recovery tools).
if opt.ignore_missing: if opt.ignore_missing:
return (0, '') return (0, "")
output = '' output = ""
if ((opt.project_header and opt.verbose) if (opt.project_header and opt.verbose) or not opt.project_header:
or not opt.project_header): output = "skipping %s/" % project.RelPath(
output = 'skipping %s/' % project.RelPath(local=opt.this_manifest_only) local=opt.this_manifest_only
return (1, output) )
return (1, output)
if opt.verbose: if opt.verbose:
stderr = subprocess.STDOUT stderr = subprocess.STDOUT
else: else:
stderr = subprocess.DEVNULL stderr = subprocess.DEVNULL
stdin = None if opt.interactive else subprocess.DEVNULL stdin = None if opt.interactive else subprocess.DEVNULL
result = subprocess.run( result = subprocess.run(
cmd, cwd=cwd, shell=shell, env=env, check=False, cmd,
encoding='utf-8', errors='replace', cwd=cwd,
stdin=stdin, stdout=subprocess.PIPE, stderr=stderr) shell=shell,
env=env,
check=False,
encoding="utf-8",
errors="replace",
stdin=stdin,
stdout=subprocess.PIPE,
stderr=stderr,
)
output = result.stdout output = result.stdout
if opt.project_header: if opt.project_header:
if output: if output:
buf = io.StringIO() buf = io.StringIO()
out = ForallColoring(config) out = ForallColoring(config)
out.redirect(buf) out.redirect(buf)
if mirror: if mirror:
project_header_path = project.name project_header_path = project.name
else: else:
project_header_path = project.RelPath(local=opt.this_manifest_only) project_header_path = project.RelPath(
out.project('project %s/' % project_header_path) local=opt.this_manifest_only
out.nl() )
buf.write(output) out.project("project %s/" % project_header_path)
output = buf.getvalue() out.nl()
return (result.returncode, output) buf.write(output)
output = buf.getvalue()
return (result.returncode, output)

42
subcmds/gitc_delete.py
View File

@ -19,28 +19,34 @@ import platform_utils
class GitcDelete(Command, GitcClientCommand): class GitcDelete(Command, GitcClientCommand):
COMMON = True COMMON = True
visible_everywhere = False visible_everywhere = False
helpSummary = "Delete a GITC Client." helpSummary = "Delete a GITC Client."
helpUsage = """ helpUsage = """
%prog %prog
""" """
helpDescription = """ helpDescription = """
This subcommand deletes the current GITC client, deleting the GITC manifest This subcommand deletes the current GITC client, deleting the GITC manifest
and all locally downloaded sources. and all locally downloaded sources.
""" """
def _Options(self, p): def _Options(self, p):
p.add_option('-f', '--force', p.add_option(
dest='force', action='store_true', "-f",
help='force the deletion (no prompt)') "--force",
dest="force",
action="store_true",
help="force the deletion (no prompt)",
)
def Execute(self, opt, args): def Execute(self, opt, args):
if not opt.force: if not opt.force:
prompt = ('This will delete GITC client: %s\nAre you sure? (yes/no) ' % prompt = (
self.gitc_manifest.gitc_client_name) "This will delete GITC client: %s\nAre you sure? (yes/no) "
response = input(prompt).lower() % self.gitc_manifest.gitc_client_name
if not response == 'yes': )
print('Response was not "yes"\n Exiting...') response = input(prompt).lower()
sys.exit(1) if not response == "yes":
platform_utils.rmtree(self.gitc_manifest.gitc_client_dir) print('Response was not "yes"\n Exiting...')
sys.exit(1)
platform_utils.rmtree(self.gitc_manifest.gitc_client_dir)

69
subcmds/gitc_init.py
View File

@ -23,13 +23,13 @@ import wrapper
class GitcInit(init.Init, GitcAvailableCommand): class GitcInit(init.Init, GitcAvailableCommand):
COMMON = True COMMON = True
MULTI_MANIFEST_SUPPORT = False MULTI_MANIFEST_SUPPORT = False
helpSummary = "Initialize a GITC Client." helpSummary = "Initialize a GITC Client."
helpUsage = """ helpUsage = """
%prog [options] [client name] %prog [options] [client name]
""" """
helpDescription = """ helpDescription = """
The '%prog' command is ran to initialize a new GITC client for use The '%prog' command is ran to initialize a new GITC client for use
with the GITC file system. with the GITC file system.
@ -47,30 +47,41 @@ The optional -f argument can be used to specify the manifest file to
use for this GITC client. use for this GITC client.
""" """
def _Options(self, p): def _Options(self, p):
super()._Options(p, gitc_init=True) super()._Options(p, gitc_init=True)
def Execute(self, opt, args): def Execute(self, opt, args):
gitc_client = gitc_utils.parse_clientdir(os.getcwd()) gitc_client = gitc_utils.parse_clientdir(os.getcwd())
if not gitc_client or (opt.gitc_client and gitc_client != opt.gitc_client): if not gitc_client or (
print('fatal: Please update your repo command. See go/gitc for instructions.', opt.gitc_client and gitc_client != opt.gitc_client
file=sys.stderr) ):
sys.exit(1) print(
self.client_dir = os.path.join(gitc_utils.get_gitc_manifest_dir(), "fatal: Please update your repo command. See go/gitc for "
gitc_client) "instructions.",
super().Execute(opt, args) file=sys.stderr,
)
sys.exit(1)
self.client_dir = os.path.join(
gitc_utils.get_gitc_manifest_dir(), gitc_client
)
super().Execute(opt, args)
manifest_file = self.manifest.manifestFile manifest_file = self.manifest.manifestFile
if opt.manifest_file: if opt.manifest_file:
if not os.path.exists(opt.manifest_file): if not os.path.exists(opt.manifest_file):
print('fatal: Specified manifest file %s does not exist.' % print(
opt.manifest_file) "fatal: Specified manifest file %s does not exist."
sys.exit(1) % opt.manifest_file
manifest_file = opt.manifest_file )
sys.exit(1)
manifest_file = opt.manifest_file
manifest = GitcManifest(self.repodir, os.path.join(self.client_dir, manifest = GitcManifest(
'.manifest')) self.repodir, os.path.join(self.client_dir, ".manifest")
manifest.Override(manifest_file) )
gitc_utils.generate_gitc_manifest(None, manifest) manifest.Override(manifest_file)
print('Please run `cd %s` to view your GITC client.' % gitc_utils.generate_gitc_manifest(None, manifest)
os.path.join(wrapper.Wrapper().GITC_FS_ROOT_DIR, gitc_client)) print(
"Please run `cd %s` to view your GITC client."
% os.path.join(wrapper.Wrapper().GITC_FS_ROOT_DIR, gitc_client)
)

481
subcmds/grep.py
View File

@ -22,19 +22,19 @@ from git_command import GitCommand
class GrepColoring(Coloring): class GrepColoring(Coloring):
def __init__(self, config): def __init__(self, config):
Coloring.__init__(self, config, 'grep') Coloring.__init__(self, config, "grep")
self.project = self.printer('project', attr='bold') self.project = self.printer("project", attr="bold")
self.fail = self.printer('fail', fg='red') self.fail = self.printer("fail", fg="red")
class Grep(PagedCommand): class Grep(PagedCommand):
COMMON = True COMMON = True
helpSummary = "Print lines matching a pattern" helpSummary = "Print lines matching a pattern"
helpUsage = """ helpUsage = """
%prog {pattern | -e pattern} [<project>...] %prog {pattern | -e pattern} [<project>...]
""" """
helpDescription = """ helpDescription = """
Search for the specified patterns in all project files. Search for the specified patterns in all project files.
# Boolean Options # Boolean Options
@ -62,215 +62,304 @@ contain a line that matches both expressions:
repo grep --all-match -e NODE -e Unexpected repo grep --all-match -e NODE -e Unexpected
""" """
PARALLEL_JOBS = DEFAULT_LOCAL_JOBS PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
@staticmethod @staticmethod
def _carry_option(_option, opt_str, value, parser): def _carry_option(_option, opt_str, value, parser):
pt = getattr(parser.values, 'cmd_argv', None) pt = getattr(parser.values, "cmd_argv", None)
if pt is None: if pt is None:
pt = [] pt = []
setattr(parser.values, 'cmd_argv', pt) setattr(parser.values, "cmd_argv", pt)
if opt_str == '-(': if opt_str == "-(":
pt.append('(') pt.append("(")
elif opt_str == '-)': elif opt_str == "-)":
pt.append(')') pt.append(")")
else: else:
pt.append(opt_str) pt.append(opt_str)
if value is not None: if value is not None:
pt.append(value) pt.append(value)
def _CommonOptions(self, p): def _CommonOptions(self, p):
"""Override common options slightly.""" """Override common options slightly."""
super()._CommonOptions(p, opt_v=False) super()._CommonOptions(p, opt_v=False)
def _Options(self, p): def _Options(self, p):
g = p.add_option_group('Sources') g = p.add_option_group("Sources")
g.add_option('--cached', g.add_option(
action='callback', callback=self._carry_option, "--cached",
help='Search the index, instead of the work tree') action="callback",
g.add_option('-r', '--revision', callback=self._carry_option,
dest='revision', action='append', metavar='TREEish', help="Search the index, instead of the work tree",
help='Search TREEish, instead of the work tree') )
g.add_option(
"-r",
"--revision",
dest="revision",
action="append",
metavar="TREEish",
help="Search TREEish, instead of the work tree",
)
g = p.add_option_group('Pattern') g = p.add_option_group("Pattern")
g.add_option('-e', g.add_option(
action='callback', callback=self._carry_option, "-e",
metavar='PATTERN', type='str', action="callback",
help='Pattern to search for') callback=self._carry_option,
g.add_option('-i', '--ignore-case', metavar="PATTERN",
action='callback', callback=self._carry_option, type="str",
help='Ignore case differences') help="Pattern to search for",
g.add_option('-a', '--text', )
action='callback', callback=self._carry_option, g.add_option(
help="Process binary files as if they were text") "-i",
g.add_option('-I', "--ignore-case",
action='callback', callback=self._carry_option, action="callback",
help="Don't match the pattern in binary files") callback=self._carry_option,
g.add_option('-w', '--word-regexp', help="Ignore case differences",
action='callback', callback=self._carry_option, )
help='Match the pattern only at word boundaries') g.add_option(
g.add_option('-v', '--invert-match', "-a",
action='callback', callback=self._carry_option, "--text",
help='Select non-matching lines') action="callback",
g.add_option('-G', '--basic-regexp', callback=self._carry_option,
action='callback', callback=self._carry_option, help="Process binary files as if they were text",
help='Use POSIX basic regexp for patterns (default)') )
g.add_option('-E', '--extended-regexp', g.add_option(
action='callback', callback=self._carry_option, "-I",
help='Use POSIX extended regexp for patterns') action="callback",
g.add_option('-F', '--fixed-strings', callback=self._carry_option,
action='callback', callback=self._carry_option, help="Don't match the pattern in binary files",
help='Use fixed strings (not regexp) for pattern') )
g.add_option(
"-w",
"--word-regexp",
action="callback",
callback=self._carry_option,
help="Match the pattern only at word boundaries",
)
g.add_option(
"-v",
"--invert-match",
action="callback",
callback=self._carry_option,
help="Select non-matching lines",
)
g.add_option(
"-G",
"--basic-regexp",
action="callback",
callback=self._carry_option,
help="Use POSIX basic regexp for patterns (default)",
)
g.add_option(
"-E",
"--extended-regexp",
action="callback",
callback=self._carry_option,
help="Use POSIX extended regexp for patterns",
)
g.add_option(
"-F",
"--fixed-strings",
action="callback",
callback=self._carry_option,
help="Use fixed strings (not regexp) for pattern",
)
g = p.add_option_group('Pattern Grouping') g = p.add_option_group("Pattern Grouping")
g.add_option('--all-match', g.add_option(
action='callback', callback=self._carry_option, "--all-match",
help='Limit match to lines that have all patterns') action="callback",
g.add_option('--and', '--or', '--not', callback=self._carry_option,
action='callback', callback=self._carry_option, help="Limit match to lines that have all patterns",
help='Boolean operators to combine patterns') )
g.add_option('-(', '-)', g.add_option(
action='callback', callback=self._carry_option, "--and",
help='Boolean operator grouping') "--or",
"--not",
action="callback",
callback=self._carry_option,
help="Boolean operators to combine patterns",
)
g.add_option(
"-(",
"-)",
action="callback",
callback=self._carry_option,
help="Boolean operator grouping",
)
g = p.add_option_group('Output') g = p.add_option_group("Output")
g.add_option('-n', g.add_option(
action='callback', callback=self._carry_option, "-n",
help='Prefix the line number to matching lines') action="callback",
g.add_option('-C', callback=self._carry_option,
action='callback', callback=self._carry_option, help="Prefix the line number to matching lines",
metavar='CONTEXT', type='str', )
help='Show CONTEXT lines around match') g.add_option(
g.add_option('-B', "-C",
action='callback', callback=self._carry_option, action="callback",
metavar='CONTEXT', type='str', callback=self._carry_option,
help='Show CONTEXT lines before match') metavar="CONTEXT",
g.add_option('-A', type="str",
action='callback', callback=self._carry_option, help="Show CONTEXT lines around match",
metavar='CONTEXT', type='str', )
help='Show CONTEXT lines after match') g.add_option(
g.add_option('-l', '--name-only', '--files-with-matches', "-B",
action='callback', callback=self._carry_option, action="callback",
help='Show only file names containing matching lines') callback=self._carry_option,
g.add_option('-L', '--files-without-match', metavar="CONTEXT",
action='callback', callback=self._carry_option, type="str",
help='Show only file names not containing matching lines') help="Show CONTEXT lines before match",
)
g.add_option(
"-A",
action="callback",
callback=self._carry_option,
metavar="CONTEXT",
type="str",
help="Show CONTEXT lines after match",
)
g.add_option(
"-l",
"--name-only",
"--files-with-matches",
action="callback",
callback=self._carry_option,
help="Show only file names containing matching lines",
)
g.add_option(
"-L",
"--files-without-match",
action="callback",
callback=self._carry_option,
help="Show only file names not containing matching lines",
)
def _ExecuteOne(self, cmd_argv, project): def _ExecuteOne(self, cmd_argv, project):
"""Process one project.""" """Process one project."""
try: try:
p = GitCommand(project, p = GitCommand(
cmd_argv, project,
bare=False, cmd_argv,
capture_stdout=True, bare=False,
capture_stderr=True) capture_stdout=True,
except GitError as e: capture_stderr=True,
return (project, -1, None, str(e)) )
except GitError as e:
return (project, -1, None, str(e))
return (project, p.Wait(), p.stdout, p.stderr) return (project, p.Wait(), p.stdout, p.stderr)
@staticmethod @staticmethod
def _ProcessResults(full_name, have_rev, opt, _pool, out, results): def _ProcessResults(full_name, have_rev, opt, _pool, out, results):
git_failed = False git_failed = False
bad_rev = False bad_rev = False
have_match = False have_match = False
_RelPath = lambda p: p.RelPath(local=opt.this_manifest_only) _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
for project, rc, stdout, stderr in results: for project, rc, stdout, stderr in results:
if rc < 0: if rc < 0:
git_failed = True git_failed = True
out.project('--- project %s ---' % _RelPath(project)) out.project("--- project %s ---" % _RelPath(project))
out.nl() out.nl()
out.fail('%s', stderr) out.fail("%s", stderr)
out.nl() out.nl()
continue continue
if rc: if rc:
# no results # no results
if stderr: if stderr:
if have_rev and 'fatal: ambiguous argument' in stderr: if have_rev and "fatal: ambiguous argument" in stderr:
bad_rev = True bad_rev = True
else: else:
out.project('--- project %s ---' % _RelPath(project)) out.project("--- project %s ---" % _RelPath(project))
out.nl() out.nl()
out.fail('%s', stderr.strip()) out.fail("%s", stderr.strip())
out.nl() out.nl()
continue continue
have_match = True have_match = True
# We cut the last element, to avoid a blank line. # We cut the last element, to avoid a blank line.
r = stdout.split('\n') r = stdout.split("\n")
r = r[0:-1] r = r[0:-1]
if have_rev and full_name: if have_rev and full_name:
for line in r: for line in r:
rev, line = line.split(':', 1) rev, line = line.split(":", 1)
out.write("%s", rev) out.write("%s", rev)
out.write(':') out.write(":")
out.project(_RelPath(project)) out.project(_RelPath(project))
out.write('/') out.write("/")
out.write("%s", line) out.write("%s", line)
out.nl() out.nl()
elif full_name: elif full_name:
for line in r: for line in r:
out.project(_RelPath(project)) out.project(_RelPath(project))
out.write('/') out.write("/")
out.write("%s", line) out.write("%s", line)
out.nl() out.nl()
else: else:
for line in r: for line in r:
print(line) print(line)
return (git_failed, bad_rev, have_match) return (git_failed, bad_rev, have_match)
def Execute(self, opt, args): def Execute(self, opt, args):
out = GrepColoring(self.manifest.manifestProject.config) out = GrepColoring(self.manifest.manifestProject.config)
cmd_argv = ['grep'] cmd_argv = ["grep"]
if out.is_on: if out.is_on:
cmd_argv.append('--color') cmd_argv.append("--color")
cmd_argv.extend(getattr(opt, 'cmd_argv', [])) cmd_argv.extend(getattr(opt, "cmd_argv", []))
if '-e' not in cmd_argv: if "-e" not in cmd_argv:
if not args: if not args:
self.Usage() self.Usage()
cmd_argv.append('-e') cmd_argv.append("-e")
cmd_argv.append(args[0]) cmd_argv.append(args[0])
args = args[1:] args = args[1:]
projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only) projects = self.GetProjects(
args, all_manifests=not opt.this_manifest_only
)
full_name = False full_name = False
if len(projects) > 1: if len(projects) > 1:
cmd_argv.append('--full-name') cmd_argv.append("--full-name")
full_name = True full_name = True
have_rev = False have_rev = False
if opt.revision: if opt.revision:
if '--cached' in cmd_argv: if "--cached" in cmd_argv:
print('fatal: cannot combine --cached and --revision', file=sys.stderr) print(
sys.exit(1) "fatal: cannot combine --cached and --revision",
have_rev = True file=sys.stderr,
cmd_argv.extend(opt.revision) )
cmd_argv.append('--') sys.exit(1)
have_rev = True
cmd_argv.extend(opt.revision)
cmd_argv.append("--")
git_failed, bad_rev, have_match = self.ExecuteInParallel( git_failed, bad_rev, have_match = self.ExecuteInParallel(
opt.jobs, opt.jobs,
functools.partial(self._ExecuteOne, cmd_argv), functools.partial(self._ExecuteOne, cmd_argv),
projects, projects,
callback=functools.partial(self._ProcessResults, full_name, have_rev, opt), callback=functools.partial(
output=out, self._ProcessResults, full_name, have_rev, opt
ordered=True) ),
output=out,
ordered=True,
)
if git_failed: if git_failed:
sys.exit(1) sys.exit(1)
elif have_match: elif have_match:
sys.exit(0) sys.exit(0)
elif have_rev and bad_rev: elif have_rev and bad_rev:
for r in opt.revision: for r in opt.revision:
print("error: can't search revision %s" % r, file=sys.stderr) print("error: can't search revision %s" % r, file=sys.stderr)
sys.exit(1) sys.exit(1)
else: else:
sys.exit(1) sys.exit(1)

282
subcmds/help.py
View File

@ -18,163 +18,193 @@ import textwrap
from subcmds import all_commands from subcmds import all_commands
from color import Coloring from color import Coloring
from command import PagedCommand, MirrorSafeCommand, GitcAvailableCommand, GitcClientCommand from command import (
PagedCommand,
MirrorSafeCommand,
GitcAvailableCommand,
GitcClientCommand,
)
import gitc_utils import gitc_utils
from wrapper import Wrapper from wrapper import Wrapper
class Help(PagedCommand, MirrorSafeCommand): class Help(PagedCommand, MirrorSafeCommand):
COMMON = False COMMON = False
helpSummary = "Display detailed help on a command" helpSummary = "Display detailed help on a command"
helpUsage = """ helpUsage = """
%prog [--all|command] %prog [--all|command]
""" """
helpDescription = """ helpDescription = """
Displays detailed usage information about a command. Displays detailed usage information about a command.
""" """
def _PrintCommands(self, commandNames): def _PrintCommands(self, commandNames):
"""Helper to display |commandNames| summaries.""" """Helper to display |commandNames| summaries."""
maxlen = 0 maxlen = 0
for name in commandNames: for name in commandNames:
maxlen = max(maxlen, len(name)) maxlen = max(maxlen, len(name))
fmt = ' %%-%ds %%s' % maxlen fmt = " %%-%ds %%s" % maxlen
for name in commandNames: for name in commandNames:
command = all_commands[name]() command = all_commands[name]()
try: try:
summary = command.helpSummary.strip() summary = command.helpSummary.strip()
except AttributeError: except AttributeError:
summary = '' summary = ""
print(fmt % (name, summary)) print(fmt % (name, summary))
def _PrintAllCommands(self): def _PrintAllCommands(self):
print('usage: repo COMMAND [ARGS]') print("usage: repo COMMAND [ARGS]")
self.PrintAllCommandsBody() self.PrintAllCommandsBody()
def PrintAllCommandsBody(self): def PrintAllCommandsBody(self):
print('The complete list of recognized repo commands is:') print("The complete list of recognized repo commands is:")
commandNames = list(sorted(all_commands)) commandNames = list(sorted(all_commands))
self._PrintCommands(commandNames) self._PrintCommands(commandNames)
print("See 'repo help <command>' for more information on a " print(
'specific command.') "See 'repo help <command>' for more information on a "
print('Bug reports:', Wrapper().BUG_URL) "specific command."
)
print("Bug reports:", Wrapper().BUG_URL)
def _PrintCommonCommands(self): def _PrintCommonCommands(self):
print('usage: repo COMMAND [ARGS]') print("usage: repo COMMAND [ARGS]")
self.PrintCommonCommandsBody() self.PrintCommonCommandsBody()
def PrintCommonCommandsBody(self): def PrintCommonCommandsBody(self):
print('The most commonly used repo commands are:') print("The most commonly used repo commands are:")
def gitc_supported(cmd): def gitc_supported(cmd):
if not isinstance(cmd, GitcAvailableCommand) and not isinstance(cmd, GitcClientCommand): if not isinstance(cmd, GitcAvailableCommand) and not isinstance(
return True cmd, GitcClientCommand
if self.client.isGitcClient: ):
return True return True
if isinstance(cmd, GitcClientCommand): if self.client.isGitcClient:
return False return True
if gitc_utils.get_gitc_manifest_dir(): if isinstance(cmd, GitcClientCommand):
return True return False
return False if gitc_utils.get_gitc_manifest_dir():
return True
return False
commandNames = list(sorted([name commandNames = list(
for name, command in all_commands.items() sorted(
if command.COMMON and gitc_supported(command)])) [
self._PrintCommands(commandNames) name
for name, command in all_commands.items()
if command.COMMON and gitc_supported(command)
]
)
)
self._PrintCommands(commandNames)
print( print(
"See 'repo help <command>' for more information on a specific command.\n" "See 'repo help <command>' for more information on a specific "
"See 'repo help --all' for a complete list of recognized commands.") "command.\nSee 'repo help --all' for a complete list of recognized "
print('Bug reports:', Wrapper().BUG_URL) "commands."
)
print("Bug reports:", Wrapper().BUG_URL)
def _PrintCommandHelp(self, cmd, header_prefix=''): def _PrintCommandHelp(self, cmd, header_prefix=""):
class _Out(Coloring): class _Out(Coloring):
def __init__(self, gc): def __init__(self, gc):
Coloring.__init__(self, gc, 'help') Coloring.__init__(self, gc, "help")
self.heading = self.printer('heading', attr='bold') self.heading = self.printer("heading", attr="bold")
self._first = True self._first = True
def _PrintSection(self, heading, bodyAttr): def _PrintSection(self, heading, bodyAttr):
try: try:
body = getattr(cmd, bodyAttr) body = getattr(cmd, bodyAttr)
except AttributeError: except AttributeError:
return return
if body == '' or body is None: if body == "" or body is None:
return return
if not self._first: if not self._first:
self.nl() self.nl()
self._first = False self._first = False
self.heading('%s%s', header_prefix, heading) self.heading("%s%s", header_prefix, heading)
self.nl() self.nl()
self.nl() self.nl()
me = 'repo %s' % cmd.NAME me = "repo %s" % cmd.NAME
body = body.strip() body = body.strip()
body = body.replace('%prog', me) body = body.replace("%prog", me)
# Extract the title, but skip any trailing {#anchors}. # Extract the title, but skip any trailing {#anchors}.
asciidoc_hdr = re.compile(r'^\n?#+ ([^{]+)(\{#.+\})?$') asciidoc_hdr = re.compile(r"^\n?#+ ([^{]+)(\{#.+\})?$")
for para in body.split("\n\n"): for para in body.split("\n\n"):
if para.startswith(' '): if para.startswith(" "):
self.write('%s', para) self.write("%s", para)
self.nl() self.nl()
self.nl() self.nl()
continue continue
m = asciidoc_hdr.match(para) m = asciidoc_hdr.match(para)
if m: if m:
self.heading('%s%s', header_prefix, m.group(1)) self.heading("%s%s", header_prefix, m.group(1))
self.nl() self.nl()
self.nl() self.nl()
continue continue
lines = textwrap.wrap(para.replace(' ', ' '), width=80, lines = textwrap.wrap(
break_long_words=False, break_on_hyphens=False) para.replace(" ", " "),
for line in lines: width=80,
self.write('%s', line) break_long_words=False,
self.nl() break_on_hyphens=False,
self.nl() )
for line in lines:
self.write("%s", line)
self.nl()
self.nl()
out = _Out(self.client.globalConfig) out = _Out(self.client.globalConfig)
out._PrintSection('Summary', 'helpSummary') out._PrintSection("Summary", "helpSummary")
cmd.OptionParser.print_help() cmd.OptionParser.print_help()
out._PrintSection('Description', 'helpDescription') out._PrintSection("Description", "helpDescription")
def _PrintAllCommandHelp(self): def _PrintAllCommandHelp(self):
for name in sorted(all_commands): for name in sorted(all_commands):
cmd = all_commands[name](manifest=self.manifest) cmd = all_commands[name](manifest=self.manifest)
self._PrintCommandHelp(cmd, header_prefix='[%s] ' % (name,)) self._PrintCommandHelp(cmd, header_prefix="[%s] " % (name,))
def _Options(self, p): def _Options(self, p):
p.add_option('-a', '--all', p.add_option(
dest='show_all', action='store_true', "-a",
help='show the complete list of commands') "--all",
p.add_option('--help-all', dest="show_all",
dest='show_all_help', action='store_true', action="store_true",
help='show the --help of all commands') help="show the complete list of commands",
)
p.add_option(
"--help-all",
dest="show_all_help",
action="store_true",
help="show the --help of all commands",
)
def Execute(self, opt, args): def Execute(self, opt, args):
if len(args) == 0: if len(args) == 0:
if opt.show_all_help: if opt.show_all_help:
self._PrintAllCommandHelp() self._PrintAllCommandHelp()
elif opt.show_all: elif opt.show_all:
self._PrintAllCommands() self._PrintAllCommands()
else: else:
self._PrintCommonCommands() self._PrintCommonCommands()
elif len(args) == 1: elif len(args) == 1:
name = args[0] name = args[0]
try: try:
cmd = all_commands[name](manifest=self.manifest) cmd = all_commands[name](manifest=self.manifest)
except KeyError: except KeyError:
print("repo: '%s' is not a repo command." % name, file=sys.stderr) print(
sys.exit(1) "repo: '%s' is not a repo command." % name, file=sys.stderr
)
sys.exit(1)
self._PrintCommandHelp(cmd) self._PrintCommandHelp(cmd)
else: else:
self._PrintCommandHelp(self) self._PrintCommandHelp(self)

399
subcmds/info.py
View File

@ -20,203 +20,234 @@ from git_refs import R_M, R_HEADS
class _Coloring(Coloring): class _Coloring(Coloring):
def __init__(self, config): def __init__(self, config):
Coloring.__init__(self, config, "status") Coloring.__init__(self, config, "status")
class Info(PagedCommand): class Info(PagedCommand):
COMMON = True COMMON = True
helpSummary = "Get info on the manifest branch, current branch or unmerged branches" helpSummary = (
helpUsage = "%prog [-dl] [-o [-c]] [<project>...]" "Get info on the manifest branch, current branch or unmerged branches"
)
helpUsage = "%prog [-dl] [-o [-c]] [<project>...]"
def _Options(self, p): def _Options(self, p):
p.add_option('-d', '--diff', p.add_option(
dest='all', action='store_true', "-d",
help="show full info and commit diff including remote branches") "--diff",
p.add_option('-o', '--overview', dest="all",
dest='overview', action='store_true', action="store_true",
help='show overview of all local commits') help="show full info and commit diff including remote branches",
p.add_option('-c', '--current-branch', )
dest="current_branch", action="store_true", p.add_option(
help="consider only checked out branches") "-o",
p.add_option('--no-current-branch', "--overview",
dest='current_branch', action='store_false', dest="overview",
help='consider all local branches') action="store_true",
# Turn this into a warning & remove this someday. help="show overview of all local commits",
p.add_option('-b', )
dest='current_branch', action='store_true', p.add_option(
help=optparse.SUPPRESS_HELP) "-c",
p.add_option('-l', '--local-only', "--current-branch",
dest="local", action="store_true", dest="current_branch",
help="disable all remote operations") action="store_true",
help="consider only checked out branches",
)
p.add_option(
"--no-current-branch",
dest="current_branch",
action="store_false",
help="consider all local branches",
)
# Turn this into a warning & remove this someday.
p.add_option(
"-b",
dest="current_branch",
action="store_true",
help=optparse.SUPPRESS_HELP,
)
p.add_option(
"-l",
"--local-only",
dest="local",
action="store_true",
help="disable all remote operations",
)
def Execute(self, opt, args): def Execute(self, opt, args):
self.out = _Coloring(self.client.globalConfig) self.out = _Coloring(self.client.globalConfig)
self.heading = self.out.printer('heading', attr='bold') self.heading = self.out.printer("heading", attr="bold")
self.headtext = self.out.nofmt_printer('headtext', fg='yellow') self.headtext = self.out.nofmt_printer("headtext", fg="yellow")
self.redtext = self.out.printer('redtext', fg='red') self.redtext = self.out.printer("redtext", fg="red")
self.sha = self.out.printer("sha", fg='yellow') self.sha = self.out.printer("sha", fg="yellow")
self.text = self.out.nofmt_printer('text') self.text = self.out.nofmt_printer("text")
self.dimtext = self.out.printer('dimtext', attr='dim') self.dimtext = self.out.printer("dimtext", attr="dim")
self.opt = opt self.opt = opt
if not opt.this_manifest_only: if not opt.this_manifest_only:
self.manifest = self.manifest.outer_client self.manifest = self.manifest.outer_client
manifestConfig = self.manifest.manifestProject.config manifestConfig = self.manifest.manifestProject.config
mergeBranch = manifestConfig.GetBranch("default").merge mergeBranch = manifestConfig.GetBranch("default").merge
manifestGroups = self.manifest.GetGroupsStr() manifestGroups = self.manifest.GetGroupsStr()
self.heading("Manifest branch: ") self.heading("Manifest branch: ")
if self.manifest.default.revisionExpr: if self.manifest.default.revisionExpr:
self.headtext(self.manifest.default.revisionExpr) self.headtext(self.manifest.default.revisionExpr)
self.out.nl() self.out.nl()
self.heading("Manifest merge branch: ") self.heading("Manifest merge branch: ")
self.headtext(mergeBranch) self.headtext(mergeBranch)
self.out.nl() self.out.nl()
self.heading("Manifest groups: ") self.heading("Manifest groups: ")
self.headtext(manifestGroups) self.headtext(manifestGroups)
self.out.nl()
self.printSeparator()
if not opt.overview:
self._printDiffInfo(opt, args)
else:
self._printCommitOverview(opt, args)
def printSeparator(self):
self.text("----------------------------")
self.out.nl()
def _printDiffInfo(self, opt, args):
# We let exceptions bubble up to main as they'll be well structured.
projs = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
for p in projs:
self.heading("Project: ")
self.headtext(p.name)
self.out.nl()
self.heading("Mount path: ")
self.headtext(p.worktree)
self.out.nl()
self.heading("Current revision: ")
self.headtext(p.GetRevisionId())
self.out.nl()
currentBranch = p.CurrentBranch
if currentBranch:
self.heading('Current branch: ')
self.headtext(currentBranch)
self.out.nl() self.out.nl()
self.heading("Manifest revision: ") self.printSeparator()
self.headtext(p.revisionExpr)
self.out.nl()
localBranches = list(p.GetBranches().keys()) if not opt.overview:
self.heading("Local Branches: ") self._printDiffInfo(opt, args)
self.redtext(str(len(localBranches))) else:
if localBranches: self._printCommitOverview(opt, args)
self.text(" [")
self.text(", ".join(localBranches))
self.text("]")
self.out.nl()
if self.opt.all: def printSeparator(self):
self.findRemoteLocalDiff(p) self.text("----------------------------")
self.printSeparator()
def findRemoteLocalDiff(self, project):
# Fetch all the latest commits.
if not self.opt.local:
project.Sync_NetworkHalf(quiet=True, current_branch_only=True)
branch = self.manifest.manifestProject.config.GetBranch('default').merge
if branch.startswith(R_HEADS):
branch = branch[len(R_HEADS):]
logTarget = R_M + branch
bareTmp = project.bare_git._bare
project.bare_git._bare = False
localCommits = project.bare_git.rev_list(
'--abbrev=8',
'--abbrev-commit',
'--pretty=oneline',
logTarget + "..",
'--')
originCommits = project.bare_git.rev_list(
'--abbrev=8',
'--abbrev-commit',
'--pretty=oneline',
".." + logTarget,
'--')
project.bare_git._bare = bareTmp
self.heading("Local Commits: ")
self.redtext(str(len(localCommits)))
self.dimtext(" (on current branch)")
self.out.nl()
for c in localCommits:
split = c.split()
self.sha(split[0] + " ")
self.text(" ".join(split[1:]))
self.out.nl()
self.printSeparator()
self.heading("Remote Commits: ")
self.redtext(str(len(originCommits)))
self.out.nl()
for c in originCommits:
split = c.split()
self.sha(split[0] + " ")
self.text(" ".join(split[1:]))
self.out.nl()
def _printCommitOverview(self, opt, args):
all_branches = []
for project in self.GetProjects(args, all_manifests=not opt.this_manifest_only):
br = [project.GetUploadableBranch(x)
for x in project.GetBranches()]
br = [x for x in br if x]
if self.opt.current_branch:
br = [x for x in br if x.name == project.CurrentBranch]
all_branches.extend(br)
if not all_branches:
return
self.out.nl()
self.heading('Projects Overview')
project = None
for branch in all_branches:
if project != branch.project:
project = branch.project
self.out.nl()
self.headtext(project.RelPath(local=opt.this_manifest_only))
self.out.nl() self.out.nl()
commits = branch.commits def _printDiffInfo(self, opt, args):
date = branch.date # We let exceptions bubble up to main as they'll be well structured.
self.text('%s %-33s (%2d commit%s, %s)' % ( projs = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
branch.name == project.CurrentBranch and '*' or ' ',
branch.name,
len(commits),
len(commits) != 1 and 's' or '',
date))
self.out.nl()
for commit in commits: for p in projs:
split = commit.split() self.heading("Project: ")
self.text('{0:38}{1} '.format('', '-')) self.headtext(p.name)
self.sha(split[0] + " ") self.out.nl()
self.text(" ".join(split[1:]))
self.heading("Mount path: ")
self.headtext(p.worktree)
self.out.nl()
self.heading("Current revision: ")
self.headtext(p.GetRevisionId())
self.out.nl()
currentBranch = p.CurrentBranch
if currentBranch:
self.heading("Current branch: ")
self.headtext(currentBranch)
self.out.nl()
self.heading("Manifest revision: ")
self.headtext(p.revisionExpr)
self.out.nl()
localBranches = list(p.GetBranches().keys())
self.heading("Local Branches: ")
self.redtext(str(len(localBranches)))
if localBranches:
self.text(" [")
self.text(", ".join(localBranches))
self.text("]")
self.out.nl()
if self.opt.all:
self.findRemoteLocalDiff(p)
self.printSeparator()
def findRemoteLocalDiff(self, project):
# Fetch all the latest commits.
if not self.opt.local:
project.Sync_NetworkHalf(quiet=True, current_branch_only=True)
branch = self.manifest.manifestProject.config.GetBranch("default").merge
if branch.startswith(R_HEADS):
branch = branch[len(R_HEADS) :]
logTarget = R_M + branch
bareTmp = project.bare_git._bare
project.bare_git._bare = False
localCommits = project.bare_git.rev_list(
"--abbrev=8",
"--abbrev-commit",
"--pretty=oneline",
logTarget + "..",
"--",
)
originCommits = project.bare_git.rev_list(
"--abbrev=8",
"--abbrev-commit",
"--pretty=oneline",
".." + logTarget,
"--",
)
project.bare_git._bare = bareTmp
self.heading("Local Commits: ")
self.redtext(str(len(localCommits)))
self.dimtext(" (on current branch)")
self.out.nl() self.out.nl()
for c in localCommits:
split = c.split()
self.sha(split[0] + " ")
self.text(" ".join(split[1:]))
self.out.nl()
self.printSeparator()
self.heading("Remote Commits: ")
self.redtext(str(len(originCommits)))
self.out.nl()
for c in originCommits:
split = c.split()
self.sha(split[0] + " ")
self.text(" ".join(split[1:]))
self.out.nl()
def _printCommitOverview(self, opt, args):
all_branches = []
for project in self.GetProjects(
args, all_manifests=not opt.this_manifest_only
):
br = [project.GetUploadableBranch(x) for x in project.GetBranches()]
br = [x for x in br if x]
if self.opt.current_branch:
br = [x for x in br if x.name == project.CurrentBranch]
all_branches.extend(br)
if not all_branches:
return
self.out.nl()
self.heading("Projects Overview")
project = None
for branch in all_branches:
if project != branch.project:
project = branch.project
self.out.nl()
self.headtext(project.RelPath(local=opt.this_manifest_only))
self.out.nl()
commits = branch.commits
date = branch.date
self.text(
"%s %-33s (%2d commit%s, %s)"
% (
branch.name == project.CurrentBranch and "*" or " ",
branch.name,
len(commits),
len(commits) != 1 and "s" or "",
date,
)
)
self.out.nl()
for commit in commits:
split = commit.split()
self.text("{0:38}{1} ".format("", "-"))
self.sha(split[0] + " ")
self.text(" ".join(split[1:]))
self.out.nl()

490
subcmds/init.py
View File

@ -22,13 +22,13 @@ from wrapper import Wrapper
class Init(InteractiveCommand, MirrorSafeCommand): class Init(InteractiveCommand, MirrorSafeCommand):
COMMON = True COMMON = True
MULTI_MANIFEST_SUPPORT = True MULTI_MANIFEST_SUPPORT = True
helpSummary = "Initialize a repo client checkout in the current directory" helpSummary = "Initialize a repo client checkout in the current directory"
helpUsage = """ helpUsage = """
%prog [options] [manifest url] %prog [options] [manifest url]
""" """
helpDescription = """ helpDescription = """
The '%prog' command is run once to install and initialize repo. The '%prog' command is run once to install and initialize repo.
The latest repo source code and manifest collection is downloaded The latest repo source code and manifest collection is downloaded
from the server and is installed in the .repo/ directory in the from the server and is installed in the .repo/ directory in the
@ -77,243 +77,303 @@ manifest, a subsequent `repo sync` (or `repo sync -d`) is necessary
to update the working directory files. to update the working directory files.
""" """
def _CommonOptions(self, p): def _CommonOptions(self, p):
"""Disable due to re-use of Wrapper().""" """Disable due to re-use of Wrapper()."""
def _Options(self, p, gitc_init=False): def _Options(self, p, gitc_init=False):
Wrapper().InitParser(p, gitc_init=gitc_init) Wrapper().InitParser(p, gitc_init=gitc_init)
m = p.add_option_group('Multi-manifest') m = p.add_option_group("Multi-manifest")
m.add_option('--outer-manifest', action='store_true', default=True, m.add_option(
help='operate starting at the outermost manifest') "--outer-manifest",
m.add_option('--no-outer-manifest', dest='outer_manifest', action="store_true",
action='store_false', help='do not operate on outer manifests') default=True,
m.add_option('--this-manifest-only', action='store_true', default=None, help="operate starting at the outermost manifest",
help='only operate on this (sub)manifest') )
m.add_option('--no-this-manifest-only', '--all-manifests', m.add_option(
dest='this_manifest_only', action='store_false', "--no-outer-manifest",
help='operate on this manifest and its submanifests') dest="outer_manifest",
action="store_false",
help="do not operate on outer manifests",
)
m.add_option(
"--this-manifest-only",
action="store_true",
default=None,
help="only operate on this (sub)manifest",
)
m.add_option(
"--no-this-manifest-only",
"--all-manifests",
dest="this_manifest_only",
action="store_false",
help="operate on this manifest and its submanifests",
)
def _RegisteredEnvironmentOptions(self): def _RegisteredEnvironmentOptions(self):
return {'REPO_MANIFEST_URL': 'manifest_url', return {
'REPO_MIRROR_LOCATION': 'reference'} "REPO_MANIFEST_URL": "manifest_url",
"REPO_MIRROR_LOCATION": "reference",
}
def _SyncManifest(self, opt): def _SyncManifest(self, opt):
"""Call manifestProject.Sync with arguments from opt. """Call manifestProject.Sync with arguments from opt.
Args: Args:
opt: options from optparse. opt: options from optparse.
""" """
# Normally this value is set when instantiating the project, but the # Normally this value is set when instantiating the project, but the
# manifest project is special and is created when instantiating the # manifest project is special and is created when instantiating the
# manifest which happens before we parse options. # manifest which happens before we parse options.
self.manifest.manifestProject.clone_depth = opt.manifest_depth self.manifest.manifestProject.clone_depth = opt.manifest_depth
if not self.manifest.manifestProject.Sync( if not self.manifest.manifestProject.Sync(
manifest_url=opt.manifest_url, manifest_url=opt.manifest_url,
manifest_branch=opt.manifest_branch, manifest_branch=opt.manifest_branch,
standalone_manifest=opt.standalone_manifest, standalone_manifest=opt.standalone_manifest,
groups=opt.groups, groups=opt.groups,
platform=opt.platform, platform=opt.platform,
mirror=opt.mirror, mirror=opt.mirror,
dissociate=opt.dissociate, dissociate=opt.dissociate,
reference=opt.reference, reference=opt.reference,
worktree=opt.worktree, worktree=opt.worktree,
submodules=opt.submodules, submodules=opt.submodules,
archive=opt.archive, archive=opt.archive,
partial_clone=opt.partial_clone, partial_clone=opt.partial_clone,
clone_filter=opt.clone_filter, clone_filter=opt.clone_filter,
partial_clone_exclude=opt.partial_clone_exclude, partial_clone_exclude=opt.partial_clone_exclude,
clone_bundle=opt.clone_bundle, clone_bundle=opt.clone_bundle,
git_lfs=opt.git_lfs, git_lfs=opt.git_lfs,
use_superproject=opt.use_superproject, use_superproject=opt.use_superproject,
verbose=opt.verbose, verbose=opt.verbose,
current_branch_only=opt.current_branch_only, current_branch_only=opt.current_branch_only,
tags=opt.tags, tags=opt.tags,
depth=opt.depth, depth=opt.depth,
git_event_log=self.git_event_log, git_event_log=self.git_event_log,
manifest_name=opt.manifest_name): manifest_name=opt.manifest_name,
sys.exit(1) ):
sys.exit(1)
def _Prompt(self, prompt, value): def _Prompt(self, prompt, value):
print('%-10s [%s]: ' % (prompt, value), end='', flush=True) print("%-10s [%s]: " % (prompt, value), end="", flush=True)
a = sys.stdin.readline().strip() a = sys.stdin.readline().strip()
if a == '': if a == "":
return value return value
return a return a
def _ShouldConfigureUser(self, opt, existing_checkout): def _ShouldConfigureUser(self, opt, existing_checkout):
gc = self.client.globalConfig gc = self.client.globalConfig
mp = self.manifest.manifestProject mp = self.manifest.manifestProject
# If we don't have local settings, get from global. # If we don't have local settings, get from global.
if not mp.config.Has('user.name') or not mp.config.Has('user.email'): if not mp.config.Has("user.name") or not mp.config.Has("user.email"):
if not gc.Has('user.name') or not gc.Has('user.email'): if not gc.Has("user.name") or not gc.Has("user.email"):
return True return True
mp.config.SetString('user.name', gc.GetString('user.name')) mp.config.SetString("user.name", gc.GetString("user.name"))
mp.config.SetString('user.email', gc.GetString('user.email')) mp.config.SetString("user.email", gc.GetString("user.email"))
if not opt.quiet and not existing_checkout or opt.verbose: if not opt.quiet and not existing_checkout or opt.verbose:
print() print()
print('Your identity is: %s <%s>' % (mp.config.GetString('user.name'), print(
mp.config.GetString('user.email'))) "Your identity is: %s <%s>"
print("If you want to change this, please re-run 'repo init' with --config-name") % (
return False mp.config.GetString("user.name"),
mp.config.GetString("user.email"),
)
)
print(
"If you want to change this, please re-run 'repo init' with "
"--config-name"
)
return False
def _ConfigureUser(self, opt): def _ConfigureUser(self, opt):
mp = self.manifest.manifestProject mp = self.manifest.manifestProject
while True:
if not opt.quiet:
print()
name = self._Prompt("Your Name", mp.UserName)
email = self._Prompt("Your Email", mp.UserEmail)
if not opt.quiet:
print()
print("Your identity is: %s <%s>" % (name, email))
print("is this correct [y/N]? ", end="", flush=True)
a = sys.stdin.readline().strip().lower()
if a in ("yes", "y", "t", "true"):
break
if name != mp.UserName:
mp.config.SetString("user.name", name)
if email != mp.UserEmail:
mp.config.SetString("user.email", email)
def _HasColorSet(self, gc):
for n in ["ui", "diff", "status"]:
if gc.Has("color.%s" % n):
return True
return False
def _ConfigureColor(self):
gc = self.client.globalConfig
if self._HasColorSet(gc):
return
class _Test(Coloring):
def __init__(self):
Coloring.__init__(self, gc, "test color display")
self._on = True
out = _Test()
while True:
if not opt.quiet:
print() print()
name = self._Prompt('Your Name', mp.UserName) print("Testing colorized output (for 'repo diff', 'repo status'):")
email = self._Prompt('Your Email', mp.UserEmail)
for c in ["black", "red", "green", "yellow", "blue", "magenta", "cyan"]:
out.write(" ")
out.printer(fg=c)(" %-6s ", c)
out.write(" ")
out.printer(fg="white", bg="black")(" %s " % "white")
out.nl()
for c in ["bold", "dim", "ul", "reverse"]:
out.write(" ")
out.printer(fg="black", attr=c)(" %-6s ", c)
out.nl()
print(
"Enable color display in this user account (y/N)? ",
end="",
flush=True,
)
a = sys.stdin.readline().strip().lower()
if a in ("y", "yes", "t", "true", "on"):
gc.SetString("color.ui", "auto")
def _DisplayResult(self):
if self.manifest.IsMirror:
init_type = "mirror "
else:
init_type = ""
if not opt.quiet:
print() print()
print('Your identity is: %s <%s>' % (name, email)) print(
print('is this correct [y/N]? ', end='', flush=True) "repo %shas been initialized in %s"
a = sys.stdin.readline().strip().lower() % (init_type, self.manifest.topdir)
if a in ('yes', 'y', 't', 'true'): )
break
if name != mp.UserName: current_dir = os.getcwd()
mp.config.SetString('user.name', name) if current_dir != self.manifest.topdir:
if email != mp.UserEmail: print(
mp.config.SetString('user.email', email) "If this is not the directory in which you want to initialize "
"repo, please run:"
)
print(" rm -r %s" % os.path.join(self.manifest.topdir, ".repo"))
print("and try again.")
def _HasColorSet(self, gc): def ValidateOptions(self, opt, args):
for n in ['ui', 'diff', 'status']: if opt.reference:
if gc.Has('color.%s' % n): opt.reference = os.path.expanduser(opt.reference)
return True
return False
def _ConfigureColor(self): # Check this here, else manifest will be tagged "not new" and init won't
gc = self.client.globalConfig # be possible anymore without removing the .repo/manifests directory.
if self._HasColorSet(gc): if opt.mirror:
return if opt.archive:
self.OptionParser.error(
"--mirror and --archive cannot be used " "together."
)
if opt.use_superproject is not None:
self.OptionParser.error(
"--mirror and --use-superproject cannot be "
"used together."
)
if opt.archive and opt.use_superproject is not None:
self.OptionParser.error(
"--archive and --use-superproject cannot be used " "together."
)
class _Test(Coloring): if opt.standalone_manifest and (
def __init__(self): opt.manifest_branch or opt.manifest_name != "default.xml"
Coloring.__init__(self, gc, 'test color display') ):
self._on = True self.OptionParser.error(
out = _Test() "--manifest-branch and --manifest-name cannot"
" be used with --standalone-manifest."
)
print() if args:
print("Testing colorized output (for 'repo diff', 'repo status'):") if opt.manifest_url:
self.OptionParser.error(
"--manifest-url option and URL argument both specified: "
"only use one to select the manifest URL."
)
for c in ['black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan']: opt.manifest_url = args.pop(0)
out.write(' ')
out.printer(fg=c)(' %-6s ', c)
out.write(' ')
out.printer(fg='white', bg='black')(' %s ' % 'white')
out.nl()
for c in ['bold', 'dim', 'ul', 'reverse']: if args:
out.write(' ') self.OptionParser.error("too many arguments to init")
out.printer(fg='black', attr=c)(' %-6s ', c)
out.nl()
print('Enable color display in this user account (y/N)? ', end='', flush=True) def Execute(self, opt, args):
a = sys.stdin.readline().strip().lower() git_require(MIN_GIT_VERSION_HARD, fail=True)
if a in ('y', 'yes', 't', 'true', 'on'): if not git_require(MIN_GIT_VERSION_SOFT):
gc.SetString('color.ui', 'auto') print(
"repo: warning: git-%s+ will soon be required; please upgrade "
"your version of git to maintain support."
% (".".join(str(x) for x in MIN_GIT_VERSION_SOFT),),
file=sys.stderr,
)
def _DisplayResult(self): rp = self.manifest.repoProject
if self.manifest.IsMirror:
init_type = 'mirror '
else:
init_type = ''
print() # Handle new --repo-url requests.
print('repo %shas been initialized in %s' % (init_type, self.manifest.topdir)) if opt.repo_url:
remote = rp.GetRemote("origin")
remote.url = opt.repo_url
remote.Save()
current_dir = os.getcwd() # Handle new --repo-rev requests.
if current_dir != self.manifest.topdir: if opt.repo_rev:
print('If this is not the directory in which you want to initialize ' wrapper = Wrapper()
'repo, please run:') try:
print(' rm -r %s' % os.path.join(self.manifest.topdir, '.repo')) remote_ref, rev = wrapper.check_repo_rev(
print('and try again.') rp.gitdir,
opt.repo_rev,
repo_verify=opt.repo_verify,
quiet=opt.quiet,
)
except wrapper.CloneFailure:
print(
"fatal: double check your --repo-rev setting.",
file=sys.stderr,
)
sys.exit(1)
branch = rp.GetBranch("default")
branch.merge = remote_ref
rp.work_git.reset("--hard", rev)
branch.Save()
def ValidateOptions(self, opt, args): if opt.worktree:
if opt.reference: # Older versions of git supported worktree, but had dangerous gc
opt.reference = os.path.expanduser(opt.reference) # bugs.
git_require((2, 15, 0), fail=True, msg="git gc worktree corruption")
# Check this here, else manifest will be tagged "not new" and init won't be # Provide a short notice that we're reinitializing an existing checkout.
# possible anymore without removing the .repo/manifests directory. # Sometimes developers might not realize that they're in one, or that
if opt.mirror: # repo doesn't do nested checkouts.
if opt.archive: existing_checkout = self.manifest.manifestProject.Exists
self.OptionParser.error('--mirror and --archive cannot be used ' if not opt.quiet and existing_checkout:
'together.') print(
if opt.use_superproject is not None: "repo: reusing existing repo client checkout in",
self.OptionParser.error('--mirror and --use-superproject cannot be ' self.manifest.topdir,
'used together.') )
if opt.archive and opt.use_superproject is not None:
self.OptionParser.error('--archive and --use-superproject cannot be used '
'together.')
if opt.standalone_manifest and (opt.manifest_branch or self._SyncManifest(opt)
opt.manifest_name != 'default.xml'):
self.OptionParser.error('--manifest-branch and --manifest-name cannot'
' be used with --standalone-manifest.')
if args: if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror:
if opt.manifest_url: if opt.config_name or self._ShouldConfigureUser(
self.OptionParser.error( opt, existing_checkout
'--manifest-url option and URL argument both specified: only use ' ):
'one to select the manifest URL.') self._ConfigureUser(opt)
self._ConfigureColor()
opt.manifest_url = args.pop(0) if not opt.quiet:
self._DisplayResult()
if args:
self.OptionParser.error('too many arguments to init')
def Execute(self, opt, args):
git_require(MIN_GIT_VERSION_HARD, fail=True)
if not git_require(MIN_GIT_VERSION_SOFT):
print('repo: warning: git-%s+ will soon be required; please upgrade your '
'version of git to maintain support.'
% ('.'.join(str(x) for x in MIN_GIT_VERSION_SOFT),),
file=sys.stderr)
rp = self.manifest.repoProject
# Handle new --repo-url requests.
if opt.repo_url:
remote = rp.GetRemote('origin')
remote.url = opt.repo_url
remote.Save()
# Handle new --repo-rev requests.
if opt.repo_rev:
wrapper = Wrapper()
try:
remote_ref, rev = wrapper.check_repo_rev(
rp.gitdir, opt.repo_rev, repo_verify=opt.repo_verify, quiet=opt.quiet)
except wrapper.CloneFailure:
print('fatal: double check your --repo-rev setting.', file=sys.stderr)
sys.exit(1)
branch = rp.GetBranch('default')
branch.merge = remote_ref
rp.work_git.reset('--hard', rev)
branch.Save()
if opt.worktree:
# Older versions of git supported worktree, but had dangerous gc bugs.
git_require((2, 15, 0), fail=True, msg='git gc worktree corruption')
# Provide a short notice that we're reinitializing an existing checkout.
# Sometimes developers might not realize that they're in one, or that
# repo doesn't do nested checkouts.
existing_checkout = self.manifest.manifestProject.Exists
if not opt.quiet and existing_checkout:
print('repo: reusing existing repo client checkout in', self.manifest.topdir)
self._SyncManifest(opt)
if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror:
if opt.config_name or self._ShouldConfigureUser(opt, existing_checkout):
self._ConfigureUser(opt)
self._ConfigureColor()
if not opt.quiet:
self._DisplayResult()

158
subcmds/list.py
View File

@ -18,13 +18,13 @@ from command import Command, MirrorSafeCommand
class List(Command, MirrorSafeCommand): class List(Command, MirrorSafeCommand):
COMMON = True COMMON = True
helpSummary = "List projects and their associated directories" helpSummary = "List projects and their associated directories"
helpUsage = """ helpUsage = """
%prog [-f] [<project>...] %prog [-f] [<project>...]
%prog [-f] -r str1 [str2]... %prog [-f] -r str1 [str2]...
""" """
helpDescription = """ helpDescription = """
List all projects; pass '.' to list the project for the cwd. List all projects; pass '.' to list the project for the cwd.
By default, only projects that currently exist in the checkout are shown. If By default, only projects that currently exist in the checkout are shown. If
@ -35,69 +35,103 @@ groups, then also pass --groups all.
This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'. This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
""" """
def _Options(self, p): def _Options(self, p):
p.add_option('-r', '--regex', p.add_option(
dest='regex', action='store_true', "-r",
help='filter the project list based on regex or wildcard matching of strings') "--regex",
p.add_option('-g', '--groups', dest="regex",
dest='groups', action="store_true",
help='filter the project list based on the groups the project is in') help="filter the project list based on regex or wildcard matching "
p.add_option('-a', '--all', "of strings",
action='store_true', )
help='show projects regardless of checkout state') p.add_option(
p.add_option('-n', '--name-only', "-g",
dest='name_only', action='store_true', "--groups",
help='display only the name of the repository') dest="groups",
p.add_option('-p', '--path-only', help="filter the project list based on the groups the project is "
dest='path_only', action='store_true', "in",
help='display only the path of the repository') )
p.add_option('-f', '--fullpath', p.add_option(
dest='fullpath', action='store_true', "-a",
help='display the full work tree path instead of the relative path') "--all",
p.add_option('--relative-to', metavar='PATH', action="store_true",
help='display paths relative to this one (default: top of repo client checkout)') help="show projects regardless of checkout state",
)
p.add_option(
"-n",
"--name-only",
dest="name_only",
action="store_true",
help="display only the name of the repository",
)
p.add_option(
"-p",
"--path-only",
dest="path_only",
action="store_true",
help="display only the path of the repository",
)
p.add_option(
"-f",
"--fullpath",
dest="fullpath",
action="store_true",
help="display the full work tree path instead of the relative path",
)
p.add_option(
"--relative-to",
metavar="PATH",
help="display paths relative to this one (default: top of repo "
"client checkout)",
)
def ValidateOptions(self, opt, args): def ValidateOptions(self, opt, args):
if opt.fullpath and opt.name_only: if opt.fullpath and opt.name_only:
self.OptionParser.error('cannot combine -f and -n') self.OptionParser.error("cannot combine -f and -n")
# Resolve any symlinks so the output is stable. # Resolve any symlinks so the output is stable.
if opt.relative_to: if opt.relative_to:
opt.relative_to = os.path.realpath(opt.relative_to) opt.relative_to = os.path.realpath(opt.relative_to)
def Execute(self, opt, args): def Execute(self, opt, args):
"""List all projects and the associated directories. """List all projects and the associated directories.
This may be possible to do with 'repo forall', but repo newbies have This may be possible to do with 'repo forall', but repo newbies have
trouble figuring that out. The idea here is that it should be more trouble figuring that out. The idea here is that it should be more
discoverable. discoverable.
Args: Args:
opt: The options. opt: The options.
args: Positional args. Can be a list of projects to list, or empty. args: Positional args. Can be a list of projects to list, or empty.
""" """
if not opt.regex: if not opt.regex:
projects = self.GetProjects(args, groups=opt.groups, missing_ok=opt.all, projects = self.GetProjects(
all_manifests=not opt.this_manifest_only) args,
else: groups=opt.groups,
projects = self.FindProjects(args, all_manifests=not opt.this_manifest_only) missing_ok=opt.all,
all_manifests=not opt.this_manifest_only,
)
else:
projects = self.FindProjects(
args, all_manifests=not opt.this_manifest_only
)
def _getpath(x): def _getpath(x):
if opt.fullpath: if opt.fullpath:
return x.worktree return x.worktree
if opt.relative_to: if opt.relative_to:
return os.path.relpath(x.worktree, opt.relative_to) return os.path.relpath(x.worktree, opt.relative_to)
return x.RelPath(local=opt.this_manifest_only) return x.RelPath(local=opt.this_manifest_only)
lines = [] lines = []
for project in projects: for project in projects:
if opt.name_only and not opt.path_only: if opt.name_only and not opt.path_only:
lines.append("%s" % (project.name)) lines.append("%s" % (project.name))
elif opt.path_only and not opt.name_only: elif opt.path_only and not opt.name_only:
lines.append("%s" % (_getpath(project))) lines.append("%s" % (_getpath(project)))
else: else:
lines.append("%s : %s" % (_getpath(project), project.name)) lines.append("%s : %s" % (_getpath(project), project.name))
if lines: if lines:
lines.sort() lines.sort()
print('\n'.join(lines)) print("\n".join(lines))

214
subcmds/manifest.py
View File

@ -20,12 +20,12 @@ from command import PagedCommand
class Manifest(PagedCommand): class Manifest(PagedCommand):
COMMON = False COMMON = False
helpSummary = "Manifest inspection utility" helpSummary = "Manifest inspection utility"
helpUsage = """ helpUsage = """
%prog [-o {-|NAME.xml}] [-m MANIFEST.xml] [-r] %prog [-o {-|NAME.xml}] [-m MANIFEST.xml] [-r]
""" """
_helpDescription = """ _helpDescription = """
With the -o option, exports the current manifest for inspection. With the -o option, exports the current manifest for inspection.
The manifest and (if present) local_manifests/ are combined The manifest and (if present) local_manifests/ are combined
@ -40,92 +40,136 @@ when the manifest was generated. The 'dest-branch' attribute is set
to indicate the remote ref to push changes to via 'repo upload'. to indicate the remote ref to push changes to via 'repo upload'.
""" """
@property @property
def helpDescription(self): def helpDescription(self):
helptext = self._helpDescription + '\n' helptext = self._helpDescription + "\n"
r = os.path.dirname(__file__) r = os.path.dirname(__file__)
r = os.path.dirname(r) r = os.path.dirname(r)
with open(os.path.join(r, 'docs', 'manifest-format.md')) as fd: with open(os.path.join(r, "docs", "manifest-format.md")) as fd:
for line in fd: for line in fd:
helptext += line helptext += line
return helptext return helptext
def _Options(self, p): def _Options(self, p):
p.add_option('-r', '--revision-as-HEAD', p.add_option(
dest='peg_rev', action='store_true', "-r",
help='save revisions as current HEAD') "--revision-as-HEAD",
p.add_option('-m', '--manifest-name', dest="peg_rev",
help='temporary manifest to use for this sync', metavar='NAME.xml') action="store_true",
p.add_option('--suppress-upstream-revision', dest='peg_rev_upstream', help="save revisions as current HEAD",
default=True, action='store_false', )
help='if in -r mode, do not write the upstream field ' p.add_option(
'(only of use if the branch names for a sha1 manifest are ' "-m",
'sensitive)') "--manifest-name",
p.add_option('--suppress-dest-branch', dest='peg_rev_dest_branch', help="temporary manifest to use for this sync",
default=True, action='store_false', metavar="NAME.xml",
help='if in -r mode, do not write the dest-branch field ' )
'(only of use if the branch names for a sha1 manifest are ' p.add_option(
'sensitive)') "--suppress-upstream-revision",
p.add_option('--json', default=False, action='store_true', dest="peg_rev_upstream",
help='output manifest in JSON format (experimental)') default=True,
p.add_option('--pretty', default=False, action='store_true', action="store_false",
help='format output for humans to read') help="if in -r mode, do not write the upstream field "
p.add_option('--no-local-manifests', default=False, action='store_true', "(only of use if the branch names for a sha1 manifest are "
dest='ignore_local_manifests', help='ignore local manifests') "sensitive)",
p.add_option('-o', '--output-file', )
dest='output_file', p.add_option(
default='-', "--suppress-dest-branch",
help='file to save the manifest to. (Filename prefix for multi-tree.)', dest="peg_rev_dest_branch",
metavar='-|NAME.xml') default=True,
action="store_false",
help="if in -r mode, do not write the dest-branch field "
"(only of use if the branch names for a sha1 manifest are "
"sensitive)",
)
p.add_option(
"--json",
default=False,
action="store_true",
help="output manifest in JSON format (experimental)",
)
p.add_option(
"--pretty",
default=False,
action="store_true",
help="format output for humans to read",
)
p.add_option(
"--no-local-manifests",
default=False,
action="store_true",
dest="ignore_local_manifests",
help="ignore local manifests",
)
p.add_option(
"-o",
"--output-file",
dest="output_file",
default="-",
help="file to save the manifest to. (Filename prefix for "
"multi-tree.)",
metavar="-|NAME.xml",
)
def _Output(self, opt): def _Output(self, opt):
# If alternate manifest is specified, override the manifest file that we're using. # If alternate manifest is specified, override the manifest file that
if opt.manifest_name: # we're using.
self.manifest.Override(opt.manifest_name, False) if opt.manifest_name:
self.manifest.Override(opt.manifest_name, False)
for manifest in self.ManifestList(opt): for manifest in self.ManifestList(opt):
output_file = opt.output_file output_file = opt.output_file
if output_file == '-': if output_file == "-":
fd = sys.stdout fd = sys.stdout
else: else:
if manifest.path_prefix: if manifest.path_prefix:
output_file = f'{opt.output_file}:{manifest.path_prefix.replace("/", "%2f")}' output_file = (
fd = open(output_file, 'w') f"{opt.output_file}:"
f'{manifest.path_prefix.replace("/", "%2f")}'
)
fd = open(output_file, "w")
manifest.SetUseLocalManifests(not opt.ignore_local_manifests) manifest.SetUseLocalManifests(not opt.ignore_local_manifests)
if opt.json: if opt.json:
print('warning: --json is experimental!', file=sys.stderr) print("warning: --json is experimental!", file=sys.stderr)
doc = manifest.ToDict(peg_rev=opt.peg_rev, doc = manifest.ToDict(
peg_rev_upstream=opt.peg_rev_upstream, peg_rev=opt.peg_rev,
peg_rev_dest_branch=opt.peg_rev_dest_branch) peg_rev_upstream=opt.peg_rev_upstream,
peg_rev_dest_branch=opt.peg_rev_dest_branch,
)
json_settings = { json_settings = {
# JSON style guide says Uunicode characters are fully allowed. # JSON style guide says Unicode characters are fully
'ensure_ascii': False, # allowed.
# We use 2 space indent to match JSON style guide. "ensure_ascii": False,
'indent': 2 if opt.pretty else None, # We use 2 space indent to match JSON style guide.
'separators': (',', ': ') if opt.pretty else (',', ':'), "indent": 2 if opt.pretty else None,
'sort_keys': True, "separators": (",", ": ") if opt.pretty else (",", ":"),
} "sort_keys": True,
fd.write(json.dumps(doc, **json_settings)) }
else: fd.write(json.dumps(doc, **json_settings))
manifest.Save(fd, else:
peg_rev=opt.peg_rev, manifest.Save(
peg_rev_upstream=opt.peg_rev_upstream, fd,
peg_rev_dest_branch=opt.peg_rev_dest_branch) peg_rev=opt.peg_rev,
if output_file != '-': peg_rev_upstream=opt.peg_rev_upstream,
fd.close() peg_rev_dest_branch=opt.peg_rev_dest_branch,
if manifest.path_prefix: )
print(f'Saved {manifest.path_prefix} submanifest to {output_file}', if output_file != "-":
file=sys.stderr) fd.close()
else: if manifest.path_prefix:
print(f'Saved manifest to {output_file}', file=sys.stderr) print(
f"Saved {manifest.path_prefix} submanifest to "
f"{output_file}",
file=sys.stderr,
)
else:
print(f"Saved manifest to {output_file}", file=sys.stderr)
def ValidateOptions(self, opt, args):
if args:
self.Usage()
def ValidateOptions(self, opt, args): def Execute(self, opt, args):
if args: self._Output(opt)
self.Usage()
def Execute(self, opt, args):
self._Output(opt)

124
subcmds/overview.py
View File

@ -19,12 +19,12 @@ from command import PagedCommand
class Overview(PagedCommand): class Overview(PagedCommand):
COMMON = True COMMON = True
helpSummary = "Display overview of unmerged project branches" helpSummary = "Display overview of unmerged project branches"
helpUsage = """ helpUsage = """
%prog [--current-branch] [<project>...] %prog [--current-branch] [<project>...]
""" """
helpDescription = """ helpDescription = """
The '%prog' command is used to display an overview of the projects branches, The '%prog' command is used to display an overview of the projects branches,
and list any local commits that have not yet been merged into the project. and list any local commits that have not yet been merged into the project.
@ -33,59 +33,77 @@ branches currently checked out in each project. By default, all branches
are displayed. are displayed.
""" """
def _Options(self, p): def _Options(self, p):
p.add_option('-c', '--current-branch', p.add_option(
dest="current_branch", action="store_true", "-c",
help="consider only checked out branches") "--current-branch",
p.add_option('--no-current-branch', dest="current_branch",
dest='current_branch', action='store_false', action="store_true",
help='consider all local branches') help="consider only checked out branches",
# Turn this into a warning & remove this someday. )
p.add_option('-b', p.add_option(
dest='current_branch', action='store_true', "--no-current-branch",
help=optparse.SUPPRESS_HELP) dest="current_branch",
action="store_false",
help="consider all local branches",
)
# Turn this into a warning & remove this someday.
p.add_option(
"-b",
dest="current_branch",
action="store_true",
help=optparse.SUPPRESS_HELP,
)
def Execute(self, opt, args): def Execute(self, opt, args):
all_branches = [] all_branches = []
for project in self.GetProjects(args, all_manifests=not opt.this_manifest_only): for project in self.GetProjects(
br = [project.GetUploadableBranch(x) args, all_manifests=not opt.this_manifest_only
for x in project.GetBranches()] ):
br = [x for x in br if x] br = [project.GetUploadableBranch(x) for x in project.GetBranches()]
if opt.current_branch: br = [x for x in br if x]
br = [x for x in br if x.name == project.CurrentBranch] if opt.current_branch:
all_branches.extend(br) br = [x for x in br if x.name == project.CurrentBranch]
all_branches.extend(br)
if not all_branches: if not all_branches:
return return
class Report(Coloring): class Report(Coloring):
def __init__(self, config): def __init__(self, config):
Coloring.__init__(self, config, 'status') Coloring.__init__(self, config, "status")
self.project = self.printer('header', attr='bold') self.project = self.printer("header", attr="bold")
self.text = self.printer('text') self.text = self.printer("text")
out = Report(all_branches[0].project.config) out = Report(all_branches[0].project.config)
out.text("Deprecated. See repo info -o.") out.text("Deprecated. See repo info -o.")
out.nl()
out.project('Projects Overview')
out.nl()
project = None
for branch in all_branches:
if project != branch.project:
project = branch.project
out.nl() out.nl()
out.project('project %s/' % project.RelPath(local=opt.this_manifest_only)) out.project("Projects Overview")
out.nl() out.nl()
commits = branch.commits project = None
date = branch.date
print('%s %-33s (%2d commit%s, %s)' % ( for branch in all_branches:
branch.name == project.CurrentBranch and '*' or ' ', if project != branch.project:
branch.name, project = branch.project
len(commits), out.nl()
len(commits) != 1 and 's' or ' ', out.project(
date)) "project %s/"
for commit in commits: % project.RelPath(local=opt.this_manifest_only)
print('%-35s - %s' % ('', commit)) )
out.nl()
commits = branch.commits
date = branch.date
print(
"%s %-33s (%2d commit%s, %s)"
% (
branch.name == project.CurrentBranch and "*" or " ",
branch.name,
len(commits),
len(commits) != 1 and "s" or " ",
date,
)
)
for commit in commits:
print("%-35s - %s" % ("", commit))

109
subcmds/prune.py
View File

@ -19,63 +19,76 @@ from command import DEFAULT_LOCAL_JOBS, PagedCommand
class Prune(PagedCommand): class Prune(PagedCommand):
COMMON = True COMMON = True
helpSummary = "Prune (delete) already merged topics" helpSummary = "Prune (delete) already merged topics"
helpUsage = """ helpUsage = """
%prog [<project>...] %prog [<project>...]
""" """
PARALLEL_JOBS = DEFAULT_LOCAL_JOBS PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
def _ExecuteOne(self, project): def _ExecuteOne(self, project):
"""Process one project.""" """Process one project."""
return project.PruneHeads() return project.PruneHeads()
def Execute(self, opt, args): def Execute(self, opt, args):
projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only) projects = self.GetProjects(
args, all_manifests=not opt.this_manifest_only
)
# NB: Should be able to refactor this module to display summary as results # NB: Should be able to refactor this module to display summary as
# come back from children. # results come back from children.
def _ProcessResults(_pool, _output, results): def _ProcessResults(_pool, _output, results):
return list(itertools.chain.from_iterable(results)) return list(itertools.chain.from_iterable(results))
all_branches = self.ExecuteInParallel( all_branches = self.ExecuteInParallel(
opt.jobs, opt.jobs,
self._ExecuteOne, self._ExecuteOne,
projects, projects,
callback=_ProcessResults, callback=_ProcessResults,
ordered=True) ordered=True,
)
if not all_branches: if not all_branches:
return return
class Report(Coloring): class Report(Coloring):
def __init__(self, config): def __init__(self, config):
Coloring.__init__(self, config, 'status') Coloring.__init__(self, config, "status")
self.project = self.printer('header', attr='bold') self.project = self.printer("header", attr="bold")
out = Report(all_branches[0].project.config) out = Report(all_branches[0].project.config)
out.project('Pending Branches') out.project("Pending Branches")
out.nl()
project = None
for branch in all_branches:
if project != branch.project:
project = branch.project
out.nl()
out.project('project %s/' % project.RelPath(local=opt.this_manifest_only))
out.nl() out.nl()
print('%s %-33s ' % ( project = None
branch.name == project.CurrentBranch and '*' or ' ',
branch.name), end='')
if not branch.base_exists: for branch in all_branches:
print('(ignoring: tracking branch is gone: %s)' % (branch.base,)) if project != branch.project:
else: project = branch.project
commits = branch.commits out.nl()
date = branch.date out.project(
print('(%2d commit%s, %s)' % ( "project %s/"
len(commits), % project.RelPath(local=opt.this_manifest_only)
len(commits) != 1 and 's' or ' ', )
date)) out.nl()
print(
"%s %-33s "
% (
branch.name == project.CurrentBranch and "*" or " ",
branch.name,
),
end="",
)
if not branch.base_exists:
print(
"(ignoring: tracking branch is gone: %s)" % (branch.base,)
)
else:
commits = branch.commits
date = branch.date
print(
"(%2d commit%s, %s)"
% (len(commits), len(commits) != 1 and "s" or " ", date)
)

277
subcmds/rebase.py
View File

@ -20,146 +20,193 @@ from git_command import GitCommand
class RebaseColoring(Coloring): class RebaseColoring(Coloring):
def __init__(self, config): def __init__(self, config):
Coloring.__init__(self, config, 'rebase') Coloring.__init__(self, config, "rebase")
self.project = self.printer('project', attr='bold') self.project = self.printer("project", attr="bold")
self.fail = self.printer('fail', fg='red') self.fail = self.printer("fail", fg="red")
class Rebase(Command): class Rebase(Command):
COMMON = True COMMON = True
helpSummary = "Rebase local branches on upstream branch" helpSummary = "Rebase local branches on upstream branch"
helpUsage = """ helpUsage = """
%prog {[<project>...] | -i <project>...} %prog {[<project>...] | -i <project>...}
""" """
helpDescription = """ helpDescription = """
'%prog' uses git rebase to move local changes in the current topic branch to '%prog' uses git rebase to move local changes in the current topic branch to
the HEAD of the upstream history, useful when you have made commits in a topic the HEAD of the upstream history, useful when you have made commits in a topic
branch but need to incorporate new upstream changes "underneath" them. branch but need to incorporate new upstream changes "underneath" them.
""" """
def _Options(self, p): def _Options(self, p):
g = p.get_option_group('--quiet') g = p.get_option_group("--quiet")
g.add_option('-i', '--interactive', g.add_option(
dest="interactive", action="store_true", "-i",
help="interactive rebase (single project only)") "--interactive",
dest="interactive",
action="store_true",
help="interactive rebase (single project only)",
)
p.add_option('--fail-fast', p.add_option(
dest='fail_fast', action='store_true', "--fail-fast",
help='stop rebasing after first error is hit') dest="fail_fast",
p.add_option('-f', '--force-rebase', action="store_true",
dest='force_rebase', action='store_true', help="stop rebasing after first error is hit",
help='pass --force-rebase to git rebase') )
p.add_option('--no-ff', p.add_option(
dest='ff', default=True, action='store_false', "-f",
help='pass --no-ff to git rebase') "--force-rebase",
p.add_option('--autosquash', dest="force_rebase",
dest='autosquash', action='store_true', action="store_true",
help='pass --autosquash to git rebase') help="pass --force-rebase to git rebase",
p.add_option('--whitespace', )
dest='whitespace', action='store', metavar='WS', p.add_option(
help='pass --whitespace to git rebase') "--no-ff",
p.add_option('--auto-stash', dest="ff",
dest='auto_stash', action='store_true', default=True,
help='stash local modifications before starting') action="store_false",
p.add_option('-m', '--onto-manifest', help="pass --no-ff to git rebase",
dest='onto_manifest', action='store_true', )
help='rebase onto the manifest version instead of upstream ' p.add_option(
'HEAD (this helps to make sure the local tree stays ' "--autosquash",
'consistent if you previously synced to a manifest)') dest="autosquash",
action="store_true",
help="pass --autosquash to git rebase",
)
p.add_option(
"--whitespace",
dest="whitespace",
action="store",
metavar="WS",
help="pass --whitespace to git rebase",
)
p.add_option(
"--auto-stash",
dest="auto_stash",
action="store_true",
help="stash local modifications before starting",
)
p.add_option(
"-m",
"--onto-manifest",
dest="onto_manifest",
action="store_true",
help="rebase onto the manifest version instead of upstream "
"HEAD (this helps to make sure the local tree stays "
"consistent if you previously synced to a manifest)",
)
def Execute(self, opt, args): def Execute(self, opt, args):
all_projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only) all_projects = self.GetProjects(
one_project = len(all_projects) == 1 args, all_manifests=not opt.this_manifest_only
)
one_project = len(all_projects) == 1
if opt.interactive and not one_project: if opt.interactive and not one_project:
print('error: interactive rebase not supported with multiple projects', print(
file=sys.stderr) "error: interactive rebase not supported with multiple "
if len(args) == 1: "projects",
print('note: project %s is mapped to more than one path' % (args[0],), file=sys.stderr,
file=sys.stderr) )
return 1 if len(args) == 1:
print(
"note: project %s is mapped to more than one path"
% (args[0],),
file=sys.stderr,
)
return 1
# Setup the common git rebase args that we use for all projects. # Setup the common git rebase args that we use for all projects.
common_args = ['rebase'] common_args = ["rebase"]
if opt.whitespace: if opt.whitespace:
common_args.append('--whitespace=%s' % opt.whitespace) common_args.append("--whitespace=%s" % opt.whitespace)
if opt.quiet: if opt.quiet:
common_args.append('--quiet') common_args.append("--quiet")
if opt.force_rebase: if opt.force_rebase:
common_args.append('--force-rebase') common_args.append("--force-rebase")
if not opt.ff: if not opt.ff:
common_args.append('--no-ff') common_args.append("--no-ff")
if opt.autosquash: if opt.autosquash:
common_args.append('--autosquash') common_args.append("--autosquash")
if opt.interactive: if opt.interactive:
common_args.append('-i') common_args.append("-i")
config = self.manifest.manifestProject.config config = self.manifest.manifestProject.config
out = RebaseColoring(config) out = RebaseColoring(config)
out.redirect(sys.stdout) out.redirect(sys.stdout)
_RelPath = lambda p: p.RelPath(local=opt.this_manifest_only) _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
ret = 0 ret = 0
for project in all_projects: for project in all_projects:
if ret and opt.fail_fast: if ret and opt.fail_fast:
break break
cb = project.CurrentBranch cb = project.CurrentBranch
if not cb: if not cb:
if one_project: if one_project:
print("error: project %s has a detached HEAD" % _RelPath(project), print(
file=sys.stderr) "error: project %s has a detached HEAD"
return 1 % _RelPath(project),
# ignore branches with detatched HEADs file=sys.stderr,
continue )
return 1
# Ignore branches with detached HEADs.
continue
upbranch = project.GetBranch(cb) upbranch = project.GetBranch(cb)
if not upbranch.LocalMerge: if not upbranch.LocalMerge:
if one_project: if one_project:
print("error: project %s does not track any remote branches" print(
% _RelPath(project), file=sys.stderr) "error: project %s does not track any remote branches"
return 1 % _RelPath(project),
# ignore branches without remotes file=sys.stderr,
continue )
return 1
# Ignore branches without remotes.
continue
args = common_args[:] args = common_args[:]
if opt.onto_manifest: if opt.onto_manifest:
args.append('--onto') args.append("--onto")
args.append(project.revisionExpr) args.append(project.revisionExpr)
args.append(upbranch.LocalMerge) args.append(upbranch.LocalMerge)
out.project('project %s: rebasing %s -> %s', out.project(
_RelPath(project), cb, upbranch.LocalMerge) "project %s: rebasing %s -> %s",
out.nl() _RelPath(project),
out.flush() cb,
upbranch.LocalMerge,
)
out.nl()
out.flush()
needs_stash = False needs_stash = False
if opt.auto_stash: if opt.auto_stash:
stash_args = ["update-index", "--refresh", "-q"] stash_args = ["update-index", "--refresh", "-q"]
if GitCommand(project, stash_args).Wait() != 0: if GitCommand(project, stash_args).Wait() != 0:
needs_stash = True needs_stash = True
# Dirty index, requires stash... # Dirty index, requires stash...
stash_args = ["stash"] stash_args = ["stash"]
if GitCommand(project, stash_args).Wait() != 0: if GitCommand(project, stash_args).Wait() != 0:
ret += 1 ret += 1
continue continue
if GitCommand(project, args).Wait() != 0: if GitCommand(project, args).Wait() != 0:
ret += 1 ret += 1
continue continue
if needs_stash: if needs_stash:
stash_args.append('pop') stash_args.append("pop")
stash_args.append('--quiet') stash_args.append("--quiet")
if GitCommand(project, stash_args).Wait() != 0: if GitCommand(project, stash_args).Wait() != 0:
ret += 1 ret += 1
if ret: if ret:
out.fail('%i projects had errors', ret) out.fail("%i projects had errors", ret)
out.nl() out.nl()
return ret return ret

55
subcmds/selfupdate.py
View File

@ -21,12 +21,12 @@ from subcmds.sync import _PostRepoFetch
class Selfupdate(Command, MirrorSafeCommand): class Selfupdate(Command, MirrorSafeCommand):
COMMON = False COMMON = False
helpSummary = "Update repo to the latest version" helpSummary = "Update repo to the latest version"
helpUsage = """ helpUsage = """
%prog %prog
""" """
helpDescription = """ helpDescription = """
The '%prog' command upgrades repo to the latest version, if a The '%prog' command upgrades repo to the latest version, if a
newer version is available. newer version is available.
@ -34,28 +34,33 @@ Normally this is done automatically by 'repo sync' and does not
need to be performed by an end-user. need to be performed by an end-user.
""" """
def _Options(self, p): def _Options(self, p):
g = p.add_option_group('repo Version options') g = p.add_option_group("repo Version options")
g.add_option('--no-repo-verify', g.add_option(
dest='repo_verify', default=True, action='store_false', "--no-repo-verify",
help='do not verify repo source code') dest="repo_verify",
g.add_option('--repo-upgraded', default=True,
dest='repo_upgraded', action='store_true', action="store_false",
help=SUPPRESS_HELP) help="do not verify repo source code",
)
g.add_option(
"--repo-upgraded",
dest="repo_upgraded",
action="store_true",
help=SUPPRESS_HELP,
)
def Execute(self, opt, args): def Execute(self, opt, args):
rp = self.manifest.repoProject rp = self.manifest.repoProject
rp.PreSync() rp.PreSync()
if opt.repo_upgraded: if opt.repo_upgraded:
_PostRepoUpgrade(self.manifest) _PostRepoUpgrade(self.manifest)
else: else:
if not rp.Sync_NetworkHalf().success: if not rp.Sync_NetworkHalf().success:
print("error: can't update repo", file=sys.stderr) print("error: can't update repo", file=sys.stderr)
sys.exit(1) sys.exit(1)
rp.bare_git.gc('--auto') rp.bare_git.gc("--auto")
_PostRepoFetch(rp, _PostRepoFetch(rp, repo_verify=opt.repo_verify, verbose=True)
repo_verify=opt.repo_verify,
verbose=True)

18
subcmds/smartsync.py
View File

@ -16,18 +16,18 @@ from subcmds.sync import Sync
class Smartsync(Sync): class Smartsync(Sync):
COMMON = True COMMON = True
helpSummary = "Update working tree to the latest known good revision" helpSummary = "Update working tree to the latest known good revision"
helpUsage = """ helpUsage = """
%prog [<project>...] %prog [<project>...]
""" """
helpDescription = """ helpDescription = """
The '%prog' command is a shortcut for sync -s. The '%prog' command is a shortcut for sync -s.
""" """
def _Options(self, p): def _Options(self, p):
Sync._Options(self, p, show_smart=False) Sync._Options(self, p, show_smart=False)
def Execute(self, opt, args): def Execute(self, opt, args):
opt.smart_sync = True opt.smart_sync = True
Sync.Execute(self, opt, args) Sync.Execute(self, opt, args)

161
subcmds/stage.py
View File

@ -20,98 +20,111 @@ from git_command import GitCommand
class _ProjectList(Coloring): class _ProjectList(Coloring):
def __init__(self, gc): def __init__(self, gc):
Coloring.__init__(self, gc, 'interactive') Coloring.__init__(self, gc, "interactive")
self.prompt = self.printer('prompt', fg='blue', attr='bold') self.prompt = self.printer("prompt", fg="blue", attr="bold")
self.header = self.printer('header', attr='bold') self.header = self.printer("header", attr="bold")
self.help = self.printer('help', fg='red', attr='bold') self.help = self.printer("help", fg="red", attr="bold")
class Stage(InteractiveCommand): class Stage(InteractiveCommand):
COMMON = True COMMON = True
helpSummary = "Stage file(s) for commit" helpSummary = "Stage file(s) for commit"
helpUsage = """ helpUsage = """
%prog -i [<project>...] %prog -i [<project>...]
""" """
helpDescription = """ helpDescription = """
The '%prog' command stages files to prepare the next commit. The '%prog' command stages files to prepare the next commit.
""" """
def _Options(self, p): def _Options(self, p):
g = p.get_option_group('--quiet') g = p.get_option_group("--quiet")
g.add_option('-i', '--interactive', g.add_option(
dest='interactive', action='store_true', "-i",
help='use interactive staging') "--interactive",
dest="interactive",
action="store_true",
help="use interactive staging",
)
def Execute(self, opt, args): def Execute(self, opt, args):
if opt.interactive: if opt.interactive:
self._Interactive(opt, args) self._Interactive(opt, args)
else: else:
self.Usage() self.Usage()
def _Interactive(self, opt, args): def _Interactive(self, opt, args):
all_projects = [ all_projects = [
p for p in self.GetProjects(args, all_manifests=not opt.this_manifest_only) p
if p.IsDirty()] for p in self.GetProjects(
if not all_projects: args, all_manifests=not opt.this_manifest_only
print('no projects have uncommitted modifications', file=sys.stderr) )
return if p.IsDirty()
]
if not all_projects:
print("no projects have uncommitted modifications", file=sys.stderr)
return
out = _ProjectList(self.manifest.manifestProject.config) out = _ProjectList(self.manifest.manifestProject.config)
while True: while True:
out.header(' %s', 'project') out.header(" %s", "project")
out.nl() out.nl()
for i in range(len(all_projects)): for i in range(len(all_projects)):
project = all_projects[i] project = all_projects[i]
out.write('%3d: %s', i + 1, out.write(
project.RelPath(local=opt.this_manifest_only) + '/') "%3d: %s",
out.nl() i + 1,
out.nl() project.RelPath(local=opt.this_manifest_only) + "/",
)
out.nl()
out.nl()
out.write('%3d: (', 0) out.write("%3d: (", 0)
out.prompt('q') out.prompt("q")
out.write('uit)') out.write("uit)")
out.nl() out.nl()
out.prompt('project> ') out.prompt("project> ")
out.flush() out.flush()
try: try:
a = sys.stdin.readline() a = sys.stdin.readline()
except KeyboardInterrupt: except KeyboardInterrupt:
out.nl() out.nl()
break break
if a == '': if a == "":
out.nl() out.nl()
break break
a = a.strip() a = a.strip()
if a.lower() in ('q', 'quit', 'exit'): if a.lower() in ("q", "quit", "exit"):
break break
if not a: if not a:
continue continue
try: try:
a_index = int(a) a_index = int(a)
except ValueError: except ValueError:
a_index = None a_index = None
if a_index is not None: if a_index is not None:
if a_index == 0: if a_index == 0:
break break
if 0 < a_index and a_index <= len(all_projects): if 0 < a_index and a_index <= len(all_projects):
_AddI(all_projects[a_index - 1]) _AddI(all_projects[a_index - 1])
continue continue
projects = [ projects = [
p for p in all_projects p
if a in [p.name, p.RelPath(local=opt.this_manifest_only)]] for p in all_projects
if len(projects) == 1: if a in [p.name, p.RelPath(local=opt.this_manifest_only)]
_AddI(projects[0]) ]
continue if len(projects) == 1:
print('Bye.') _AddI(projects[0])
continue
print("Bye.")
def _AddI(project): def _AddI(project):
p = GitCommand(project, ['add', '--interactive'], bare=False) p = GitCommand(project, ["add", "--interactive"], bare=False)
p.Wait() p.Wait()

224
subcmds/start.py
View File

@ -25,119 +25,147 @@ from project import SyncBuffer
class Start(Command): class Start(Command):
COMMON = True COMMON = True
helpSummary = "Start a new branch for development" helpSummary = "Start a new branch for development"
helpUsage = """ helpUsage = """
%prog <newbranchname> [--all | <project>...] %prog <newbranchname> [--all | <project>...]
""" """
helpDescription = """ helpDescription = """
'%prog' begins a new branch of development, starting from the '%prog' begins a new branch of development, starting from the
revision specified in the manifest. revision specified in the manifest.
""" """
PARALLEL_JOBS = DEFAULT_LOCAL_JOBS PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
def _Options(self, p): def _Options(self, p):
p.add_option('--all', p.add_option(
dest='all', action='store_true', "--all",
help='begin branch in all projects') dest="all",
p.add_option('-r', '--rev', '--revision', dest='revision', action="store_true",
help='point branch at this revision instead of upstream') help="begin branch in all projects",
p.add_option('--head', '--HEAD', )
dest='revision', action='store_const', const='HEAD', p.add_option(
help='abbreviation for --rev HEAD') "-r",
"--rev",
"--revision",
dest="revision",
help="point branch at this revision instead of upstream",
)
p.add_option(
"--head",
"--HEAD",
dest="revision",
action="store_const",
const="HEAD",
help="abbreviation for --rev HEAD",
)
def ValidateOptions(self, opt, args): def ValidateOptions(self, opt, args):
if not args: if not args:
self.Usage() self.Usage()
nb = args[0] nb = args[0]
if not git.check_ref_format('heads/%s' % nb): if not git.check_ref_format("heads/%s" % nb):
self.OptionParser.error("'%s' is not a valid name" % nb) self.OptionParser.error("'%s' is not a valid name" % nb)
def _ExecuteOne(self, revision, nb, project): def _ExecuteOne(self, revision, nb, project):
"""Start one project.""" """Start one project."""
# If the current revision is immutable, such as a SHA1, a tag or # If the current revision is immutable, such as a SHA1, a tag or
# a change, then we can't push back to it. Substitute with # a change, then we can't push back to it. Substitute with
# dest_branch, if defined; or with manifest default revision instead. # dest_branch, if defined; or with manifest default revision instead.
branch_merge = '' branch_merge = ""
if IsImmutable(project.revisionExpr): if IsImmutable(project.revisionExpr):
if project.dest_branch: if project.dest_branch:
branch_merge = project.dest_branch branch_merge = project.dest_branch
else: else:
branch_merge = self.manifest.default.revisionExpr branch_merge = self.manifest.default.revisionExpr
try: try:
ret = project.StartBranch( ret = project.StartBranch(
nb, branch_merge=branch_merge, revision=revision) nb, branch_merge=branch_merge, revision=revision
except Exception as e: )
print('error: unable to checkout %s: %s' % (project.name, e), file=sys.stderr) except Exception as e:
ret = False print(
return (ret, project) "error: unable to checkout %s: %s" % (project.name, e),
file=sys.stderr,
)
ret = False
return (ret, project)
def Execute(self, opt, args): def Execute(self, opt, args):
nb = args[0] nb = args[0]
err = [] err = []
projects = [] projects = []
if not opt.all: if not opt.all:
projects = args[1:] projects = args[1:]
if len(projects) < 1: if len(projects) < 1:
projects = ['.'] # start it in the local project by default projects = ["."] # start it in the local project by default
all_projects = self.GetProjects(projects, all_projects = self.GetProjects(
missing_ok=bool(self.gitc_manifest), projects,
all_manifests=not opt.this_manifest_only) missing_ok=bool(self.gitc_manifest),
all_manifests=not opt.this_manifest_only,
)
# This must happen after we find all_projects, since GetProjects may need # This must happen after we find all_projects, since GetProjects may
# the local directory, which will disappear once we save the GITC manifest. # need the local directory, which will disappear once we save the GITC
if self.gitc_manifest: # manifest.
gitc_projects = self.GetProjects(projects, manifest=self.gitc_manifest, if self.gitc_manifest:
missing_ok=True) gitc_projects = self.GetProjects(
for project in gitc_projects: projects, manifest=self.gitc_manifest, missing_ok=True
if project.old_revision: )
project.already_synced = True for project in gitc_projects:
else: if project.old_revision:
project.already_synced = False project.already_synced = True
project.old_revision = project.revisionExpr else:
project.revisionExpr = None project.already_synced = False
# Save the GITC manifest. project.old_revision = project.revisionExpr
gitc_utils.save_manifest(self.gitc_manifest) project.revisionExpr = None
# Save the GITC manifest.
gitc_utils.save_manifest(self.gitc_manifest)
# Make sure we have a valid CWD # Make sure we have a valid CWD.
if not os.path.exists(os.getcwd()): if not os.path.exists(os.getcwd()):
os.chdir(self.manifest.topdir) os.chdir(self.manifest.topdir)
pm = Progress('Syncing %s' % nb, len(all_projects), quiet=opt.quiet) pm = Progress("Syncing %s" % nb, len(all_projects), quiet=opt.quiet)
for project in all_projects: for project in all_projects:
gitc_project = self.gitc_manifest.paths[project.relpath] gitc_project = self.gitc_manifest.paths[project.relpath]
# Sync projects that have not been opened. # Sync projects that have not been opened.
if not gitc_project.already_synced: if not gitc_project.already_synced:
proj_localdir = os.path.join(self.gitc_manifest.gitc_client_dir, proj_localdir = os.path.join(
project.relpath) self.gitc_manifest.gitc_client_dir, project.relpath
project.worktree = proj_localdir )
if not os.path.exists(proj_localdir): project.worktree = proj_localdir
os.makedirs(proj_localdir) if not os.path.exists(proj_localdir):
project.Sync_NetworkHalf() os.makedirs(proj_localdir)
sync_buf = SyncBuffer(self.manifest.manifestProject.config) project.Sync_NetworkHalf()
project.Sync_LocalHalf(sync_buf) sync_buf = SyncBuffer(self.manifest.manifestProject.config)
project.revisionId = gitc_project.old_revision project.Sync_LocalHalf(sync_buf)
pm.update() project.revisionId = gitc_project.old_revision
pm.end() pm.update()
pm.end()
def _ProcessResults(_pool, pm, results): def _ProcessResults(_pool, pm, results):
for (result, project) in results: for result, project in results:
if not result: if not result:
err.append(project) err.append(project)
pm.update() pm.update()
self.ExecuteInParallel( self.ExecuteInParallel(
opt.jobs, opt.jobs,
functools.partial(self._ExecuteOne, opt.revision, nb), functools.partial(self._ExecuteOne, opt.revision, nb),
all_projects, all_projects,
callback=_ProcessResults, callback=_ProcessResults,
output=Progress('Starting %s' % (nb,), len(all_projects), quiet=opt.quiet)) output=Progress(
"Starting %s" % (nb,), len(all_projects), quiet=opt.quiet
),
)
if err: if err:
for p in err: for p in err:
print("error: %s/: cannot start %s" % (p.RelPath(local=opt.this_manifest_only), nb), print(
file=sys.stderr) "error: %s/: cannot start %s"
sys.exit(1) % (p.RelPath(local=opt.this_manifest_only), nb),
file=sys.stderr,
)
sys.exit(1)

207
subcmds/status.py
View File

@ -24,12 +24,12 @@ import platform_utils
class Status(PagedCommand): class Status(PagedCommand):
COMMON = True COMMON = True
helpSummary = "Show the working tree status" helpSummary = "Show the working tree status"
helpUsage = """ helpUsage = """
%prog [<project>...] %prog [<project>...]
""" """
helpDescription = """ helpDescription = """
'%prog' compares the working tree to the staging area (aka index), '%prog' compares the working tree to the staging area (aka index),
and the most recent commit on this branch (HEAD), in each project and the most recent commit on this branch (HEAD), in each project
specified. A summary is displayed, one line per file where there specified. A summary is displayed, one line per file where there
@ -76,109 +76,128 @@ the following meanings:
d: deleted ( in index, not in work tree ) d: deleted ( in index, not in work tree )
""" """
PARALLEL_JOBS = DEFAULT_LOCAL_JOBS PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
def _Options(self, p): def _Options(self, p):
p.add_option('-o', '--orphans', p.add_option(
dest='orphans', action='store_true', "-o",
help="include objects in working directory outside of repo projects") "--orphans",
dest="orphans",
action="store_true",
help="include objects in working directory outside of repo "
"projects",
)
def _StatusHelper(self, quiet, local, project): def _StatusHelper(self, quiet, local, project):
"""Obtains the status for a specific project. """Obtains the status for a specific project.
Obtains the status for a project, redirecting the output to Obtains the status for a project, redirecting the output to
the specified object. the specified object.
Args: Args:
quiet: Where to output the status. quiet: Where to output the status.
local: a boolean, if True, the path is relative to the local local: a boolean, if True, the path is relative to the local
(sub)manifest. If false, the path is relative to the (sub)manifest. If false, the path is relative to the outermost
outermost manifest. manifest.
project: Project to get status of. project: Project to get status of.
Returns: Returns:
The status of the project. The status of the project.
""" """
buf = io.StringIO() buf = io.StringIO()
ret = project.PrintWorkTreeStatus(quiet=quiet, output_redir=buf, ret = project.PrintWorkTreeStatus(
local=local) quiet=quiet, output_redir=buf, local=local
return (ret, buf.getvalue()) )
return (ret, buf.getvalue())
def _FindOrphans(self, dirs, proj_dirs, proj_dirs_parents, outstring): def _FindOrphans(self, dirs, proj_dirs, proj_dirs_parents, outstring):
"""find 'dirs' that are present in 'proj_dirs_parents' but not in 'proj_dirs'""" """find 'dirs' that are present in 'proj_dirs_parents' but not in 'proj_dirs'""" # noqa: E501
status_header = ' --\t' status_header = " --\t"
for item in dirs: for item in dirs:
if not platform_utils.isdir(item): if not platform_utils.isdir(item):
outstring.append(''.join([status_header, item])) outstring.append("".join([status_header, item]))
continue continue
if item in proj_dirs: if item in proj_dirs:
continue continue
if item in proj_dirs_parents: if item in proj_dirs_parents:
self._FindOrphans(glob.glob('%s/.*' % item) + self._FindOrphans(
glob.glob('%s/*' % item), glob.glob("%s/.*" % item) + glob.glob("%s/*" % item),
proj_dirs, proj_dirs_parents, outstring) proj_dirs,
continue proj_dirs_parents,
outstring.append(''.join([status_header, item, '/'])) outstring,
)
continue
outstring.append("".join([status_header, item, "/"]))
def Execute(self, opt, args): def Execute(self, opt, args):
all_projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only) all_projects = self.GetProjects(
args, all_manifests=not opt.this_manifest_only
)
def _ProcessResults(_pool, _output, results): def _ProcessResults(_pool, _output, results):
ret = 0 ret = 0
for (state, output) in results: for state, output in results:
if output: if output:
print(output, end='') print(output, end="")
if state == 'CLEAN': if state == "CLEAN":
ret += 1 ret += 1
return ret return ret
counter = self.ExecuteInParallel( counter = self.ExecuteInParallel(
opt.jobs, opt.jobs,
functools.partial(self._StatusHelper, opt.quiet, opt.this_manifest_only), functools.partial(
all_projects, self._StatusHelper, opt.quiet, opt.this_manifest_only
callback=_ProcessResults, ),
ordered=True) all_projects,
callback=_ProcessResults,
ordered=True,
)
if not opt.quiet and len(all_projects) == counter: if not opt.quiet and len(all_projects) == counter:
print('nothing to commit (working directory clean)') print("nothing to commit (working directory clean)")
if opt.orphans: if opt.orphans:
proj_dirs = set() proj_dirs = set()
proj_dirs_parents = set() proj_dirs_parents = set()
for project in self.GetProjects(None, missing_ok=True, all_manifests=not opt.this_manifest_only): for project in self.GetProjects(
relpath = project.RelPath(local=opt.this_manifest_only) None, missing_ok=True, all_manifests=not opt.this_manifest_only
proj_dirs.add(relpath) ):
(head, _tail) = os.path.split(relpath) relpath = project.RelPath(local=opt.this_manifest_only)
while head != "": proj_dirs.add(relpath)
proj_dirs_parents.add(head) (head, _tail) = os.path.split(relpath)
(head, _tail) = os.path.split(head) while head != "":
proj_dirs.add('.repo') proj_dirs_parents.add(head)
(head, _tail) = os.path.split(head)
proj_dirs.add(".repo")
class StatusColoring(Coloring): class StatusColoring(Coloring):
def __init__(self, config): def __init__(self, config):
Coloring.__init__(self, config, 'status') Coloring.__init__(self, config, "status")
self.project = self.printer('header', attr='bold') self.project = self.printer("header", attr="bold")
self.untracked = self.printer('untracked', fg='red') self.untracked = self.printer("untracked", fg="red")
orig_path = os.getcwd() orig_path = os.getcwd()
try: try:
os.chdir(self.manifest.topdir) os.chdir(self.manifest.topdir)
outstring = [] outstring = []
self._FindOrphans(glob.glob('.*') + self._FindOrphans(
glob.glob('*'), glob.glob(".*") + glob.glob("*"),
proj_dirs, proj_dirs_parents, outstring) proj_dirs,
proj_dirs_parents,
outstring,
)
if outstring: if outstring:
output = StatusColoring(self.client.globalConfig) output = StatusColoring(self.client.globalConfig)
output.project('Objects not within a project (orphans)') output.project("Objects not within a project (orphans)")
output.nl() output.nl()
for entry in outstring: for entry in outstring:
output.untracked(entry) output.untracked(entry)
output.nl() output.nl()
else: else:
print('No orphan files or directories') print("No orphan files or directories")
finally: finally:
# Restore CWD. # Restore CWD.
os.chdir(orig_path) os.chdir(orig_path)

2978
subcmds/sync.py
View File

File diff suppressed because it is too large Load Diff

1102
subcmds/upload.py
View File

File diff suppressed because it is too large Load Diff

75
subcmds/version.py
View File

@ -22,45 +22,52 @@ from wrapper import Wrapper
class Version(Command, MirrorSafeCommand): class Version(Command, MirrorSafeCommand):
wrapper_version = None wrapper_version = None
wrapper_path = None wrapper_path = None
COMMON = False COMMON = False
helpSummary = "Display the version of repo" helpSummary = "Display the version of repo"
helpUsage = """ helpUsage = """
%prog %prog
""" """
def Execute(self, opt, args): def Execute(self, opt, args):
rp = self.manifest.repoProject rp = self.manifest.repoProject
rem = rp.GetRemote() rem = rp.GetRemote()
branch = rp.GetBranch('default') branch = rp.GetBranch("default")
# These might not be the same. Report them both. # These might not be the same. Report them both.
src_ver = RepoSourceVersion() src_ver = RepoSourceVersion()
rp_ver = rp.bare_git.describe(HEAD) rp_ver = rp.bare_git.describe(HEAD)
print('repo version %s' % rp_ver) print("repo version %s" % rp_ver)
print(' (from %s)' % rem.url) print(" (from %s)" % rem.url)
print(' (tracking %s)' % branch.merge) print(" (tracking %s)" % branch.merge)
print(' (%s)' % rp.bare_git.log('-1', '--format=%cD', HEAD)) print(" (%s)" % rp.bare_git.log("-1", "--format=%cD", HEAD))
if self.wrapper_path is not None: if self.wrapper_path is not None:
print('repo launcher version %s' % self.wrapper_version) print("repo launcher version %s" % self.wrapper_version)
print(' (from %s)' % self.wrapper_path) print(" (from %s)" % self.wrapper_path)
if src_ver != rp_ver: if src_ver != rp_ver:
print(' (currently at %s)' % src_ver) print(" (currently at %s)" % src_ver)
print('repo User-Agent %s' % user_agent.repo) print("repo User-Agent %s" % user_agent.repo)
print('git %s' % git.version_tuple().full) print("git %s" % git.version_tuple().full)
print('git User-Agent %s' % user_agent.git) print("git User-Agent %s" % user_agent.git)
print('Python %s' % sys.version) print("Python %s" % sys.version)
uname = platform.uname() uname = platform.uname()
if sys.version_info.major < 3: if sys.version_info.major < 3:
# Python 3 returns a named tuple, but Python 2 is simpler. # Python 3 returns a named tuple, but Python 2 is simpler.
print(uname) print(uname)
else: else:
print('OS %s %s (%s)' % (uname.system, uname.release, uname.version)) print(
print('CPU %s (%s)' % "OS %s %s (%s)" % (uname.system, uname.release, uname.version)
(uname.machine, uname.processor if uname.processor else 'unknown')) )
print('Bug reports:', Wrapper().BUG_URL) print(
"CPU %s (%s)"
% (
uname.machine,
uname.processor if uname.processor else "unknown",
)
)
print("Bug reports:", Wrapper().BUG_URL)

4
tests/conftest.py
View File

@ -21,5 +21,5 @@ import repo_trace
@pytest.fixture(autouse=True) @pytest.fixture(autouse=True)
def disable_repo_trace(tmp_path): def disable_repo_trace(tmp_path):
"""Set an environment marker to relax certain strict checks for test code.""" """Set an environment marker to relax certain strict checks for test code.""" # noqa: E501
repo_trace._TRACE_FILE = str(tmp_path / 'TRACE_FILE_from_test') repo_trace._TRACE_FILE = str(tmp_path / "TRACE_FILE_from_test")

44
tests/test_editor.py
View File

@ -20,37 +20,37 @@ from editor import Editor
class EditorTestCase(unittest.TestCase): class EditorTestCase(unittest.TestCase):
"""Take care of resetting Editor state across tests.""" """Take care of resetting Editor state across tests."""
def setUp(self): def setUp(self):
self.setEditor(None) self.setEditor(None)
def tearDown(self): def tearDown(self):
self.setEditor(None) self.setEditor(None)
@staticmethod @staticmethod
def setEditor(editor): def setEditor(editor):
Editor._editor = editor Editor._editor = editor
class GetEditor(EditorTestCase): class GetEditor(EditorTestCase):
"""Check GetEditor behavior.""" """Check GetEditor behavior."""
def test_basic(self): def test_basic(self):
"""Basic checking of _GetEditor.""" """Basic checking of _GetEditor."""
self.setEditor(':') self.setEditor(":")
self.assertEqual(':', Editor._GetEditor()) self.assertEqual(":", Editor._GetEditor())
class EditString(EditorTestCase): class EditString(EditorTestCase):
"""Check EditString behavior.""" """Check EditString behavior."""
def test_no_editor(self): def test_no_editor(self):
"""Check behavior when no editor is available.""" """Check behavior when no editor is available."""
self.setEditor(':') self.setEditor(":")
self.assertEqual('foo', Editor.EditString('foo')) self.assertEqual("foo", Editor.EditString("foo"))
def test_cat_editor(self): def test_cat_editor(self):
"""Check behavior when editor is `cat`.""" """Check behavior when editor is `cat`."""
self.setEditor('cat') self.setEditor("cat")
self.assertEqual('foo', Editor.EditString('foo')) self.assertEqual("foo", Editor.EditString("foo"))

54
tests/test_error.py
View File

@ -22,32 +22,34 @@ import error
class PickleTests(unittest.TestCase): class PickleTests(unittest.TestCase):
"""Make sure all our custom exceptions can be pickled.""" """Make sure all our custom exceptions can be pickled."""
def getExceptions(self): def getExceptions(self):
"""Return all our custom exceptions.""" """Return all our custom exceptions."""
for name in dir(error): for name in dir(error):
cls = getattr(error, name) cls = getattr(error, name)
if isinstance(cls, type) and issubclass(cls, Exception): if isinstance(cls, type) and issubclass(cls, Exception):
yield cls yield cls
def testExceptionLookup(self): def testExceptionLookup(self):
"""Make sure our introspection logic works.""" """Make sure our introspection logic works."""
classes = list(self.getExceptions()) classes = list(self.getExceptions())
self.assertIn(error.HookError, classes) self.assertIn(error.HookError, classes)
# Don't assert the exact number to avoid being a change-detector test. # Don't assert the exact number to avoid being a change-detector test.
self.assertGreater(len(classes), 10) self.assertGreater(len(classes), 10)
def testPickle(self): def testPickle(self):
"""Try to pickle all the exceptions.""" """Try to pickle all the exceptions."""
for cls in self.getExceptions(): for cls in self.getExceptions():
args = inspect.getfullargspec(cls.__init__).args[1:] args = inspect.getfullargspec(cls.__init__).args[1:]
obj = cls(*args) obj = cls(*args)
p = pickle.dumps(obj) p = pickle.dumps(obj)
try: try:
newobj = pickle.loads(p) newobj = pickle.loads(p)
except Exception as e: # pylint: disable=broad-except except Exception as e: # pylint: disable=broad-except
self.fail('Class %s is unable to be pickled: %s\n' self.fail(
'Incomplete super().__init__(...) call?' % (cls, e)) "Class %s is unable to be pickled: %s\n"
self.assertIsInstance(newobj, cls) "Incomplete super().__init__(...) call?" % (cls, e)
self.assertEqual(str(obj), str(newobj)) )
self.assertIsInstance(newobj, cls)
self.assertEqual(str(obj), str(newobj))

202
tests/test_git_command.py
View File

@ -19,138 +19,146 @@ import os
import unittest import unittest
try: try:
from unittest import mock from unittest import mock
except ImportError: except ImportError:
import mock import mock
import git_command import git_command
import wrapper import wrapper
class GitCommandTest(unittest.TestCase): class GitCommandTest(unittest.TestCase):
"""Tests the GitCommand class (via git_command.git).""" """Tests the GitCommand class (via git_command.git)."""
def setUp(self): def setUp(self):
def realpath_mock(val):
return val
def realpath_mock(val): mock.patch.object(
return val os.path, "realpath", side_effect=realpath_mock
).start()
mock.patch.object(os.path, 'realpath', side_effect=realpath_mock).start() def tearDown(self):
mock.patch.stopall()
def tearDown(self): def test_alternative_setting_when_matching(self):
mock.patch.stopall() r = git_command._build_env(
objdir=os.path.join("zap", "objects"), gitdir="zap"
)
def test_alternative_setting_when_matching(self): self.assertIsNone(r.get("GIT_ALTERNATE_OBJECT_DIRECTORIES"))
r = git_command._build_env( self.assertEqual(
objdir = os.path.join('zap', 'objects'), r.get("GIT_OBJECT_DIRECTORY"), os.path.join("zap", "objects")
gitdir = 'zap' )
)
self.assertIsNone(r.get('GIT_ALTERNATE_OBJECT_DIRECTORIES')) def test_alternative_setting_when_different(self):
self.assertEqual(r.get('GIT_OBJECT_DIRECTORY'), os.path.join('zap', 'objects')) r = git_command._build_env(
objdir=os.path.join("wow", "objects"), gitdir="zap"
)
def test_alternative_setting_when_different(self): self.assertEqual(
r = git_command._build_env( r.get("GIT_ALTERNATE_OBJECT_DIRECTORIES"),
objdir = os.path.join('wow', 'objects'), os.path.join("zap", "objects"),
gitdir = 'zap' )
) self.assertEqual(
r.get("GIT_OBJECT_DIRECTORY"), os.path.join("wow", "objects")
self.assertEqual(r.get('GIT_ALTERNATE_OBJECT_DIRECTORIES'), os.path.join('zap', 'objects')) )
self.assertEqual(r.get('GIT_OBJECT_DIRECTORY'), os.path.join('wow', 'objects'))
class GitCallUnitTest(unittest.TestCase): class GitCallUnitTest(unittest.TestCase):
"""Tests the _GitCall class (via git_command.git).""" """Tests the _GitCall class (via git_command.git)."""
def test_version_tuple(self): def test_version_tuple(self):
"""Check git.version_tuple() handling.""" """Check git.version_tuple() handling."""
ver = git_command.git.version_tuple() ver = git_command.git.version_tuple()
self.assertIsNotNone(ver) self.assertIsNotNone(ver)
# We don't dive too deep into the values here to avoid having to update # We don't dive too deep into the values here to avoid having to update
# whenever git versions change. We do check relative to this min version # whenever git versions change. We do check relative to this min
# as this is what `repo` itself requires via MIN_GIT_VERSION. # version as this is what `repo` itself requires via MIN_GIT_VERSION.
MIN_GIT_VERSION = (2, 10, 2) MIN_GIT_VERSION = (2, 10, 2)
self.assertTrue(isinstance(ver.major, int)) self.assertTrue(isinstance(ver.major, int))
self.assertTrue(isinstance(ver.minor, int)) self.assertTrue(isinstance(ver.minor, int))
self.assertTrue(isinstance(ver.micro, int)) self.assertTrue(isinstance(ver.micro, int))
self.assertGreater(ver.major, MIN_GIT_VERSION[0] - 1) self.assertGreater(ver.major, MIN_GIT_VERSION[0] - 1)
self.assertGreaterEqual(ver.micro, 0) self.assertGreaterEqual(ver.micro, 0)
self.assertGreaterEqual(ver.major, 0) self.assertGreaterEqual(ver.major, 0)
self.assertGreaterEqual(ver, MIN_GIT_VERSION) self.assertGreaterEqual(ver, MIN_GIT_VERSION)
self.assertLess(ver, (9999, 9999, 9999)) self.assertLess(ver, (9999, 9999, 9999))
self.assertNotEqual('', ver.full) self.assertNotEqual("", ver.full)
class UserAgentUnitTest(unittest.TestCase): class UserAgentUnitTest(unittest.TestCase):
"""Tests the UserAgent function.""" """Tests the UserAgent function."""
def test_smoke_os(self): def test_smoke_os(self):
"""Make sure UA OS setting returns something useful.""" """Make sure UA OS setting returns something useful."""
os_name = git_command.user_agent.os os_name = git_command.user_agent.os
# We can't dive too deep because of OS/tool differences, but we can check # We can't dive too deep because of OS/tool differences, but we can
# the general form. # check the general form.
m = re.match(r'^[^ ]+$', os_name) m = re.match(r"^[^ ]+$", os_name)
self.assertIsNotNone(m) self.assertIsNotNone(m)
def test_smoke_repo(self): def test_smoke_repo(self):
"""Make sure repo UA returns something useful.""" """Make sure repo UA returns something useful."""
ua = git_command.user_agent.repo ua = git_command.user_agent.repo
# We can't dive too deep because of OS/tool differences, but we can check # We can't dive too deep because of OS/tool differences, but we can
# the general form. # check the general form.
m = re.match(r'^git-repo/[^ ]+ ([^ ]+) git/[^ ]+ Python/[0-9.]+', ua) m = re.match(r"^git-repo/[^ ]+ ([^ ]+) git/[^ ]+ Python/[0-9.]+", ua)
self.assertIsNotNone(m) self.assertIsNotNone(m)
def test_smoke_git(self): def test_smoke_git(self):
"""Make sure git UA returns something useful.""" """Make sure git UA returns something useful."""
ua = git_command.user_agent.git ua = git_command.user_agent.git
# We can't dive too deep because of OS/tool differences, but we can check # We can't dive too deep because of OS/tool differences, but we can
# the general form. # check the general form.
m = re.match(r'^git/[^ ]+ ([^ ]+) git-repo/[^ ]+', ua) m = re.match(r"^git/[^ ]+ ([^ ]+) git-repo/[^ ]+", ua)
self.assertIsNotNone(m) self.assertIsNotNone(m)
class GitRequireTests(unittest.TestCase): class GitRequireTests(unittest.TestCase):
"""Test the git_require helper.""" """Test the git_require helper."""
def setUp(self): def setUp(self):
self.wrapper = wrapper.Wrapper() self.wrapper = wrapper.Wrapper()
ver = self.wrapper.GitVersion(1, 2, 3, 4) ver = self.wrapper.GitVersion(1, 2, 3, 4)
mock.patch.object(git_command.git, 'version_tuple', return_value=ver).start() mock.patch.object(
git_command.git, "version_tuple", return_value=ver
).start()
def tearDown(self): def tearDown(self):
mock.patch.stopall() mock.patch.stopall()
def test_older_nonfatal(self): def test_older_nonfatal(self):
"""Test non-fatal require calls with old versions.""" """Test non-fatal require calls with old versions."""
self.assertFalse(git_command.git_require((2,))) self.assertFalse(git_command.git_require((2,)))
self.assertFalse(git_command.git_require((1, 3))) self.assertFalse(git_command.git_require((1, 3)))
self.assertFalse(git_command.git_require((1, 2, 4))) self.assertFalse(git_command.git_require((1, 2, 4)))
self.assertFalse(git_command.git_require((1, 2, 3, 5))) self.assertFalse(git_command.git_require((1, 2, 3, 5)))
def test_newer_nonfatal(self): def test_newer_nonfatal(self):
"""Test non-fatal require calls with newer versions.""" """Test non-fatal require calls with newer versions."""
self.assertTrue(git_command.git_require((0,))) self.assertTrue(git_command.git_require((0,)))
self.assertTrue(git_command.git_require((1, 0))) self.assertTrue(git_command.git_require((1, 0)))
self.assertTrue(git_command.git_require((1, 2, 0))) self.assertTrue(git_command.git_require((1, 2, 0)))
self.assertTrue(git_command.git_require((1, 2, 3, 0))) self.assertTrue(git_command.git_require((1, 2, 3, 0)))
def test_equal_nonfatal(self): def test_equal_nonfatal(self):
"""Test require calls with equal values.""" """Test require calls with equal values."""
self.assertTrue(git_command.git_require((1, 2, 3, 4), fail=False)) self.assertTrue(git_command.git_require((1, 2, 3, 4), fail=False))
self.assertTrue(git_command.git_require((1, 2, 3, 4), fail=True)) self.assertTrue(git_command.git_require((1, 2, 3, 4), fail=True))
def test_older_fatal(self): def test_older_fatal(self):
"""Test fatal require calls with old versions.""" """Test fatal require calls with old versions."""
with self.assertRaises(SystemExit) as e: with self.assertRaises(SystemExit) as e:
git_command.git_require((2,), fail=True) git_command.git_require((2,), fail=True)
self.assertNotEqual(0, e.code) self.assertNotEqual(0, e.code)
def test_older_fatal_msg(self): def test_older_fatal_msg(self):
"""Test fatal require calls with old versions and message.""" """Test fatal require calls with old versions and message."""
with self.assertRaises(SystemExit) as e: with self.assertRaises(SystemExit) as e:
git_command.git_require((2,), fail=True, msg='so sad') git_command.git_require((2,), fail=True, msg="so sad")
self.assertNotEqual(0, e.code) self.assertNotEqual(0, e.code)

264
tests/test_git_config.py
View File

@ -22,167 +22,169 @@ import git_config
def fixture(*paths): def fixture(*paths):
"""Return a path relative to test/fixtures. """Return a path relative to test/fixtures."""
""" return os.path.join(os.path.dirname(__file__), "fixtures", *paths)
return os.path.join(os.path.dirname(__file__), 'fixtures', *paths)
class GitConfigReadOnlyTests(unittest.TestCase): class GitConfigReadOnlyTests(unittest.TestCase):
"""Read-only tests of the GitConfig class.""" """Read-only tests of the GitConfig class."""
def setUp(self): def setUp(self):
"""Create a GitConfig object using the test.gitconfig fixture. """Create a GitConfig object using the test.gitconfig fixture."""
""" config_fixture = fixture("test.gitconfig")
config_fixture = fixture('test.gitconfig') self.config = git_config.GitConfig(config_fixture)
self.config = git_config.GitConfig(config_fixture)
def test_GetString_with_empty_config_values(self): def test_GetString_with_empty_config_values(self):
""" """
Test config entries with no value. Test config entries with no value.
[section] [section]
empty empty
""" """
val = self.config.GetString('section.empty') val = self.config.GetString("section.empty")
self.assertEqual(val, None) self.assertEqual(val, None)
def test_GetString_with_true_value(self): def test_GetString_with_true_value(self):
""" """
Test config entries with a string value. Test config entries with a string value.
[section] [section]
nonempty = true nonempty = true
""" """
val = self.config.GetString('section.nonempty') val = self.config.GetString("section.nonempty")
self.assertEqual(val, 'true') self.assertEqual(val, "true")
def test_GetString_from_missing_file(self): def test_GetString_from_missing_file(self):
""" """
Test missing config file Test missing config file
""" """
config_fixture = fixture('not.present.gitconfig') config_fixture = fixture("not.present.gitconfig")
config = git_config.GitConfig(config_fixture) config = git_config.GitConfig(config_fixture)
val = config.GetString('empty') val = config.GetString("empty")
self.assertEqual(val, None) self.assertEqual(val, None)
def test_GetBoolean_undefined(self): def test_GetBoolean_undefined(self):
"""Test GetBoolean on key that doesn't exist.""" """Test GetBoolean on key that doesn't exist."""
self.assertIsNone(self.config.GetBoolean('section.missing')) self.assertIsNone(self.config.GetBoolean("section.missing"))
def test_GetBoolean_invalid(self): def test_GetBoolean_invalid(self):
"""Test GetBoolean on invalid boolean value.""" """Test GetBoolean on invalid boolean value."""
self.assertIsNone(self.config.GetBoolean('section.boolinvalid')) self.assertIsNone(self.config.GetBoolean("section.boolinvalid"))
def test_GetBoolean_true(self): def test_GetBoolean_true(self):
"""Test GetBoolean on valid true boolean.""" """Test GetBoolean on valid true boolean."""
self.assertTrue(self.config.GetBoolean('section.booltrue')) self.assertTrue(self.config.GetBoolean("section.booltrue"))
def test_GetBoolean_false(self): def test_GetBoolean_false(self):
"""Test GetBoolean on valid false boolean.""" """Test GetBoolean on valid false boolean."""
self.assertFalse(self.config.GetBoolean('section.boolfalse')) self.assertFalse(self.config.GetBoolean("section.boolfalse"))
def test_GetInt_undefined(self): def test_GetInt_undefined(self):
"""Test GetInt on key that doesn't exist.""" """Test GetInt on key that doesn't exist."""
self.assertIsNone(self.config.GetInt('section.missing')) self.assertIsNone(self.config.GetInt("section.missing"))
def test_GetInt_invalid(self): def test_GetInt_invalid(self):
"""Test GetInt on invalid integer value.""" """Test GetInt on invalid integer value."""
self.assertIsNone(self.config.GetBoolean('section.intinvalid')) self.assertIsNone(self.config.GetBoolean("section.intinvalid"))
def test_GetInt_valid(self): def test_GetInt_valid(self):
"""Test GetInt on valid integers.""" """Test GetInt on valid integers."""
TESTS = ( TESTS = (
('inthex', 16), ("inthex", 16),
('inthexk', 16384), ("inthexk", 16384),
('int', 10), ("int", 10),
('intk', 10240), ("intk", 10240),
('intm', 10485760), ("intm", 10485760),
('intg', 10737418240), ("intg", 10737418240),
) )
for key, value in TESTS: for key, value in TESTS:
self.assertEqual(value, self.config.GetInt('section.%s' % (key,))) self.assertEqual(value, self.config.GetInt("section.%s" % (key,)))
class GitConfigReadWriteTests(unittest.TestCase): class GitConfigReadWriteTests(unittest.TestCase):
"""Read/write tests of the GitConfig class.""" """Read/write tests of the GitConfig class."""
def setUp(self): def setUp(self):
self.tmpfile = tempfile.NamedTemporaryFile() self.tmpfile = tempfile.NamedTemporaryFile()
self.config = self.get_config() self.config = self.get_config()
def get_config(self): def get_config(self):
"""Get a new GitConfig instance.""" """Get a new GitConfig instance."""
return git_config.GitConfig(self.tmpfile.name) return git_config.GitConfig(self.tmpfile.name)
def test_SetString(self): def test_SetString(self):
"""Test SetString behavior.""" """Test SetString behavior."""
# Set a value. # Set a value.
self.assertIsNone(self.config.GetString('foo.bar')) self.assertIsNone(self.config.GetString("foo.bar"))
self.config.SetString('foo.bar', 'val') self.config.SetString("foo.bar", "val")
self.assertEqual('val', self.config.GetString('foo.bar')) self.assertEqual("val", self.config.GetString("foo.bar"))
# Make sure the value was actually written out. # Make sure the value was actually written out.
config = self.get_config() config = self.get_config()
self.assertEqual('val', config.GetString('foo.bar')) self.assertEqual("val", config.GetString("foo.bar"))
# Update the value. # Update the value.
self.config.SetString('foo.bar', 'valll') self.config.SetString("foo.bar", "valll")
self.assertEqual('valll', self.config.GetString('foo.bar')) self.assertEqual("valll", self.config.GetString("foo.bar"))
config = self.get_config() config = self.get_config()
self.assertEqual('valll', config.GetString('foo.bar')) self.assertEqual("valll", config.GetString("foo.bar"))
# Delete the value. # Delete the value.
self.config.SetString('foo.bar', None) self.config.SetString("foo.bar", None)
self.assertIsNone(self.config.GetString('foo.bar')) self.assertIsNone(self.config.GetString("foo.bar"))
config = self.get_config() config = self.get_config()
self.assertIsNone(config.GetString('foo.bar')) self.assertIsNone(config.GetString("foo.bar"))
def test_SetBoolean(self): def test_SetBoolean(self):
"""Test SetBoolean behavior.""" """Test SetBoolean behavior."""
# Set a true value. # Set a true value.
self.assertIsNone(self.config.GetBoolean('foo.bar')) self.assertIsNone(self.config.GetBoolean("foo.bar"))
for val in (True, 1): for val in (True, 1):
self.config.SetBoolean('foo.bar', val) self.config.SetBoolean("foo.bar", val)
self.assertTrue(self.config.GetBoolean('foo.bar')) self.assertTrue(self.config.GetBoolean("foo.bar"))
# Make sure the value was actually written out. # Make sure the value was actually written out.
config = self.get_config() config = self.get_config()
self.assertTrue(config.GetBoolean('foo.bar')) self.assertTrue(config.GetBoolean("foo.bar"))
self.assertEqual('true', config.GetString('foo.bar')) self.assertEqual("true", config.GetString("foo.bar"))
# Set a false value. # Set a false value.
for val in (False, 0): for val in (False, 0):
self.config.SetBoolean('foo.bar', val) self.config.SetBoolean("foo.bar", val)
self.assertFalse(self.config.GetBoolean('foo.bar')) self.assertFalse(self.config.GetBoolean("foo.bar"))
# Make sure the value was actually written out. # Make sure the value was actually written out.
config = self.get_config() config = self.get_config()
self.assertFalse(config.GetBoolean('foo.bar')) self.assertFalse(config.GetBoolean("foo.bar"))
self.assertEqual('false', config.GetString('foo.bar')) self.assertEqual("false", config.GetString("foo.bar"))
# Delete the value. # Delete the value.
self.config.SetBoolean('foo.bar', None) self.config.SetBoolean("foo.bar", None)
self.assertIsNone(self.config.GetBoolean('foo.bar')) self.assertIsNone(self.config.GetBoolean("foo.bar"))
config = self.get_config() config = self.get_config()
self.assertIsNone(config.GetBoolean('foo.bar')) self.assertIsNone(config.GetBoolean("foo.bar"))
def test_GetSyncAnalysisStateData(self): def test_GetSyncAnalysisStateData(self):
"""Test config entries with a sync state analysis data.""" """Test config entries with a sync state analysis data."""
superproject_logging_data = {} superproject_logging_data = {}
superproject_logging_data['test'] = False superproject_logging_data["test"] = False
options = type('options', (object,), {})() options = type("options", (object,), {})()
options.verbose = 'true' options.verbose = "true"
options.mp_update = 'false' options.mp_update = "false"
TESTS = ( TESTS = (
('superproject.test', 'false'), ("superproject.test", "false"),
('options.verbose', 'true'), ("options.verbose", "true"),
('options.mpupdate', 'false'), ("options.mpupdate", "false"),
('main.version', '1'), ("main.version", "1"),
) )
self.config.UpdateSyncAnalysisState(options, superproject_logging_data) self.config.UpdateSyncAnalysisState(options, superproject_logging_data)
sync_data = self.config.GetSyncAnalysisStateData() sync_data = self.config.GetSyncAnalysisStateData()
for key, value in TESTS: for key, value in TESTS:
self.assertEqual(sync_data[f'{git_config.SYNC_STATE_PREFIX}{key}'], value) self.assertEqual(
self.assertTrue(sync_data[f'{git_config.SYNC_STATE_PREFIX}main.synctime']) sync_data[f"{git_config.SYNC_STATE_PREFIX}{key}"], value
)
self.assertTrue(
sync_data[f"{git_config.SYNC_STATE_PREFIX}main.synctime"]
)

742
tests/test_git_superproject.py
View File

@ -28,297 +28,369 @@ from test_manifest_xml import sort_attributes
class SuperprojectTestCase(unittest.TestCase): class SuperprojectTestCase(unittest.TestCase):
"""TestCase for the Superproject module.""" """TestCase for the Superproject module."""
PARENT_SID_KEY = 'GIT_TRACE2_PARENT_SID' PARENT_SID_KEY = "GIT_TRACE2_PARENT_SID"
PARENT_SID_VALUE = 'parent_sid' PARENT_SID_VALUE = "parent_sid"
SELF_SID_REGEX = r'repo-\d+T\d+Z-.*' SELF_SID_REGEX = r"repo-\d+T\d+Z-.*"
FULL_SID_REGEX = r'^%s/%s' % (PARENT_SID_VALUE, SELF_SID_REGEX) FULL_SID_REGEX = r"^%s/%s" % (PARENT_SID_VALUE, SELF_SID_REGEX)
def setUp(self): def setUp(self):
"""Set up superproject every time.""" """Set up superproject every time."""
self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests') self.tempdirobj = tempfile.TemporaryDirectory(prefix="repo_tests")
self.tempdir = self.tempdirobj.name self.tempdir = self.tempdirobj.name
self.repodir = os.path.join(self.tempdir, '.repo') self.repodir = os.path.join(self.tempdir, ".repo")
self.manifest_file = os.path.join( self.manifest_file = os.path.join(
self.repodir, manifest_xml.MANIFEST_FILE_NAME) self.repodir, manifest_xml.MANIFEST_FILE_NAME
os.mkdir(self.repodir) )
self.platform = platform.system().lower() os.mkdir(self.repodir)
self.platform = platform.system().lower()
# By default we initialize with the expected case where # By default we initialize with the expected case where
# repo launches us (so GIT_TRACE2_PARENT_SID is set). # repo launches us (so GIT_TRACE2_PARENT_SID is set).
env = { env = {
self.PARENT_SID_KEY: self.PARENT_SID_VALUE, self.PARENT_SID_KEY: self.PARENT_SID_VALUE,
} }
self.git_event_log = git_trace2_event_log.EventLog(env=env) self.git_event_log = git_trace2_event_log.EventLog(env=env)
# The manifest parsing really wants a git repo currently. # The manifest parsing really wants a git repo currently.
gitdir = os.path.join(self.repodir, 'manifests.git') gitdir = os.path.join(self.repodir, "manifests.git")
os.mkdir(gitdir) os.mkdir(gitdir)
with open(os.path.join(gitdir, 'config'), 'w') as fp: with open(os.path.join(gitdir, "config"), "w") as fp:
fp.write("""[remote "origin"] fp.write(
"""[remote "origin"]
url = https://localhost:0/manifest url = https://localhost:0/manifest
""") """
)
manifest = self.getXmlManifest(""" manifest = self.getXmlManifest(
"""
<manifest> <manifest>
<remote name="default-remote" fetch="http://localhost" /> <remote name="default-remote" fetch="http://localhost" />
<default remote="default-remote" revision="refs/heads/main" /> <default remote="default-remote" revision="refs/heads/main" />
<superproject name="superproject"/> <superproject name="superproject"/>
<project path="art" name="platform/art" groups="notdefault,platform-""" + self.platform + """ <project path="art" name="platform/art" groups="notdefault,platform-"""
+ self.platform
+ """
" /></manifest> " /></manifest>
""") """
self._superproject = git_superproject.Superproject( )
manifest, name='superproject', self._superproject = git_superproject.Superproject(
remote=manifest.remotes.get('default-remote').ToRemoteSpec('superproject'), manifest,
revision='refs/heads/main') name="superproject",
remote=manifest.remotes.get("default-remote").ToRemoteSpec(
"superproject"
),
revision="refs/heads/main",
)
def tearDown(self): def tearDown(self):
"""Tear down superproject every time.""" """Tear down superproject every time."""
self.tempdirobj.cleanup() self.tempdirobj.cleanup()
def getXmlManifest(self, data): def getXmlManifest(self, data):
"""Helper to initialize a manifest for testing.""" """Helper to initialize a manifest for testing."""
with open(self.manifest_file, 'w') as fp: with open(self.manifest_file, "w") as fp:
fp.write(data) fp.write(data)
return manifest_xml.XmlManifest(self.repodir, self.manifest_file) return manifest_xml.XmlManifest(self.repodir, self.manifest_file)
def verifyCommonKeys(self, log_entry, expected_event_name, full_sid=True): def verifyCommonKeys(self, log_entry, expected_event_name, full_sid=True):
"""Helper function to verify common event log keys.""" """Helper function to verify common event log keys."""
self.assertIn('event', log_entry) self.assertIn("event", log_entry)
self.assertIn('sid', log_entry) self.assertIn("sid", log_entry)
self.assertIn('thread', log_entry) self.assertIn("thread", log_entry)
self.assertIn('time', log_entry) self.assertIn("time", log_entry)
# Do basic data format validation. # Do basic data format validation.
self.assertEqual(expected_event_name, log_entry['event']) self.assertEqual(expected_event_name, log_entry["event"])
if full_sid: if full_sid:
self.assertRegex(log_entry['sid'], self.FULL_SID_REGEX) self.assertRegex(log_entry["sid"], self.FULL_SID_REGEX)
else: else:
self.assertRegex(log_entry['sid'], self.SELF_SID_REGEX) self.assertRegex(log_entry["sid"], self.SELF_SID_REGEX)
self.assertRegex(log_entry['time'], r'^\d+-\d+-\d+T\d+:\d+:\d+\.\d+Z$') self.assertRegex(log_entry["time"], r"^\d+-\d+-\d+T\d+:\d+:\d+\.\d+Z$")
def readLog(self, log_path): def readLog(self, log_path):
"""Helper function to read log data into a list.""" """Helper function to read log data into a list."""
log_data = [] log_data = []
with open(log_path, mode='rb') as f: with open(log_path, mode="rb") as f:
for line in f: for line in f:
log_data.append(json.loads(line)) log_data.append(json.loads(line))
return log_data return log_data
def verifyErrorEvent(self): def verifyErrorEvent(self):
"""Helper to verify that error event is written.""" """Helper to verify that error event is written."""
with tempfile.TemporaryDirectory(prefix='event_log_tests') as tempdir: with tempfile.TemporaryDirectory(prefix="event_log_tests") as tempdir:
log_path = self.git_event_log.Write(path=tempdir) log_path = self.git_event_log.Write(path=tempdir)
self.log_data = self.readLog(log_path) self.log_data = self.readLog(log_path)
self.assertEqual(len(self.log_data), 2) self.assertEqual(len(self.log_data), 2)
error_event = self.log_data[1] error_event = self.log_data[1]
self.verifyCommonKeys(self.log_data[0], expected_event_name='version') self.verifyCommonKeys(self.log_data[0], expected_event_name="version")
self.verifyCommonKeys(error_event, expected_event_name='error') self.verifyCommonKeys(error_event, expected_event_name="error")
# Check for 'error' event specific fields. # Check for 'error' event specific fields.
self.assertIn('msg', error_event) self.assertIn("msg", error_event)
self.assertIn('fmt', error_event) self.assertIn("fmt", error_event)
def test_superproject_get_superproject_no_superproject(self): def test_superproject_get_superproject_no_superproject(self):
"""Test with no url.""" """Test with no url."""
manifest = self.getXmlManifest(""" manifest = self.getXmlManifest(
"""
<manifest> <manifest>
</manifest> </manifest>
""") """
self.assertIsNone(manifest.superproject) )
self.assertIsNone(manifest.superproject)
def test_superproject_get_superproject_invalid_url(self): def test_superproject_get_superproject_invalid_url(self):
"""Test with an invalid url.""" """Test with an invalid url."""
manifest = self.getXmlManifest(""" manifest = self.getXmlManifest(
"""
<manifest> <manifest>
<remote name="test-remote" fetch="localhost" /> <remote name="test-remote" fetch="localhost" />
<default remote="test-remote" revision="refs/heads/main" /> <default remote="test-remote" revision="refs/heads/main" />
<superproject name="superproject"/> <superproject name="superproject"/>
</manifest> </manifest>
""") """
superproject = git_superproject.Superproject( )
manifest, name='superproject', superproject = git_superproject.Superproject(
remote=manifest.remotes.get('test-remote').ToRemoteSpec('superproject'), manifest,
revision='refs/heads/main') name="superproject",
sync_result = superproject.Sync(self.git_event_log) remote=manifest.remotes.get("test-remote").ToRemoteSpec(
self.assertFalse(sync_result.success) "superproject"
self.assertTrue(sync_result.fatal) ),
revision="refs/heads/main",
def test_superproject_get_superproject_invalid_branch(self): )
"""Test with an invalid branch.""" sync_result = superproject.Sync(self.git_event_log)
manifest = self.getXmlManifest("""
<manifest>
<remote name="test-remote" fetch="localhost" />
<default remote="test-remote" revision="refs/heads/main" />
<superproject name="superproject"/>
</manifest>
""")
self._superproject = git_superproject.Superproject(
manifest, name='superproject',
remote=manifest.remotes.get('test-remote').ToRemoteSpec('superproject'),
revision='refs/heads/main')
with mock.patch.object(self._superproject, '_branch', 'junk'):
sync_result = self._superproject.Sync(self.git_event_log)
self.assertFalse(sync_result.success)
self.assertTrue(sync_result.fatal)
self.verifyErrorEvent()
def test_superproject_get_superproject_mock_init(self):
"""Test with _Init failing."""
with mock.patch.object(self._superproject, '_Init', return_value=False):
sync_result = self._superproject.Sync(self.git_event_log)
self.assertFalse(sync_result.success)
self.assertTrue(sync_result.fatal)
def test_superproject_get_superproject_mock_fetch(self):
"""Test with _Fetch failing."""
with mock.patch.object(self._superproject, '_Init', return_value=True):
os.mkdir(self._superproject._superproject_path)
with mock.patch.object(self._superproject, '_Fetch', return_value=False):
sync_result = self._superproject.Sync(self.git_event_log)
self.assertFalse(sync_result.success) self.assertFalse(sync_result.success)
self.assertTrue(sync_result.fatal) self.assertTrue(sync_result.fatal)
def test_superproject_get_all_project_commit_ids_mock_ls_tree(self): def test_superproject_get_superproject_invalid_branch(self):
"""Test with LsTree being a mock.""" """Test with an invalid branch."""
data = ('120000 blob 158258bdf146f159218e2b90f8b699c4d85b5804\tAndroid.bp\x00' manifest = self.getXmlManifest(
'160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00' """
'160000 commit e9d25da64d8d365dbba7c8ee00fe8c4473fe9a06\tbootable/recovery\x00' <manifest>
'120000 blob acc2cbdf438f9d2141f0ae424cec1d8fc4b5d97f\tbootstrap.bash\x00' <remote name="test-remote" fetch="localhost" />
'160000 commit ade9b7a0d874e25fff4bf2552488825c6f111928\tbuild/bazel\x00') <default remote="test-remote" revision="refs/heads/main" />
with mock.patch.object(self._superproject, '_Init', return_value=True): <superproject name="superproject"/>
with mock.patch.object(self._superproject, '_Fetch', return_value=True): </manifest>
with mock.patch.object(self._superproject, '_LsTree', return_value=data): """
commit_ids_result = self._superproject._GetAllProjectsCommitIds() )
self.assertEqual(commit_ids_result.commit_ids, { self._superproject = git_superproject.Superproject(
'art': '2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea', manifest,
'bootable/recovery': 'e9d25da64d8d365dbba7c8ee00fe8c4473fe9a06', name="superproject",
'build/bazel': 'ade9b7a0d874e25fff4bf2552488825c6f111928' remote=manifest.remotes.get("test-remote").ToRemoteSpec(
}) "superproject"
self.assertFalse(commit_ids_result.fatal) ),
revision="refs/heads/main",
)
with mock.patch.object(self._superproject, "_branch", "junk"):
sync_result = self._superproject.Sync(self.git_event_log)
self.assertFalse(sync_result.success)
self.assertTrue(sync_result.fatal)
self.verifyErrorEvent()
def test_superproject_write_manifest_file(self): def test_superproject_get_superproject_mock_init(self):
"""Test with writing manifest to a file after setting revisionId.""" """Test with _Init failing."""
self.assertEqual(len(self._superproject._manifest.projects), 1) with mock.patch.object(self._superproject, "_Init", return_value=False):
project = self._superproject._manifest.projects[0] sync_result = self._superproject.Sync(self.git_event_log)
project.SetRevisionId('ABCDEF') self.assertFalse(sync_result.success)
# Create temporary directory so that it can write the file. self.assertTrue(sync_result.fatal)
os.mkdir(self._superproject._superproject_path)
manifest_path = self._superproject._WriteManifestFile()
self.assertIsNotNone(manifest_path)
with open(manifest_path, 'r') as fp:
manifest_xml_data = fp.read()
self.assertEqual(
sort_attributes(manifest_xml_data),
'<?xml version="1.0" ?><manifest>'
'<remote fetch="http://localhost" name="default-remote"/>'
'<default remote="default-remote" revision="refs/heads/main"/>'
'<project groups="notdefault,platform-' + self.platform + '" '
'name="platform/art" path="art" revision="ABCDEF" upstream="refs/heads/main"/>'
'<superproject name="superproject"/>'
'</manifest>')
def test_superproject_update_project_revision_id(self): def test_superproject_get_superproject_mock_fetch(self):
"""Test with LsTree being a mock.""" """Test with _Fetch failing."""
self.assertEqual(len(self._superproject._manifest.projects), 1) with mock.patch.object(self._superproject, "_Init", return_value=True):
projects = self._superproject._manifest.projects os.mkdir(self._superproject._superproject_path)
data = ('160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00' with mock.patch.object(
'160000 commit e9d25da64d8d365dbba7c8ee00fe8c4473fe9a06\tbootable/recovery\x00') self._superproject, "_Fetch", return_value=False
with mock.patch.object(self._superproject, '_Init', return_value=True): ):
with mock.patch.object(self._superproject, '_Fetch', return_value=True): sync_result = self._superproject.Sync(self.git_event_log)
with mock.patch.object(self._superproject, self.assertFalse(sync_result.success)
'_LsTree', self.assertTrue(sync_result.fatal)
return_value=data):
# Create temporary directory so that it can write the file. def test_superproject_get_all_project_commit_ids_mock_ls_tree(self):
os.mkdir(self._superproject._superproject_path) """Test with LsTree being a mock."""
update_result = self._superproject.UpdateProjectsRevisionId(projects, self.git_event_log) data = (
self.assertIsNotNone(update_result.manifest_path) "120000 blob 158258bdf146f159218e2b90f8b699c4d85b5804\tAndroid.bp\x00"
self.assertFalse(update_result.fatal) "160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00"
with open(update_result.manifest_path, 'r') as fp: "160000 commit e9d25da64d8d365dbba7c8ee00fe8c4473fe9a06\tbootable/recovery\x00"
"120000 blob acc2cbdf438f9d2141f0ae424cec1d8fc4b5d97f\tbootstrap.bash\x00"
"160000 commit ade9b7a0d874e25fff4bf2552488825c6f111928\tbuild/bazel\x00"
)
with mock.patch.object(self._superproject, "_Init", return_value=True):
with mock.patch.object(
self._superproject, "_Fetch", return_value=True
):
with mock.patch.object(
self._superproject, "_LsTree", return_value=data
):
commit_ids_result = (
self._superproject._GetAllProjectsCommitIds()
)
self.assertEqual(
commit_ids_result.commit_ids,
{
"art": "2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea",
"bootable/recovery": "e9d25da64d8d365dbba7c8ee00fe8c4473fe9a06",
"build/bazel": "ade9b7a0d874e25fff4bf2552488825c6f111928",
},
)
self.assertFalse(commit_ids_result.fatal)
def test_superproject_write_manifest_file(self):
"""Test with writing manifest to a file after setting revisionId."""
self.assertEqual(len(self._superproject._manifest.projects), 1)
project = self._superproject._manifest.projects[0]
project.SetRevisionId("ABCDEF")
# Create temporary directory so that it can write the file.
os.mkdir(self._superproject._superproject_path)
manifest_path = self._superproject._WriteManifestFile()
self.assertIsNotNone(manifest_path)
with open(manifest_path, "r") as fp:
manifest_xml_data = fp.read() manifest_xml_data = fp.read()
self.assertEqual( self.assertEqual(
sort_attributes(manifest_xml_data), sort_attributes(manifest_xml_data),
'<?xml version="1.0" ?><manifest>' '<?xml version="1.0" ?><manifest>'
'<remote fetch="http://localhost" name="default-remote"/>' '<remote fetch="http://localhost" name="default-remote"/>'
'<default remote="default-remote" revision="refs/heads/main"/>' '<default remote="default-remote" revision="refs/heads/main"/>'
'<project groups="notdefault,platform-' + self.platform + '" ' '<project groups="notdefault,platform-' + self.platform + '" '
'name="platform/art" path="art" ' 'name="platform/art" path="art" revision="ABCDEF" upstream="refs/heads/main"/>'
'revision="2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea" upstream="refs/heads/main"/>' '<superproject name="superproject"/>'
'<superproject name="superproject"/>' "</manifest>",
'</manifest>') )
def test_superproject_update_project_revision_id_no_superproject_tag(self): def test_superproject_update_project_revision_id(self):
"""Test update of commit ids of a manifest without superproject tag.""" """Test with LsTree being a mock."""
manifest = self.getXmlManifest(""" self.assertEqual(len(self._superproject._manifest.projects), 1)
projects = self._superproject._manifest.projects
data = (
"160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00"
"160000 commit e9d25da64d8d365dbba7c8ee00fe8c4473fe9a06\tbootable/recovery\x00"
)
with mock.patch.object(self._superproject, "_Init", return_value=True):
with mock.patch.object(
self._superproject, "_Fetch", return_value=True
):
with mock.patch.object(
self._superproject, "_LsTree", return_value=data
):
# Create temporary directory so that it can write the file.
os.mkdir(self._superproject._superproject_path)
update_result = self._superproject.UpdateProjectsRevisionId(
projects, self.git_event_log
)
self.assertIsNotNone(update_result.manifest_path)
self.assertFalse(update_result.fatal)
with open(update_result.manifest_path, "r") as fp:
manifest_xml_data = fp.read()
self.assertEqual(
sort_attributes(manifest_xml_data),
'<?xml version="1.0" ?><manifest>'
'<remote fetch="http://localhost" name="default-remote"/>'
'<default remote="default-remote" revision="refs/heads/main"/>'
'<project groups="notdefault,platform-'
+ self.platform
+ '" '
'name="platform/art" path="art" '
'revision="2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea" upstream="refs/heads/main"/>'
'<superproject name="superproject"/>'
"</manifest>",
)
def test_superproject_update_project_revision_id_no_superproject_tag(self):
"""Test update of commit ids of a manifest without superproject tag."""
manifest = self.getXmlManifest(
"""
<manifest> <manifest>
<remote name="default-remote" fetch="http://localhost" /> <remote name="default-remote" fetch="http://localhost" />
<default remote="default-remote" revision="refs/heads/main" /> <default remote="default-remote" revision="refs/heads/main" />
<project name="test-name"/> <project name="test-name"/>
</manifest> </manifest>
""") """
self.maxDiff = None )
self.assertIsNone(manifest.superproject) self.maxDiff = None
self.assertEqual( self.assertIsNone(manifest.superproject)
sort_attributes(manifest.ToXml().toxml()), self.assertEqual(
'<?xml version="1.0" ?><manifest>' sort_attributes(manifest.ToXml().toxml()),
'<remote fetch="http://localhost" name="default-remote"/>' '<?xml version="1.0" ?><manifest>'
'<default remote="default-remote" revision="refs/heads/main"/>' '<remote fetch="http://localhost" name="default-remote"/>'
'<project name="test-name"/>' '<default remote="default-remote" revision="refs/heads/main"/>'
'</manifest>') '<project name="test-name"/>'
"</manifest>",
)
def test_superproject_update_project_revision_id_from_local_manifest_group(self): def test_superproject_update_project_revision_id_from_local_manifest_group(
"""Test update of commit ids of a manifest that have local manifest no superproject group.""" self,
local_group = manifest_xml.LOCAL_MANIFEST_GROUP_PREFIX + ':local' ):
manifest = self.getXmlManifest(""" """Test update of commit ids of a manifest that have local manifest no superproject group."""
local_group = manifest_xml.LOCAL_MANIFEST_GROUP_PREFIX + ":local"
manifest = self.getXmlManifest(
"""
<manifest> <manifest>
<remote name="default-remote" fetch="http://localhost" /> <remote name="default-remote" fetch="http://localhost" />
<remote name="goog" fetch="http://localhost2" /> <remote name="goog" fetch="http://localhost2" />
<default remote="default-remote" revision="refs/heads/main" /> <default remote="default-remote" revision="refs/heads/main" />
<superproject name="superproject"/> <superproject name="superproject"/>
<project path="vendor/x" name="platform/vendor/x" remote="goog" <project path="vendor/x" name="platform/vendor/x" remote="goog"
groups=\"""" + local_group + """ groups=\""""
+ local_group
+ """
" revision="master-with-vendor" clone-depth="1" /> " revision="master-with-vendor" clone-depth="1" />
<project path="art" name="platform/art" groups="notdefault,platform-""" + self.platform + """ <project path="art" name="platform/art" groups="notdefault,platform-"""
+ self.platform
+ """
" /></manifest> " /></manifest>
""") """
self.maxDiff = None )
self._superproject = git_superproject.Superproject( self.maxDiff = None
manifest, name='superproject', self._superproject = git_superproject.Superproject(
remote=manifest.remotes.get('default-remote').ToRemoteSpec('superproject'), manifest,
revision='refs/heads/main') name="superproject",
self.assertEqual(len(self._superproject._manifest.projects), 2) remote=manifest.remotes.get("default-remote").ToRemoteSpec(
projects = self._superproject._manifest.projects "superproject"
data = ('160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00') ),
with mock.patch.object(self._superproject, '_Init', return_value=True): revision="refs/heads/main",
with mock.patch.object(self._superproject, '_Fetch', return_value=True): )
with mock.patch.object(self._superproject, self.assertEqual(len(self._superproject._manifest.projects), 2)
'_LsTree', projects = self._superproject._manifest.projects
return_value=data): data = "160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00"
# Create temporary directory so that it can write the file. with mock.patch.object(self._superproject, "_Init", return_value=True):
os.mkdir(self._superproject._superproject_path) with mock.patch.object(
update_result = self._superproject.UpdateProjectsRevisionId(projects, self.git_event_log) self._superproject, "_Fetch", return_value=True
self.assertIsNotNone(update_result.manifest_path) ):
self.assertFalse(update_result.fatal) with mock.patch.object(
with open(update_result.manifest_path, 'r') as fp: self._superproject, "_LsTree", return_value=data
manifest_xml_data = fp.read() ):
# Verify platform/vendor/x's project revision hasn't changed. # Create temporary directory so that it can write the file.
self.assertEqual( os.mkdir(self._superproject._superproject_path)
sort_attributes(manifest_xml_data), update_result = self._superproject.UpdateProjectsRevisionId(
'<?xml version="1.0" ?><manifest>' projects, self.git_event_log
'<remote fetch="http://localhost" name="default-remote"/>' )
'<remote fetch="http://localhost2" name="goog"/>' self.assertIsNotNone(update_result.manifest_path)
'<default remote="default-remote" revision="refs/heads/main"/>' self.assertFalse(update_result.fatal)
'<project groups="notdefault,platform-' + self.platform + '" ' with open(update_result.manifest_path, "r") as fp:
'name="platform/art" path="art" ' manifest_xml_data = fp.read()
'revision="2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea" upstream="refs/heads/main"/>' # Verify platform/vendor/x's project revision hasn't
'<superproject name="superproject"/>' # changed.
'</manifest>') self.assertEqual(
sort_attributes(manifest_xml_data),
'<?xml version="1.0" ?><manifest>'
'<remote fetch="http://localhost" name="default-remote"/>'
'<remote fetch="http://localhost2" name="goog"/>'
'<default remote="default-remote" revision="refs/heads/main"/>'
'<project groups="notdefault,platform-'
+ self.platform
+ '" '
'name="platform/art" path="art" '
'revision="2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea" upstream="refs/heads/main"/>'
'<superproject name="superproject"/>'
"</manifest>",
)
def test_superproject_update_project_revision_id_with_pinned_manifest(self): def test_superproject_update_project_revision_id_with_pinned_manifest(self):
"""Test update of commit ids of a pinned manifest.""" """Test update of commit ids of a pinned manifest."""
manifest = self.getXmlManifest(""" manifest = self.getXmlManifest(
"""
<manifest> <manifest>
<remote name="default-remote" fetch="http://localhost" /> <remote name="default-remote" fetch="http://localhost" />
<default remote="default-remote" revision="refs/heads/main" /> <default remote="default-remote" revision="refs/heads/main" />
@ -326,80 +398,132 @@ class SuperprojectTestCase(unittest.TestCase):
<project path="vendor/x" name="platform/vendor/x" revision="" /> <project path="vendor/x" name="platform/vendor/x" revision="" />
<project path="vendor/y" name="platform/vendor/y" <project path="vendor/y" name="platform/vendor/y"
revision="52d3c9f7c107839ece2319d077de0cd922aa9d8f" /> revision="52d3c9f7c107839ece2319d077de0cd922aa9d8f" />
<project path="art" name="platform/art" groups="notdefault,platform-""" + self.platform + """ <project path="art" name="platform/art" groups="notdefault,platform-"""
+ self.platform
+ """
" /></manifest> " /></manifest>
""") """
self.maxDiff = None )
self._superproject = git_superproject.Superproject( self.maxDiff = None
manifest, name='superproject', self._superproject = git_superproject.Superproject(
remote=manifest.remotes.get('default-remote').ToRemoteSpec('superproject'), manifest,
revision='refs/heads/main') name="superproject",
self.assertEqual(len(self._superproject._manifest.projects), 3) remote=manifest.remotes.get("default-remote").ToRemoteSpec(
projects = self._superproject._manifest.projects "superproject"
data = ('160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00' ),
'160000 commit e9d25da64d8d365dbba7c8ee00fe8c4473fe9a06\tvendor/x\x00') revision="refs/heads/main",
with mock.patch.object(self._superproject, '_Init', return_value=True): )
with mock.patch.object(self._superproject, '_Fetch', return_value=True): self.assertEqual(len(self._superproject._manifest.projects), 3)
with mock.patch.object(self._superproject, projects = self._superproject._manifest.projects
'_LsTree', data = (
return_value=data): "160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00"
# Create temporary directory so that it can write the file. "160000 commit e9d25da64d8d365dbba7c8ee00fe8c4473fe9a06\tvendor/x\x00"
os.mkdir(self._superproject._superproject_path) )
update_result = self._superproject.UpdateProjectsRevisionId(projects, self.git_event_log) with mock.patch.object(self._superproject, "_Init", return_value=True):
self.assertIsNotNone(update_result.manifest_path) with mock.patch.object(
self.assertFalse(update_result.fatal) self._superproject, "_Fetch", return_value=True
with open(update_result.manifest_path, 'r') as fp: ):
manifest_xml_data = fp.read() with mock.patch.object(
# Verify platform/vendor/x's project revision hasn't changed. self._superproject, "_LsTree", return_value=data
self.assertEqual( ):
sort_attributes(manifest_xml_data), # Create temporary directory so that it can write the file.
'<?xml version="1.0" ?><manifest>' os.mkdir(self._superproject._superproject_path)
'<remote fetch="http://localhost" name="default-remote"/>' update_result = self._superproject.UpdateProjectsRevisionId(
'<default remote="default-remote" revision="refs/heads/main"/>' projects, self.git_event_log
'<project groups="notdefault,platform-' + self.platform + '" ' )
'name="platform/art" path="art" ' self.assertIsNotNone(update_result.manifest_path)
'revision="2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea" upstream="refs/heads/main"/>' self.assertFalse(update_result.fatal)
'<project name="platform/vendor/x" path="vendor/x" ' with open(update_result.manifest_path, "r") as fp:
'revision="e9d25da64d8d365dbba7c8ee00fe8c4473fe9a06" upstream="refs/heads/main"/>' manifest_xml_data = fp.read()
'<project name="platform/vendor/y" path="vendor/y" ' # Verify platform/vendor/x's project revision hasn't
'revision="52d3c9f7c107839ece2319d077de0cd922aa9d8f"/>' # changed.
'<superproject name="superproject"/>' self.assertEqual(
'</manifest>') sort_attributes(manifest_xml_data),
'<?xml version="1.0" ?><manifest>'
'<remote fetch="http://localhost" name="default-remote"/>'
'<default remote="default-remote" revision="refs/heads/main"/>'
'<project groups="notdefault,platform-'
+ self.platform
+ '" '
'name="platform/art" path="art" '
'revision="2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea" upstream="refs/heads/main"/>'
'<project name="platform/vendor/x" path="vendor/x" '
'revision="e9d25da64d8d365dbba7c8ee00fe8c4473fe9a06" upstream="refs/heads/main"/>'
'<project name="platform/vendor/y" path="vendor/y" '
'revision="52d3c9f7c107839ece2319d077de0cd922aa9d8f"/>'
'<superproject name="superproject"/>'
"</manifest>",
)
def test_Fetch(self): def test_Fetch(self):
manifest = self.getXmlManifest(""" manifest = self.getXmlManifest(
"""
<manifest> <manifest>
<remote name="default-remote" fetch="http://localhost" /> <remote name="default-remote" fetch="http://localhost" />
<default remote="default-remote" revision="refs/heads/main" /> <default remote="default-remote" revision="refs/heads/main" />
<superproject name="superproject"/> <superproject name="superproject"/>
" /></manifest> " /></manifest>
""") """
self.maxDiff = None )
self._superproject = git_superproject.Superproject( self.maxDiff = None
manifest, name='superproject', self._superproject = git_superproject.Superproject(
remote=manifest.remotes.get('default-remote').ToRemoteSpec('superproject'), manifest,
revision='refs/heads/main') name="superproject",
os.mkdir(self._superproject._superproject_path) remote=manifest.remotes.get("default-remote").ToRemoteSpec(
os.mkdir(self._superproject._work_git) "superproject"
with mock.patch.object(self._superproject, '_Init', return_value=True): ),
with mock.patch('git_superproject.GitCommand', autospec=True) as mock_git_command: revision="refs/heads/main",
with mock.patch('git_superproject.GitRefs.get', autospec=True) as mock_git_refs: )
instance = mock_git_command.return_value os.mkdir(self._superproject._superproject_path)
instance.Wait.return_value = 0 os.mkdir(self._superproject._work_git)
mock_git_refs.side_effect = ['', '1234'] with mock.patch.object(self._superproject, "_Init", return_value=True):
with mock.patch(
"git_superproject.GitCommand", autospec=True
) as mock_git_command:
with mock.patch(
"git_superproject.GitRefs.get", autospec=True
) as mock_git_refs:
instance = mock_git_command.return_value
instance.Wait.return_value = 0
mock_git_refs.side_effect = ["", "1234"]
self.assertTrue(self._superproject._Fetch()) self.assertTrue(self._superproject._Fetch())
self.assertEqual(mock_git_command.call_args.args,(None, [ self.assertEqual(
'fetch', 'http://localhost/superproject', '--depth', '1', mock_git_command.call_args.args,
'--force', '--no-tags', '--filter', 'blob:none', (
'refs/heads/main:refs/heads/main' None,
])) [
"fetch",
"http://localhost/superproject",
"--depth",
"1",
"--force",
"--no-tags",
"--filter",
"blob:none",
"refs/heads/main:refs/heads/main",
],
),
)
# If branch for revision exists, set as --negotiation-tip. # If branch for revision exists, set as --negotiation-tip.
self.assertTrue(self._superproject._Fetch()) self.assertTrue(self._superproject._Fetch())
self.assertEqual(mock_git_command.call_args.args,(None, [ self.assertEqual(
'fetch', 'http://localhost/superproject', '--depth', '1', mock_git_command.call_args.args,
'--force', '--no-tags', '--filter', 'blob:none', (
'--negotiation-tip', '1234', None,
'refs/heads/main:refs/heads/main' [
])) "fetch",
"http://localhost/superproject",
"--depth",
"1",
"--force",
"--no-tags",
"--filter",
"blob:none",
"--negotiation-tip",
"1234",
"refs/heads/main:refs/heads/main",
],
),
)

695
tests/test_git_trace2_event_log.py
View File

@ -27,361 +27,382 @@ import platform_utils
def serverLoggingThread(socket_path, server_ready, received_traces): def serverLoggingThread(socket_path, server_ready, received_traces):
"""Helper function to receive logs over a Unix domain socket. """Helper function to receive logs over a Unix domain socket.
Appends received messages on the provided socket and appends to received_traces. Appends received messages on the provided socket and appends to
received_traces.
Args: Args:
socket_path: path to a Unix domain socket on which to listen for traces socket_path: path to a Unix domain socket on which to listen for traces
server_ready: a threading.Condition used to signal to the caller that this thread is ready to server_ready: a threading.Condition used to signal to the caller that
accept connections this thread is ready to accept connections
received_traces: a list to which received traces will be appended (after decoding to a utf-8 received_traces: a list to which received traces will be appended (after
string). decoding to a utf-8 string).
""" """
platform_utils.remove(socket_path, missing_ok=True) platform_utils.remove(socket_path, missing_ok=True)
data = b'' data = b""
with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock: with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock:
sock.bind(socket_path) sock.bind(socket_path)
sock.listen(0) sock.listen(0)
with server_ready: with server_ready:
server_ready.notify() server_ready.notify()
with sock.accept()[0] as conn: with sock.accept()[0] as conn:
while True: while True:
recved = conn.recv(4096) recved = conn.recv(4096)
if not recved: if not recved:
break break
data += recved data += recved
received_traces.extend(data.decode('utf-8').splitlines()) received_traces.extend(data.decode("utf-8").splitlines())
class EventLogTestCase(unittest.TestCase): class EventLogTestCase(unittest.TestCase):
"""TestCase for the EventLog module.""" """TestCase for the EventLog module."""
PARENT_SID_KEY = 'GIT_TRACE2_PARENT_SID' PARENT_SID_KEY = "GIT_TRACE2_PARENT_SID"
PARENT_SID_VALUE = 'parent_sid' PARENT_SID_VALUE = "parent_sid"
SELF_SID_REGEX = r'repo-\d+T\d+Z-.*' SELF_SID_REGEX = r"repo-\d+T\d+Z-.*"
FULL_SID_REGEX = r'^%s/%s' % (PARENT_SID_VALUE, SELF_SID_REGEX) FULL_SID_REGEX = r"^%s/%s" % (PARENT_SID_VALUE, SELF_SID_REGEX)
def setUp(self): def setUp(self):
"""Load the event_log module every time.""" """Load the event_log module every time."""
self._event_log_module = None self._event_log_module = None
# By default we initialize with the expected case where # By default we initialize with the expected case where
# repo launches us (so GIT_TRACE2_PARENT_SID is set). # repo launches us (so GIT_TRACE2_PARENT_SID is set).
env = { env = {
self.PARENT_SID_KEY: self.PARENT_SID_VALUE, self.PARENT_SID_KEY: self.PARENT_SID_VALUE,
} }
self._event_log_module = git_trace2_event_log.EventLog(env=env) self._event_log_module = git_trace2_event_log.EventLog(env=env)
self._log_data = None self._log_data = None
def verifyCommonKeys(self, log_entry, expected_event_name=None, full_sid=True): def verifyCommonKeys(
"""Helper function to verify common event log keys.""" self, log_entry, expected_event_name=None, full_sid=True
self.assertIn('event', log_entry) ):
self.assertIn('sid', log_entry) """Helper function to verify common event log keys."""
self.assertIn('thread', log_entry) self.assertIn("event", log_entry)
self.assertIn('time', log_entry) self.assertIn("sid", log_entry)
self.assertIn("thread", log_entry)
self.assertIn("time", log_entry)
# Do basic data format validation. # Do basic data format validation.
if expected_event_name: if expected_event_name:
self.assertEqual(expected_event_name, log_entry['event']) self.assertEqual(expected_event_name, log_entry["event"])
if full_sid: if full_sid:
self.assertRegex(log_entry['sid'], self.FULL_SID_REGEX) self.assertRegex(log_entry["sid"], self.FULL_SID_REGEX)
else: else:
self.assertRegex(log_entry['sid'], self.SELF_SID_REGEX) self.assertRegex(log_entry["sid"], self.SELF_SID_REGEX)
self.assertRegex(log_entry['time'], r'^\d+-\d+-\d+T\d+:\d+:\d+\.\d+Z$') self.assertRegex(log_entry["time"], r"^\d+-\d+-\d+T\d+:\d+:\d+\.\d+Z$")
def readLog(self, log_path): def readLog(self, log_path):
"""Helper function to read log data into a list.""" """Helper function to read log data into a list."""
log_data = [] log_data = []
with open(log_path, mode='rb') as f: with open(log_path, mode="rb") as f:
for line in f: for line in f:
log_data.append(json.loads(line)) log_data.append(json.loads(line))
return log_data return log_data
def remove_prefix(self, s, prefix): def remove_prefix(self, s, prefix):
"""Return a copy string after removing |prefix| from |s|, if present or the original string.""" """Return a copy string after removing |prefix| from |s|, if present or
if s.startswith(prefix): the original string."""
return s[len(prefix):] if s.startswith(prefix):
else: return s[len(prefix) :]
return s else:
return s
def test_initial_state_with_parent_sid(self): def test_initial_state_with_parent_sid(self):
"""Test initial state when 'GIT_TRACE2_PARENT_SID' is set by parent.""" """Test initial state when 'GIT_TRACE2_PARENT_SID' is set by parent."""
self.assertRegex(self._event_log_module.full_sid, self.FULL_SID_REGEX) self.assertRegex(self._event_log_module.full_sid, self.FULL_SID_REGEX)
def test_initial_state_no_parent_sid(self): def test_initial_state_no_parent_sid(self):
"""Test initial state when 'GIT_TRACE2_PARENT_SID' is not set.""" """Test initial state when 'GIT_TRACE2_PARENT_SID' is not set."""
# Setup an empty environment dict (no parent sid). # Setup an empty environment dict (no parent sid).
self._event_log_module = git_trace2_event_log.EventLog(env={}) self._event_log_module = git_trace2_event_log.EventLog(env={})
self.assertRegex(self._event_log_module.full_sid, self.SELF_SID_REGEX) self.assertRegex(self._event_log_module.full_sid, self.SELF_SID_REGEX)
def test_version_event(self): def test_version_event(self):
"""Test 'version' event data is valid. """Test 'version' event data is valid.
Verify that the 'version' event is written even when no other Verify that the 'version' event is written even when no other
events are addded. events are addded.
Expected event log: Expected event log:
<version event> <version event>
""" """
with tempfile.TemporaryDirectory(prefix='event_log_tests') as tempdir: with tempfile.TemporaryDirectory(prefix="event_log_tests") as tempdir:
log_path = self._event_log_module.Write(path=tempdir) log_path = self._event_log_module.Write(path=tempdir)
self._log_data = self.readLog(log_path) self._log_data = self.readLog(log_path)
# A log with no added events should only have the version entry. # A log with no added events should only have the version entry.
self.assertEqual(len(self._log_data), 1) self.assertEqual(len(self._log_data), 1)
version_event = self._log_data[0] version_event = self._log_data[0]
self.verifyCommonKeys(version_event, expected_event_name='version') self.verifyCommonKeys(version_event, expected_event_name="version")
# Check for 'version' event specific fields. # Check for 'version' event specific fields.
self.assertIn('evt', version_event) self.assertIn("evt", version_event)
self.assertIn('exe', version_event) self.assertIn("exe", version_event)
# Verify "evt" version field is a string. # Verify "evt" version field is a string.
self.assertIsInstance(version_event['evt'], str) self.assertIsInstance(version_event["evt"], str)
def test_start_event(self): def test_start_event(self):
"""Test and validate 'start' event data is valid. """Test and validate 'start' event data is valid.
Expected event log:
<version event>
<start event>
"""
self._event_log_module.StartEvent()
with tempfile.TemporaryDirectory(prefix='event_log_tests') as tempdir:
log_path = self._event_log_module.Write(path=tempdir)
self._log_data = self.readLog(log_path)
self.assertEqual(len(self._log_data), 2)
start_event = self._log_data[1]
self.verifyCommonKeys(self._log_data[0], expected_event_name='version')
self.verifyCommonKeys(start_event, expected_event_name='start')
# Check for 'start' event specific fields.
self.assertIn('argv', start_event)
self.assertTrue(isinstance(start_event['argv'], list))
def test_exit_event_result_none(self):
"""Test 'exit' event data is valid when result is None.
We expect None result to be converted to 0 in the exit event data.
Expected event log:
<version event>
<exit event>
"""
self._event_log_module.ExitEvent(None)
with tempfile.TemporaryDirectory(prefix='event_log_tests') as tempdir:
log_path = self._event_log_module.Write(path=tempdir)
self._log_data = self.readLog(log_path)
self.assertEqual(len(self._log_data), 2)
exit_event = self._log_data[1]
self.verifyCommonKeys(self._log_data[0], expected_event_name='version')
self.verifyCommonKeys(exit_event, expected_event_name='exit')
# Check for 'exit' event specific fields.
self.assertIn('code', exit_event)
# 'None' result should convert to 0 (successful) return code.
self.assertEqual(exit_event['code'], 0)
def test_exit_event_result_integer(self):
"""Test 'exit' event data is valid when result is an integer.
Expected event log:
<version event>
<exit event>
"""
self._event_log_module.ExitEvent(2)
with tempfile.TemporaryDirectory(prefix='event_log_tests') as tempdir:
log_path = self._event_log_module.Write(path=tempdir)
self._log_data = self.readLog(log_path)
self.assertEqual(len(self._log_data), 2)
exit_event = self._log_data[1]
self.verifyCommonKeys(self._log_data[0], expected_event_name='version')
self.verifyCommonKeys(exit_event, expected_event_name='exit')
# Check for 'exit' event specific fields.
self.assertIn('code', exit_event)
self.assertEqual(exit_event['code'], 2)
def test_command_event(self):
"""Test and validate 'command' event data is valid.
Expected event log:
<version event>
<command event>
"""
name = 'repo'
subcommands = ['init' 'this']
self._event_log_module.CommandEvent(name='repo', subcommands=subcommands)
with tempfile.TemporaryDirectory(prefix='event_log_tests') as tempdir:
log_path = self._event_log_module.Write(path=tempdir)
self._log_data = self.readLog(log_path)
self.assertEqual(len(self._log_data), 2)
command_event = self._log_data[1]
self.verifyCommonKeys(self._log_data[0], expected_event_name='version')
self.verifyCommonKeys(command_event, expected_event_name='command')
# Check for 'command' event specific fields.
self.assertIn('name', command_event)
self.assertIn('subcommands', command_event)
self.assertEqual(command_event['name'], name)
self.assertEqual(command_event['subcommands'], subcommands)
def test_def_params_event_repo_config(self):
"""Test 'def_params' event data outputs only repo config keys.
Expected event log:
<version event>
<def_param event>
<def_param event>
"""
config = {
'git.foo': 'bar',
'repo.partialclone': 'true',
'repo.partialclonefilter': 'blob:none',
}
self._event_log_module.DefParamRepoEvents(config)
with tempfile.TemporaryDirectory(prefix='event_log_tests') as tempdir:
log_path = self._event_log_module.Write(path=tempdir)
self._log_data = self.readLog(log_path)
self.assertEqual(len(self._log_data), 3)
def_param_events = self._log_data[1:]
self.verifyCommonKeys(self._log_data[0], expected_event_name='version')
for event in def_param_events:
self.verifyCommonKeys(event, expected_event_name='def_param')
# Check for 'def_param' event specific fields.
self.assertIn('param', event)
self.assertIn('value', event)
self.assertTrue(event['param'].startswith('repo.'))
def test_def_params_event_no_repo_config(self):
"""Test 'def_params' event data won't output non-repo config keys.
Expected event log:
<version event>
"""
config = {
'git.foo': 'bar',
'git.core.foo2': 'baz',
}
self._event_log_module.DefParamRepoEvents(config)
with tempfile.TemporaryDirectory(prefix='event_log_tests') as tempdir:
log_path = self._event_log_module.Write(path=tempdir)
self._log_data = self.readLog(log_path)
self.assertEqual(len(self._log_data), 1)
self.verifyCommonKeys(self._log_data[0], expected_event_name='version')
def test_data_event_config(self):
"""Test 'data' event data outputs all config keys.
Expected event log:
<version event>
<data event>
<data event>
"""
config = {
'git.foo': 'bar',
'repo.partialclone': 'false',
'repo.syncstate.superproject.hassuperprojecttag': 'true',
'repo.syncstate.superproject.sys.argv': ['--', 'sync', 'protobuf'],
}
prefix_value = 'prefix'
self._event_log_module.LogDataConfigEvents(config, prefix_value)
with tempfile.TemporaryDirectory(prefix='event_log_tests') as tempdir:
log_path = self._event_log_module.Write(path=tempdir)
self._log_data = self.readLog(log_path)
self.assertEqual(len(self._log_data), 5)
data_events = self._log_data[1:]
self.verifyCommonKeys(self._log_data[0], expected_event_name='version')
for event in data_events:
self.verifyCommonKeys(event)
# Check for 'data' event specific fields.
self.assertIn('key', event)
self.assertIn('value', event)
key = event['key']
key = self.remove_prefix(key, f'{prefix_value}/')
value = event['value']
self.assertEqual(self._event_log_module.GetDataEventName(value), event['event'])
self.assertTrue(key in config and value == config[key])
def test_error_event(self):
"""Test and validate 'error' event data is valid.
Expected event log:
<version event>
<error event>
"""
msg = 'invalid option: --cahced'
fmt = 'invalid option: %s'
self._event_log_module.ErrorEvent(msg, fmt)
with tempfile.TemporaryDirectory(prefix='event_log_tests') as tempdir:
log_path = self._event_log_module.Write(path=tempdir)
self._log_data = self.readLog(log_path)
self.assertEqual(len(self._log_data), 2)
error_event = self._log_data[1]
self.verifyCommonKeys(self._log_data[0], expected_event_name='version')
self.verifyCommonKeys(error_event, expected_event_name='error')
# Check for 'error' event specific fields.
self.assertIn('msg', error_event)
self.assertIn('fmt', error_event)
self.assertEqual(error_event['msg'], msg)
self.assertEqual(error_event['fmt'], fmt)
def test_write_with_filename(self):
"""Test Write() with a path to a file exits with None."""
self.assertIsNone(self._event_log_module.Write(path='path/to/file'))
def test_write_with_git_config(self):
"""Test Write() uses the git config path when 'git config' call succeeds."""
with tempfile.TemporaryDirectory(prefix='event_log_tests') as tempdir:
with mock.patch.object(self._event_log_module,
'_GetEventTargetPath', return_value=tempdir):
self.assertEqual(os.path.dirname(self._event_log_module.Write()), tempdir)
def test_write_no_git_config(self):
"""Test Write() with no git config variable present exits with None."""
with mock.patch.object(self._event_log_module,
'_GetEventTargetPath', return_value=None):
self.assertIsNone(self._event_log_module.Write())
def test_write_non_string(self):
"""Test Write() with non-string type for |path| throws TypeError."""
with self.assertRaises(TypeError):
self._event_log_module.Write(path=1234)
def test_write_socket(self):
"""Test Write() with Unix domain socket for |path| and validate received traces."""
received_traces = []
with tempfile.TemporaryDirectory(prefix='test_server_sockets') as tempdir:
socket_path = os.path.join(tempdir, "server.sock")
server_ready = threading.Condition()
# Start "server" listening on Unix domain socket at socket_path.
try:
server_thread = threading.Thread(
target=serverLoggingThread,
args=(socket_path, server_ready, received_traces))
server_thread.start()
with server_ready:
server_ready.wait(timeout=120)
Expected event log:
<version event>
<start event>
"""
self._event_log_module.StartEvent() self._event_log_module.StartEvent()
path = self._event_log_module.Write(path=f'af_unix:{socket_path}') with tempfile.TemporaryDirectory(prefix="event_log_tests") as tempdir:
finally: log_path = self._event_log_module.Write(path=tempdir)
server_thread.join(timeout=5) self._log_data = self.readLog(log_path)
self.assertEqual(path, f'af_unix:stream:{socket_path}') self.assertEqual(len(self._log_data), 2)
self.assertEqual(len(received_traces), 2) start_event = self._log_data[1]
version_event = json.loads(received_traces[0]) self.verifyCommonKeys(self._log_data[0], expected_event_name="version")
start_event = json.loads(received_traces[1]) self.verifyCommonKeys(start_event, expected_event_name="start")
self.verifyCommonKeys(version_event, expected_event_name='version') # Check for 'start' event specific fields.
self.verifyCommonKeys(start_event, expected_event_name='start') self.assertIn("argv", start_event)
# Check for 'start' event specific fields. self.assertTrue(isinstance(start_event["argv"], list))
self.assertIn('argv', start_event)
self.assertIsInstance(start_event['argv'], list) def test_exit_event_result_none(self):
"""Test 'exit' event data is valid when result is None.
We expect None result to be converted to 0 in the exit event data.
Expected event log:
<version event>
<exit event>
"""
self._event_log_module.ExitEvent(None)
with tempfile.TemporaryDirectory(prefix="event_log_tests") as tempdir:
log_path = self._event_log_module.Write(path=tempdir)
self._log_data = self.readLog(log_path)
self.assertEqual(len(self._log_data), 2)
exit_event = self._log_data[1]
self.verifyCommonKeys(self._log_data[0], expected_event_name="version")
self.verifyCommonKeys(exit_event, expected_event_name="exit")
# Check for 'exit' event specific fields.
self.assertIn("code", exit_event)
# 'None' result should convert to 0 (successful) return code.
self.assertEqual(exit_event["code"], 0)
def test_exit_event_result_integer(self):
"""Test 'exit' event data is valid when result is an integer.
Expected event log:
<version event>
<exit event>
"""
self._event_log_module.ExitEvent(2)
with tempfile.TemporaryDirectory(prefix="event_log_tests") as tempdir:
log_path = self._event_log_module.Write(path=tempdir)
self._log_data = self.readLog(log_path)
self.assertEqual(len(self._log_data), 2)
exit_event = self._log_data[1]
self.verifyCommonKeys(self._log_data[0], expected_event_name="version")
self.verifyCommonKeys(exit_event, expected_event_name="exit")
# Check for 'exit' event specific fields.
self.assertIn("code", exit_event)
self.assertEqual(exit_event["code"], 2)
def test_command_event(self):
"""Test and validate 'command' event data is valid.
Expected event log:
<version event>
<command event>
"""
name = "repo"
subcommands = ["init" "this"]
self._event_log_module.CommandEvent(
name="repo", subcommands=subcommands
)
with tempfile.TemporaryDirectory(prefix="event_log_tests") as tempdir:
log_path = self._event_log_module.Write(path=tempdir)
self._log_data = self.readLog(log_path)
self.assertEqual(len(self._log_data), 2)
command_event = self._log_data[1]
self.verifyCommonKeys(self._log_data[0], expected_event_name="version")
self.verifyCommonKeys(command_event, expected_event_name="command")
# Check for 'command' event specific fields.
self.assertIn("name", command_event)
self.assertIn("subcommands", command_event)
self.assertEqual(command_event["name"], name)
self.assertEqual(command_event["subcommands"], subcommands)
def test_def_params_event_repo_config(self):
"""Test 'def_params' event data outputs only repo config keys.
Expected event log:
<version event>
<def_param event>
<def_param event>
"""
config = {
"git.foo": "bar",
"repo.partialclone": "true",
"repo.partialclonefilter": "blob:none",
}
self._event_log_module.DefParamRepoEvents(config)
with tempfile.TemporaryDirectory(prefix="event_log_tests") as tempdir:
log_path = self._event_log_module.Write(path=tempdir)
self._log_data = self.readLog(log_path)
self.assertEqual(len(self._log_data), 3)
def_param_events = self._log_data[1:]
self.verifyCommonKeys(self._log_data[0], expected_event_name="version")
for event in def_param_events:
self.verifyCommonKeys(event, expected_event_name="def_param")
# Check for 'def_param' event specific fields.
self.assertIn("param", event)
self.assertIn("value", event)
self.assertTrue(event["param"].startswith("repo."))
def test_def_params_event_no_repo_config(self):
"""Test 'def_params' event data won't output non-repo config keys.
Expected event log:
<version event>
"""
config = {
"git.foo": "bar",
"git.core.foo2": "baz",
}
self._event_log_module.DefParamRepoEvents(config)
with tempfile.TemporaryDirectory(prefix="event_log_tests") as tempdir:
log_path = self._event_log_module.Write(path=tempdir)
self._log_data = self.readLog(log_path)
self.assertEqual(len(self._log_data), 1)
self.verifyCommonKeys(self._log_data[0], expected_event_name="version")
def test_data_event_config(self):
"""Test 'data' event data outputs all config keys.
Expected event log:
<version event>
<data event>
<data event>
"""
config = {
"git.foo": "bar",
"repo.partialclone": "false",
"repo.syncstate.superproject.hassuperprojecttag": "true",
"repo.syncstate.superproject.sys.argv": ["--", "sync", "protobuf"],
}
prefix_value = "prefix"
self._event_log_module.LogDataConfigEvents(config, prefix_value)
with tempfile.TemporaryDirectory(prefix="event_log_tests") as tempdir:
log_path = self._event_log_module.Write(path=tempdir)
self._log_data = self.readLog(log_path)
self.assertEqual(len(self._log_data), 5)
data_events = self._log_data[1:]
self.verifyCommonKeys(self._log_data[0], expected_event_name="version")
for event in data_events:
self.verifyCommonKeys(event)
# Check for 'data' event specific fields.
self.assertIn("key", event)
self.assertIn("value", event)
key = event["key"]
key = self.remove_prefix(key, f"{prefix_value}/")
value = event["value"]
self.assertEqual(
self._event_log_module.GetDataEventName(value), event["event"]
)
self.assertTrue(key in config and value == config[key])
def test_error_event(self):
"""Test and validate 'error' event data is valid.
Expected event log:
<version event>
<error event>
"""
msg = "invalid option: --cahced"
fmt = "invalid option: %s"
self._event_log_module.ErrorEvent(msg, fmt)
with tempfile.TemporaryDirectory(prefix="event_log_tests") as tempdir:
log_path = self._event_log_module.Write(path=tempdir)
self._log_data = self.readLog(log_path)
self.assertEqual(len(self._log_data), 2)
error_event = self._log_data[1]
self.verifyCommonKeys(self._log_data[0], expected_event_name="version")
self.verifyCommonKeys(error_event, expected_event_name="error")
# Check for 'error' event specific fields.
self.assertIn("msg", error_event)
self.assertIn("fmt", error_event)
self.assertEqual(error_event["msg"], msg)
self.assertEqual(error_event["fmt"], fmt)
def test_write_with_filename(self):
"""Test Write() with a path to a file exits with None."""
self.assertIsNone(self._event_log_module.Write(path="path/to/file"))
def test_write_with_git_config(self):
"""Test Write() uses the git config path when 'git config' call
succeeds."""
with tempfile.TemporaryDirectory(prefix="event_log_tests") as tempdir:
with mock.patch.object(
self._event_log_module,
"_GetEventTargetPath",
return_value=tempdir,
):
self.assertEqual(
os.path.dirname(self._event_log_module.Write()), tempdir
)
def test_write_no_git_config(self):
"""Test Write() with no git config variable present exits with None."""
with mock.patch.object(
self._event_log_module, "_GetEventTargetPath", return_value=None
):
self.assertIsNone(self._event_log_module.Write())
def test_write_non_string(self):
"""Test Write() with non-string type for |path| throws TypeError."""
with self.assertRaises(TypeError):
self._event_log_module.Write(path=1234)
def test_write_socket(self):
"""Test Write() with Unix domain socket for |path| and validate received
traces."""
received_traces = []
with tempfile.TemporaryDirectory(
prefix="test_server_sockets"
) as tempdir:
socket_path = os.path.join(tempdir, "server.sock")
server_ready = threading.Condition()
# Start "server" listening on Unix domain socket at socket_path.
try:
server_thread = threading.Thread(
target=serverLoggingThread,
args=(socket_path, server_ready, received_traces),
)
server_thread.start()
with server_ready:
server_ready.wait(timeout=120)
self._event_log_module.StartEvent()
path = self._event_log_module.Write(
path=f"af_unix:{socket_path}"
)
finally:
server_thread.join(timeout=5)
self.assertEqual(path, f"af_unix:stream:{socket_path}")
self.assertEqual(len(received_traces), 2)
version_event = json.loads(received_traces[0])
start_event = json.loads(received_traces[1])
self.verifyCommonKeys(version_event, expected_event_name="version")
self.verifyCommonKeys(start_event, expected_event_name="start")
# Check for 'start' event specific fields.
self.assertIn("argv", start_event)
self.assertIsInstance(start_event["argv"], list)

63
tests/test_hooks.py
View File

@ -17,39 +17,38 @@
import hooks import hooks
import unittest import unittest
class RepoHookShebang(unittest.TestCase): class RepoHookShebang(unittest.TestCase):
"""Check shebang parsing in RepoHook.""" """Check shebang parsing in RepoHook."""
def test_no_shebang(self): def test_no_shebang(self):
"""Lines w/out shebangs should be rejected.""" """Lines w/out shebangs should be rejected."""
DATA = ( DATA = ("", "#\n# foo\n", "# Bad shebang in script\n#!/foo\n")
'', for data in DATA:
'#\n# foo\n', self.assertIsNone(hooks.RepoHook._ExtractInterpFromShebang(data))
'# Bad shebang in script\n#!/foo\n'
)
for data in DATA:
self.assertIsNone(hooks.RepoHook._ExtractInterpFromShebang(data))
def test_direct_interp(self): def test_direct_interp(self):
"""Lines whose shebang points directly to the interpreter.""" """Lines whose shebang points directly to the interpreter."""
DATA = ( DATA = (
('#!/foo', '/foo'), ("#!/foo", "/foo"),
('#! /foo', '/foo'), ("#! /foo", "/foo"),
('#!/bin/foo ', '/bin/foo'), ("#!/bin/foo ", "/bin/foo"),
('#! /usr/foo ', '/usr/foo'), ("#! /usr/foo ", "/usr/foo"),
('#! /usr/foo -args', '/usr/foo'), ("#! /usr/foo -args", "/usr/foo"),
) )
for shebang, interp in DATA: for shebang, interp in DATA:
self.assertEqual(hooks.RepoHook._ExtractInterpFromShebang(shebang), self.assertEqual(
interp) hooks.RepoHook._ExtractInterpFromShebang(shebang), interp
)
def test_env_interp(self): def test_env_interp(self):
"""Lines whose shebang launches through `env`.""" """Lines whose shebang launches through `env`."""
DATA = ( DATA = (
('#!/usr/bin/env foo', 'foo'), ("#!/usr/bin/env foo", "foo"),
('#!/bin/env foo', 'foo'), ("#!/bin/env foo", "foo"),
('#! /bin/env /bin/foo ', '/bin/foo'), ("#! /bin/env /bin/foo ", "/bin/foo"),
) )
for shebang, interp in DATA: for shebang, interp in DATA:
self.assertEqual(hooks.RepoHook._ExtractInterpFromShebang(shebang), self.assertEqual(
interp) hooks.RepoHook._ExtractInterpFromShebang(shebang), interp
)

1304
tests/test_manifest_xml.py
View File

File diff suppressed because it is too large Load Diff

42
tests/test_platform_utils.py
View File

@ -22,29 +22,31 @@ import platform_utils
class RemoveTests(unittest.TestCase): class RemoveTests(unittest.TestCase):
"""Check remove() helper.""" """Check remove() helper."""
def testMissingOk(self): def testMissingOk(self):
"""Check missing_ok handling.""" """Check missing_ok handling."""
with tempfile.TemporaryDirectory() as tmpdir: with tempfile.TemporaryDirectory() as tmpdir:
path = os.path.join(tmpdir, 'test') path = os.path.join(tmpdir, "test")
# Should not fail. # Should not fail.
platform_utils.remove(path, missing_ok=True) platform_utils.remove(path, missing_ok=True)
# Should fail. # Should fail.
self.assertRaises(OSError, platform_utils.remove, path) self.assertRaises(OSError, platform_utils.remove, path)
self.assertRaises(OSError, platform_utils.remove, path, missing_ok=False) self.assertRaises(
OSError, platform_utils.remove, path, missing_ok=False
)
# Should not fail if it exists. # Should not fail if it exists.
open(path, 'w').close() open(path, "w").close()
platform_utils.remove(path, missing_ok=True) platform_utils.remove(path, missing_ok=True)
self.assertFalse(os.path.exists(path)) self.assertFalse(os.path.exists(path))
open(path, 'w').close() open(path, "w").close()
platform_utils.remove(path) platform_utils.remove(path)
self.assertFalse(os.path.exists(path)) self.assertFalse(os.path.exists(path))
open(path, 'w').close() open(path, "w").close()
platform_utils.remove(path, missing_ok=False) platform_utils.remove(path, missing_ok=False)
self.assertFalse(os.path.exists(path)) self.assertFalse(os.path.exists(path))

769
tests/test_project.py
View File

@ -31,452 +31,493 @@ import project
@contextlib.contextmanager @contextlib.contextmanager
def TempGitTree(): def TempGitTree():
"""Create a new empty git checkout for testing.""" """Create a new empty git checkout for testing."""
with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir: with tempfile.TemporaryDirectory(prefix="repo-tests") as tempdir:
# Tests need to assume, that main is default branch at init, # Tests need to assume, that main is default branch at init,
# which is not supported in config until 2.28. # which is not supported in config until 2.28.
cmd = ['git', 'init'] cmd = ["git", "init"]
if git_command.git_require((2, 28, 0)): if git_command.git_require((2, 28, 0)):
cmd += ['--initial-branch=main'] cmd += ["--initial-branch=main"]
else: else:
# Use template dir for init. # Use template dir for init.
templatedir = tempfile.mkdtemp(prefix='.test-template') templatedir = tempfile.mkdtemp(prefix=".test-template")
with open(os.path.join(templatedir, 'HEAD'), 'w') as fp: with open(os.path.join(templatedir, "HEAD"), "w") as fp:
fp.write('ref: refs/heads/main\n') fp.write("ref: refs/heads/main\n")
cmd += ['--template', templatedir] cmd += ["--template", templatedir]
subprocess.check_call(cmd, cwd=tempdir) subprocess.check_call(cmd, cwd=tempdir)
yield tempdir yield tempdir
class FakeProject(object): class FakeProject(object):
"""A fake for Project for basic functionality.""" """A fake for Project for basic functionality."""
def __init__(self, worktree): def __init__(self, worktree):
self.worktree = worktree self.worktree = worktree
self.gitdir = os.path.join(worktree, '.git') self.gitdir = os.path.join(worktree, ".git")
self.name = 'fakeproject' self.name = "fakeproject"
self.work_git = project.Project._GitGetByExec( self.work_git = project.Project._GitGetByExec(
self, bare=False, gitdir=self.gitdir) self, bare=False, gitdir=self.gitdir
self.bare_git = project.Project._GitGetByExec( )
self, bare=True, gitdir=self.gitdir) self.bare_git = project.Project._GitGetByExec(
self.config = git_config.GitConfig.ForRepository(gitdir=self.gitdir) self, bare=True, gitdir=self.gitdir
)
self.config = git_config.GitConfig.ForRepository(gitdir=self.gitdir)
class ReviewableBranchTests(unittest.TestCase): class ReviewableBranchTests(unittest.TestCase):
"""Check ReviewableBranch behavior.""" """Check ReviewableBranch behavior."""
def test_smoke(self): def test_smoke(self):
"""A quick run through everything.""" """A quick run through everything."""
with TempGitTree() as tempdir: with TempGitTree() as tempdir:
fakeproj = FakeProject(tempdir) fakeproj = FakeProject(tempdir)
# Generate some commits. # Generate some commits.
with open(os.path.join(tempdir, 'readme'), 'w') as fp: with open(os.path.join(tempdir, "readme"), "w") as fp:
fp.write('txt') fp.write("txt")
fakeproj.work_git.add('readme') fakeproj.work_git.add("readme")
fakeproj.work_git.commit('-mAdd file') fakeproj.work_git.commit("-mAdd file")
fakeproj.work_git.checkout('-b', 'work') fakeproj.work_git.checkout("-b", "work")
fakeproj.work_git.rm('-f', 'readme') fakeproj.work_git.rm("-f", "readme")
fakeproj.work_git.commit('-mDel file') fakeproj.work_git.commit("-mDel file")
# Start off with the normal details. # Start off with the normal details.
rb = project.ReviewableBranch( rb = project.ReviewableBranch(
fakeproj, fakeproj.config.GetBranch('work'), 'main') fakeproj, fakeproj.config.GetBranch("work"), "main"
self.assertEqual('work', rb.name) )
self.assertEqual(1, len(rb.commits)) self.assertEqual("work", rb.name)
self.assertIn('Del file', rb.commits[0]) self.assertEqual(1, len(rb.commits))
d = rb.unabbrev_commits self.assertIn("Del file", rb.commits[0])
self.assertEqual(1, len(d)) d = rb.unabbrev_commits
short, long = next(iter(d.items())) self.assertEqual(1, len(d))
self.assertTrue(long.startswith(short)) short, long = next(iter(d.items()))
self.assertTrue(rb.base_exists) self.assertTrue(long.startswith(short))
# Hard to assert anything useful about this. self.assertTrue(rb.base_exists)
self.assertTrue(rb.date) # Hard to assert anything useful about this.
self.assertTrue(rb.date)
# Now delete the tracking branch! # Now delete the tracking branch!
fakeproj.work_git.branch('-D', 'main') fakeproj.work_git.branch("-D", "main")
rb = project.ReviewableBranch( rb = project.ReviewableBranch(
fakeproj, fakeproj.config.GetBranch('work'), 'main') fakeproj, fakeproj.config.GetBranch("work"), "main"
self.assertEqual(0, len(rb.commits)) )
self.assertFalse(rb.base_exists) self.assertEqual(0, len(rb.commits))
# Hard to assert anything useful about this. self.assertFalse(rb.base_exists)
self.assertTrue(rb.date) # Hard to assert anything useful about this.
self.assertTrue(rb.date)
class CopyLinkTestCase(unittest.TestCase): class CopyLinkTestCase(unittest.TestCase):
"""TestCase for stub repo client checkouts. """TestCase for stub repo client checkouts.
It'll have a layout like this: It'll have a layout like this:
tempdir/ # self.tempdir tempdir/ # self.tempdir
checkout/ # self.topdir checkout/ # self.topdir
git-project/ # self.worktree git-project/ # self.worktree
Attributes: Attributes:
tempdir: A dedicated temporary directory. tempdir: A dedicated temporary directory.
worktree: The top of the repo client checkout. worktree: The top of the repo client checkout.
topdir: The top of a project checkout. topdir: The top of a project checkout.
""" """
def setUp(self): def setUp(self):
self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests') self.tempdirobj = tempfile.TemporaryDirectory(prefix="repo_tests")
self.tempdir = self.tempdirobj.name self.tempdir = self.tempdirobj.name
self.topdir = os.path.join(self.tempdir, 'checkout') self.topdir = os.path.join(self.tempdir, "checkout")
self.worktree = os.path.join(self.topdir, 'git-project') self.worktree = os.path.join(self.topdir, "git-project")
os.makedirs(self.topdir) os.makedirs(self.topdir)
os.makedirs(self.worktree) os.makedirs(self.worktree)
def tearDown(self): def tearDown(self):
self.tempdirobj.cleanup() self.tempdirobj.cleanup()
@staticmethod @staticmethod
def touch(path): def touch(path):
with open(path, 'w'): with open(path, "w"):
pass pass
def assertExists(self, path, msg=None): def assertExists(self, path, msg=None):
"""Make sure |path| exists.""" """Make sure |path| exists."""
if os.path.exists(path): if os.path.exists(path):
return return
if msg is None: if msg is None:
msg = ['path is missing: %s' % path] msg = ["path is missing: %s" % path]
while path != '/': while path != "/":
path = os.path.dirname(path) path = os.path.dirname(path)
if not path: if not path:
# If we're given something like "foo", abort once we get to "". # If we're given something like "foo", abort once we get to
break # "".
result = os.path.exists(path) break
msg.append('\tos.path.exists(%s): %s' % (path, result)) result = os.path.exists(path)
if result: msg.append("\tos.path.exists(%s): %s" % (path, result))
msg.append('\tcontents: %r' % os.listdir(path)) if result:
break msg.append("\tcontents: %r" % os.listdir(path))
msg = '\n'.join(msg) break
msg = "\n".join(msg)
raise self.failureException(msg) raise self.failureException(msg)
class CopyFile(CopyLinkTestCase): class CopyFile(CopyLinkTestCase):
"""Check _CopyFile handling.""" """Check _CopyFile handling."""
def CopyFile(self, src, dest): def CopyFile(self, src, dest):
return project._CopyFile(self.worktree, src, self.topdir, dest) return project._CopyFile(self.worktree, src, self.topdir, dest)
def test_basic(self): def test_basic(self):
"""Basic test of copying a file from a project to the toplevel.""" """Basic test of copying a file from a project to the toplevel."""
src = os.path.join(self.worktree, 'foo.txt') src = os.path.join(self.worktree, "foo.txt")
self.touch(src) self.touch(src)
cf = self.CopyFile('foo.txt', 'foo') cf = self.CopyFile("foo.txt", "foo")
cf._Copy() cf._Copy()
self.assertExists(os.path.join(self.topdir, 'foo')) self.assertExists(os.path.join(self.topdir, "foo"))
def test_src_subdir(self): def test_src_subdir(self):
"""Copy a file from a subdir of a project.""" """Copy a file from a subdir of a project."""
src = os.path.join(self.worktree, 'bar', 'foo.txt') src = os.path.join(self.worktree, "bar", "foo.txt")
os.makedirs(os.path.dirname(src)) os.makedirs(os.path.dirname(src))
self.touch(src) self.touch(src)
cf = self.CopyFile('bar/foo.txt', 'new.txt') cf = self.CopyFile("bar/foo.txt", "new.txt")
cf._Copy() cf._Copy()
self.assertExists(os.path.join(self.topdir, 'new.txt')) self.assertExists(os.path.join(self.topdir, "new.txt"))
def test_dest_subdir(self): def test_dest_subdir(self):
"""Copy a file to a subdir of a checkout.""" """Copy a file to a subdir of a checkout."""
src = os.path.join(self.worktree, 'foo.txt') src = os.path.join(self.worktree, "foo.txt")
self.touch(src) self.touch(src)
cf = self.CopyFile('foo.txt', 'sub/dir/new.txt') cf = self.CopyFile("foo.txt", "sub/dir/new.txt")
self.assertFalse(os.path.exists(os.path.join(self.topdir, 'sub'))) self.assertFalse(os.path.exists(os.path.join(self.topdir, "sub")))
cf._Copy() cf._Copy()
self.assertExists(os.path.join(self.topdir, 'sub', 'dir', 'new.txt')) self.assertExists(os.path.join(self.topdir, "sub", "dir", "new.txt"))
def test_update(self): def test_update(self):
"""Make sure changed files get copied again.""" """Make sure changed files get copied again."""
src = os.path.join(self.worktree, 'foo.txt') src = os.path.join(self.worktree, "foo.txt")
dest = os.path.join(self.topdir, 'bar') dest = os.path.join(self.topdir, "bar")
with open(src, 'w') as f: with open(src, "w") as f:
f.write('1st') f.write("1st")
cf = self.CopyFile('foo.txt', 'bar') cf = self.CopyFile("foo.txt", "bar")
cf._Copy() cf._Copy()
self.assertExists(dest) self.assertExists(dest)
with open(dest) as f: with open(dest) as f:
self.assertEqual(f.read(), '1st') self.assertEqual(f.read(), "1st")
with open(src, 'w') as f: with open(src, "w") as f:
f.write('2nd!') f.write("2nd!")
cf._Copy() cf._Copy()
with open(dest) as f: with open(dest) as f:
self.assertEqual(f.read(), '2nd!') self.assertEqual(f.read(), "2nd!")
def test_src_block_symlink(self): def test_src_block_symlink(self):
"""Do not allow reading from a symlinked path.""" """Do not allow reading from a symlinked path."""
src = os.path.join(self.worktree, 'foo.txt') src = os.path.join(self.worktree, "foo.txt")
sym = os.path.join(self.worktree, 'sym') sym = os.path.join(self.worktree, "sym")
self.touch(src) self.touch(src)
platform_utils.symlink('foo.txt', sym) platform_utils.symlink("foo.txt", sym)
self.assertExists(sym) self.assertExists(sym)
cf = self.CopyFile('sym', 'foo') cf = self.CopyFile("sym", "foo")
self.assertRaises(error.ManifestInvalidPathError, cf._Copy) self.assertRaises(error.ManifestInvalidPathError, cf._Copy)
def test_src_block_symlink_traversal(self): def test_src_block_symlink_traversal(self):
"""Do not allow reading through a symlink dir.""" """Do not allow reading through a symlink dir."""
realfile = os.path.join(self.tempdir, 'file.txt') realfile = os.path.join(self.tempdir, "file.txt")
self.touch(realfile) self.touch(realfile)
src = os.path.join(self.worktree, 'bar', 'file.txt') src = os.path.join(self.worktree, "bar", "file.txt")
platform_utils.symlink(self.tempdir, os.path.join(self.worktree, 'bar')) platform_utils.symlink(self.tempdir, os.path.join(self.worktree, "bar"))
self.assertExists(src) self.assertExists(src)
cf = self.CopyFile('bar/file.txt', 'foo') cf = self.CopyFile("bar/file.txt", "foo")
self.assertRaises(error.ManifestInvalidPathError, cf._Copy) self.assertRaises(error.ManifestInvalidPathError, cf._Copy)
def test_src_block_copy_from_dir(self): def test_src_block_copy_from_dir(self):
"""Do not allow copying from a directory.""" """Do not allow copying from a directory."""
src = os.path.join(self.worktree, 'dir') src = os.path.join(self.worktree, "dir")
os.makedirs(src) os.makedirs(src)
cf = self.CopyFile('dir', 'foo') cf = self.CopyFile("dir", "foo")
self.assertRaises(error.ManifestInvalidPathError, cf._Copy) self.assertRaises(error.ManifestInvalidPathError, cf._Copy)
def test_dest_block_symlink(self): def test_dest_block_symlink(self):
"""Do not allow writing to a symlink.""" """Do not allow writing to a symlink."""
src = os.path.join(self.worktree, 'foo.txt') src = os.path.join(self.worktree, "foo.txt")
self.touch(src) self.touch(src)
platform_utils.symlink('dest', os.path.join(self.topdir, 'sym')) platform_utils.symlink("dest", os.path.join(self.topdir, "sym"))
cf = self.CopyFile('foo.txt', 'sym') cf = self.CopyFile("foo.txt", "sym")
self.assertRaises(error.ManifestInvalidPathError, cf._Copy) self.assertRaises(error.ManifestInvalidPathError, cf._Copy)
def test_dest_block_symlink_traversal(self): def test_dest_block_symlink_traversal(self):
"""Do not allow writing through a symlink dir.""" """Do not allow writing through a symlink dir."""
src = os.path.join(self.worktree, 'foo.txt') src = os.path.join(self.worktree, "foo.txt")
self.touch(src) self.touch(src)
platform_utils.symlink(tempfile.gettempdir(), platform_utils.symlink(
os.path.join(self.topdir, 'sym')) tempfile.gettempdir(), os.path.join(self.topdir, "sym")
cf = self.CopyFile('foo.txt', 'sym/foo.txt') )
self.assertRaises(error.ManifestInvalidPathError, cf._Copy) cf = self.CopyFile("foo.txt", "sym/foo.txt")
self.assertRaises(error.ManifestInvalidPathError, cf._Copy)
def test_src_block_copy_to_dir(self): def test_src_block_copy_to_dir(self):
"""Do not allow copying to a directory.""" """Do not allow copying to a directory."""
src = os.path.join(self.worktree, 'foo.txt') src = os.path.join(self.worktree, "foo.txt")
self.touch(src) self.touch(src)
os.makedirs(os.path.join(self.topdir, 'dir')) os.makedirs(os.path.join(self.topdir, "dir"))
cf = self.CopyFile('foo.txt', 'dir') cf = self.CopyFile("foo.txt", "dir")
self.assertRaises(error.ManifestInvalidPathError, cf._Copy) self.assertRaises(error.ManifestInvalidPathError, cf._Copy)
class LinkFile(CopyLinkTestCase): class LinkFile(CopyLinkTestCase):
"""Check _LinkFile handling.""" """Check _LinkFile handling."""
def LinkFile(self, src, dest): def LinkFile(self, src, dest):
return project._LinkFile(self.worktree, src, self.topdir, dest) return project._LinkFile(self.worktree, src, self.topdir, dest)
def test_basic(self): def test_basic(self):
"""Basic test of linking a file from a project into the toplevel.""" """Basic test of linking a file from a project into the toplevel."""
src = os.path.join(self.worktree, 'foo.txt') src = os.path.join(self.worktree, "foo.txt")
self.touch(src) self.touch(src)
lf = self.LinkFile('foo.txt', 'foo') lf = self.LinkFile("foo.txt", "foo")
lf._Link() lf._Link()
dest = os.path.join(self.topdir, 'foo') dest = os.path.join(self.topdir, "foo")
self.assertExists(dest) self.assertExists(dest)
self.assertTrue(os.path.islink(dest)) self.assertTrue(os.path.islink(dest))
self.assertEqual(os.path.join('git-project', 'foo.txt'), os.readlink(dest)) self.assertEqual(
os.path.join("git-project", "foo.txt"), os.readlink(dest)
)
def test_src_subdir(self): def test_src_subdir(self):
"""Link to a file in a subdir of a project.""" """Link to a file in a subdir of a project."""
src = os.path.join(self.worktree, 'bar', 'foo.txt') src = os.path.join(self.worktree, "bar", "foo.txt")
os.makedirs(os.path.dirname(src)) os.makedirs(os.path.dirname(src))
self.touch(src) self.touch(src)
lf = self.LinkFile('bar/foo.txt', 'foo') lf = self.LinkFile("bar/foo.txt", "foo")
lf._Link() lf._Link()
self.assertExists(os.path.join(self.topdir, 'foo')) self.assertExists(os.path.join(self.topdir, "foo"))
def test_src_self(self): def test_src_self(self):
"""Link to the project itself.""" """Link to the project itself."""
dest = os.path.join(self.topdir, 'foo', 'bar') dest = os.path.join(self.topdir, "foo", "bar")
lf = self.LinkFile('.', 'foo/bar') lf = self.LinkFile(".", "foo/bar")
lf._Link() lf._Link()
self.assertExists(dest) self.assertExists(dest)
self.assertEqual(os.path.join('..', 'git-project'), os.readlink(dest)) self.assertEqual(os.path.join("..", "git-project"), os.readlink(dest))
def test_dest_subdir(self): def test_dest_subdir(self):
"""Link a file to a subdir of a checkout.""" """Link a file to a subdir of a checkout."""
src = os.path.join(self.worktree, 'foo.txt') src = os.path.join(self.worktree, "foo.txt")
self.touch(src) self.touch(src)
lf = self.LinkFile('foo.txt', 'sub/dir/foo/bar') lf = self.LinkFile("foo.txt", "sub/dir/foo/bar")
self.assertFalse(os.path.exists(os.path.join(self.topdir, 'sub'))) self.assertFalse(os.path.exists(os.path.join(self.topdir, "sub")))
lf._Link() lf._Link()
self.assertExists(os.path.join(self.topdir, 'sub', 'dir', 'foo', 'bar')) self.assertExists(os.path.join(self.topdir, "sub", "dir", "foo", "bar"))
def test_src_block_relative(self): def test_src_block_relative(self):
"""Do not allow relative symlinks.""" """Do not allow relative symlinks."""
BAD_SOURCES = ( BAD_SOURCES = (
'./', "./",
'..', "..",
'../', "../",
'foo/.', "foo/.",
'foo/./bar', "foo/./bar",
'foo/..', "foo/..",
'foo/../foo', "foo/../foo",
) )
for src in BAD_SOURCES: for src in BAD_SOURCES:
lf = self.LinkFile(src, 'foo') lf = self.LinkFile(src, "foo")
self.assertRaises(error.ManifestInvalidPathError, lf._Link) self.assertRaises(error.ManifestInvalidPathError, lf._Link)
def test_update(self): def test_update(self):
"""Make sure changed targets get updated.""" """Make sure changed targets get updated."""
dest = os.path.join(self.topdir, 'sym') dest = os.path.join(self.topdir, "sym")
src = os.path.join(self.worktree, 'foo.txt') src = os.path.join(self.worktree, "foo.txt")
self.touch(src) self.touch(src)
lf = self.LinkFile('foo.txt', 'sym') lf = self.LinkFile("foo.txt", "sym")
lf._Link() lf._Link()
self.assertEqual(os.path.join('git-project', 'foo.txt'), os.readlink(dest)) self.assertEqual(
os.path.join("git-project", "foo.txt"), os.readlink(dest)
)
# Point the symlink somewhere else. # Point the symlink somewhere else.
os.unlink(dest) os.unlink(dest)
platform_utils.symlink(self.tempdir, dest) platform_utils.symlink(self.tempdir, dest)
lf._Link() lf._Link()
self.assertEqual(os.path.join('git-project', 'foo.txt'), os.readlink(dest)) self.assertEqual(
os.path.join("git-project", "foo.txt"), os.readlink(dest)
)
class MigrateWorkTreeTests(unittest.TestCase): class MigrateWorkTreeTests(unittest.TestCase):
"""Check _MigrateOldWorkTreeGitDir handling.""" """Check _MigrateOldWorkTreeGitDir handling."""
_SYMLINKS = { _SYMLINKS = {
'config', 'description', 'hooks', 'info', 'logs', 'objects', "config",
'packed-refs', 'refs', 'rr-cache', 'shallow', 'svn', "description",
} "hooks",
_FILES = { "info",
'COMMIT_EDITMSG', 'FETCH_HEAD', 'HEAD', 'index', 'ORIG_HEAD', "logs",
'unknown-file-should-be-migrated', "objects",
} "packed-refs",
_CLEAN_FILES = { "refs",
'a-vim-temp-file~', '#an-emacs-temp-file#', "rr-cache",
} "shallow",
"svn",
}
_FILES = {
"COMMIT_EDITMSG",
"FETCH_HEAD",
"HEAD",
"index",
"ORIG_HEAD",
"unknown-file-should-be-migrated",
}
_CLEAN_FILES = {
"a-vim-temp-file~",
"#an-emacs-temp-file#",
}
@classmethod @classmethod
@contextlib.contextmanager @contextlib.contextmanager
def _simple_layout(cls): def _simple_layout(cls):
"""Create a simple repo client checkout to test against.""" """Create a simple repo client checkout to test against."""
with tempfile.TemporaryDirectory() as tempdir: with tempfile.TemporaryDirectory() as tempdir:
tempdir = Path(tempdir) tempdir = Path(tempdir)
gitdir = tempdir / '.repo/projects/src/test.git' gitdir = tempdir / ".repo/projects/src/test.git"
gitdir.mkdir(parents=True) gitdir.mkdir(parents=True)
cmd = ['git', 'init', '--bare', str(gitdir)] cmd = ["git", "init", "--bare", str(gitdir)]
subprocess.check_call(cmd) subprocess.check_call(cmd)
dotgit = tempdir / 'src/test/.git' dotgit = tempdir / "src/test/.git"
dotgit.mkdir(parents=True) dotgit.mkdir(parents=True)
for name in cls._SYMLINKS: for name in cls._SYMLINKS:
(dotgit / name).symlink_to(f'../../../.repo/projects/src/test.git/{name}') (dotgit / name).symlink_to(
for name in cls._FILES | cls._CLEAN_FILES: f"../../../.repo/projects/src/test.git/{name}"
(dotgit / name).write_text(name) )
for name in cls._FILES | cls._CLEAN_FILES:
(dotgit / name).write_text(name)
yield tempdir yield tempdir
def test_standard(self): def test_standard(self):
"""Migrate a standard checkout that we expect.""" """Migrate a standard checkout that we expect."""
with self._simple_layout() as tempdir: with self._simple_layout() as tempdir:
dotgit = tempdir / 'src/test/.git' dotgit = tempdir / "src/test/.git"
project.Project._MigrateOldWorkTreeGitDir(str(dotgit)) project.Project._MigrateOldWorkTreeGitDir(str(dotgit))
# Make sure the dir was transformed into a symlink. # Make sure the dir was transformed into a symlink.
self.assertTrue(dotgit.is_symlink()) self.assertTrue(dotgit.is_symlink())
self.assertEqual(os.readlink(dotgit), os.path.normpath('../../.repo/projects/src/test.git')) self.assertEqual(
os.readlink(dotgit),
os.path.normpath("../../.repo/projects/src/test.git"),
)
# Make sure files were moved over. # Make sure files were moved over.
gitdir = tempdir / '.repo/projects/src/test.git' gitdir = tempdir / ".repo/projects/src/test.git"
for name in self._FILES: for name in self._FILES:
self.assertEqual(name, (gitdir / name).read_text()) self.assertEqual(name, (gitdir / name).read_text())
# Make sure files were removed. # Make sure files were removed.
for name in self._CLEAN_FILES: for name in self._CLEAN_FILES:
self.assertFalse((gitdir / name).exists()) self.assertFalse((gitdir / name).exists())
def test_unknown(self): def test_unknown(self):
"""A checkout with unknown files should abort.""" """A checkout with unknown files should abort."""
with self._simple_layout() as tempdir: with self._simple_layout() as tempdir:
dotgit = tempdir / 'src/test/.git' dotgit = tempdir / "src/test/.git"
(tempdir / '.repo/projects/src/test.git/random-file').write_text('one') (tempdir / ".repo/projects/src/test.git/random-file").write_text(
(dotgit / 'random-file').write_text('two') "one"
with self.assertRaises(error.GitError): )
project.Project._MigrateOldWorkTreeGitDir(str(dotgit)) (dotgit / "random-file").write_text("two")
with self.assertRaises(error.GitError):
project.Project._MigrateOldWorkTreeGitDir(str(dotgit))
# Make sure no content was actually changed. # Make sure no content was actually changed.
self.assertTrue(dotgit.is_dir()) self.assertTrue(dotgit.is_dir())
for name in self._FILES: for name in self._FILES:
self.assertTrue((dotgit / name).is_file()) self.assertTrue((dotgit / name).is_file())
for name in self._CLEAN_FILES: for name in self._CLEAN_FILES:
self.assertTrue((dotgit / name).is_file()) self.assertTrue((dotgit / name).is_file())
for name in self._SYMLINKS: for name in self._SYMLINKS:
self.assertTrue((dotgit / name).is_symlink()) self.assertTrue((dotgit / name).is_symlink())
class ManifestPropertiesFetchedCorrectly(unittest.TestCase): class ManifestPropertiesFetchedCorrectly(unittest.TestCase):
"""Ensure properties are fetched properly.""" """Ensure properties are fetched properly."""
def setUpManifest(self, tempdir): def setUpManifest(self, tempdir):
repodir = os.path.join(tempdir, '.repo') repodir = os.path.join(tempdir, ".repo")
manifest_dir = os.path.join(repodir, 'manifests') manifest_dir = os.path.join(repodir, "manifests")
manifest_file = os.path.join( manifest_file = os.path.join(repodir, manifest_xml.MANIFEST_FILE_NAME)
repodir, manifest_xml.MANIFEST_FILE_NAME) os.mkdir(repodir)
local_manifest_dir = os.path.join( os.mkdir(manifest_dir)
repodir, manifest_xml.LOCAL_MANIFESTS_DIR_NAME) manifest = manifest_xml.XmlManifest(repodir, manifest_file)
os.mkdir(repodir)
os.mkdir(manifest_dir)
manifest = manifest_xml.XmlManifest(repodir, manifest_file)
return project.ManifestProject( return project.ManifestProject(
manifest, 'test/manifest', os.path.join(tempdir, '.git'), tempdir) manifest, "test/manifest", os.path.join(tempdir, ".git"), tempdir
)
def test_manifest_config_properties(self): def test_manifest_config_properties(self):
"""Test we are fetching the manifest config properties correctly.""" """Test we are fetching the manifest config properties correctly."""
with TempGitTree() as tempdir: with TempGitTree() as tempdir:
fakeproj = self.setUpManifest(tempdir) fakeproj = self.setUpManifest(tempdir)
# Set property using the expected Set method, then ensure # Set property using the expected Set method, then ensure
# the porperty functions are using the correct Get methods. # the porperty functions are using the correct Get methods.
fakeproj.config.SetString( fakeproj.config.SetString(
'manifest.standalone', 'https://chicken/manifest.git') "manifest.standalone", "https://chicken/manifest.git"
self.assertEqual( )
fakeproj.standalone_manifest_url, 'https://chicken/manifest.git') self.assertEqual(
fakeproj.standalone_manifest_url, "https://chicken/manifest.git"
)
fakeproj.config.SetString('manifest.groups', 'test-group, admin-group') fakeproj.config.SetString(
self.assertEqual(fakeproj.manifest_groups, 'test-group, admin-group') "manifest.groups", "test-group, admin-group"
)
self.assertEqual(
fakeproj.manifest_groups, "test-group, admin-group"
)
fakeproj.config.SetString('repo.reference', 'mirror/ref') fakeproj.config.SetString("repo.reference", "mirror/ref")
self.assertEqual(fakeproj.reference, 'mirror/ref') self.assertEqual(fakeproj.reference, "mirror/ref")
fakeproj.config.SetBoolean('repo.dissociate', False) fakeproj.config.SetBoolean("repo.dissociate", False)
self.assertFalse(fakeproj.dissociate) self.assertFalse(fakeproj.dissociate)
fakeproj.config.SetBoolean('repo.archive', False) fakeproj.config.SetBoolean("repo.archive", False)
self.assertFalse(fakeproj.archive) self.assertFalse(fakeproj.archive)
fakeproj.config.SetBoolean('repo.mirror', False) fakeproj.config.SetBoolean("repo.mirror", False)
self.assertFalse(fakeproj.mirror) self.assertFalse(fakeproj.mirror)
fakeproj.config.SetBoolean('repo.worktree', False) fakeproj.config.SetBoolean("repo.worktree", False)
self.assertFalse(fakeproj.use_worktree) self.assertFalse(fakeproj.use_worktree)
fakeproj.config.SetBoolean('repo.clonebundle', False) fakeproj.config.SetBoolean("repo.clonebundle", False)
self.assertFalse(fakeproj.clone_bundle) self.assertFalse(fakeproj.clone_bundle)
fakeproj.config.SetBoolean('repo.submodules', False) fakeproj.config.SetBoolean("repo.submodules", False)
self.assertFalse(fakeproj.submodules) self.assertFalse(fakeproj.submodules)
fakeproj.config.SetBoolean('repo.git-lfs', False) fakeproj.config.SetBoolean("repo.git-lfs", False)
self.assertFalse(fakeproj.git_lfs) self.assertFalse(fakeproj.git_lfs)
fakeproj.config.SetBoolean('repo.superproject', False) fakeproj.config.SetBoolean("repo.superproject", False)
self.assertFalse(fakeproj.use_superproject) self.assertFalse(fakeproj.use_superproject)
fakeproj.config.SetBoolean('repo.partialclone', False) fakeproj.config.SetBoolean("repo.partialclone", False)
self.assertFalse(fakeproj.partial_clone) self.assertFalse(fakeproj.partial_clone)
fakeproj.config.SetString('repo.depth', '48') fakeproj.config.SetString("repo.depth", "48")
self.assertEqual(fakeproj.depth, '48') self.assertEqual(fakeproj.depth, "48")
fakeproj.config.SetString('repo.clonefilter', 'blob:limit=10M') fakeproj.config.SetString("repo.clonefilter", "blob:limit=10M")
self.assertEqual(fakeproj.clone_filter, 'blob:limit=10M') self.assertEqual(fakeproj.clone_filter, "blob:limit=10M")
fakeproj.config.SetString('repo.partialcloneexclude', 'third_party/big_repo') fakeproj.config.SetString(
self.assertEqual(fakeproj.partial_clone_exclude, 'third_party/big_repo') "repo.partialcloneexclude", "third_party/big_repo"
)
self.assertEqual(
fakeproj.partial_clone_exclude, "third_party/big_repo"
)
fakeproj.config.SetString('manifest.platform', 'auto') fakeproj.config.SetString("manifest.platform", "auto")
self.assertEqual(fakeproj.manifest_platform, 'auto') self.assertEqual(fakeproj.manifest_platform, "auto")

56
tests/test_repo_trace.py
View File

@ -22,35 +22,39 @@ import repo_trace
class TraceTests(unittest.TestCase): class TraceTests(unittest.TestCase):
"""Check Trace behavior.""" """Check Trace behavior."""
def testTrace_MaxSizeEnforced(self): def testTrace_MaxSizeEnforced(self):
content = 'git chicken' content = "git chicken"
with repo_trace.Trace(content, first_trace=True): with repo_trace.Trace(content, first_trace=True):
pass pass
first_trace_size = os.path.getsize(repo_trace._TRACE_FILE) first_trace_size = os.path.getsize(repo_trace._TRACE_FILE)
with repo_trace.Trace(content): with repo_trace.Trace(content):
pass pass
self.assertGreater( self.assertGreater(
os.path.getsize(repo_trace._TRACE_FILE), first_trace_size) os.path.getsize(repo_trace._TRACE_FILE), first_trace_size
)
# Check we clear everything is the last chunk is larger than _MAX_SIZE. # Check we clear everything is the last chunk is larger than _MAX_SIZE.
with mock.patch('repo_trace._MAX_SIZE', 0): with mock.patch("repo_trace._MAX_SIZE", 0):
with repo_trace.Trace(content, first_trace=True): with repo_trace.Trace(content, first_trace=True):
pass pass
self.assertEqual(first_trace_size, self.assertEqual(
os.path.getsize(repo_trace._TRACE_FILE)) first_trace_size, os.path.getsize(repo_trace._TRACE_FILE)
)
# Check we only clear the chunks we need to. # Check we only clear the chunks we need to.
repo_trace._MAX_SIZE = (first_trace_size + 1) / (1024 * 1024) repo_trace._MAX_SIZE = (first_trace_size + 1) / (1024 * 1024)
with repo_trace.Trace(content, first_trace=True): with repo_trace.Trace(content, first_trace=True):
pass pass
self.assertEqual(first_trace_size * 2, self.assertEqual(
os.path.getsize(repo_trace._TRACE_FILE)) first_trace_size * 2, os.path.getsize(repo_trace._TRACE_FILE)
)
with repo_trace.Trace(content, first_trace=True): with repo_trace.Trace(content, first_trace=True):
pass pass
self.assertEqual(first_trace_size * 2, self.assertEqual(
os.path.getsize(repo_trace._TRACE_FILE)) first_trace_size * 2, os.path.getsize(repo_trace._TRACE_FILE)
)

90
tests/test_ssh.py
View File

@ -23,52 +23,56 @@ import ssh
class SshTests(unittest.TestCase): class SshTests(unittest.TestCase):
"""Tests the ssh functions.""" """Tests the ssh functions."""
def test_parse_ssh_version(self): def test_parse_ssh_version(self):
"""Check _parse_ssh_version() handling.""" """Check _parse_ssh_version() handling."""
ver = ssh._parse_ssh_version('Unknown\n') ver = ssh._parse_ssh_version("Unknown\n")
self.assertEqual(ver, ()) self.assertEqual(ver, ())
ver = ssh._parse_ssh_version('OpenSSH_1.0\n') ver = ssh._parse_ssh_version("OpenSSH_1.0\n")
self.assertEqual(ver, (1, 0)) self.assertEqual(ver, (1, 0))
ver = ssh._parse_ssh_version('OpenSSH_6.6.1p1 Ubuntu-2ubuntu2.13, OpenSSL 1.0.1f 6 Jan 2014\n') ver = ssh._parse_ssh_version(
self.assertEqual(ver, (6, 6, 1)) "OpenSSH_6.6.1p1 Ubuntu-2ubuntu2.13, OpenSSL 1.0.1f 6 Jan 2014\n"
ver = ssh._parse_ssh_version('OpenSSH_7.6p1 Ubuntu-4ubuntu0.3, OpenSSL 1.0.2n 7 Dec 2017\n') )
self.assertEqual(ver, (7, 6)) self.assertEqual(ver, (6, 6, 1))
ver = ssh._parse_ssh_version(
"OpenSSH_7.6p1 Ubuntu-4ubuntu0.3, OpenSSL 1.0.2n 7 Dec 2017\n"
)
self.assertEqual(ver, (7, 6))
def test_version(self): def test_version(self):
"""Check version() handling.""" """Check version() handling."""
with mock.patch('ssh._run_ssh_version', return_value='OpenSSH_1.2\n'): with mock.patch("ssh._run_ssh_version", return_value="OpenSSH_1.2\n"):
self.assertEqual(ssh.version(), (1, 2)) self.assertEqual(ssh.version(), (1, 2))
def test_context_manager_empty(self): def test_context_manager_empty(self):
"""Verify context manager with no clients works correctly.""" """Verify context manager with no clients works correctly."""
with multiprocessing.Manager() as manager: with multiprocessing.Manager() as manager:
with ssh.ProxyManager(manager): with ssh.ProxyManager(manager):
pass pass
def test_context_manager_child_cleanup(self): def test_context_manager_child_cleanup(self):
"""Verify orphaned clients & masters get cleaned up.""" """Verify orphaned clients & masters get cleaned up."""
with multiprocessing.Manager() as manager: with multiprocessing.Manager() as manager:
with ssh.ProxyManager(manager) as ssh_proxy: with ssh.ProxyManager(manager) as ssh_proxy:
client = subprocess.Popen(['sleep', '964853320']) client = subprocess.Popen(["sleep", "964853320"])
ssh_proxy.add_client(client) ssh_proxy.add_client(client)
master = subprocess.Popen(['sleep', '964853321']) master = subprocess.Popen(["sleep", "964853321"])
ssh_proxy.add_master(master) ssh_proxy.add_master(master)
# If the process still exists, these will throw timeout errors. # If the process still exists, these will throw timeout errors.
client.wait(0) client.wait(0)
master.wait(0) master.wait(0)
def test_ssh_sock(self): def test_ssh_sock(self):
"""Check sock() function.""" """Check sock() function."""
manager = multiprocessing.Manager() manager = multiprocessing.Manager()
proxy = ssh.ProxyManager(manager) proxy = ssh.ProxyManager(manager)
with mock.patch('tempfile.mkdtemp', return_value='/tmp/foo'): with mock.patch("tempfile.mkdtemp", return_value="/tmp/foo"):
# old ssh version uses port # Old ssh version uses port.
with mock.patch('ssh.version', return_value=(6, 6)): with mock.patch("ssh.version", return_value=(6, 6)):
self.assertTrue(proxy.sock().endswith('%p')) self.assertTrue(proxy.sock().endswith("%p"))
proxy._sock_path = None proxy._sock_path = None
# new ssh version uses hash # New ssh version uses hash.
with mock.patch('ssh.version', return_value=(6, 7)): with mock.patch("ssh.version", return_value=(6, 7)):
self.assertTrue(proxy.sock().endswith('%C')) self.assertTrue(proxy.sock().endswith("%C"))

86
tests/test_subcmds.py
View File

@ -21,53 +21,57 @@ import subcmds
class AllCommands(unittest.TestCase): class AllCommands(unittest.TestCase):
"""Check registered all_commands.""" """Check registered all_commands."""
def test_required_basic(self): def test_required_basic(self):
"""Basic checking of registered commands.""" """Basic checking of registered commands."""
# NB: We don't test all subcommands as we want to avoid "change detection" # NB: We don't test all subcommands as we want to avoid "change
# tests, so we just look for the most common/important ones here that are # detection" tests, so we just look for the most common/important ones
# unlikely to ever change. # here that are unlikely to ever change.
for cmd in {'cherry-pick', 'help', 'init', 'start', 'sync', 'upload'}: for cmd in {"cherry-pick", "help", "init", "start", "sync", "upload"}:
self.assertIn(cmd, subcmds.all_commands) self.assertIn(cmd, subcmds.all_commands)
def test_naming(self): def test_naming(self):
"""Verify we don't add things that we shouldn't.""" """Verify we don't add things that we shouldn't."""
for cmd in subcmds.all_commands: for cmd in subcmds.all_commands:
# Reject filename suffixes like "help.py". # Reject filename suffixes like "help.py".
self.assertNotIn('.', cmd) self.assertNotIn(".", cmd)
# Make sure all '_' were converted to '-'. # Make sure all '_' were converted to '-'.
self.assertNotIn('_', cmd) self.assertNotIn("_", cmd)
# Reject internal python paths like "__init__". # Reject internal python paths like "__init__".
self.assertFalse(cmd.startswith('__')) self.assertFalse(cmd.startswith("__"))
def test_help_desc_style(self): def test_help_desc_style(self):
"""Force some consistency in option descriptions. """Force some consistency in option descriptions.
Python's optparse & argparse has a few default options like --help. Their Python's optparse & argparse has a few default options like --help.
option description text uses lowercase sentence fragments, so enforce our Their option description text uses lowercase sentence fragments, so
options follow the same style so UI is consistent. enforce our options follow the same style so UI is consistent.
We enforce: We enforce:
* Text starts with lowercase. * Text starts with lowercase.
* Text doesn't end with period. * Text doesn't end with period.
""" """
for name, cls in subcmds.all_commands.items(): for name, cls in subcmds.all_commands.items():
cmd = cls() cmd = cls()
parser = cmd.OptionParser parser = cmd.OptionParser
for option in parser.option_list: for option in parser.option_list:
if option.help == optparse.SUPPRESS_HELP: if option.help == optparse.SUPPRESS_HELP:
continue continue
c = option.help[0] c = option.help[0]
self.assertEqual( self.assertEqual(
c.lower(), c, c.lower(),
msg=f'subcmds/{name}.py: {option.get_opt_string()}: help text ' c,
f'should start with lowercase: "{option.help}"') msg=f"subcmds/{name}.py: {option.get_opt_string()}: "
f'help text should start with lowercase: "{option.help}"',
)
self.assertNotEqual( self.assertNotEqual(
option.help[-1], '.', option.help[-1],
msg=f'subcmds/{name}.py: {option.get_opt_string()}: help text ' ".",
f'should not end in a period: "{option.help}"') msg=f"subcmds/{name}.py: {option.get_opt_string()}: "
f'help text should not end in a period: "{option.help}"',
)

45
tests/test_subcmds_init.py
View File

@ -20,30 +20,27 @@ from subcmds import init
class InitCommand(unittest.TestCase): class InitCommand(unittest.TestCase):
"""Check registered all_commands.""" """Check registered all_commands."""
def setUp(self): def setUp(self):
self.cmd = init.Init() self.cmd = init.Init()
def test_cli_parser_good(self): def test_cli_parser_good(self):
"""Check valid command line options.""" """Check valid command line options."""
ARGV = ( ARGV = ([],)
[], for argv in ARGV:
) opts, args = self.cmd.OptionParser.parse_args(argv)
for argv in ARGV: self.cmd.ValidateOptions(opts, args)
opts, args = self.cmd.OptionParser.parse_args(argv)
self.cmd.ValidateOptions(opts, args)
def test_cli_parser_bad(self): def test_cli_parser_bad(self):
"""Check invalid command line options.""" """Check invalid command line options."""
ARGV = ( ARGV = (
# Too many arguments. # Too many arguments.
['url', 'asdf'], ["url", "asdf"],
# Conflicting options.
# Conflicting options. ["--mirror", "--archive"],
['--mirror', '--archive'], )
) for argv in ARGV:
for argv in ARGV: opts, args = self.cmd.OptionParser.parse_args(argv)
opts, args = self.cmd.OptionParser.parse_args(argv) with self.assertRaises(SystemExit):
with self.assertRaises(SystemExit): self.cmd.ValidateOptions(opts, args)
self.cmd.ValidateOptions(opts, args)

201
tests/test_subcmds_sync.py
View File

@ -23,111 +23,138 @@ import command
from subcmds import sync from subcmds import sync
@pytest.mark.parametrize('use_superproject, cli_args, result', [ @pytest.mark.parametrize(
(True, ['--current-branch'], True), "use_superproject, cli_args, result",
(True, ['--no-current-branch'], True), [
(True, [], True), (True, ["--current-branch"], True),
(False, ['--current-branch'], True), (True, ["--no-current-branch"], True),
(False, ['--no-current-branch'], False), (True, [], True),
(False, [], None), (False, ["--current-branch"], True),
]) (False, ["--no-current-branch"], False),
(False, [], None),
],
)
def test_get_current_branch_only(use_superproject, cli_args, result): def test_get_current_branch_only(use_superproject, cli_args, result):
"""Test Sync._GetCurrentBranchOnly logic. """Test Sync._GetCurrentBranchOnly logic.
Sync._GetCurrentBranchOnly should return True if a superproject is requested, Sync._GetCurrentBranchOnly should return True if a superproject is
and otherwise the value of the current_branch_only option. requested, and otherwise the value of the current_branch_only option.
""" """
cmd = sync.Sync() cmd = sync.Sync()
opts, _ = cmd.OptionParser.parse_args(cli_args) opts, _ = cmd.OptionParser.parse_args(cli_args)
with mock.patch('git_superproject.UseSuperproject', with mock.patch(
return_value=use_superproject): "git_superproject.UseSuperproject", return_value=use_superproject
assert cmd._GetCurrentBranchOnly(opts, cmd.manifest) == result ):
assert cmd._GetCurrentBranchOnly(opts, cmd.manifest) == result
# Used to patch os.cpu_count() for reliable results. # Used to patch os.cpu_count() for reliable results.
OS_CPU_COUNT = 24 OS_CPU_COUNT = 24
@pytest.mark.parametrize('argv, jobs_manifest, jobs, jobs_net, jobs_check', [
# No user or manifest settings. @pytest.mark.parametrize(
([], None, OS_CPU_COUNT, 1, command.DEFAULT_LOCAL_JOBS), "argv, jobs_manifest, jobs, jobs_net, jobs_check",
# No user settings, so manifest settings control. [
([], 3, 3, 3, 3), # No user or manifest settings.
# User settings, but no manifest. ([], None, OS_CPU_COUNT, 1, command.DEFAULT_LOCAL_JOBS),
(['--jobs=4'], None, 4, 4, 4), # No user settings, so manifest settings control.
(['--jobs=4', '--jobs-network=5'], None, 4, 5, 4), ([], 3, 3, 3, 3),
(['--jobs=4', '--jobs-checkout=6'], None, 4, 4, 6), # User settings, but no manifest.
(['--jobs=4', '--jobs-network=5', '--jobs-checkout=6'], None, 4, 5, 6), (["--jobs=4"], None, 4, 4, 4),
(['--jobs-network=5'], None, OS_CPU_COUNT, 5, command.DEFAULT_LOCAL_JOBS), (["--jobs=4", "--jobs-network=5"], None, 4, 5, 4),
(['--jobs-checkout=6'], None, OS_CPU_COUNT, 1, 6), (["--jobs=4", "--jobs-checkout=6"], None, 4, 4, 6),
(['--jobs-network=5', '--jobs-checkout=6'], None, OS_CPU_COUNT, 5, 6), (["--jobs=4", "--jobs-network=5", "--jobs-checkout=6"], None, 4, 5, 6),
# User settings with manifest settings. (
(['--jobs=4'], 3, 4, 4, 4), ["--jobs-network=5"],
(['--jobs=4', '--jobs-network=5'], 3, 4, 5, 4), None,
(['--jobs=4', '--jobs-checkout=6'], 3, 4, 4, 6), OS_CPU_COUNT,
(['--jobs=4', '--jobs-network=5', '--jobs-checkout=6'], 3, 4, 5, 6), 5,
(['--jobs-network=5'], 3, 3, 5, 3), command.DEFAULT_LOCAL_JOBS,
(['--jobs-checkout=6'], 3, 3, 3, 6), ),
(['--jobs-network=5', '--jobs-checkout=6'], 3, 3, 5, 6), (["--jobs-checkout=6"], None, OS_CPU_COUNT, 1, 6),
# Settings that exceed rlimits get capped. (["--jobs-network=5", "--jobs-checkout=6"], None, OS_CPU_COUNT, 5, 6),
(['--jobs=1000000'], None, 83, 83, 83), # User settings with manifest settings.
([], 1000000, 83, 83, 83), (["--jobs=4"], 3, 4, 4, 4),
]) (["--jobs=4", "--jobs-network=5"], 3, 4, 5, 4),
(["--jobs=4", "--jobs-checkout=6"], 3, 4, 4, 6),
(["--jobs=4", "--jobs-network=5", "--jobs-checkout=6"], 3, 4, 5, 6),
(["--jobs-network=5"], 3, 3, 5, 3),
(["--jobs-checkout=6"], 3, 3, 3, 6),
(["--jobs-network=5", "--jobs-checkout=6"], 3, 3, 5, 6),
# Settings that exceed rlimits get capped.
(["--jobs=1000000"], None, 83, 83, 83),
([], 1000000, 83, 83, 83),
],
)
def test_cli_jobs(argv, jobs_manifest, jobs, jobs_net, jobs_check): def test_cli_jobs(argv, jobs_manifest, jobs, jobs_net, jobs_check):
"""Tests --jobs option behavior.""" """Tests --jobs option behavior."""
mp = mock.MagicMock() mp = mock.MagicMock()
mp.manifest.default.sync_j = jobs_manifest mp.manifest.default.sync_j = jobs_manifest
cmd = sync.Sync() cmd = sync.Sync()
opts, args = cmd.OptionParser.parse_args(argv) opts, args = cmd.OptionParser.parse_args(argv)
cmd.ValidateOptions(opts, args) cmd.ValidateOptions(opts, args)
with mock.patch.object(sync, '_rlimit_nofile', return_value=(256, 256)): with mock.patch.object(sync, "_rlimit_nofile", return_value=(256, 256)):
with mock.patch.object(os, 'cpu_count', return_value=OS_CPU_COUNT): with mock.patch.object(os, "cpu_count", return_value=OS_CPU_COUNT):
cmd._ValidateOptionsWithManifest(opts, mp) cmd._ValidateOptionsWithManifest(opts, mp)
assert opts.jobs == jobs assert opts.jobs == jobs
assert opts.jobs_network == jobs_net assert opts.jobs_network == jobs_net
assert opts.jobs_checkout == jobs_check assert opts.jobs_checkout == jobs_check
class GetPreciousObjectsState(unittest.TestCase): class GetPreciousObjectsState(unittest.TestCase):
"""Tests for _GetPreciousObjectsState.""" """Tests for _GetPreciousObjectsState."""
def setUp(self): def setUp(self):
"""Common setup.""" """Common setup."""
self.cmd = sync.Sync() self.cmd = sync.Sync()
self.project = p = mock.MagicMock(use_git_worktrees=False, self.project = p = mock.MagicMock(
UseAlternates=False) use_git_worktrees=False, UseAlternates=False
p.manifest.GetProjectsWithName.return_value = [p] )
p.manifest.GetProjectsWithName.return_value = [p]
self.opt = mock.Mock(spec_set=['this_manifest_only']) self.opt = mock.Mock(spec_set=["this_manifest_only"])
self.opt.this_manifest_only = False self.opt.this_manifest_only = False
def test_worktrees(self): def test_worktrees(self):
"""False for worktrees.""" """False for worktrees."""
self.project.use_git_worktrees = True self.project.use_git_worktrees = True
self.assertFalse(self.cmd._GetPreciousObjectsState(self.project, self.opt)) self.assertFalse(
self.cmd._GetPreciousObjectsState(self.project, self.opt)
)
def test_not_shared(self): def test_not_shared(self):
"""Singleton project.""" """Singleton project."""
self.assertFalse(self.cmd._GetPreciousObjectsState(self.project, self.opt)) self.assertFalse(
self.cmd._GetPreciousObjectsState(self.project, self.opt)
)
def test_shared(self): def test_shared(self):
"""Shared project.""" """Shared project."""
self.project.manifest.GetProjectsWithName.return_value = [ self.project.manifest.GetProjectsWithName.return_value = [
self.project, self.project self.project,
] self.project,
self.assertTrue(self.cmd._GetPreciousObjectsState(self.project, self.opt)) ]
self.assertTrue(
self.cmd._GetPreciousObjectsState(self.project, self.opt)
)
def test_shared_with_alternates(self): def test_shared_with_alternates(self):
"""Shared project, with alternates.""" """Shared project, with alternates."""
self.project.manifest.GetProjectsWithName.return_value = [ self.project.manifest.GetProjectsWithName.return_value = [
self.project, self.project self.project,
] self.project,
self.project.UseAlternates = True ]
self.assertFalse(self.cmd._GetPreciousObjectsState(self.project, self.opt)) self.project.UseAlternates = True
self.assertFalse(
self.cmd._GetPreciousObjectsState(self.project, self.opt)
)
def test_not_found(self): def test_not_found(self):
"""Project not found in manifest.""" """Project not found in manifest."""
self.project.manifest.GetProjectsWithName.return_value = [] self.project.manifest.GetProjectsWithName.return_value = []
self.assertFalse(self.cmd._GetPreciousObjectsState(self.project, self.opt)) self.assertFalse(
self.cmd._GetPreciousObjectsState(self.project, self.opt)
)

10
tests/test_update_manpages.py
View File

@ -20,9 +20,9 @@ from release import update_manpages
class UpdateManpagesTest(unittest.TestCase): class UpdateManpagesTest(unittest.TestCase):
"""Tests the update-manpages code.""" """Tests the update-manpages code."""
def test_replace_regex(self): def test_replace_regex(self):
"""Check that replace_regex works.""" """Check that replace_regex works."""
data = '\n\033[1mSummary\033[m\n' data = "\n\033[1mSummary\033[m\n"
self.assertEqual(update_manpages.replace_regex(data),'\nSummary\n') self.assertEqual(update_manpages.replace_regex(data), "\nSummary\n")

907
tests/test_wrapper.py
View File

File diff suppressed because it is too large Load Diff

1
tox.ini
View File

@ -27,6 +27,7 @@ python =
[testenv] [testenv]
deps = deps =
black
pytest pytest
pytest-timeout pytest-timeout
commands = {envpython} run_tests {posargs} commands = {envpython} run_tests {posargs}

14
wrapper.py
View File

@ -19,14 +19,14 @@ import os
def WrapperPath(): def WrapperPath():
return os.path.join(os.path.dirname(__file__), 'repo') return os.path.join(os.path.dirname(__file__), "repo")
@functools.lru_cache(maxsize=None) @functools.lru_cache(maxsize=None)
def Wrapper(): def Wrapper():
modname = 'wrapper' modname = "wrapper"
loader = importlib.machinery.SourceFileLoader(modname, WrapperPath()) loader = importlib.machinery.SourceFileLoader(modname, WrapperPath())
spec = importlib.util.spec_from_loader(modname, loader) spec = importlib.util.spec_from_loader(modname, loader)
module = importlib.util.module_from_spec(spec) module = importlib.util.module_from_spec(spec)
loader.exec_module(module) loader.exec_module(module)
return module return module