diff --git a/repo b/repo index 79237752..d2a4df2e 100755 --- a/repo +++ b/repo @@ -41,112 +41,136 @@ MIN_PYTHON_VERSION_HARD = (3, 5) # Keep basic logic in sync with repo_trace.py. class Trace(object): - """Trace helper logic.""" + """Trace helper logic.""" - REPO_TRACE = 'REPO_TRACE' + REPO_TRACE = "REPO_TRACE" - def __init__(self): - self.set(os.environ.get(self.REPO_TRACE) == '1') + def __init__(self): + self.set(os.environ.get(self.REPO_TRACE) == "1") - def set(self, value): - self.enabled = bool(value) + def set(self, value): + self.enabled = bool(value) - def print(self, *args, **kwargs): - if self.enabled: - print(*args, **kwargs) + def print(self, *args, **kwargs): + if self.enabled: + print(*args, **kwargs) trace = Trace() def exec_command(cmd): - """Execute |cmd| or return None on failure.""" - trace.print(':', ' '.join(cmd)) - try: - if platform.system() == 'Windows': - ret = subprocess.call(cmd) - sys.exit(ret) - else: - os.execvp(cmd[0], cmd) - except Exception: - pass + """Execute |cmd| or return None on failure.""" + trace.print(":", " ".join(cmd)) + try: + if platform.system() == "Windows": + ret = subprocess.call(cmd) + sys.exit(ret) + else: + os.execvp(cmd[0], cmd) + except Exception: + pass def check_python_version(): - """Make sure the active Python version is recent enough.""" - def reexec(prog): - exec_command([prog] + sys.argv) + """Make sure the active Python version is recent enough.""" - ver = sys.version_info - major = ver.major - minor = ver.minor + def reexec(prog): + exec_command([prog] + sys.argv) - # Abort on very old Python 2 versions. - if (major, minor) < (2, 7): - print('repo: error: Your Python version is too old. ' - 'Please use Python {}.{} or newer instead.'.format( - *MIN_PYTHON_VERSION_SOFT), file=sys.stderr) - sys.exit(1) + ver = sys.version_info + major = ver.major + minor = ver.minor - # Try to re-exec the version specific Python 3 if needed. - if (major, minor) < MIN_PYTHON_VERSION_SOFT: - # Python makes releases ~once a year, so try our min version +10 to help - # bridge the gap. This is the fallback anyways so perf isn't critical. - min_major, min_minor = MIN_PYTHON_VERSION_SOFT - for inc in range(0, 10): - reexec('python{}.{}'.format(min_major, min_minor + inc)) + # Abort on very old Python 2 versions. + if (major, minor) < (2, 7): + print( + "repo: error: Your Python version is too old. " + "Please use Python {}.{} or newer instead.".format( + *MIN_PYTHON_VERSION_SOFT + ), + file=sys.stderr, + ) + sys.exit(1) - # Fallback to older versions if possible. - for inc in range(MIN_PYTHON_VERSION_SOFT[1] - MIN_PYTHON_VERSION_HARD[1], 0, -1): - # Don't downgrade, and don't reexec ourselves (which would infinite loop). - if (min_major, min_minor - inc) <= (major, minor): - break - reexec('python{}.{}'.format(min_major, min_minor - inc)) + # Try to re-exec the version specific Python 3 if needed. + if (major, minor) < MIN_PYTHON_VERSION_SOFT: + # Python makes releases ~once a year, so try our min version +10 to help + # bridge the gap. This is the fallback anyways so perf isn't critical. + min_major, min_minor = MIN_PYTHON_VERSION_SOFT + for inc in range(0, 10): + reexec("python{}.{}".format(min_major, min_minor + inc)) - # Try the generic Python 3 wrapper, but only if it's new enough. If it - # isn't, we want to just give up below and make the user resolve things. - try: - proc = subprocess.Popen( - ['python3', '-c', 'import sys; ' - 'print(sys.version_info.major, sys.version_info.minor)'], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (output, _) = proc.communicate() - python3_ver = tuple(int(x) for x in output.decode('utf-8').split()) - except (OSError, subprocess.CalledProcessError): - python3_ver = None + # Fallback to older versions if possible. + for inc in range( + MIN_PYTHON_VERSION_SOFT[1] - MIN_PYTHON_VERSION_HARD[1], 0, -1 + ): + # Don't downgrade, and don't reexec ourselves (which would infinite loop). + if (min_major, min_minor - inc) <= (major, minor): + break + reexec("python{}.{}".format(min_major, min_minor - inc)) - # If the python3 version looks like it's new enough, give it a try. - if (python3_ver and python3_ver >= MIN_PYTHON_VERSION_HARD - and python3_ver != (major, minor)): - reexec('python3') + # Try the generic Python 3 wrapper, but only if it's new enough. If it + # isn't, we want to just give up below and make the user resolve things. + try: + proc = subprocess.Popen( + [ + "python3", + "-c", + "import sys; " + "print(sys.version_info.major, sys.version_info.minor)", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + (output, _) = proc.communicate() + python3_ver = tuple(int(x) for x in output.decode("utf-8").split()) + except (OSError, subprocess.CalledProcessError): + python3_ver = None - # We're still here, so diagnose things for the user. - if major < 3: - print('repo: error: Python 2 is no longer supported; ' - 'Please upgrade to Python {}.{}+.'.format(*MIN_PYTHON_VERSION_HARD), - file=sys.stderr) - sys.exit(1) - elif (major, minor) < MIN_PYTHON_VERSION_HARD: - print('repo: error: Python 3 version is too old; ' - 'Please use Python {}.{} or newer.'.format(*MIN_PYTHON_VERSION_HARD), - file=sys.stderr) - sys.exit(1) + # If the python3 version looks like it's new enough, give it a try. + if ( + python3_ver + and python3_ver >= MIN_PYTHON_VERSION_HARD + and python3_ver != (major, minor) + ): + reexec("python3") + + # We're still here, so diagnose things for the user. + if major < 3: + print( + "repo: error: Python 2 is no longer supported; " + "Please upgrade to Python {}.{}+.".format( + *MIN_PYTHON_VERSION_HARD + ), + file=sys.stderr, + ) + sys.exit(1) + elif (major, minor) < MIN_PYTHON_VERSION_HARD: + print( + "repo: error: Python 3 version is too old; " + "Please use Python {}.{} or newer.".format( + *MIN_PYTHON_VERSION_HARD + ), + file=sys.stderr, + ) + sys.exit(1) -if __name__ == '__main__': - check_python_version() +if __name__ == "__main__": + check_python_version() # repo default configuration # -REPO_URL = os.environ.get('REPO_URL', None) +REPO_URL = os.environ.get("REPO_URL", None) if not REPO_URL: - REPO_URL = 'https://gerrit.googlesource.com/git-repo' -REPO_REV = os.environ.get('REPO_REV') + REPO_URL = "https://gerrit.googlesource.com/git-repo" +REPO_REV = os.environ.get("REPO_REV") if not REPO_REV: - REPO_REV = 'stable' + REPO_REV = "stable" # URL to file bug reports for repo tool issues. -BUG_URL = 'https://issues.gerritcodereview.com/issues/new?component=1370071' +BUG_URL = "https://issues.gerritcodereview.com/issues/new?component=1370071" # increment this whenever we make important changes to this script VERSION = (2, 36) @@ -231,19 +255,19 @@ cZ7aFsJF4PtcDrfdejyAxbtsSHI= -----END PGP PUBLIC KEY BLOCK----- """ -GIT = 'git' # our git command +GIT = "git" # our git command # NB: The version of git that the repo launcher requires may be much older than # the version of git that the main repo source tree requires. Keeping this at # an older version also makes it easier for users to upgrade/rollback as needed. # # git-1.7 is in (EOL) Ubuntu Precise. -MIN_GIT_VERSION = (1, 7, 2) # minimum supported git version -repodir = '.repo' # name of repo's private directory -S_repo = 'repo' # special repo repository -S_manifests = 'manifests' # special manifest repository -REPO_MAIN = S_repo + '/main.py' # main script -GITC_CONFIG_FILE = '/gitc/.config' -GITC_FS_ROOT_DIR = '/gitc/manifest-rw/' +MIN_GIT_VERSION = (1, 7, 2) # minimum supported git version +repodir = ".repo" # name of repo's private directory +S_repo = "repo" # special repo repository +S_manifests = "manifests" # special manifest repository +REPO_MAIN = S_repo + "/main.py" # main script +GITC_CONFIG_FILE = "/gitc/.config" +GITC_FS_ROOT_DIR = "/gitc/manifest-rw/" import collections @@ -256,1074 +280,1270 @@ import stat if sys.version_info[0] == 3: - import urllib.error - import urllib.request + import urllib.error + import urllib.request else: - import imp + import imp - import urllib2 - urllib = imp.new_module('urllib') - urllib.request = urllib2 - urllib.error = urllib2 + import urllib2 + + urllib = imp.new_module("urllib") + urllib.request = urllib2 + urllib.error = urllib2 -repo_config_dir = os.getenv('REPO_CONFIG_DIR', os.path.expanduser('~')) -home_dot_repo = os.path.join(repo_config_dir, '.repoconfig') -gpg_dir = os.path.join(home_dot_repo, 'gnupg') +repo_config_dir = os.getenv("REPO_CONFIG_DIR", os.path.expanduser("~")) +home_dot_repo = os.path.join(repo_config_dir, ".repoconfig") +gpg_dir = os.path.join(home_dot_repo, "gnupg") def GetParser(gitc_init=False): - """Setup the CLI parser.""" - if gitc_init: - sys.exit('repo: fatal: GITC not supported.') - else: - usage = 'repo init [options] [-u] url' + """Setup the CLI parser.""" + if gitc_init: + sys.exit("repo: fatal: GITC not supported.") + else: + usage = "repo init [options] [-u] url" - parser = optparse.OptionParser(usage=usage) - InitParser(parser) - return parser + parser = optparse.OptionParser(usage=usage) + InitParser(parser) + return parser def InitParser(parser): - """Setup the CLI parser.""" - # NB: Keep in sync with command.py:_CommonOptions(). + """Setup the CLI parser.""" + # NB: Keep in sync with command.py:_CommonOptions(). - # Logging. - group = parser.add_option_group('Logging options') - group.add_option('-v', '--verbose', - dest='output_mode', action='store_true', - help='show all output') - group.add_option('-q', '--quiet', - dest='output_mode', action='store_false', - help='only show errors') + # Logging. + group = parser.add_option_group("Logging options") + group.add_option( + "-v", + "--verbose", + dest="output_mode", + action="store_true", + help="show all output", + ) + group.add_option( + "-q", + "--quiet", + dest="output_mode", + action="store_false", + help="only show errors", + ) - # Manifest. - group = parser.add_option_group('Manifest options') - group.add_option('-u', '--manifest-url', - help='manifest repository location', metavar='URL') - group.add_option('-b', '--manifest-branch', metavar='REVISION', - help='manifest branch or revision (use HEAD for default)') - group.add_option('-m', '--manifest-name', default='default.xml', - help='initial manifest file', metavar='NAME.xml') - group.add_option('-g', '--groups', default='default', - help='restrict manifest projects to ones with specified ' - 'group(s) [default|all|G1,G2,G3|G4,-G5,-G6]', - metavar='GROUP') - group.add_option('-p', '--platform', default='auto', - help='restrict manifest projects to ones with a specified ' - 'platform group [auto|all|none|linux|darwin|...]', - metavar='PLATFORM') - group.add_option('--submodules', action='store_true', - help='sync any submodules associated with the manifest repo') - group.add_option('--standalone-manifest', action='store_true', - help='download the manifest as a static file ' - 'rather then create a git checkout of ' - 'the manifest repo') - group.add_option('--manifest-depth', type='int', default=0, metavar='DEPTH', - help='create a shallow clone of the manifest repo with ' - 'given depth (0 for full clone); see git clone ' - '(default: %default)') + # Manifest. + group = parser.add_option_group("Manifest options") + group.add_option( + "-u", + "--manifest-url", + help="manifest repository location", + metavar="URL", + ) + group.add_option( + "-b", + "--manifest-branch", + metavar="REVISION", + help="manifest branch or revision (use HEAD for default)", + ) + group.add_option( + "-m", + "--manifest-name", + default="default.xml", + help="initial manifest file", + metavar="NAME.xml", + ) + group.add_option( + "-g", + "--groups", + default="default", + help="restrict manifest projects to ones with specified " + "group(s) [default|all|G1,G2,G3|G4,-G5,-G6]", + metavar="GROUP", + ) + group.add_option( + "-p", + "--platform", + default="auto", + help="restrict manifest projects to ones with a specified " + "platform group [auto|all|none|linux|darwin|...]", + metavar="PLATFORM", + ) + group.add_option( + "--submodules", + action="store_true", + help="sync any submodules associated with the manifest repo", + ) + group.add_option( + "--standalone-manifest", + action="store_true", + help="download the manifest as a static file " + "rather then create a git checkout of " + "the manifest repo", + ) + group.add_option( + "--manifest-depth", + type="int", + default=0, + metavar="DEPTH", + help="create a shallow clone of the manifest repo with " + "given depth (0 for full clone); see git clone " + "(default: %default)", + ) - # Options that only affect manifest project, and not any of the projects - # specified in the manifest itself. - group = parser.add_option_group('Manifest (only) checkout options') + # Options that only affect manifest project, and not any of the projects + # specified in the manifest itself. + group = parser.add_option_group("Manifest (only) checkout options") - group.add_option('--current-branch', '-c', default=True, - dest='current_branch_only', action='store_true', - help='fetch only current manifest branch from server (default)') - group.add_option('--no-current-branch', - dest='current_branch_only', action='store_false', - help='fetch all manifest branches from server') - group.add_option('--tags', - action='store_true', - help='fetch tags in the manifest') - group.add_option('--no-tags', - dest='tags', action='store_false', - help="don't fetch tags in the manifest") + group.add_option( + "--current-branch", + "-c", + default=True, + dest="current_branch_only", + action="store_true", + help="fetch only current manifest branch from server (default)", + ) + group.add_option( + "--no-current-branch", + dest="current_branch_only", + action="store_false", + help="fetch all manifest branches from server", + ) + group.add_option( + "--tags", action="store_true", help="fetch tags in the manifest" + ) + group.add_option( + "--no-tags", + dest="tags", + action="store_false", + help="don't fetch tags in the manifest", + ) - # These are fundamentally different ways of structuring the checkout. - group = parser.add_option_group('Checkout modes') - group.add_option('--mirror', action='store_true', - help='create a replica of the remote repositories ' - 'rather than a client working directory') - group.add_option('--archive', action='store_true', - help='checkout an archive instead of a git repository for ' - 'each project. See git archive.') - group.add_option('--worktree', action='store_true', - help='use git-worktree to manage projects') + # These are fundamentally different ways of structuring the checkout. + group = parser.add_option_group("Checkout modes") + group.add_option( + "--mirror", + action="store_true", + help="create a replica of the remote repositories " + "rather than a client working directory", + ) + group.add_option( + "--archive", + action="store_true", + help="checkout an archive instead of a git repository for " + "each project. See git archive.", + ) + group.add_option( + "--worktree", + action="store_true", + help="use git-worktree to manage projects", + ) - # These are fundamentally different ways of structuring the checkout. - group = parser.add_option_group('Project checkout optimizations') - group.add_option('--reference', - help='location of mirror directory', metavar='DIR') - group.add_option('--dissociate', action='store_true', - help='dissociate from reference mirrors after clone') - group.add_option('--depth', type='int', default=None, - help='create a shallow clone with given depth; ' - 'see git clone') - group.add_option('--partial-clone', action='store_true', - help='perform partial clone (https://git-scm.com/' - 'docs/gitrepository-layout#_code_partialclone_code)') - group.add_option('--no-partial-clone', action='store_false', - help='disable use of partial clone (https://git-scm.com/' - 'docs/gitrepository-layout#_code_partialclone_code)') - group.add_option('--partial-clone-exclude', action='store', - help='exclude the specified projects (a comma-delimited ' - 'project names) from partial clone (https://git-scm.com' - '/docs/gitrepository-layout#_code_partialclone_code)') - group.add_option('--clone-filter', action='store', default='blob:none', - help='filter for use with --partial-clone ' - '[default: %default]') - group.add_option('--use-superproject', action='store_true', default=None, - help='use the manifest superproject to sync projects; implies -c') - group.add_option('--no-use-superproject', action='store_false', - dest='use_superproject', - help='disable use of manifest superprojects') - group.add_option('--clone-bundle', action='store_true', - help='enable use of /clone.bundle on HTTP/HTTPS ' - '(default if not --partial-clone)') - group.add_option('--no-clone-bundle', - dest='clone_bundle', action='store_false', - help='disable use of /clone.bundle on HTTP/HTTPS (default if --partial-clone)') - group.add_option('--git-lfs', action='store_true', - help='enable Git LFS support') - group.add_option('--no-git-lfs', - dest='git_lfs', action='store_false', - help='disable Git LFS support') + # These are fundamentally different ways of structuring the checkout. + group = parser.add_option_group("Project checkout optimizations") + group.add_option( + "--reference", help="location of mirror directory", metavar="DIR" + ) + group.add_option( + "--dissociate", + action="store_true", + help="dissociate from reference mirrors after clone", + ) + group.add_option( + "--depth", + type="int", + default=None, + help="create a shallow clone with given depth; " "see git clone", + ) + group.add_option( + "--partial-clone", + action="store_true", + help="perform partial clone (https://git-scm.com/" + "docs/gitrepository-layout#_code_partialclone_code)", + ) + group.add_option( + "--no-partial-clone", + action="store_false", + help="disable use of partial clone (https://git-scm.com/" + "docs/gitrepository-layout#_code_partialclone_code)", + ) + group.add_option( + "--partial-clone-exclude", + action="store", + help="exclude the specified projects (a comma-delimited " + "project names) from partial clone (https://git-scm.com" + "/docs/gitrepository-layout#_code_partialclone_code)", + ) + group.add_option( + "--clone-filter", + action="store", + default="blob:none", + help="filter for use with --partial-clone " "[default: %default]", + ) + group.add_option( + "--use-superproject", + action="store_true", + default=None, + help="use the manifest superproject to sync projects; implies -c", + ) + group.add_option( + "--no-use-superproject", + action="store_false", + dest="use_superproject", + help="disable use of manifest superprojects", + ) + group.add_option( + "--clone-bundle", + action="store_true", + help="enable use of /clone.bundle on HTTP/HTTPS " + "(default if not --partial-clone)", + ) + group.add_option( + "--no-clone-bundle", + dest="clone_bundle", + action="store_false", + help="disable use of /clone.bundle on HTTP/HTTPS (default if --partial-clone)", + ) + group.add_option( + "--git-lfs", action="store_true", help="enable Git LFS support" + ) + group.add_option( + "--no-git-lfs", + dest="git_lfs", + action="store_false", + help="disable Git LFS support", + ) - # Tool. - group = parser.add_option_group('repo Version options') - group.add_option('--repo-url', metavar='URL', - help='repo repository location ($REPO_URL)') - group.add_option('--repo-rev', metavar='REV', - help='repo branch or revision ($REPO_REV)') - group.add_option('--repo-branch', dest='repo_rev', - help=optparse.SUPPRESS_HELP) - group.add_option('--no-repo-verify', - dest='repo_verify', default=True, action='store_false', - help='do not verify repo source code') + # Tool. + group = parser.add_option_group("repo Version options") + group.add_option( + "--repo-url", metavar="URL", help="repo repository location ($REPO_URL)" + ) + group.add_option( + "--repo-rev", metavar="REV", help="repo branch or revision ($REPO_REV)" + ) + group.add_option( + "--repo-branch", dest="repo_rev", help=optparse.SUPPRESS_HELP + ) + group.add_option( + "--no-repo-verify", + dest="repo_verify", + default=True, + action="store_false", + help="do not verify repo source code", + ) - # Other. - group = parser.add_option_group('Other options') - group.add_option('--config-name', - action='store_true', default=False, - help='Always prompt for name/e-mail') + # Other. + group = parser.add_option_group("Other options") + group.add_option( + "--config-name", + action="store_true", + default=False, + help="Always prompt for name/e-mail", + ) - return parser + return parser # This is a poor replacement for subprocess.run until we require Python 3.6+. RunResult = collections.namedtuple( - 'RunResult', ('returncode', 'stdout', 'stderr')) + "RunResult", ("returncode", "stdout", "stderr") +) class RunError(Exception): - """Error when running a command failed.""" + """Error when running a command failed.""" def run_command(cmd, **kwargs): - """Run |cmd| and return its output.""" - check = kwargs.pop('check', False) - if kwargs.pop('capture_output', False): - kwargs.setdefault('stdout', subprocess.PIPE) - kwargs.setdefault('stderr', subprocess.PIPE) - cmd_input = kwargs.pop('input', None) + """Run |cmd| and return its output.""" + check = kwargs.pop("check", False) + if kwargs.pop("capture_output", False): + kwargs.setdefault("stdout", subprocess.PIPE) + kwargs.setdefault("stderr", subprocess.PIPE) + cmd_input = kwargs.pop("input", None) - def decode(output): - """Decode |output| to text.""" - if output is None: - return output - try: - return output.decode('utf-8') - except UnicodeError: - print('repo: warning: Invalid UTF-8 output:\ncmd: %r\n%r' % (cmd, output), - file=sys.stderr) - return output.decode('utf-8', 'backslashreplace') + def decode(output): + """Decode |output| to text.""" + if output is None: + return output + try: + return output.decode("utf-8") + except UnicodeError: + print( + "repo: warning: Invalid UTF-8 output:\ncmd: %r\n%r" + % (cmd, output), + file=sys.stderr, + ) + return output.decode("utf-8", "backslashreplace") - # Run & package the results. - proc = subprocess.Popen(cmd, **kwargs) - (stdout, stderr) = proc.communicate(input=cmd_input) - dbg = ': ' + ' '.join(cmd) - if cmd_input is not None: - dbg += ' 0<|' - if stdout == subprocess.PIPE: - dbg += ' 1>|' - if stderr == subprocess.PIPE: - dbg += ' 2>|' - elif stderr == subprocess.STDOUT: - dbg += ' 2>&1' - trace.print(dbg) - ret = RunResult(proc.returncode, decode(stdout), decode(stderr)) + # Run & package the results. + proc = subprocess.Popen(cmd, **kwargs) + (stdout, stderr) = proc.communicate(input=cmd_input) + dbg = ": " + " ".join(cmd) + if cmd_input is not None: + dbg += " 0<|" + if stdout == subprocess.PIPE: + dbg += " 1>|" + if stderr == subprocess.PIPE: + dbg += " 2>|" + elif stderr == subprocess.STDOUT: + dbg += " 2>&1" + trace.print(dbg) + ret = RunResult(proc.returncode, decode(stdout), decode(stderr)) - # If things failed, print useful debugging output. - if check and ret.returncode: - print('repo: error: "%s" failed with exit status %s' % - (cmd[0], ret.returncode), file=sys.stderr) - print(' cwd: %s\n cmd: %r' % - (kwargs.get('cwd', os.getcwd()), cmd), file=sys.stderr) + # If things failed, print useful debugging output. + if check and ret.returncode: + print( + 'repo: error: "%s" failed with exit status %s' + % (cmd[0], ret.returncode), + file=sys.stderr, + ) + print( + " cwd: %s\n cmd: %r" % (kwargs.get("cwd", os.getcwd()), cmd), + file=sys.stderr, + ) - def _print_output(name, output): - if output: - print(' %s:\n >> %s' % (name, '\n >> '.join(output.splitlines())), - file=sys.stderr) + def _print_output(name, output): + if output: + print( + " %s:\n >> %s" + % (name, "\n >> ".join(output.splitlines())), + file=sys.stderr, + ) - _print_output('stdout', ret.stdout) - _print_output('stderr', ret.stderr) - raise RunError(ret) + _print_output("stdout", ret.stdout) + _print_output("stderr", ret.stderr) + raise RunError(ret) - return ret + return ret _gitc_manifest_dir = None def get_gitc_manifest_dir(): - global _gitc_manifest_dir - if _gitc_manifest_dir is None: - _gitc_manifest_dir = '' - try: - with open(GITC_CONFIG_FILE, 'r') as gitc_config: - for line in gitc_config: - match = re.match('gitc_dir=(?P.*)', line) - if match: - _gitc_manifest_dir = match.group('gitc_manifest_dir') - except IOError: - pass - return _gitc_manifest_dir + global _gitc_manifest_dir + if _gitc_manifest_dir is None: + _gitc_manifest_dir = "" + try: + with open(GITC_CONFIG_FILE, "r") as gitc_config: + for line in gitc_config: + match = re.match("gitc_dir=(?P.*)", line) + if match: + _gitc_manifest_dir = match.group("gitc_manifest_dir") + except IOError: + pass + return _gitc_manifest_dir def gitc_parse_clientdir(gitc_fs_path): - """Parse a path in the GITC FS and return its client name. + """Parse a path in the GITC FS and return its client name. - Args: - gitc_fs_path: A subdirectory path within the GITC_FS_ROOT_DIR. + Args: + gitc_fs_path: A subdirectory path within the GITC_FS_ROOT_DIR. - Returns: - The GITC client name. - """ - if gitc_fs_path == GITC_FS_ROOT_DIR: - return None - if not gitc_fs_path.startswith(GITC_FS_ROOT_DIR): - manifest_dir = get_gitc_manifest_dir() - if manifest_dir == '': - return None - if manifest_dir[-1] != '/': - manifest_dir += '/' - if gitc_fs_path == manifest_dir: - return None - if not gitc_fs_path.startswith(manifest_dir): - return None - return gitc_fs_path.split(manifest_dir)[1].split('/')[0] - return gitc_fs_path.split(GITC_FS_ROOT_DIR)[1].split('/')[0] + Returns: + The GITC client name. + """ + if gitc_fs_path == GITC_FS_ROOT_DIR: + return None + if not gitc_fs_path.startswith(GITC_FS_ROOT_DIR): + manifest_dir = get_gitc_manifest_dir() + if manifest_dir == "": + return None + if manifest_dir[-1] != "/": + manifest_dir += "/" + if gitc_fs_path == manifest_dir: + return None + if not gitc_fs_path.startswith(manifest_dir): + return None + return gitc_fs_path.split(manifest_dir)[1].split("/")[0] + return gitc_fs_path.split(GITC_FS_ROOT_DIR)[1].split("/")[0] class CloneFailure(Exception): - """Indicate the remote clone of repo itself failed. - """ + """Indicate the remote clone of repo itself failed.""" def check_repo_verify(repo_verify, quiet=False): - """Check the --repo-verify state.""" - if not repo_verify: - print('repo: warning: verification of repo code has been disabled;\n' - 'repo will not be able to verify the integrity of itself.\n', - file=sys.stderr) - return False + """Check the --repo-verify state.""" + if not repo_verify: + print( + "repo: warning: verification of repo code has been disabled;\n" + "repo will not be able to verify the integrity of itself.\n", + file=sys.stderr, + ) + return False - if NeedSetupGnuPG(): - return SetupGnuPG(quiet) + if NeedSetupGnuPG(): + return SetupGnuPG(quiet) - return True + return True def check_repo_rev(dst, rev, repo_verify=True, quiet=False): - """Check that |rev| is valid.""" - do_verify = check_repo_verify(repo_verify, quiet=quiet) - remote_ref, local_rev = resolve_repo_rev(dst, rev) - if not quiet and not remote_ref.startswith('refs/heads/'): - print('warning: repo is not tracking a remote branch, so it will not ' - 'receive updates', file=sys.stderr) - if do_verify: - rev = verify_rev(dst, remote_ref, local_rev, quiet) - else: - rev = local_rev - return (remote_ref, rev) + """Check that |rev| is valid.""" + do_verify = check_repo_verify(repo_verify, quiet=quiet) + remote_ref, local_rev = resolve_repo_rev(dst, rev) + if not quiet and not remote_ref.startswith("refs/heads/"): + print( + "warning: repo is not tracking a remote branch, so it will not " + "receive updates", + file=sys.stderr, + ) + if do_verify: + rev = verify_rev(dst, remote_ref, local_rev, quiet) + else: + rev = local_rev + return (remote_ref, rev) def _Init(args, gitc_init=False): - """Installs repo by cloning it over the network. - """ - parser = GetParser(gitc_init=gitc_init) - opt, args = parser.parse_args(args) - if args: - if not opt.manifest_url: - opt.manifest_url = args.pop(0) + """Installs repo by cloning it over the network.""" + parser = GetParser(gitc_init=gitc_init) + opt, args = parser.parse_args(args) if args: - parser.print_usage() - sys.exit(1) - opt.quiet = opt.output_mode is False - opt.verbose = opt.output_mode is True + if not opt.manifest_url: + opt.manifest_url = args.pop(0) + if args: + parser.print_usage() + sys.exit(1) + opt.quiet = opt.output_mode is False + opt.verbose = opt.output_mode is True - if opt.clone_bundle is None: - opt.clone_bundle = False if opt.partial_clone else True + if opt.clone_bundle is None: + opt.clone_bundle = False if opt.partial_clone else True - url = opt.repo_url or REPO_URL - rev = opt.repo_rev or REPO_REV + url = opt.repo_url or REPO_URL + rev = opt.repo_rev or REPO_REV - try: - os.mkdir(repodir) - except OSError as e: - if e.errno != errno.EEXIST: - print('fatal: cannot make %s directory: %s' - % (repodir, e.strerror), file=sys.stderr) - # Don't raise CloneFailure; that would delete the - # name. Instead exit immediately. - # - sys.exit(1) + try: + os.mkdir(repodir) + except OSError as e: + if e.errno != errno.EEXIST: + print( + "fatal: cannot make %s directory: %s" % (repodir, e.strerror), + file=sys.stderr, + ) + # Don't raise CloneFailure; that would delete the + # name. Instead exit immediately. + # + sys.exit(1) - _CheckGitVersion() - try: - if not opt.quiet: - print('Downloading Repo source from', url) - dst_final = os.path.abspath(os.path.join(repodir, S_repo)) - dst = dst_final + '.tmp' - shutil.rmtree(dst, ignore_errors=True) - _Clone(url, dst, opt.clone_bundle, opt.quiet, opt.verbose) + _CheckGitVersion() + try: + if not opt.quiet: + print("Downloading Repo source from", url) + dst_final = os.path.abspath(os.path.join(repodir, S_repo)) + dst = dst_final + ".tmp" + shutil.rmtree(dst, ignore_errors=True) + _Clone(url, dst, opt.clone_bundle, opt.quiet, opt.verbose) - remote_ref, rev = check_repo_rev(dst, rev, opt.repo_verify, quiet=opt.quiet) - _Checkout(dst, remote_ref, rev, opt.quiet) + remote_ref, rev = check_repo_rev( + dst, rev, opt.repo_verify, quiet=opt.quiet + ) + _Checkout(dst, remote_ref, rev, opt.quiet) - if not os.path.isfile(os.path.join(dst, 'repo')): - print("fatal: '%s' does not look like a git-repo repository, is " - "--repo-url set correctly?" % url, file=sys.stderr) - raise CloneFailure() + if not os.path.isfile(os.path.join(dst, "repo")): + print( + "fatal: '%s' does not look like a git-repo repository, is " + "--repo-url set correctly?" % url, + file=sys.stderr, + ) + raise CloneFailure() - os.rename(dst, dst_final) + os.rename(dst, dst_final) - except CloneFailure: - print('fatal: double check your --repo-rev setting.', file=sys.stderr) - if opt.quiet: - print('fatal: repo init failed; run without --quiet to see why', - file=sys.stderr) - raise + except CloneFailure: + print("fatal: double check your --repo-rev setting.", file=sys.stderr) + if opt.quiet: + print( + "fatal: repo init failed; run without --quiet to see why", + file=sys.stderr, + ) + raise def run_git(*args, **kwargs): - """Run git and return execution details.""" - kwargs.setdefault('capture_output', True) - kwargs.setdefault('check', True) - try: - return run_command([GIT] + list(args), **kwargs) - except OSError as e: - print(file=sys.stderr) - print('repo: error: "%s" is not available' % GIT, file=sys.stderr) - print('repo: error: %s' % e, file=sys.stderr) - print(file=sys.stderr) - print('Please make sure %s is installed and in your path.' % GIT, - file=sys.stderr) - sys.exit(1) - except RunError: - raise CloneFailure() + """Run git and return execution details.""" + kwargs.setdefault("capture_output", True) + kwargs.setdefault("check", True) + try: + return run_command([GIT] + list(args), **kwargs) + except OSError as e: + print(file=sys.stderr) + print('repo: error: "%s" is not available' % GIT, file=sys.stderr) + print("repo: error: %s" % e, file=sys.stderr) + print(file=sys.stderr) + print( + "Please make sure %s is installed and in your path." % GIT, + file=sys.stderr, + ) + sys.exit(1) + except RunError: + raise CloneFailure() # The git version info broken down into components for easy analysis. # Similar to Python's sys.version_info. GitVersion = collections.namedtuple( - 'GitVersion', ('major', 'minor', 'micro', 'full')) + "GitVersion", ("major", "minor", "micro", "full") +) def ParseGitVersion(ver_str=None): - if ver_str is None: - # Load the version ourselves. - ver_str = run_git('--version').stdout + if ver_str is None: + # Load the version ourselves. + ver_str = run_git("--version").stdout - if not ver_str.startswith('git version '): - return None + if not ver_str.startswith("git version "): + return None - full_version = ver_str[len('git version '):].strip() - num_ver_str = full_version.split('-')[0] - to_tuple = [] - for num_str in num_ver_str.split('.')[:3]: - if num_str.isdigit(): - to_tuple.append(int(num_str)) - else: - to_tuple.append(0) - to_tuple.append(full_version) - return GitVersion(*to_tuple) + full_version = ver_str[len("git version ") :].strip() + num_ver_str = full_version.split("-")[0] + to_tuple = [] + for num_str in num_ver_str.split(".")[:3]: + if num_str.isdigit(): + to_tuple.append(int(num_str)) + else: + to_tuple.append(0) + to_tuple.append(full_version) + return GitVersion(*to_tuple) def _CheckGitVersion(): - ver_act = ParseGitVersion() - if ver_act is None: - print('fatal: unable to detect git version', file=sys.stderr) - raise CloneFailure() + ver_act = ParseGitVersion() + if ver_act is None: + print("fatal: unable to detect git version", file=sys.stderr) + raise CloneFailure() - if ver_act < MIN_GIT_VERSION: - need = '.'.join(map(str, MIN_GIT_VERSION)) - print('fatal: git %s or later required; found %s' % (need, ver_act.full), - file=sys.stderr) - raise CloneFailure() + if ver_act < MIN_GIT_VERSION: + need = ".".join(map(str, MIN_GIT_VERSION)) + print( + "fatal: git %s or later required; found %s" % (need, ver_act.full), + file=sys.stderr, + ) + raise CloneFailure() def SetGitTrace2ParentSid(env=None): - """Set up GIT_TRACE2_PARENT_SID for git tracing.""" - # We roughly follow the format git itself uses in trace2/tr2_sid.c. - # (1) Be unique (2) be valid filename (3) be fixed length. - # - # Since we always export this variable, we try to avoid more expensive calls. - # e.g. We don't attempt hostname lookups or hashing the results. - if env is None: - env = os.environ + """Set up GIT_TRACE2_PARENT_SID for git tracing.""" + # We roughly follow the format git itself uses in trace2/tr2_sid.c. + # (1) Be unique (2) be valid filename (3) be fixed length. + # + # Since we always export this variable, we try to avoid more expensive calls. + # e.g. We don't attempt hostname lookups or hashing the results. + if env is None: + env = os.environ - KEY = 'GIT_TRACE2_PARENT_SID' + KEY = "GIT_TRACE2_PARENT_SID" - now = datetime.datetime.utcnow() - value = 'repo-%s-P%08x' % (now.strftime('%Y%m%dT%H%M%SZ'), os.getpid()) + now = datetime.datetime.utcnow() + value = "repo-%s-P%08x" % (now.strftime("%Y%m%dT%H%M%SZ"), os.getpid()) - # If it's already set, then append ourselves. - if KEY in env: - value = env[KEY] + '/' + value + # If it's already set, then append ourselves. + if KEY in env: + value = env[KEY] + "/" + value - _setenv(KEY, value, env=env) + _setenv(KEY, value, env=env) def _setenv(key, value, env=None): - """Set |key| in the OS environment |env| to |value|.""" - if env is None: - env = os.environ - # Environment handling across systems is messy. - try: - env[key] = value - except UnicodeEncodeError: - env[key] = value.encode() + """Set |key| in the OS environment |env| to |value|.""" + if env is None: + env = os.environ + # Environment handling across systems is messy. + try: + env[key] = value + except UnicodeEncodeError: + env[key] = value.encode() def NeedSetupGnuPG(): - if not os.path.isdir(home_dot_repo): - return True + if not os.path.isdir(home_dot_repo): + return True - kv = os.path.join(home_dot_repo, 'keyring-version') - if not os.path.exists(kv): - return True + kv = os.path.join(home_dot_repo, "keyring-version") + if not os.path.exists(kv): + return True - kv = open(kv).read() - if not kv: - return True + kv = open(kv).read() + if not kv: + return True - kv = tuple(map(int, kv.split('.'))) - if kv < KEYRING_VERSION: - return True - return False + kv = tuple(map(int, kv.split("."))) + if kv < KEYRING_VERSION: + return True + return False def SetupGnuPG(quiet): - try: - os.mkdir(home_dot_repo) - except OSError as e: - if e.errno != errno.EEXIST: - print('fatal: cannot make %s directory: %s' - % (home_dot_repo, e.strerror), file=sys.stderr) - sys.exit(1) + try: + os.mkdir(home_dot_repo) + except OSError as e: + if e.errno != errno.EEXIST: + print( + "fatal: cannot make %s directory: %s" + % (home_dot_repo, e.strerror), + file=sys.stderr, + ) + sys.exit(1) - try: - os.mkdir(gpg_dir, stat.S_IRWXU) - except OSError as e: - if e.errno != errno.EEXIST: - print('fatal: cannot make %s directory: %s' % (gpg_dir, e.strerror), - file=sys.stderr) - sys.exit(1) + try: + os.mkdir(gpg_dir, stat.S_IRWXU) + except OSError as e: + if e.errno != errno.EEXIST: + print( + "fatal: cannot make %s directory: %s" % (gpg_dir, e.strerror), + file=sys.stderr, + ) + sys.exit(1) - if not quiet: - print('repo: Updating release signing keys to keyset ver %s' % - ('.'.join(str(x) for x in KEYRING_VERSION),)) - # NB: We use --homedir (and cwd below) because some environments (Windows) do - # not correctly handle full native paths. We avoid the issue by changing to - # the right dir with cwd=gpg_dir before executing gpg, and then telling gpg to - # use the cwd (.) as its homedir which leaves the path resolution logic to it. - cmd = ['gpg', '--homedir', '.', '--import'] - try: - # gpg can be pretty chatty. Always capture the output and if something goes - # wrong, the builtin check failure will dump stdout & stderr for debugging. - run_command(cmd, stdin=subprocess.PIPE, capture_output=True, - cwd=gpg_dir, check=True, - input=MAINTAINER_KEYS.encode('utf-8')) - except OSError: if not quiet: - print('warning: gpg (GnuPG) is not available.', file=sys.stderr) - print('warning: Installing it is strongly encouraged.', file=sys.stderr) - print(file=sys.stderr) - return False + print( + "repo: Updating release signing keys to keyset ver %s" + % (".".join(str(x) for x in KEYRING_VERSION),) + ) + # NB: We use --homedir (and cwd below) because some environments (Windows) do + # not correctly handle full native paths. We avoid the issue by changing to + # the right dir with cwd=gpg_dir before executing gpg, and then telling gpg to + # use the cwd (.) as its homedir which leaves the path resolution logic to it. + cmd = ["gpg", "--homedir", ".", "--import"] + try: + # gpg can be pretty chatty. Always capture the output and if something goes + # wrong, the builtin check failure will dump stdout & stderr for debugging. + run_command( + cmd, + stdin=subprocess.PIPE, + capture_output=True, + cwd=gpg_dir, + check=True, + input=MAINTAINER_KEYS.encode("utf-8"), + ) + except OSError: + if not quiet: + print("warning: gpg (GnuPG) is not available.", file=sys.stderr) + print( + "warning: Installing it is strongly encouraged.", + file=sys.stderr, + ) + print(file=sys.stderr) + return False - with open(os.path.join(home_dot_repo, 'keyring-version'), 'w') as fd: - fd.write('.'.join(map(str, KEYRING_VERSION)) + '\n') - return True + with open(os.path.join(home_dot_repo, "keyring-version"), "w") as fd: + fd.write(".".join(map(str, KEYRING_VERSION)) + "\n") + return True def _SetConfig(cwd, name, value): - """Set a git configuration option to the specified value. - """ - run_git('config', name, value, cwd=cwd) + """Set a git configuration option to the specified value.""" + run_git("config", name, value, cwd=cwd) def _GetRepoConfig(name): - """Read a repo configuration option.""" - config = os.path.join(home_dot_repo, 'config') - if not os.path.exists(config): - return None + """Read a repo configuration option.""" + config = os.path.join(home_dot_repo, "config") + if not os.path.exists(config): + return None - cmd = ['config', '--file', config, '--get', name] - ret = run_git(*cmd, check=False) - if ret.returncode == 0: - return ret.stdout - elif ret.returncode == 1: - return None - else: - print('repo: error: git %s failed:\n%s' % (' '.join(cmd), ret.stderr), - file=sys.stderr) - raise RunError() + cmd = ["config", "--file", config, "--get", name] + ret = run_git(*cmd, check=False) + if ret.returncode == 0: + return ret.stdout + elif ret.returncode == 1: + return None + else: + print( + "repo: error: git %s failed:\n%s" % (" ".join(cmd), ret.stderr), + file=sys.stderr, + ) + raise RunError() def _InitHttp(): - handlers = [] + handlers = [] - mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm() - try: - import netrc - n = netrc.netrc() - for host in n.hosts: - p = n.hosts[host] - mgr.add_password(p[1], 'http://%s/' % host, p[0], p[2]) - mgr.add_password(p[1], 'https://%s/' % host, p[0], p[2]) - except Exception: - pass - handlers.append(urllib.request.HTTPBasicAuthHandler(mgr)) - handlers.append(urllib.request.HTTPDigestAuthHandler(mgr)) + mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm() + try: + import netrc - if 'http_proxy' in os.environ: - url = os.environ['http_proxy'] - handlers.append(urllib.request.ProxyHandler({'http': url, 'https': url})) - if 'REPO_CURL_VERBOSE' in os.environ: - handlers.append(urllib.request.HTTPHandler(debuglevel=1)) - handlers.append(urllib.request.HTTPSHandler(debuglevel=1)) - urllib.request.install_opener(urllib.request.build_opener(*handlers)) + n = netrc.netrc() + for host in n.hosts: + p = n.hosts[host] + mgr.add_password(p[1], "http://%s/" % host, p[0], p[2]) + mgr.add_password(p[1], "https://%s/" % host, p[0], p[2]) + except Exception: + pass + handlers.append(urllib.request.HTTPBasicAuthHandler(mgr)) + handlers.append(urllib.request.HTTPDigestAuthHandler(mgr)) + + if "http_proxy" in os.environ: + url = os.environ["http_proxy"] + handlers.append( + urllib.request.ProxyHandler({"http": url, "https": url}) + ) + if "REPO_CURL_VERBOSE" in os.environ: + handlers.append(urllib.request.HTTPHandler(debuglevel=1)) + handlers.append(urllib.request.HTTPSHandler(debuglevel=1)) + urllib.request.install_opener(urllib.request.build_opener(*handlers)) def _Fetch(url, cwd, src, quiet, verbose): - cmd = ['fetch'] - if not verbose: - cmd.append('--quiet') - err = None - if not quiet and sys.stdout.isatty(): - cmd.append('--progress') - elif not verbose: - err = subprocess.PIPE - cmd.append(src) - cmd.append('+refs/heads/*:refs/remotes/origin/*') - cmd.append('+refs/tags/*:refs/tags/*') - run_git(*cmd, stderr=err, capture_output=False, cwd=cwd) + cmd = ["fetch"] + if not verbose: + cmd.append("--quiet") + err = None + if not quiet and sys.stdout.isatty(): + cmd.append("--progress") + elif not verbose: + err = subprocess.PIPE + cmd.append(src) + cmd.append("+refs/heads/*:refs/remotes/origin/*") + cmd.append("+refs/tags/*:refs/tags/*") + run_git(*cmd, stderr=err, capture_output=False, cwd=cwd) def _DownloadBundle(url, cwd, quiet, verbose): - if not url.endswith('/'): - url += '/' - url += 'clone.bundle' + if not url.endswith("/"): + url += "/" + url += "clone.bundle" - ret = run_git('config', '--get-regexp', 'url.*.insteadof', cwd=cwd, - check=False) - for line in ret.stdout.splitlines(): - m = re.compile(r'^url\.(.*)\.insteadof (.*)$').match(line) - if m: - new_url = m.group(1) - old_url = m.group(2) - if url.startswith(old_url): - url = new_url + url[len(old_url):] - break + ret = run_git( + "config", "--get-regexp", "url.*.insteadof", cwd=cwd, check=False + ) + for line in ret.stdout.splitlines(): + m = re.compile(r"^url\.(.*)\.insteadof (.*)$").match(line) + if m: + new_url = m.group(1) + old_url = m.group(2) + if url.startswith(old_url): + url = new_url + url[len(old_url) :] + break - if not url.startswith('http:') and not url.startswith('https:'): - return False + if not url.startswith("http:") and not url.startswith("https:"): + return False - dest = open(os.path.join(cwd, '.git', 'clone.bundle'), 'w+b') - try: + dest = open(os.path.join(cwd, ".git", "clone.bundle"), "w+b") try: - r = urllib.request.urlopen(url) - except urllib.error.HTTPError as e: - if e.code not in [400, 401, 403, 404, 501]: - print('warning: Cannot get %s' % url, file=sys.stderr) - print('warning: HTTP error %s' % e.code, file=sys.stderr) - return False - except urllib.error.URLError as e: - print('fatal: Cannot get %s' % url, file=sys.stderr) - print('fatal: error %s' % e.reason, file=sys.stderr) - raise CloneFailure() - try: - if verbose: - print('Downloading clone bundle %s' % url, file=sys.stderr) - while True: - buf = r.read(8192) - if not buf: - return True - dest.write(buf) + try: + r = urllib.request.urlopen(url) + except urllib.error.HTTPError as e: + if e.code not in [400, 401, 403, 404, 501]: + print("warning: Cannot get %s" % url, file=sys.stderr) + print("warning: HTTP error %s" % e.code, file=sys.stderr) + return False + except urllib.error.URLError as e: + print("fatal: Cannot get %s" % url, file=sys.stderr) + print("fatal: error %s" % e.reason, file=sys.stderr) + raise CloneFailure() + try: + if verbose: + print("Downloading clone bundle %s" % url, file=sys.stderr) + while True: + buf = r.read(8192) + if not buf: + return True + dest.write(buf) + finally: + r.close() finally: - r.close() - finally: - dest.close() + dest.close() def _ImportBundle(cwd): - path = os.path.join(cwd, '.git', 'clone.bundle') - try: - _Fetch(cwd, cwd, path, True, False) - finally: - os.remove(path) + path = os.path.join(cwd, ".git", "clone.bundle") + try: + _Fetch(cwd, cwd, path, True, False) + finally: + os.remove(path) def _Clone(url, cwd, clone_bundle, quiet, verbose): - """Clones a git repository to a new subdirectory of repodir - """ - if verbose: - print('Cloning git repository', url) + """Clones a git repository to a new subdirectory of repodir""" + if verbose: + print("Cloning git repository", url) - try: - os.mkdir(cwd) - except OSError as e: - print('fatal: cannot make %s directory: %s' % (cwd, e.strerror), - file=sys.stderr) - raise CloneFailure() + try: + os.mkdir(cwd) + except OSError as e: + print( + "fatal: cannot make %s directory: %s" % (cwd, e.strerror), + file=sys.stderr, + ) + raise CloneFailure() - run_git('init', '--quiet', cwd=cwd) + run_git("init", "--quiet", cwd=cwd) - _InitHttp() - _SetConfig(cwd, 'remote.origin.url', url) - _SetConfig(cwd, - 'remote.origin.fetch', - '+refs/heads/*:refs/remotes/origin/*') - if clone_bundle and _DownloadBundle(url, cwd, quiet, verbose): - _ImportBundle(cwd) - _Fetch(url, cwd, 'origin', quiet, verbose) + _InitHttp() + _SetConfig(cwd, "remote.origin.url", url) + _SetConfig( + cwd, "remote.origin.fetch", "+refs/heads/*:refs/remotes/origin/*" + ) + if clone_bundle and _DownloadBundle(url, cwd, quiet, verbose): + _ImportBundle(cwd) + _Fetch(url, cwd, "origin", quiet, verbose) def resolve_repo_rev(cwd, committish): - """Figure out what REPO_REV represents. + """Figure out what REPO_REV represents. - We support: - * refs/heads/xxx: Branch. - * refs/tags/xxx: Tag. - * xxx: Branch or tag or commit. + We support: + * refs/heads/xxx: Branch. + * refs/tags/xxx: Tag. + * xxx: Branch or tag or commit. - Args: - cwd: The git checkout to run in. - committish: The REPO_REV argument to resolve. + Args: + cwd: The git checkout to run in. + committish: The REPO_REV argument to resolve. - Returns: - A tuple of (remote ref, commit) as makes sense for the committish. - For branches, this will look like ('refs/heads/stable', ). - For tags, this will look like ('refs/tags/v1.0', ). - For commits, this will be (, ). - """ - def resolve(committish): - ret = run_git('rev-parse', '--verify', '%s^{commit}' % (committish,), - cwd=cwd, check=False) - return None if ret.returncode else ret.stdout.strip() + Returns: + A tuple of (remote ref, commit) as makes sense for the committish. + For branches, this will look like ('refs/heads/stable', ). + For tags, this will look like ('refs/tags/v1.0', ). + For commits, this will be (, ). + """ - # An explicit branch. - if committish.startswith('refs/heads/'): - remote_ref = committish - committish = committish[len('refs/heads/'):] - rev = resolve('refs/remotes/origin/%s' % committish) - if rev is None: - print('repo: error: unknown branch "%s"' % (committish,), - file=sys.stderr) - raise CloneFailure() - return (remote_ref, rev) + def resolve(committish): + ret = run_git( + "rev-parse", + "--verify", + "%s^{commit}" % (committish,), + cwd=cwd, + check=False, + ) + return None if ret.returncode else ret.stdout.strip() - # An explicit tag. - if committish.startswith('refs/tags/'): - remote_ref = committish - committish = committish[len('refs/tags/'):] - rev = resolve(remote_ref) - if rev is None: - print('repo: error: unknown tag "%s"' % (committish,), - file=sys.stderr) - raise CloneFailure() - return (remote_ref, rev) + # An explicit branch. + if committish.startswith("refs/heads/"): + remote_ref = committish + committish = committish[len("refs/heads/") :] + rev = resolve("refs/remotes/origin/%s" % committish) + if rev is None: + print( + 'repo: error: unknown branch "%s"' % (committish,), + file=sys.stderr, + ) + raise CloneFailure() + return (remote_ref, rev) - # See if it's a short branch name. - rev = resolve('refs/remotes/origin/%s' % committish) - if rev: - return ('refs/heads/%s' % (committish,), rev) + # An explicit tag. + if committish.startswith("refs/tags/"): + remote_ref = committish + committish = committish[len("refs/tags/") :] + rev = resolve(remote_ref) + if rev is None: + print( + 'repo: error: unknown tag "%s"' % (committish,), file=sys.stderr + ) + raise CloneFailure() + return (remote_ref, rev) - # See if it's a tag. - rev = resolve('refs/tags/%s' % committish) - if rev: - return ('refs/tags/%s' % (committish,), rev) + # See if it's a short branch name. + rev = resolve("refs/remotes/origin/%s" % committish) + if rev: + return ("refs/heads/%s" % (committish,), rev) - # See if it's a commit. - rev = resolve(committish) - if rev and rev.lower().startswith(committish.lower()): - return (rev, rev) + # See if it's a tag. + rev = resolve("refs/tags/%s" % committish) + if rev: + return ("refs/tags/%s" % (committish,), rev) - # Give up! - print('repo: error: unable to resolve "%s"' % (committish,), file=sys.stderr) - raise CloneFailure() + # See if it's a commit. + rev = resolve(committish) + if rev and rev.lower().startswith(committish.lower()): + return (rev, rev) + + # Give up! + print( + 'repo: error: unable to resolve "%s"' % (committish,), file=sys.stderr + ) + raise CloneFailure() def verify_rev(cwd, remote_ref, rev, quiet): - """Verify the commit has been signed by a tag.""" - ret = run_git('describe', rev, cwd=cwd) - cur = ret.stdout.strip() + """Verify the commit has been signed by a tag.""" + ret = run_git("describe", rev, cwd=cwd) + cur = ret.stdout.strip() - m = re.compile(r'^(.*)-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur) - if m: - cur = m.group(1) - if not quiet: - print(file=sys.stderr) - print("warning: '%s' is not signed; falling back to signed release '%s'" - % (remote_ref, cur), file=sys.stderr) - print(file=sys.stderr) + m = re.compile(r"^(.*)-[0-9]{1,}-g[0-9a-f]{1,}$").match(cur) + if m: + cur = m.group(1) + if not quiet: + print(file=sys.stderr) + print( + "warning: '%s' is not signed; falling back to signed release '%s'" + % (remote_ref, cur), + file=sys.stderr, + ) + print(file=sys.stderr) - env = os.environ.copy() - _setenv('GNUPGHOME', gpg_dir, env) - run_git('tag', '-v', cur, cwd=cwd, env=env) - return '%s^0' % cur + env = os.environ.copy() + _setenv("GNUPGHOME", gpg_dir, env) + run_git("tag", "-v", cur, cwd=cwd, env=env) + return "%s^0" % cur def _Checkout(cwd, remote_ref, rev, quiet): - """Checkout an upstream branch into the repository and track it. - """ - run_git('update-ref', 'refs/heads/default', rev, cwd=cwd) + """Checkout an upstream branch into the repository and track it.""" + run_git("update-ref", "refs/heads/default", rev, cwd=cwd) - _SetConfig(cwd, 'branch.default.remote', 'origin') - _SetConfig(cwd, 'branch.default.merge', remote_ref) + _SetConfig(cwd, "branch.default.remote", "origin") + _SetConfig(cwd, "branch.default.merge", remote_ref) - run_git('symbolic-ref', 'HEAD', 'refs/heads/default', cwd=cwd) + run_git("symbolic-ref", "HEAD", "refs/heads/default", cwd=cwd) - cmd = ['read-tree', '--reset', '-u'] - if not quiet: - cmd.append('-v') - cmd.append('HEAD') - run_git(*cmd, cwd=cwd) + cmd = ["read-tree", "--reset", "-u"] + if not quiet: + cmd.append("-v") + cmd.append("HEAD") + run_git(*cmd, cwd=cwd) def _FindRepo(): - """Look for a repo installation, starting at the current directory. - """ - curdir = os.getcwd() - repo = None + """Look for a repo installation, starting at the current directory.""" + curdir = os.getcwd() + repo = None - olddir = None - while curdir != olddir and not repo: - repo = os.path.join(curdir, repodir, REPO_MAIN) - if not os.path.isfile(repo): - repo = None - olddir = curdir - curdir = os.path.dirname(curdir) - return (repo, os.path.join(curdir, repodir)) + olddir = None + while curdir != olddir and not repo: + repo = os.path.join(curdir, repodir, REPO_MAIN) + if not os.path.isfile(repo): + repo = None + olddir = curdir + curdir = os.path.dirname(curdir) + return (repo, os.path.join(curdir, repodir)) class _Options(object): - help = False - version = False + help = False + version = False def _ExpandAlias(name): - """Look up user registered aliases.""" - # We don't resolve aliases for existing subcommands. This matches git. - if name in {'gitc-init', 'help', 'init'}: - return name, [] + """Look up user registered aliases.""" + # We don't resolve aliases for existing subcommands. This matches git. + if name in {"gitc-init", "help", "init"}: + return name, [] - alias = _GetRepoConfig('alias.%s' % (name,)) - if alias is None: - return name, [] + alias = _GetRepoConfig("alias.%s" % (name,)) + if alias is None: + return name, [] - args = alias.strip().split(' ', 1) - name = args[0] - if len(args) == 2: - args = shlex.split(args[1]) - else: - args = [] - return name, args + args = alias.strip().split(" ", 1) + name = args[0] + if len(args) == 2: + args = shlex.split(args[1]) + else: + args = [] + return name, args def _ParseArguments(args): - cmd = None - opt = _Options() - arg = [] + cmd = None + opt = _Options() + arg = [] - for i in range(len(args)): - a = args[i] - if a == '-h' or a == '--help': - opt.help = True - elif a == '--version': - opt.version = True - elif a == '--trace': - trace.set(True) - elif not a.startswith('-'): - cmd = a - arg = args[i + 1:] - break - return cmd, opt, arg + for i in range(len(args)): + a = args[i] + if a == "-h" or a == "--help": + opt.help = True + elif a == "--version": + opt.version = True + elif a == "--trace": + trace.set(True) + elif not a.startswith("-"): + cmd = a + arg = args[i + 1 :] + break + return cmd, opt, arg class Requirements(object): - """Helper for checking repo's system requirements.""" + """Helper for checking repo's system requirements.""" - REQUIREMENTS_NAME = 'requirements.json' + REQUIREMENTS_NAME = "requirements.json" - def __init__(self, requirements): - """Initialize. + def __init__(self, requirements): + """Initialize. - Args: - requirements: A dictionary of settings. - """ - self.requirements = requirements + Args: + requirements: A dictionary of settings. + """ + self.requirements = requirements - @classmethod - def from_dir(cls, path): - return cls.from_file(os.path.join(path, cls.REQUIREMENTS_NAME)) + @classmethod + def from_dir(cls, path): + return cls.from_file(os.path.join(path, cls.REQUIREMENTS_NAME)) - @classmethod - def from_file(cls, path): - try: - with open(path, 'rb') as f: - data = f.read() - except EnvironmentError: - # NB: EnvironmentError is used for Python 2 & 3 compatibility. - # If we couldn't open the file, assume it's an old source tree. - return None + @classmethod + def from_file(cls, path): + try: + with open(path, "rb") as f: + data = f.read() + except EnvironmentError: + # NB: EnvironmentError is used for Python 2 & 3 compatibility. + # If we couldn't open the file, assume it's an old source tree. + return None - return cls.from_data(data) + return cls.from_data(data) - @classmethod - def from_data(cls, data): - comment_line = re.compile(br'^ *#') - strip_data = b''.join(x for x in data.splitlines() if not comment_line.match(x)) - try: - json_data = json.loads(strip_data) - except Exception: # pylint: disable=broad-except - # If we couldn't parse it, assume it's incompatible. - return None + @classmethod + def from_data(cls, data): + comment_line = re.compile(rb"^ *#") + strip_data = b"".join( + x for x in data.splitlines() if not comment_line.match(x) + ) + try: + json_data = json.loads(strip_data) + except Exception: # pylint: disable=broad-except + # If we couldn't parse it, assume it's incompatible. + return None - return cls(json_data) + return cls(json_data) - def _get_soft_ver(self, pkg): - """Return the soft version for |pkg| if it exists.""" - return self.requirements.get(pkg, {}).get('soft', ()) + def _get_soft_ver(self, pkg): + """Return the soft version for |pkg| if it exists.""" + return self.requirements.get(pkg, {}).get("soft", ()) - def _get_hard_ver(self, pkg): - """Return the hard version for |pkg| if it exists.""" - return self.requirements.get(pkg, {}).get('hard', ()) + def _get_hard_ver(self, pkg): + """Return the hard version for |pkg| if it exists.""" + return self.requirements.get(pkg, {}).get("hard", ()) - @staticmethod - def _format_ver(ver): - """Return a dotted version from |ver|.""" - return '.'.join(str(x) for x in ver) + @staticmethod + def _format_ver(ver): + """Return a dotted version from |ver|.""" + return ".".join(str(x) for x in ver) - def assert_ver(self, pkg, curr_ver): - """Verify |pkg|'s |curr_ver| is new enough.""" - curr_ver = tuple(curr_ver) - soft_ver = tuple(self._get_soft_ver(pkg)) - hard_ver = tuple(self._get_hard_ver(pkg)) - if curr_ver < hard_ver: - print('repo: error: Your version of "%s" (%s) is unsupported; ' - 'Please upgrade to at least version %s to continue.' % - (pkg, self._format_ver(curr_ver), self._format_ver(soft_ver)), - file=sys.stderr) - sys.exit(1) + def assert_ver(self, pkg, curr_ver): + """Verify |pkg|'s |curr_ver| is new enough.""" + curr_ver = tuple(curr_ver) + soft_ver = tuple(self._get_soft_ver(pkg)) + hard_ver = tuple(self._get_hard_ver(pkg)) + if curr_ver < hard_ver: + print( + 'repo: error: Your version of "%s" (%s) is unsupported; ' + "Please upgrade to at least version %s to continue." + % (pkg, self._format_ver(curr_ver), self._format_ver(soft_ver)), + file=sys.stderr, + ) + sys.exit(1) - if curr_ver < soft_ver: - print('repo: warning: Your version of "%s" (%s) is no longer supported; ' - 'Please upgrade to at least version %s to avoid breakage.' % - (pkg, self._format_ver(curr_ver), self._format_ver(soft_ver)), - file=sys.stderr) + if curr_ver < soft_ver: + print( + 'repo: warning: Your version of "%s" (%s) is no longer supported; ' + "Please upgrade to at least version %s to avoid breakage." + % (pkg, self._format_ver(curr_ver), self._format_ver(soft_ver)), + file=sys.stderr, + ) - def assert_all(self): - """Assert all of the requirements are satisified.""" - # See if we need a repo launcher upgrade first. - self.assert_ver('repo', VERSION) + def assert_all(self): + """Assert all of the requirements are satisified.""" + # See if we need a repo launcher upgrade first. + self.assert_ver("repo", VERSION) - # Check python before we try to import the repo code. - self.assert_ver('python', sys.version_info) + # Check python before we try to import the repo code. + self.assert_ver("python", sys.version_info) - # Check git while we're at it. - self.assert_ver('git', ParseGitVersion()) + # Check git while we're at it. + self.assert_ver("git", ParseGitVersion()) def _Usage(): - gitc_usage = "" - if get_gitc_manifest_dir(): - gitc_usage = " gitc-init Initialize a GITC Client.\n" + gitc_usage = "" + if get_gitc_manifest_dir(): + gitc_usage = " gitc-init Initialize a GITC Client.\n" - print( - """usage: repo COMMAND [ARGS] + print( + """usage: repo COMMAND [ARGS] repo is not yet installed. Use "repo init" to install it here. The most commonly used repo commands are: init Install repo in the current working directory -""" + gitc_usage + - """ help Display detailed help on a command +""" + + gitc_usage + + """ help Display detailed help on a command For access to the full online help, install repo ("repo init"). -""") - print('Bug reports:', BUG_URL) - sys.exit(0) +""" + ) + print("Bug reports:", BUG_URL) + sys.exit(0) def _Help(args): - if args: - if args[0] in {'init', 'gitc-init'}: - parser = GetParser(gitc_init=args[0] == 'gitc-init') - parser.print_help() - sys.exit(0) + if args: + if args[0] in {"init", "gitc-init"}: + parser = GetParser(gitc_init=args[0] == "gitc-init") + parser.print_help() + sys.exit(0) + else: + print( + "error: '%s' is not a bootstrap command.\n" + ' For access to online help, install repo ("repo init").' + % args[0], + file=sys.stderr, + ) else: - print("error: '%s' is not a bootstrap command.\n" - ' For access to online help, install repo ("repo init").' - % args[0], file=sys.stderr) - else: - _Usage() - sys.exit(1) + _Usage() + sys.exit(1) def _Version(): - """Show version information.""" - print('') - print('repo launcher version %s' % ('.'.join(str(x) for x in VERSION),)) - print(' (from %s)' % (__file__,)) - print('git %s' % (ParseGitVersion().full,)) - print('Python %s' % sys.version) - uname = platform.uname() - if sys.version_info.major < 3: - # Python 3 returns a named tuple, but Python 2 is simpler. - print(uname) - else: - print('OS %s %s (%s)' % (uname.system, uname.release, uname.version)) - print('CPU %s (%s)' % - (uname.machine, uname.processor if uname.processor else 'unknown')) - print('Bug reports:', BUG_URL) - sys.exit(0) + """Show version information.""" + print("") + print("repo launcher version %s" % (".".join(str(x) for x in VERSION),)) + print(" (from %s)" % (__file__,)) + print("git %s" % (ParseGitVersion().full,)) + print("Python %s" % sys.version) + uname = platform.uname() + if sys.version_info.major < 3: + # Python 3 returns a named tuple, but Python 2 is simpler. + print(uname) + else: + print("OS %s %s (%s)" % (uname.system, uname.release, uname.version)) + print( + "CPU %s (%s)" + % (uname.machine, uname.processor if uname.processor else "unknown") + ) + print("Bug reports:", BUG_URL) + sys.exit(0) def _NotInstalled(): - print('error: repo is not installed. Use "repo init" to install it here.', - file=sys.stderr) - sys.exit(1) + print( + 'error: repo is not installed. Use "repo init" to install it here.', + file=sys.stderr, + ) + sys.exit(1) def _NoCommands(cmd): - print("""error: command '%s' requires repo to be installed first. - Use "repo init" to install it here.""" % cmd, file=sys.stderr) - sys.exit(1) + print( + """error: command '%s' requires repo to be installed first. + Use "repo init" to install it here.""" + % cmd, + file=sys.stderr, + ) + sys.exit(1) def _RunSelf(wrapper_path): - my_dir = os.path.dirname(wrapper_path) - my_main = os.path.join(my_dir, 'main.py') - my_git = os.path.join(my_dir, '.git') + my_dir = os.path.dirname(wrapper_path) + my_main = os.path.join(my_dir, "main.py") + my_git = os.path.join(my_dir, ".git") - if os.path.isfile(my_main) and os.path.isdir(my_git): - for name in ['git_config.py', - 'project.py', - 'subcmds']: - if not os.path.exists(os.path.join(my_dir, name)): - return None, None - return my_main, my_git - return None, None + if os.path.isfile(my_main) and os.path.isdir(my_git): + for name in ["git_config.py", "project.py", "subcmds"]: + if not os.path.exists(os.path.join(my_dir, name)): + return None, None + return my_main, my_git + return None, None def _SetDefaultsTo(gitdir): - global REPO_URL - global REPO_REV + global REPO_URL + global REPO_REV - REPO_URL = gitdir - ret = run_git('--git-dir=%s' % gitdir, 'symbolic-ref', 'HEAD', check=False) - if ret.returncode: - # If we're not tracking a branch (bisect/etc...), then fall back to commit. - print('repo: warning: %s has no current branch; using HEAD' % gitdir, - file=sys.stderr) - try: - ret = run_git('rev-parse', 'HEAD', cwd=gitdir) - except CloneFailure: - print('fatal: %s has invalid HEAD' % gitdir, file=sys.stderr) - sys.exit(1) + REPO_URL = gitdir + ret = run_git("--git-dir=%s" % gitdir, "symbolic-ref", "HEAD", check=False) + if ret.returncode: + # If we're not tracking a branch (bisect/etc...), then fall back to commit. + print( + "repo: warning: %s has no current branch; using HEAD" % gitdir, + file=sys.stderr, + ) + try: + ret = run_git("rev-parse", "HEAD", cwd=gitdir) + except CloneFailure: + print("fatal: %s has invalid HEAD" % gitdir, file=sys.stderr) + sys.exit(1) - REPO_REV = ret.stdout.strip() + REPO_REV = ret.stdout.strip() def main(orig_args): - cmd, opt, args = _ParseArguments(orig_args) + cmd, opt, args = _ParseArguments(orig_args) - # We run this early as we run some git commands ourselves. - SetGitTrace2ParentSid() + # We run this early as we run some git commands ourselves. + SetGitTrace2ParentSid() - repo_main, rel_repo_dir = None, None - # Don't use the local repo copy, make sure to switch to the gitc client first. - if cmd != 'gitc-init': - repo_main, rel_repo_dir = _FindRepo() + repo_main, rel_repo_dir = None, None + # Don't use the local repo copy, make sure to switch to the gitc client first. + if cmd != "gitc-init": + repo_main, rel_repo_dir = _FindRepo() - wrapper_path = os.path.abspath(__file__) - my_main, my_git = _RunSelf(wrapper_path) + wrapper_path = os.path.abspath(__file__) + my_main, my_git = _RunSelf(wrapper_path) - cwd = os.getcwd() - if get_gitc_manifest_dir() and cwd.startswith(get_gitc_manifest_dir()): - print('error: repo cannot be used in the GITC local manifest directory.' - '\nIf you want to work on this GITC client please rerun this ' - 'command from the corresponding client under /gitc/', - file=sys.stderr) - sys.exit(1) - if not repo_main: - # Only expand aliases here since we'll be parsing the CLI ourselves. - # If we had repo_main, alias expansion would happen in main.py. - cmd, alias_args = _ExpandAlias(cmd) - args = alias_args + args - - if opt.help: - _Usage() - if cmd == 'help': - _Help(args) - if opt.version or cmd == 'version': - _Version() - if not cmd: - _NotInstalled() - if cmd == 'init' or cmd == 'gitc-init': - if my_git: - _SetDefaultsTo(my_git) - try: - _Init(args, gitc_init=(cmd == 'gitc-init')) - except CloneFailure: - path = os.path.join(repodir, S_repo) - print("fatal: cloning the git-repo repository failed, will remove " - "'%s' " % path, file=sys.stderr) - shutil.rmtree(path, ignore_errors=True) - shutil.rmtree(path + '.tmp', ignore_errors=True) + cwd = os.getcwd() + if get_gitc_manifest_dir() and cwd.startswith(get_gitc_manifest_dir()): + print( + "error: repo cannot be used in the GITC local manifest directory." + "\nIf you want to work on this GITC client please rerun this " + "command from the corresponding client under /gitc/", + file=sys.stderr, + ) sys.exit(1) - repo_main, rel_repo_dir = _FindRepo() - else: - _NoCommands(cmd) + if not repo_main: + # Only expand aliases here since we'll be parsing the CLI ourselves. + # If we had repo_main, alias expansion would happen in main.py. + cmd, alias_args = _ExpandAlias(cmd) + args = alias_args + args - if my_main: - repo_main = my_main + if opt.help: + _Usage() + if cmd == "help": + _Help(args) + if opt.version or cmd == "version": + _Version() + if not cmd: + _NotInstalled() + if cmd == "init" or cmd == "gitc-init": + if my_git: + _SetDefaultsTo(my_git) + try: + _Init(args, gitc_init=(cmd == "gitc-init")) + except CloneFailure: + path = os.path.join(repodir, S_repo) + print( + "fatal: cloning the git-repo repository failed, will remove " + "'%s' " % path, + file=sys.stderr, + ) + shutil.rmtree(path, ignore_errors=True) + shutil.rmtree(path + ".tmp", ignore_errors=True) + sys.exit(1) + repo_main, rel_repo_dir = _FindRepo() + else: + _NoCommands(cmd) - if not repo_main: - print("fatal: unable to find repo entry point", file=sys.stderr) - sys.exit(1) + if my_main: + repo_main = my_main - reqs = Requirements.from_dir(os.path.dirname(repo_main)) - if reqs: - reqs.assert_all() + if not repo_main: + print("fatal: unable to find repo entry point", file=sys.stderr) + sys.exit(1) - ver_str = '.'.join(map(str, VERSION)) - me = [sys.executable, repo_main, - '--repo-dir=%s' % rel_repo_dir, - '--wrapper-version=%s' % ver_str, - '--wrapper-path=%s' % wrapper_path, - '--'] - me.extend(orig_args) - exec_command(me) - print("fatal: unable to start %s" % repo_main, file=sys.stderr) - sys.exit(148) + reqs = Requirements.from_dir(os.path.dirname(repo_main)) + if reqs: + reqs.assert_all() + + ver_str = ".".join(map(str, VERSION)) + me = [ + sys.executable, + repo_main, + "--repo-dir=%s" % rel_repo_dir, + "--wrapper-version=%s" % ver_str, + "--wrapper-path=%s" % wrapper_path, + "--", + ] + me.extend(orig_args) + exec_command(me) + print("fatal: unable to start %s" % repo_main, file=sys.stderr) + sys.exit(148) -if __name__ == '__main__': - main(sys.argv[1:]) +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/run_tests b/run_tests index d61308f6..7307f827 100755 --- a/run_tests +++ b/run_tests @@ -27,8 +27,16 @@ ROOT_DIR = os.path.dirname(os.path.realpath(__file__)) def run_black(): """Returns the exit code from black.""" + # Black by default only matches .py files. We have to list standalone + # scripts manually. + extra_programs = [ + "repo", + "run_tests", + "release/update-manpages", + ] return subprocess.run( - [sys.executable, "-m", "black", "--check", ROOT_DIR], check=False + [sys.executable, "-m", "black", "--check", ROOT_DIR] + extra_programs, + check=False, ).returncode