# Copyright (C) 2008 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import functools import json import os import subprocess import sys from typing import Any, Optional from error import GitError from error import RepoExitError from git_refs import HEAD from git_trace2_event_log_base import BaseEventLog import platform_utils from repo_trace import IsTrace from repo_trace import REPO_TRACE from repo_trace import Trace from wrapper import Wrapper GIT = "git" # NB: These do not need to be kept in sync with the repo launcher script. # These may be much newer as it allows the repo launcher to roll between # different repo releases while source versions might require a newer git. # # The soft version is when we start warning users that the version is old and # we'll be dropping support for it. We'll refuse to work with versions older # than the hard version. # # git-1.7 is in (EOL) Ubuntu Precise. git-1.9 is in Ubuntu Trusty. MIN_GIT_VERSION_SOFT = (1, 9, 1) MIN_GIT_VERSION_HARD = (1, 7, 2) GIT_DIR = "GIT_DIR" LAST_GITDIR = None LAST_CWD = None DEFAULT_GIT_FAIL_MESSAGE = "git command failure" ERROR_EVENT_LOGGING_PREFIX = "RepoGitCommandError" # Common line length limit GIT_ERROR_STDOUT_LINES = 1 GIT_ERROR_STDERR_LINES = 1 INVALID_GIT_EXIT_CODE = 126 class _GitCall(object): @functools.lru_cache(maxsize=None) def version_tuple(self): ret = Wrapper().ParseGitVersion() if ret is None: msg = "fatal: unable to detect git version" print(msg, file=sys.stderr) raise GitRequireError(msg) return ret def __getattr__(self, name): name = name.replace("_", "-") def fun(*cmdv): command = [name] command.extend(cmdv) return GitCommand(None, command, add_event_log=False).Wait() == 0 return fun git = _GitCall() def RepoSourceVersion(): """Return the version of the repo.git tree.""" ver = getattr(RepoSourceVersion, "version", None) # We avoid GitCommand so we don't run into circular deps -- GitCommand needs # to initialize version info we provide. if ver is None: env = GitCommand._GetBasicEnv() proj = os.path.dirname(os.path.abspath(__file__)) env[GIT_DIR] = os.path.join(proj, ".git") result = subprocess.run( [GIT, "describe", HEAD], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, encoding="utf-8", env=env, check=False, ) if result.returncode == 0: ver = result.stdout.strip() if ver.startswith("v"): ver = ver[1:] else: ver = "unknown" setattr(RepoSourceVersion, "version", ver) return ver @functools.lru_cache(maxsize=None) def GetEventTargetPath(): """Get the 'trace2.eventtarget' path from git configuration. Returns: path: git config's 'trace2.eventtarget' path if it exists, or None """ path = None cmd = ["config", "--get", "trace2.eventtarget"] # TODO(https://crbug.com/gerrit/13706): Use GitConfig when it supports # system git config variables. p = GitCommand( None, cmd, capture_stdout=True, capture_stderr=True, bare=True, add_event_log=False, ) retval = p.Wait() if retval == 0: # Strip trailing carriage-return in path. path = p.stdout.rstrip("\n") elif retval != 1: # `git config --get` is documented to produce an exit status of `1` # if the requested variable is not present in the configuration. # Report any other return value as an error. print( "repo: error: 'git config --get' call failed with return code: " "%r, stderr: %r" % (retval, p.stderr), file=sys.stderr, ) return path class UserAgent(object): """Mange User-Agent settings when talking to external services We follow the style as documented here: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent """ _os = None _repo_ua = None _git_ua = None @property def os(self): """The operating system name.""" if self._os is None: os_name = sys.platform if os_name.lower().startswith("linux"): os_name = "Linux" elif os_name == "win32": os_name = "Win32" elif os_name == "cygwin": os_name = "Cygwin" elif os_name == "darwin": os_name = "Darwin" self._os = os_name return self._os @property def repo(self): """The UA when connecting directly from repo.""" if self._repo_ua is None: py_version = sys.version_info self._repo_ua = "git-repo/%s (%s) git/%s Python/%d.%d.%d" % ( RepoSourceVersion(), self.os, git.version_tuple().full, py_version.major, py_version.minor, py_version.micro, ) return self._repo_ua @property def git(self): """The UA when running git.""" if self._git_ua is None: self._git_ua = "git/%s (%s) git-repo/%s" % ( git.version_tuple().full, self.os, RepoSourceVersion(), ) return self._git_ua user_agent = UserAgent() def git_require(min_version, fail=False, msg=""): git_version = git.version_tuple() if min_version <= git_version: return True if fail: need = ".".join(map(str, min_version)) if msg: msg = " for " + msg error_msg = "fatal: git %s or later required%s" % (need, msg) print(error_msg, file=sys.stderr) raise GitRequireError(error_msg) return False def _build_env( _kwargs_only=(), bare: Optional[bool] = False, disable_editor: Optional[bool] = False, ssh_proxy: Optional[Any] = None, gitdir: Optional[str] = None, objdir: Optional[str] = None, ): """Constucts an env dict for command execution.""" assert _kwargs_only == (), "_build_env only accepts keyword arguments." env = GitCommand._GetBasicEnv() if disable_editor: env["GIT_EDITOR"] = ":" if ssh_proxy: env["REPO_SSH_SOCK"] = ssh_proxy.sock() env["GIT_SSH"] = ssh_proxy.proxy env["GIT_SSH_VARIANT"] = "ssh" if "http_proxy" in env and "darwin" == sys.platform: s = "'http.proxy=%s'" % (env["http_proxy"],) p = env.get("GIT_CONFIG_PARAMETERS") if p is not None: s = p + " " + s env["GIT_CONFIG_PARAMETERS"] = s if "GIT_ALLOW_PROTOCOL" not in env: env[ "GIT_ALLOW_PROTOCOL" ] = "file:git:http:https:ssh:persistent-http:persistent-https:sso:rpc" env["GIT_HTTP_USER_AGENT"] = user_agent.git if objdir: # Set to the place we want to save the objects. env["GIT_OBJECT_DIRECTORY"] = objdir alt_objects = os.path.join(gitdir, "objects") if gitdir else None if alt_objects and os.path.realpath(alt_objects) != os.path.realpath( objdir ): # Allow git to search the original place in case of local or unique # refs that git will attempt to resolve even if we aren't fetching # them. env["GIT_ALTERNATE_OBJECT_DIRECTORIES"] = alt_objects if bare and gitdir is not None: env[GIT_DIR] = gitdir return env class GitCommand(object): """Wrapper around a single git invocation.""" def __init__( self, project, cmdv, bare=False, input=None, capture_stdout=False, capture_stderr=False, merge_output=False, disable_editor=False, ssh_proxy=None, cwd=None, gitdir=None, objdir=None, verify_command=False, add_event_log=True, ): if project: if not cwd: cwd = project.worktree if not gitdir: gitdir = project.gitdir self.project = project self.cmdv = cmdv self.verify_command = verify_command # Git on Windows wants its paths only using / for reliability. if platform_utils.isWindows(): if objdir: objdir = objdir.replace("\\", "/") if gitdir: gitdir = gitdir.replace("\\", "/") env = _build_env( disable_editor=disable_editor, ssh_proxy=ssh_proxy, objdir=objdir, gitdir=gitdir, bare=bare, ) command = [GIT] if bare: cwd = None command_name = cmdv[0] command.append(command_name) # Need to use the --progress flag for fetch/clone so output will be # displayed as by default git only does progress output if stderr is a # TTY. if sys.stderr.isatty() and command_name in ("fetch", "clone"): if "--progress" not in cmdv and "--quiet" not in cmdv: command.append("--progress") command.extend(cmdv[1:]) stdin = subprocess.PIPE if input else None stdout = subprocess.PIPE if capture_stdout else None stderr = ( subprocess.STDOUT if merge_output else (subprocess.PIPE if capture_stderr else None) ) event_log = ( BaseEventLog(env=env, add_init_count=True) if add_event_log else None ) try: self._RunCommand( command, env, stdin=stdin, stdout=stdout, stderr=stderr, ssh_proxy=ssh_proxy, cwd=cwd, input=input, ) self.VerifyCommand() except GitCommandError as e: if event_log is not None: error_info = json.dumps( { "ErrorType": type(e).__name__, "Project": e.project, "CommandName": command_name, "Message": str(e), "ReturnCode": str(e.git_rc) if e.git_rc is not None else None, } ) event_log.ErrorEvent( f"{ERROR_EVENT_LOGGING_PREFIX}:{error_info}" ) event_log.Write(GetEventTargetPath()) if isinstance(e, GitPopenCommandError): raise def _RunCommand( self, command, env, stdin=None, stdout=None, stderr=None, ssh_proxy=None, cwd=None, input=None, ): dbg = "" if IsTrace(): global LAST_CWD global LAST_GITDIR if cwd and LAST_CWD != cwd: if LAST_GITDIR or LAST_CWD: dbg += "\n" dbg += ": cd %s\n" % cwd LAST_CWD = cwd if GIT_DIR in env and LAST_GITDIR != env[GIT_DIR]: if LAST_GITDIR or LAST_CWD: dbg += "\n" dbg += ": export GIT_DIR=%s\n" % env[GIT_DIR] LAST_GITDIR = env[GIT_DIR] if "GIT_OBJECT_DIRECTORY" in env: dbg += ( ": export GIT_OBJECT_DIRECTORY=%s\n" % env["GIT_OBJECT_DIRECTORY"] ) if "GIT_ALTERNATE_OBJECT_DIRECTORIES" in env: dbg += ": export GIT_ALTERNATE_OBJECT_DIRECTORIES=%s\n" % ( env["GIT_ALTERNATE_OBJECT_DIRECTORIES"] ) dbg += ": " dbg += " ".join(command) if stdin == subprocess.PIPE: dbg += " 0<|" if stdout == subprocess.PIPE: dbg += " 1>|" if stderr == subprocess.PIPE: dbg += " 2>|" elif stderr == subprocess.STDOUT: dbg += " 2>&1" with Trace( "git command %s %s with debug: %s", LAST_GITDIR, command, dbg ): try: p = subprocess.Popen( command, cwd=cwd, env=env, encoding="utf-8", errors="backslashreplace", stdin=stdin, stdout=stdout, stderr=stderr, ) except Exception as e: raise GitPopenCommandError( message="%s: %s" % (command[1], e), project=self.project.name if self.project else None, command_args=self.cmdv, ) if ssh_proxy: ssh_proxy.add_client(p) self.process = p try: self.stdout, self.stderr = p.communicate(input=input) finally: if ssh_proxy: ssh_proxy.remove_client(p) self.rc = p.wait() @staticmethod def _GetBasicEnv(): """Return a basic env for running git under. This is guaranteed to be side-effect free. """ env = os.environ.copy() for key in ( REPO_TRACE, GIT_DIR, "GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_OBJECT_DIRECTORY", "GIT_WORK_TREE", "GIT_GRAFT_FILE", "GIT_INDEX_FILE", ): env.pop(key, None) return env def VerifyCommand(self): if self.rc == 0: return None stdout = ( "\n".join(self.stdout.split("\n")[:GIT_ERROR_STDOUT_LINES]) if self.stdout else None ) stderr = ( "\n".join(self.stderr.split("\n")[:GIT_ERROR_STDERR_LINES]) if self.stderr else None ) project = self.project.name if self.project else None raise GitCommandError( project=project, command_args=self.cmdv, git_rc=self.rc, git_stdout=stdout, git_stderr=stderr, ) def Wait(self): if self.verify_command: self.VerifyCommand() return self.rc class GitRequireError(RepoExitError): """Error raised when git version is unavailable or invalid.""" def __init__(self, message, exit_code: int = INVALID_GIT_EXIT_CODE): super().__init__(message, exit_code=exit_code) class GitCommandError(GitError): """ Error raised from a failed git command. Note that GitError can refer to any Git related error (e.g. branch not specified for project.py 'UploadForReview'), while GitCommandError is raised exclusively from non-zero exit codes returned from git commands. """ def __init__( self, message: str = DEFAULT_GIT_FAIL_MESSAGE, git_rc: int = None, git_stdout: str = None, git_stderr: str = None, **kwargs, ): super().__init__( message, **kwargs, ) self.git_rc = git_rc self.git_stdout = git_stdout self.git_stderr = git_stderr def __str__(self): args = "[]" if not self.command_args else " ".join(self.command_args) error_type = type(self).__name__ return f"""{error_type}: {self.message} Project: {self.project} Args: {args} Stdout: {self.git_stdout} Stderr: {self.git_stderr}""" class GitPopenCommandError(GitError): """ Error raised when subprocess.Popen fails for a GitCommand """