sync: Fix syntax error on Python 2.4

Change-Id: I371d032d5a1ddde137721cbe2b24bfa38f20aaaa
Signed-off-by: Shawn O. Pearce <sop@google.com>

command.py

@@ -74,7 +74,7 @@ class Command(object):
         project = all.get(arg)
 
         if not project:
-          path = os.path.abspath(arg)
+          path = os.path.abspath(arg).replace('\\', '/')
 
           if not by_path:
             by_path = dict()
@@ -82,13 +82,15 @@ class Command(object):
               by_path[p.worktree] = p
 
           if os.path.exists(path):
+            oldpath = None
             while path \
-              and path != '/' \
+              and path != oldpath \
               and path != self.manifest.topdir:
               try:
                 project = by_path[path]
                 break
               except KeyError:
+                oldpath = path
                 path = os.path.dirname(path)
           else:
             try:

docs/manifest-format.txt

@@ -25,7 +25,8 @@ following DTD:
                         default?,
                         manifest-server?,
                         remove-project*,
-                        project*)>
+                        project*,
+                        repo-hooks?)>
   
     <!ELEMENT notice (#PCDATA)>
   
@@ -49,6 +50,10 @@ following DTD:
   
     <!ELEMENT remove-project (EMPTY)>
     <!ATTLIST remove-project name  CDATA #REQUIRED>
+
+    <!ELEMENT repo-hooks (EMPTY)>
+    <!ATTLIST repo-hooks in-project CDATA #REQUIRED>
+    <!ATTLIST repo-hooks enabled-list CDATA #REQUIRED>
   ]>
 
 A description of the elements and their attributes follows.

error.py

@@ -75,3 +75,10 @@ class RepoChangedException(Exception):
   """
   def __init__(self, extra_args=[]):
     self.extra_args = extra_args
+
+class HookError(Exception):
+  """Thrown if a 'repo-hook' could not be run.
+
+  The common case is that the file wasn't present when we tried to run it.
+  """
+  pass

git_command.py

@@ -112,6 +112,9 @@ def git_require(min_version, fail=False):
     sys.exit(1)
   return False
 
+def _setenv(env, name, value):
+  env[name] = value.encode()
+
 class GitCommand(object):
   def __init__(self,
                project,
@@ -124,7 +127,7 @@ class GitCommand(object):
                ssh_proxy = False,
                cwd = None,
                gitdir = None):
-    env = dict(os.environ)
+    env = os.environ.copy()
 
     for e in [REPO_TRACE,
               GIT_DIR,
@@ -137,10 +140,10 @@ class GitCommand(object):
         del env[e]
 
     if disable_editor:
-      env['GIT_EDITOR'] = ':'
+      _setenv(env, 'GIT_EDITOR', ':')
     if ssh_proxy:
-      env['REPO_SSH_SOCK'] = ssh_sock()
-      env['GIT_SSH'] = _ssh_proxy()
+      _setenv(env, 'REPO_SSH_SOCK', ssh_sock())
+      _setenv(env, 'GIT_SSH', _ssh_proxy())
 
     if project:
       if not cwd:
@@ -151,7 +154,7 @@ class GitCommand(object):
     command = [GIT]
     if bare:
       if gitdir:
-        env[GIT_DIR] = gitdir
+        _setenv(env, GIT_DIR, gitdir)
       cwd = None
     command.extend(cmdv)
 

git_config.py

@@ -18,7 +18,13 @@ import os
 import re
 import subprocess
 import sys
+try:
+  import threading as _threading
+except ImportError:
+  import dummy_threading as _threading
 import time
+import urllib2
+
 from signal import SIGTERM
 from urllib2 import urlopen, HTTPError
 from error import GitError, UploadError
@@ -361,76 +367,97 @@ class RefSpec(object):
 _master_processes = []
 _master_keys = set()
 _ssh_master = True
+_master_keys_lock = None
+
+def init_ssh():
+  """Should be called once at the start of repo to init ssh master handling.
+
+  At the moment, all we do is to create our lock.
+  """
+  global _master_keys_lock
+  assert _master_keys_lock is None, "Should only call init_ssh once"
+  _master_keys_lock = _threading.Lock()
 
 def _open_ssh(host, port=None):
   global _ssh_master
 
-  # Check to see whether we already think that the master is running; if we
-  # think it's already running, return right away.
-  if port is not None:
-    key = '%s:%s' % (host, port)
-  else:
-    key = host
-
-  if key in _master_keys:
-    return True
-
-  if not _ssh_master \
-  or 'GIT_SSH' in os.environ \
-  or sys.platform in ('win32', 'cygwin'):
-    # failed earlier, or cygwin ssh can't do this
-    #
-    return False
-
-  # We will make two calls to ssh; this is the common part of both calls.
-  command_base = ['ssh',
-                   '-o','ControlPath %s' % ssh_sock(),
-                   host]
-  if port is not None:
-    command_base[1:1] = ['-p',str(port)]
-
-  # Since the key wasn't in _master_keys, we think that master isn't running.
-  # ...but before actually starting a master, we'll double-check.  This can
-  # be important because we can't tell that that 'git@myhost.com' is the same
-  # as 'myhost.com' where "User git" is setup in the user's ~/.ssh/config file.
-  check_command = command_base + ['-O','check']
+  # Acquire the lock.  This is needed to prevent opening multiple masters for
+  # the same host when we're running "repo sync -jN" (for N > 1) _and_ the
+  # manifest <remote fetch="ssh://xyz"> specifies a different host from the
+  # one that was passed to repo init.
+  _master_keys_lock.acquire()
   try:
-    Trace(': %s', ' '.join(check_command))
-    check_process = subprocess.Popen(check_command,
-                                     stdout=subprocess.PIPE,
-                                     stderr=subprocess.PIPE)
-    check_process.communicate() # read output, but ignore it...
-    isnt_running = check_process.wait()
 
-    if not isnt_running:
-      # Our double-check found that the master _was_ infact running.  Add to
-      # the list of keys.
-      _master_keys.add(key)
+    # Check to see whether we already think that the master is running; if we
+    # think it's already running, return right away.
+    if port is not None:
+      key = '%s:%s' % (host, port)
+    else:
+      key = host
+
+    if key in _master_keys:
       return True
-  except Exception:
-    # Ignore excpetions.  We we will fall back to the normal command and print
-    # to the log there.
-    pass
 
-  command = command_base[:1] + \
-            ['-M', '-N'] + \
-            command_base[1:]
-  try:
-    Trace(': %s', ' '.join(command))
-    p = subprocess.Popen(command)
-  except Exception, e:
-    _ssh_master = False
-    print >>sys.stderr, \
-      '\nwarn: cannot enable ssh control master for %s:%s\n%s' \
-      % (host,port, str(e))
-    return False
+    if not _ssh_master \
+    or 'GIT_SSH' in os.environ \
+    or sys.platform in ('win32', 'cygwin'):
+      # failed earlier, or cygwin ssh can't do this
+      #
+      return False
 
-  _master_processes.append(p)
-  _master_keys.add(key)
-  time.sleep(1)
-  return True
+    # We will make two calls to ssh; this is the common part of both calls.
+    command_base = ['ssh',
+                     '-o','ControlPath %s' % ssh_sock(),
+                     host]
+    if port is not None:
+      command_base[1:1] = ['-p',str(port)]
+
+    # Since the key wasn't in _master_keys, we think that master isn't running.
+    # ...but before actually starting a master, we'll double-check.  This can
+    # be important because we can't tell that that 'git@myhost.com' is the same
+    # as 'myhost.com' where "User git" is setup in the user's ~/.ssh/config file.
+    check_command = command_base + ['-O','check']
+    try:
+      Trace(': %s', ' '.join(check_command))
+      check_process = subprocess.Popen(check_command,
+                                       stdout=subprocess.PIPE,
+                                       stderr=subprocess.PIPE)
+      check_process.communicate() # read output, but ignore it...
+      isnt_running = check_process.wait()
+
+      if not isnt_running:
+        # Our double-check found that the master _was_ infact running.  Add to
+        # the list of keys.
+        _master_keys.add(key)
+        return True
+    except Exception:
+      # Ignore excpetions.  We we will fall back to the normal command and print
+      # to the log there.
+      pass
+
+    command = command_base[:1] + \
+              ['-M', '-N'] + \
+              command_base[1:]
+    try:
+      Trace(': %s', ' '.join(command))
+      p = subprocess.Popen(command)
+    except Exception, e:
+      _ssh_master = False
+      print >>sys.stderr, \
+        '\nwarn: cannot enable ssh control master for %s:%s\n%s' \
+        % (host,port, str(e))
+      return False
+
+    _master_processes.append(p)
+    _master_keys.add(key)
+    time.sleep(1)
+    return True
+  finally:
+    _master_keys_lock.release()
 
 def close_ssh():
+  global _master_keys_lock
+
   terminate_ssh_clients()
 
   for p in _master_processes:
@@ -449,6 +476,9 @@ def close_ssh():
     except OSError:
       pass
 
+  # We're done with the lock, so we can delete it.
+  _master_keys_lock = None
+
 URI_SCP = re.compile(r'^([^@:]*@?[^:/]{1,}):')
 URI_ALL = re.compile(r'^([a-z][a-z+]*)://([^@/]*@?[^/]*)/')
 
@@ -535,23 +565,25 @@ class Remote(object):
         try:
           info = urlopen(u).read()
           if info == 'NOT_AVAILABLE':
-            raise UploadError('Upload over ssh unavailable')
+            raise UploadError('%s: SSH disabled' % self.review)
           if '<' in info:
             # Assume the server gave us some sort of HTML
             # response back, like maybe a login page.
             #
-            raise UploadError('Cannot read %s:\n%s' % (u, info))
+            raise UploadError('%s: Cannot parse response' % u)
 
           self._review_protocol = 'ssh'
           self._review_host = info.split(" ")[0]
           self._review_port = info.split(" ")[1]
+        except urllib2.URLError, e:
+          raise UploadError('%s: %s' % (self.review, e.reason[1]))
         except HTTPError, e:
           if e.code == 404:
             self._review_protocol = 'http-post'
             self._review_host = None
             self._review_port = None
           else:
-            raise UploadError('Cannot guess Gerrit version')
+            raise UploadError('Upload over ssh unavailable')
 
         REVIEW_CACHE[u] = (
           self._review_protocol,

main.py

@@ -28,7 +28,7 @@ import re
 import sys
 
 from trace import SetTrace
-from git_config import close_ssh
+from git_config import init_ssh, close_ssh
 from command import InteractiveCommand
 from command import MirrorSafeCommand
 from command import PagedCommand
@@ -61,6 +61,8 @@ class _Repo(object):
   def __init__(self, repodir):
     self.repodir = repodir
     self.commands = all_commands
+    # add 'branch' as an alias for 'branches'
+    all_commands['branch'] = all_commands['branches']
 
   def _Run(self, argv):
     name = None
@@ -214,6 +216,7 @@ def _Main(argv):
   repo = _Repo(opt.repodir)
   try:
     try:
+      init_ssh()
       repo._Run(argv)
     finally:
       close_ssh()

manifest_xml.py

@@ -171,6 +171,14 @@ class XmlManifest(object):
         ce.setAttribute('dest', c.dest)
         e.appendChild(ce)
 
+    if self._repo_hooks_project:
+      root.appendChild(doc.createTextNode(''))
+      e = doc.createElement('repo-hooks')
+      e.setAttribute('in-project', self._repo_hooks_project.name)
+      e.setAttribute('enabled-list',
+                     ' '.join(self._repo_hooks_project.enabled_repo_hooks))
+      root.appendChild(e)
+
     doc.writexml(fd, '', '  ', '\n', 'UTF-8')
 
   @property
@@ -188,6 +196,11 @@ class XmlManifest(object):
     self._Load()
     return self._default
 
+  @property
+  def repo_hooks_project(self):
+    self._Load()
+    return self._repo_hooks_project
+
   @property
   def notice(self):
     self._Load()
@@ -207,6 +220,7 @@ class XmlManifest(object):
     self._projects = {}
     self._remotes = {}
     self._default = None
+    self._repo_hooks_project = None
     self._notice = None
     self.branch = None
     self._manifest_server = None
@@ -239,15 +253,15 @@ class XmlManifest(object):
   def _ParseManifest(self, is_root_file):
     root = xml.dom.minidom.parse(self.manifestFile)
     if not root or not root.childNodes:
-      raise ManifestParseError, \
-            "no root node in %s" % \
-            self.manifestFile
+      raise ManifestParseError(
+          "no root node in %s" %
+          self.manifestFile)
 
     config = root.childNodes[0]
     if config.nodeName != 'manifest':
-      raise ManifestParseError, \
-            "no <manifest> in %s" % \
-            self.manifestFile
+      raise ManifestParseError(
+          "no <manifest> in %s" %
+          self.manifestFile)
 
     for node in config.childNodes:
       if node.nodeName == 'remove-project':
@@ -255,25 +269,30 @@ class XmlManifest(object):
         try:
           del self._projects[name]
         except KeyError:
-          raise ManifestParseError, \
-                'project %s not found' % \
-                (name)
+          raise ManifestParseError(
+              'project %s not found' %
+              (name))
+
+        # If the manifest removes the hooks project, treat it as if it deleted
+        # the repo-hooks element too.
+        if self._repo_hooks_project and (self._repo_hooks_project.name == name):
+          self._repo_hooks_project = None
 
     for node in config.childNodes:
       if node.nodeName == 'remote':
         remote = self._ParseRemote(node)
         if self._remotes.get(remote.name):
-          raise ManifestParseError, \
-                'duplicate remote %s in %s' % \
-                (remote.name, self.manifestFile)
+          raise ManifestParseError(
+              'duplicate remote %s in %s' %
+              (remote.name, self.manifestFile))
         self._remotes[remote.name] = remote
 
     for node in config.childNodes:
       if node.nodeName == 'default':
         if self._default is not None:
-          raise ManifestParseError, \
-                'duplicate default in %s' % \
-                (self.manifestFile)
+          raise ManifestParseError(
+              'duplicate default in %s' %
+              (self.manifestFile))
         self._default = self._ParseDefault(node)
     if self._default is None:
       self._default = _Default()
@@ -281,29 +300,52 @@ class XmlManifest(object):
     for node in config.childNodes:
       if node.nodeName == 'notice':
         if self._notice is not None:
-          raise ManifestParseError, \
-                'duplicate notice in %s' % \
-                (self.manifestFile)
+          raise ManifestParseError(
+              'duplicate notice in %s' %
+              (self.manifestFile))
         self._notice = self._ParseNotice(node)
 
     for node in config.childNodes:
       if node.nodeName == 'manifest-server':
         url = self._reqatt(node, 'url')
         if self._manifest_server is not None:
-            raise ManifestParseError, \
-                'duplicate manifest-server in %s' % \
-                (self.manifestFile)
+            raise ManifestParseError(
+                'duplicate manifest-server in %s' %
+                (self.manifestFile))
         self._manifest_server = url
 
     for node in config.childNodes:
       if node.nodeName == 'project':
         project = self._ParseProject(node)
         if self._projects.get(project.name):
-          raise ManifestParseError, \
-                'duplicate project %s in %s' % \
-                (project.name, self.manifestFile)
+          raise ManifestParseError(
+              'duplicate project %s in %s' %
+              (project.name, self.manifestFile))
         self._projects[project.name] = project
 
+    for node in config.childNodes:
+      if node.nodeName == 'repo-hooks':
+        # Get the name of the project and the (space-separated) list of enabled.
+        repo_hooks_project = self._reqatt(node, 'in-project')
+        enabled_repo_hooks = self._reqatt(node, 'enabled-list').split()
+
+        # Only one project can be the hooks project
+        if self._repo_hooks_project is not None:
+          raise ManifestParseError(
+              'duplicate repo-hooks in %s' %
+              (self.manifestFile))
+
+        # Store a reference to the Project.
+        try:
+          self._repo_hooks_project = self._projects[repo_hooks_project]
+        except KeyError:
+          raise ManifestParseError(
+              'project %s not found for repo-hooks' %
+              (repo_hooks_project))
+
+        # Store the enabled hooks in the Project object.
+        self._repo_hooks_project.enabled_repo_hooks = enabled_repo_hooks
+
   def _AddMetaProjectMirror(self, m):
     name = None
     m_url = m.GetRemote(m.remote.name).url
@@ -435,7 +477,7 @@ class XmlManifest(object):
       worktree = None
       gitdir = os.path.join(self.topdir, '%s.git' % name)
     else:
-      worktree = os.path.join(self.topdir, path)
+      worktree = os.path.join(self.topdir, path).replace('\\', '/')
       gitdir = os.path.join(self.repodir, 'projects/%s.git' % path)
 
     project = Project(manifest = self,

project.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import traceback
 import errno
 import filecmp
 import os
@@ -24,7 +25,7 @@ import urllib2
 from color import Coloring
 from git_command import GitCommand
 from git_config import GitConfig, IsId
-from error import GitError, ImportError, UploadError
+from error import GitError, HookError, ImportError, UploadError
 from error import ManifestInvalidRevisionError
 
 from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M
@@ -54,14 +55,25 @@ def not_rev(r):
 def sq(r):
   return "'" + r.replace("'", "'\''") + "'"
 
-hook_list = None
-def repo_hooks():
-  global hook_list
-  if hook_list is None:
+_project_hook_list = None
+def _ProjectHooks():
+  """List the hooks present in the 'hooks' directory.
+
+  These hooks are project hooks and are copied to the '.git/hooks' directory
+  of all subprojects.
+
+  This function caches the list of hooks (based on the contents of the
+  'repo/hooks' directory) on the first call.
+
+  Returns:
+    A list of absolute paths to all of the files in the hooks directory.
+  """
+  global _project_hook_list
+  if _project_hook_list is None:
     d = os.path.abspath(os.path.dirname(__file__))
     d = os.path.join(d , 'hooks')
-    hook_list = map(lambda x: os.path.join(d, x), os.listdir(d))
-  return hook_list
+    _project_hook_list = map(lambda x: os.path.join(d, x), os.listdir(d))
+  return _project_hook_list
 
 def relpath(dst, src):
   src = os.path.dirname(src)
@@ -223,6 +235,249 @@ class RemoteSpec(object):
     self.url = url
     self.review = review
 
+class RepoHook(object):
+  """A RepoHook contains information about a script to run as a hook.
+
+  Hooks are used to run a python script before running an upload (for instance,
+  to run presubmit checks).  Eventually, we may have hooks for other actions.
+
+  This shouldn't be confused with files in the 'repo/hooks' directory.  Those
+  files are copied into each '.git/hooks' folder for each project.  Repo-level
+  hooks are associated instead with repo actions.
+
+  Hooks are always python.  When a hook is run, we will load the hook into the
+  interpreter and execute its main() function.
+  """
+  def __init__(self,
+               hook_type,
+               hooks_project,
+               topdir,
+               abort_if_user_denies=False):
+    """RepoHook constructor.
+
+    Params:
+      hook_type: A string representing the type of hook.  This is also used
+          to figure out the name of the file containing the hook.  For
+          example: 'pre-upload'.
+      hooks_project: The project containing the repo hooks.  If you have a
+          manifest, this is manifest.repo_hooks_project.  OK if this is None,
+          which will make the hook a no-op.
+      topdir: Repo's top directory (the one containing the .repo directory).
+          Scripts will run with CWD as this directory.  If you have a manifest,
+          this is manifest.topdir
+      abort_if_user_denies: If True, we'll throw a HookError() if the user
+          doesn't allow us to run the hook.
+    """
+    self._hook_type = hook_type
+    self._hooks_project = hooks_project
+    self._topdir = topdir
+    self._abort_if_user_denies = abort_if_user_denies
+
+    # Store the full path to the script for convenience.
+    if self._hooks_project:
+      self._script_fullpath = os.path.join(self._hooks_project.worktree,
+                                           self._hook_type + '.py')
+    else:
+      self._script_fullpath = None
+
+  def _GetHash(self):
+    """Return a hash of the contents of the hooks directory.
+
+    We'll just use git to do this.  This hash has the property that if anything
+    changes in the directory we will return a different has.
+
+    SECURITY CONSIDERATION:
+      This hash only represents the contents of files in the hook directory, not
+      any other files imported or called by hooks.  Changes to imported files
+      can change the script behavior without affecting the hash.
+
+    Returns:
+      A string representing the hash.  This will always be ASCII so that it can
+      be printed to the user easily.
+    """
+    assert self._hooks_project, "Must have hooks to calculate their hash."
+
+    # We will use the work_git object rather than just calling GetRevisionId().
+    # That gives us a hash of the latest checked in version of the files that
+    # the user will actually be executing.  Specifically, GetRevisionId()
+    # doesn't appear to change even if a user checks out a different version
+    # of the hooks repo (via git checkout) nor if a user commits their own revs.
+    #
+    # NOTE: Local (non-committed) changes will not be factored into this hash.
+    # I think this is OK, since we're really only worried about warning the user
+    # about upstream changes.
+    return self._hooks_project.work_git.rev_parse('HEAD')
+
+  def _GetMustVerb(self):
+    """Return 'must' if the hook is required; 'should' if not."""
+    if self._abort_if_user_denies:
+      return 'must'
+    else:
+      return 'should'
+
+  def _CheckForHookApproval(self):
+    """Check to see whether this hook has been approved.
+
+    We'll look at the hash of all of the hooks.  If this matches the hash that
+    the user last approved, we're done.  If it doesn't, we'll ask the user
+    about approval.
+
+    Note that we ask permission for each individual hook even though we use
+    the hash of all hooks when detecting changes.  We'd like the user to be
+    able to approve / deny each hook individually.  We only use the hash of all
+    hooks because there is no other easy way to detect changes to local imports.
+
+    Returns:
+      True if this hook is approved to run; False otherwise.
+
+    Raises:
+      HookError: Raised if the user doesn't approve and abort_if_user_denies
+          was passed to the consturctor.
+    """
+    hooks_dir = self._hooks_project.worktree
+    hooks_config = self._hooks_project.config
+    git_approval_key = 'repo.hooks.%s.approvedhash' % self._hook_type
+
+    # Get the last hash that the user approved for this hook; may be None.
+    old_hash = hooks_config.GetString(git_approval_key)
+
+    # Get the current hash so we can tell if scripts changed since approval.
+    new_hash = self._GetHash()
+
+    if old_hash is not None:
+      # User previously approved hook and asked not to be prompted again.
+      if new_hash == old_hash:
+        # Approval matched.  We're done.
+        return True
+      else:
+        # Give the user a reason why we're prompting, since they last told
+        # us to "never ask again".
+        prompt = 'WARNING: Scripts have changed since %s was allowed.\n\n' % (
+            self._hook_type)
+    else:
+      prompt = ''
+
+    # Prompt the user if we're not on a tty; on a tty we'll assume "no".
+    if sys.stdout.isatty():
+      prompt += ('Repo %s run the script:\n'
+                 '  %s\n'
+                 '\n'
+                 'Do you want to allow this script to run '
+                 '(yes/yes-never-ask-again/NO)? ') % (
+                 self._GetMustVerb(), self._script_fullpath)
+      response = raw_input(prompt).lower()
+      print
+
+      # User is doing a one-time approval.
+      if response in ('y', 'yes'):
+        return True
+      elif response == 'yes-never-ask-again':
+        hooks_config.SetString(git_approval_key, new_hash)
+        return True
+
+    # For anything else, we'll assume no approval.
+    if self._abort_if_user_denies:
+      raise HookError('You must allow the %s hook or use --no-verify.' %
+                      self._hook_type)
+
+    return False
+
+  def _ExecuteHook(self, **kwargs):
+    """Actually execute the given hook.
+
+    This will run the hook's 'main' function in our python interpreter.
+
+    Args:
+      kwargs: Keyword arguments to pass to the hook.  These are often specific
+          to the hook type.  For instance, pre-upload hooks will contain
+          a project_list.
+    """
+    # Keep sys.path and CWD stashed away so that we can always restore them
+    # upon function exit.
+    orig_path = os.getcwd()
+    orig_syspath = sys.path
+
+    try:
+      # Always run hooks with CWD as topdir.
+      os.chdir(self._topdir)
+
+      # Put the hook dir as the first item of sys.path so hooks can do
+      # relative imports.  We want to replace the repo dir as [0] so
+      # hooks can't import repo files.
+      sys.path = [os.path.dirname(self._script_fullpath)] + sys.path[1:]
+
+      # Exec, storing global context in the context dict.  We catch exceptions
+      # and  convert to a HookError w/ just the failing traceback.
+      context = {}
+      try:
+        execfile(self._script_fullpath, context)
+      except Exception:
+        raise HookError('%s\nFailed to import %s hook; see traceback above.' % (
+                        traceback.format_exc(), self._hook_type))
+
+      # Running the script should have defined a main() function.
+      if 'main' not in context:
+        raise HookError('Missing main() in: "%s"' % self._script_fullpath)
+
+
+      # Add 'hook_should_take_kwargs' to the arguments to be passed to main.
+      # We don't actually want hooks to define their main with this argument--
+      # it's there to remind them that their hook should always take **kwargs.
+      # For instance, a pre-upload hook should be defined like:
+      #   def main(project_list, **kwargs):
+      #
+      # This allows us to later expand the API without breaking old hooks.
+      kwargs = kwargs.copy()
+      kwargs['hook_should_take_kwargs'] = True
+
+      # Call the main function in the hook.  If the hook should cause the
+      # build to fail, it will raise an Exception.  We'll catch that convert
+      # to a HookError w/ just the failing traceback.
+      try:
+        context['main'](**kwargs)
+      except Exception:
+        raise HookError('%s\nFailed to run main() for %s hook; see traceback '
+                        'above.' % (
+                        traceback.format_exc(), self._hook_type))
+    finally:
+      # Restore sys.path and CWD.
+      sys.path = orig_syspath
+      os.chdir(orig_path)
+
+  def Run(self, user_allows_all_hooks, **kwargs):
+    """Run the hook.
+
+    If the hook doesn't exist (because there is no hooks project or because
+    this particular hook is not enabled), this is a no-op.
+
+    Args:
+      user_allows_all_hooks: If True, we will never prompt about running the
+          hook--we'll just assume it's OK to run it.
+      kwargs: Keyword arguments to pass to the hook.  These are often specific
+          to the hook type.  For instance, pre-upload hooks will contain
+          a project_list.
+
+    Raises:
+      HookError: If there was a problem finding the hook or the user declined
+          to run a required hook (from _CheckForHookApproval).
+    """
+    # No-op if there is no hooks project or if hook is disabled.
+    if ((not self._hooks_project) or
+        (self._hook_type not in self._hooks_project.enabled_repo_hooks)):
+      return
+
+    # Bail with a nice error if we can't find the hook.
+    if not os.path.isfile(self._script_fullpath):
+      raise HookError('Couldn\'t find repo hook: "%s"' % self._script_fullpath)
+
+    # Make sure the user is OK with running the hook.
+    if (not user_allows_all_hooks) and (not self._CheckForHookApproval()):
+      return
+
+    # Run the hook with the same version of python we're using.
+    self._ExecuteHook(**kwargs)
+
+
 class Project(object):
   def __init__(self,
                manifest,
@@ -236,8 +491,11 @@ class Project(object):
     self.manifest = manifest
     self.name = name
     self.remote = remote
-    self.gitdir = gitdir
-    self.worktree = worktree
+    self.gitdir = gitdir.replace('\\', '/')
+    if worktree:
+      self.worktree = worktree.replace('\\', '/')
+    else:
+      self.worktree = None
     self.relpath = relpath
     self.revisionExpr = revisionExpr
 
@@ -261,6 +519,10 @@ class Project(object):
     self.bare_git = self._GitGetByExec(self, bare=True)
     self.bare_ref = GitRefs(gitdir)
 
+    # This will be filled in if a project is later identified to be the
+    # project containing repo hooks.
+    self.enabled_repo_hooks = []
+
   @property
   def Exists(self):
     return os.path.isdir(self.gitdir)
@@ -677,11 +939,11 @@ class Project(object):
     """Perform only the local IO portion of the sync process.
        Network access is not required.
     """
-    self._InitWorkTree()
     all = self.bare_ref.all
     self.CleanPublishedCache(all)
-
     revid = self.GetRevisionId(all)
+
+    self._InitWorkTree()
     head = self.work_git.GetHead()
     if head.startswith(R_HEADS):
       branch = head[len(R_HEADS):]
@@ -1189,10 +1451,10 @@ class Project(object):
     hooks = self._gitdir_path('hooks')
     if not os.path.exists(hooks):
       os.makedirs(hooks)
-    for stock_hook in repo_hooks():
+    for stock_hook in _ProjectHooks():
       name = os.path.basename(stock_hook)
 
-      if name in ('commit-msg') and not self.remote.review:
+      if name in ('commit-msg',) and not self.remote.review:
         # Don't install a Gerrit Code Review hook if this
         # project does not appear to use it for reviews.
         #
@@ -1285,6 +1547,11 @@ class Project(object):
       cmd.append(HEAD)
       if GitCommand(self, cmd).Wait() != 0:
         raise GitError("cannot initialize work tree")
+
+      rr_cache = os.path.join(self.gitdir, 'rr-cache')
+      if not os.path.exists(rr_cache):
+        os.makedirs(rr_cache)
+
       self._CopyFiles()
 
   def _gitdir_path(self, path):
@@ -1443,6 +1710,22 @@ class Project(object):
       return r
 
     def __getattr__(self, name):
+      """Allow arbitrary git commands using pythonic syntax.
+
+      This allows you to do things like:
+        git_obj.rev_parse('HEAD')
+
+      Since we don't have a 'rev_parse' method defined, the __getattr__ will
+      run.  We'll replace the '_' with a '-' and try to run a git command.
+      Any other arguments will be passed to the git command.
+
+      Args:
+        name: The name of the git command to call.  Any '_' characters will
+            be replaced with '-'.
+
+      Returns:
+        A callable object that will try to call git with the named command.
+      """
       name = name.replace('_', '-')
       def runner(*args):
         cmdv = [name]

repo

@@ -28,7 +28,7 @@ if __name__ == '__main__':
 del magic
 
 # increment this whenever we make important changes to this script
-VERSION = (1, 9)
+VERSION = (1, 10)
 
 # increment this if the MAINTAINER_KEYS block is modified
 KEYRING_VERSION = (1,0)
@@ -259,8 +259,8 @@ def _SetupGnuPG(quiet):
             gpg_dir, e.strerror)
       sys.exit(1)
 
-  env = dict(os.environ)
-  env['GNUPGHOME'] = gpg_dir
+  env = os.environ.copy()
+  env['GNUPGHOME'] = gpg_dir.encode()
 
   cmd = ['gpg', '--import']
   try:
@@ -378,8 +378,8 @@ def _Verify(cwd, branch, quiet):
         % (branch, cur)
       print >>sys.stderr
 
-  env = dict(os.environ)
-  env['GNUPGHOME'] = gpg_dir
+  env = os.environ.copy()
+  env['GNUPGHOME'] = gpg_dir.encode()
 
   cmd = [GIT, 'tag', '-v', cur]
   proc = subprocess.Popen(cmd,
@@ -430,10 +430,14 @@ def _FindRepo():
   dir = os.getcwd()
   repo = None
 
-  while dir != '/' and not repo:
+  olddir = None
+  while dir != '/' \
+    and dir != olddir \
+    and not repo:
     repo = os.path.join(dir, repodir, REPO_MAIN)
     if not os.path.isfile(repo):
       repo = None
+      olddir = dir
       dir = os.path.dirname(dir)
   return (repo, os.path.join(dir, repodir))
 
@@ -479,6 +483,7 @@ def _Help(args):
   if args:
     if args[0] == 'init':
       init_optparse.print_help()
+      sys.exit(0)
     else:
       print >>sys.stderr,\
       "error: '%s' is not a bootstrap command.\n"\

subcmds/download.py

@@ -36,6 +36,9 @@ makes it available in your project's local working directory.
     pass
 
   def _ParseChangeIds(self, args):
+    if not args:
+      self.Usage()
+
     to_get = []
     project = None
 

subcmds/forall.py

@@ -151,11 +151,11 @@ terminal and are not redirected.
     first = True
 
     for project in self.GetProjects(args):
-      env = dict(os.environ.iteritems())
+      env = os.environ.copy()
       def setenv(name, val):
         if val is None:
           val = ''
-        env[name] = val
+        env[name] = val.encode()
 
       setenv('REPO_PROJECT', project.name)
       setenv('REPO_PATH', project.relpath)
@@ -169,6 +169,12 @@ terminal and are not redirected.
       else:
         cwd = project.worktree
 
+      if not os.path.exists(cwd):
+        if (opt.project_header and opt.verbose) \
+        or not opt.project_header:
+          print >>sys.stderr, 'skipping %s/' % project.relpath
+        continue
+
       if opt.project_header:
         stdin = subprocess.PIPE
         stdout = subprocess.PIPE

subcmds/help.py

@@ -94,6 +94,8 @@ See 'repo help --all' for a complete list of recognized commands.
           body = getattr(cmd, bodyAttr)
         except AttributeError:
           return
+        if body == '' or body is None:
+          return
 
         self.nl()
 

subcmds/list.py

@@ -0,0 +1,48 @@
+#
+# Copyright (C) 2011 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from command import Command, MirrorSafeCommand
+
+class List(Command, MirrorSafeCommand):
+  common = True
+  helpSummary = "List projects and their associated directories"
+  helpUsage = """
+%prog [<project>...]
+"""
+  helpDescription = """
+List all projects; pass '.' to list the project for the cwd.
+
+This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
+"""
+
+  def Execute(self, opt, args):
+    """List all projects and the associated directories.
+
+    This may be possible to do with 'repo forall', but repo newbies have
+    trouble figuring that out.  The idea here is that it should be more
+    discoverable.
+
+    Args:
+      opt: The options.  We don't take any.
+      args: Positional args.  Can be a list of projects to list, or empty.
+    """
+    projects = self.GetProjects(args)
+
+    lines = []
+    for project in projects:
+      lines.append("%s : %s" % (project.relpath, project.name))
+
+    lines.sort()
+    print '\n'.join(lines)

subcmds/selfupdate.py

@@ -55,6 +55,7 @@ need to be performed by an end-user.
         print >>sys.stderr, "error: can't update repo"
         sys.exit(1)
 
+      rp.bare_git.gc('--auto')
       _PostRepoFetch(rp,
                      no_repo_verify = opt.no_repo_verify,
                      verbose = True)

subcmds/sync.py

@@ -39,6 +39,10 @@ from project import R_HEADS
 from project import SyncBuffer
 from progress import Progress
 
+class _FetchError(Exception):
+  """Internal error thrown in _FetchHelper() when we don't want stack trace."""
+  pass
+
 class Sync(Command, MirrorSafeCommand):
   jobs = 1
   common = True
@@ -135,20 +139,61 @@ later is required to fix a server side protocol bug.
                  dest='repo_upgraded', action='store_true',
                  help=SUPPRESS_HELP)
 
-  def _FetchHelper(self, opt, project, lock, fetched, pm, sem):
-      if not project.Sync_NetworkHalf(quiet=opt.quiet):
-        print >>sys.stderr, 'error: Cannot fetch %s' % project.name
-        if opt.force_broken:
-          print >>sys.stderr, 'warn: --force-broken, continuing to sync'
-        else:
-          sem.release()
-          sys.exit(1)
+  def _FetchHelper(self, opt, project, lock, fetched, pm, sem, err_event):
+      """Main function of the fetch threads when jobs are > 1.
 
-      lock.acquire()
-      fetched.add(project.gitdir)
-      pm.update()
-      lock.release()
-      sem.release()
+      Args:
+        opt: Program options returned from optparse.  See _Options().
+        project: Project object for the project to fetch.
+        lock: Lock for accessing objects that are shared amongst multiple
+            _FetchHelper() threads.
+        fetched: set object that we will add project.gitdir to when we're done
+            (with our lock held).
+        pm: Instance of a Project object.  We will call pm.update() (with our
+            lock held).
+        sem: We'll release() this semaphore when we exit so that another thread
+            can be started up.
+        err_event: We'll set this event in the case of an error (after printing
+            out info about the error).
+      """
+      # We'll set to true once we've locked the lock.
+      did_lock = False
+
+      # Encapsulate everything in a try/except/finally so that:
+      # - We always set err_event in the case of an exception.
+      # - We always make sure we call sem.release().
+      # - We always make sure we unlock the lock if we locked it.
+      try:
+        try:
+          success = project.Sync_NetworkHalf(quiet=opt.quiet)
+
+          # Lock around all the rest of the code, since printing, updating a set
+          # and Progress.update() are not thread safe.
+          lock.acquire()
+          did_lock = True
+
+          if not success:
+            print >>sys.stderr, 'error: Cannot fetch %s' % project.name
+            if opt.force_broken:
+              print >>sys.stderr, 'warn: --force-broken, continuing to sync'
+            else:
+              raise _FetchError()
+
+          fetched.add(project.gitdir)
+          pm.update()
+        except BaseException, e:
+          # Notify the _Fetch() function about all errors.
+          err_event.set()
+
+          # If we got our own _FetchError, we don't want a stack trace.
+          # However, if we got something else (something in Sync_NetworkHalf?),
+          # we'd like one (so re-raise after we've set err_event).
+          if not isinstance(e, _FetchError):
+            raise
+      finally:
+        if did_lock:
+          lock.release()
+        sem.release()
 
   def _Fetch(self, projects, opt):
     fetched = set()
@@ -169,7 +214,13 @@ later is required to fix a server side protocol bug.
       threads = set()
       lock = _threading.Lock()
       sem = _threading.Semaphore(self.jobs)
+      err_event = _threading.Event()
       for project in projects:
+        # Check for any errors before starting any new threads.
+        # ...we'll let existing threads finish, though.
+        if err_event.is_set():
+          break
+
         sem.acquire()
         t = _threading.Thread(target = self._FetchHelper,
                               args = (opt,
@@ -177,14 +228,22 @@ later is required to fix a server side protocol bug.
                                       lock,
                                       fetched,
                                       pm,
-                                      sem))
+                                      sem,
+                                      err_event))
         threads.add(t)
         t.start()
 
       for t in threads:
         t.join()
 
+      # If we saw an error, exit with code 1 so that other scripts can check.
+      if err_event.is_set():
+        print >>sys.stderr, '\nerror: Exited sync due to fetch errors'
+        sys.exit(1)
+
     pm.end()
+    for project in projects:
+      project.bare_git.gc('--auto')
     return fetched
 
   def UpdateProjectList(self):
@@ -269,7 +328,7 @@ uncommitted changes are present' % project.relpath
         if branch.startswith(R_HEADS):
           branch = branch[len(R_HEADS):]
 
-        env = dict(os.environ)
+        env = os.environ.copy()
         if (env.has_key('TARGET_PRODUCT') and
             env.has_key('TARGET_BUILD_VARIANT')):
           target = '%s-%s' % (env['TARGET_PRODUCT'],
@@ -413,9 +472,9 @@ warning: Cannot automatically authenticate repo."""
       % (project.name, rev)
     return False
 
-  env = dict(os.environ)
-  env['GIT_DIR'] = project.gitdir
-  env['GNUPGHOME'] = gpg_dir
+  env = os.environ.copy()
+  env['GIT_DIR'] = project.gitdir.encode()
+  env['GNUPGHOME'] = gpg_dir.encode()
 
   cmd = [GIT, 'tag', '-v', cur]
   proc = subprocess.Popen(cmd,

subcmds/upload.py

@@ -19,7 +19,8 @@ import sys
 
 from command import InteractiveCommand
 from editor import Editor
-from error import UploadError
+from error import HookError, UploadError
+from project import RepoHook
 
 UNUSUAL_COMMIT_THRESHOLD = 5
 
@@ -120,6 +121,29 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
                  type='string',  action='append', dest='cc',
                  help='Also send email to these email addresses.')
 
+    # Options relating to upload hook.  Note that verify and no-verify are NOT
+    # opposites of each other, which is why they store to different locations.
+    # We are using them to match 'git commit' syntax.
+    #
+    # Combinations:
+    # - no-verify=False, verify=False (DEFAULT):
+    #   If stdout is a tty, can prompt about running upload hooks if needed.
+    #   If user denies running hooks, the upload is cancelled.  If stdout is
+    #   not a tty and we would need to prompt about upload hooks, upload is
+    #   cancelled.
+    # - no-verify=False, verify=True:
+    #   Always run upload hooks with no prompt.
+    # - no-verify=True, verify=False:
+    #   Never run upload hooks, but upload anyway (AKA bypass hooks).
+    # - no-verify=True, verify=True:
+    #   Invalid
+    p.add_option('--no-verify',
+                 dest='bypass_hooks', action='store_true',
+                 help='Do not run the upload hook.')
+    p.add_option('--verify',
+                 dest='allow_all_hooks', action='store_true',
+                 help='Run the upload hook without prompting.')
+
   def _SingleBranch(self, opt, branch, people):
     project = branch.project
     name = branch.name
@@ -283,15 +307,19 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
         have_errors = True
 
     print >>sys.stderr, ''
-    print >>sys.stderr, '--------------------------------------------'
+    print >>sys.stderr, '----------------------------------------------------------------------'
 
     if have_errors:
       for branch in todo:
         if not branch.uploaded:
-          print >>sys.stderr, '[FAILED] %-15s %-15s  (%s)' % (
+          if len(str(branch.error)) <= 30:
+            fmt = ' (%s)'
+          else:
+            fmt = '\n       (%s)'
+          print >>sys.stderr, ('[FAILED] %-15s %-15s' + fmt) % (
                  branch.project.relpath + '/', \
                  branch.name, \
-                 branch.error)
+                 str(branch.error))
       print >>sys.stderr, ''
 
     for branch in todo:
@@ -309,17 +337,27 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
     reviewers = []
     cc = []
 
+    for project in project_list:
+      avail = project.GetUploadableBranches()
+      if avail:
+        pending.append((project, avail))
+
+    if pending and (not opt.bypass_hooks):
+      hook = RepoHook('pre-upload', self.manifest.repo_hooks_project,
+                      self.manifest.topdir, abort_if_user_denies=True)
+      pending_proj_names = [project.name for (project, avail) in pending]
+      try:
+        hook.Run(opt.allow_all_hooks, project_list=pending_proj_names)
+      except HookError, e:
+        print >>sys.stderr, "ERROR: %s" % str(e)
+        return
+
     if opt.reviewers:
       reviewers = _SplitEmails(opt.reviewers)
     if opt.cc:
       cc = _SplitEmails(opt.cc)
     people = (reviewers,cc)
 
-    for project in project_list:
-      avail = project.GetUploadableBranches()
-      if avail:
-        pending.append((project, avail))
-
     if not pending:
       print >>sys.stdout, "no branches ready for upload"
     elif len(pending) == 1 and len(pending[0][1]) == 1: