From 6d7508b3d52781a3f8170a4257c65e2de176cc68 Mon Sep 17 00:00:00 2001 From: Nico Sallembien Date: Thu, 1 Apr 2010 11:03:53 -0700 Subject: [PATCH 01/33] Allow 'y' as a valid response when confirming identity I prefer having to type only one character rather than all three, and it seems like other confirmation prompts use the same style. --- subcmds/init.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/subcmds/init.py b/subcmds/init.py index 75a58f11..4023ab6d 100644 --- a/subcmds/init.py +++ b/subcmds/init.py @@ -167,8 +167,9 @@ to update the working directory files. print '' print 'Your identity is: %s <%s>' % (name, email) - sys.stdout.write('is this correct [yes/no]? ') - if 'yes' == sys.stdin.readline().strip(): + sys.stdout.write('is this correct [y/n]? ') + a = sys.stdin.readline().strip() + if a in ('yes', 'y', 't', 'true'): break if name != mp.UserName: From a1bfd2cd7253b1662e08f5ec5be3d863430c756c Mon Sep 17 00:00:00 2001 From: Nico Sallembien Date: Tue, 6 Apr 2010 10:40:01 -0700 Subject: [PATCH 02/33] Add a 'smart sync' option to repo sync This option allows the user to specify a manifest server to use when syncing. This manifest server will provide a manifest pegging each project to a known green build. This allows developers to work on a known good tree that is known to build and pass tests, preventing failed builds to hamper productivity. The manifest used is not "sticky" so as to allow subsequent 'repo sync' calls to sync to the tip of the tree. Change-Id: Id0a24ece20f5a88034ad364b416a1dd2e394226d --- docs/manifest-format.txt | 25 +++++++++++++++++++ manifest_xml.py | 32 ++++++++++++++++++++++--- subcmds/sync.py | 52 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 106 insertions(+), 3 deletions(-) diff --git a/docs/manifest-format.txt b/docs/manifest-format.txt index da0e69ff..211344ee 100644 --- a/docs/manifest-format.txt +++ b/docs/manifest-format.txt @@ -22,6 +22,7 @@ following DTD: @@ -33,6 +34,9 @@ following DTD: + + + @@ -89,6 +93,27 @@ Attribute `revision`: Name of a Git branch (e.g. `master` or revision attribute will use this revision. +Element manifest-server +----------------------- + +At most one manifest-server may be specified. The url attribute +is used to specify the URL of a manifest server, which is an +XML RPC service that will return a manifest in which each project +is pegged to a known good revision for the current branch and +target. + +The manifest server should implement: + + GetApprovedManifest(branch, target) + +The target to use is defined by environment variables TARGET_PRODUCT +and TARGET_BUILD_VARIANT. These variables are used to create a string +of the form $TARGET_PRODUCT-$TARGET_BUILD_VARIANT, e.g. passion-userdebug. +If one of those variables or both are not present, the program will call +GetApprovedManifest without the target paramater and the manifest server +should choose a reasonable default target. + + Element project --------------- diff --git a/manifest_xml.py b/manifest_xml.py index 7d02f9d6..d0c9debe 100644 --- a/manifest_xml.py +++ b/manifest_xml.py @@ -65,8 +65,8 @@ class XmlManifest(object): self._Unload() - def Link(self, name): - """Update the repo metadata to use a different manifest. + def Override(self, name): + """Use a different manifest, just for the current instantiation. """ path = os.path.join(self.manifestProject.worktree, name) if not os.path.isfile(path): @@ -80,6 +80,11 @@ class XmlManifest(object): finally: self.manifestFile = old + def Link(self, name): + """Update the repo metadata to use a different manifest. + """ + self.Override(name) + try: if os.path.exists(self.manifestFile): os.remove(self.manifestFile) @@ -123,6 +128,12 @@ class XmlManifest(object): root.appendChild(e) root.appendChild(doc.createTextNode('')) + if self._manifest_server: + e = doc.createElement('manifest-server') + e.setAttribute('url', self._manifest_server) + root.appendChild(e) + root.appendChild(doc.createTextNode('')) + sort_projects = list(self.projects.keys()) sort_projects.sort() @@ -168,6 +179,11 @@ class XmlManifest(object): self._Load() return self._default + @property + def manifest_server(self): + self._Load() + return self._manifest_server + @property def IsMirror(self): return self.manifestProject.config.GetBoolean('repo.mirror') @@ -178,6 +194,7 @@ class XmlManifest(object): self._remotes = {} self._default = None self.branch = None + self._manifest_server = None def _Load(self): if not self._loaded: @@ -246,6 +263,15 @@ class XmlManifest(object): if self._default is None: self._default = _Default() + for node in config.childNodes: + if node.nodeName == 'manifest-server': + url = self._reqatt(node, 'url') + if self._manifest_server is not None: + raise ManifestParseError, \ + 'duplicate manifest-server in %s' % \ + (self.manifestFile) + self._manifest_server = url + for node in config.childNodes: if node.nodeName == 'project': project = self._ParseProject(node) @@ -315,7 +341,7 @@ class XmlManifest(object): def _ParseProject(self, node): """ reads a element from the manifest file - """ + """ name = self._reqatt(node, 'name') remote = self._get_remote(node) diff --git a/subcmds/sync.py b/subcmds/sync.py index ceb81eaa..deff171a 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -17,9 +17,11 @@ from optparse import SUPPRESS_HELP import os import re import shutil +import socket import subprocess import sys import time +import xmlrpclib from git_command import GIT from project import HEAD @@ -57,6 +59,10 @@ back to the manifest revision. This option is especially helpful if the project is currently on a topic branch, but the manifest revision is temporarily needed. +The -s/--smart-sync option can be used to sync to a known good +build as specified by the manifest-server element in the current +manifest. + SSH Connections --------------- @@ -97,6 +103,9 @@ later is required to fix a server side protocol bug. p.add_option('-d','--detach', dest='detach_head', action='store_true', help='detach projects back to manifest revision') + p.add_option('-s', '--smart-sync', + dest='smart_sync', action='store_true', + help='smart sync using manifest from a known good build') g = p.add_option_group('repo Version options') g.add_option('--no-repo-verify', @@ -182,6 +191,49 @@ uncommitted changes are present' % project.relpath print >>sys.stderr, 'error: cannot combine -n and -l' sys.exit(1) + if opt.smart_sync: + if not self.manifest.manifest_server: + print >>sys.stderr, \ + 'error: cannot smart sync: no manifest server defined in manifest' + sys.exit(1) + try: + server = xmlrpclib.Server(self.manifest.manifest_server) + p = self.manifest.manifestProject + b = p.GetBranch(p.CurrentBranch) + branch = b.merge + + env = dict(os.environ) + if (env.has_key('TARGET_PRODUCT') and + env.has_key('TARGET_BUILD_VARIANT')): + target = '%s-%s' % (env['TARGET_PRODUCT'], + env['TARGET_BUILD_VARIANT']) + [success, manifest_str] = server.GetApprovedManifest(branch, target) + else: + [success, manifest_str] = server.GetApprovedManifest(branch) + + if success: + manifest_name = "smart_sync_override.xml" + manifest_path = os.path.join(self.manifest.manifestProject.worktree, + manifest_name) + try: + f = open(manifest_path, 'w') + try: + f.write(manifest_str) + self.manifest.Override(manifest_name) + finally: + f.close() + except IOError: + print >>sys.stderr, 'error: cannot write manifest to %s' % \ + manifest_path + sys.exit(1) + else: + print >>sys.stderr, 'error: %s' % manifest_str + sys.exit(1) + except socket.error: + print >>sys.stderr, 'error: cannot connect to manifest server %s' % ( + self.manifest.manifest_server) + sys.exit(1) + rp = self.manifest.repoProject rp.PreSync() From f3fdf823cf9785e4ceca3e8416b719282d84b6d0 Mon Sep 17 00:00:00 2001 From: Anthony Date: Sat, 26 Sep 2009 13:38:52 -0400 Subject: [PATCH 03/33] sync: Safely skip already deleted projects Do not error if a project is missing on the filesystem, is deleted from manifest.xml, but still exists in project.list. Change-Id: I1d13e435473c83091e27e4df571504ef493282dd --- subcmds/sync.py | 54 ++++++++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/subcmds/sync.py b/subcmds/sync.py index deff171a..02dd82df 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -147,32 +147,36 @@ later is required to fix a server side protocol bug. if not path: continue if path not in new_project_paths: - project = Project( - manifest = self.manifest, - name = path, - remote = RemoteSpec('origin'), - gitdir = os.path.join(self.manifest.topdir, - path, '.git'), - worktree = os.path.join(self.manifest.topdir, path), - relpath = path, - revisionExpr = 'HEAD', - revisionId = None) - if project.IsDirty(): - print >>sys.stderr, 'error: Cannot remove project "%s": \ + """If the path has already been deleted, we don't need to do it + """ + if os.path.exists(self.manifest.topdir + '/' + path): + project = Project( + manifest = self.manifest, + name = path, + remote = RemoteSpec('origin'), + gitdir = os.path.join(self.manifest.topdir, + path, '.git'), + worktree = os.path.join(self.manifest.topdir, path), + relpath = path, + revisionExpr = 'HEAD', + revisionId = None) + + if project.IsDirty(): + print >>sys.stderr, 'error: Cannot remove project "%s": \ uncommitted changes are present' % project.relpath - print >>sys.stderr, ' commit changes, then run sync again' - return -1 - else: - print >>sys.stderr, 'Deleting obsolete path %s' % project.worktree - shutil.rmtree(project.worktree) - # Try deleting parent subdirs if they are empty - dir = os.path.dirname(project.worktree) - while dir != self.manifest.topdir: - try: - os.rmdir(dir) - except OSError: - break - dir = os.path.dirname(dir) + print >>sys.stderr, ' commit changes, then run sync again' + return -1 + else: + print >>sys.stderr, 'Deleting obsolete path %s' % project.worktree + shutil.rmtree(project.worktree) + # Try deleting parent subdirs if they are empty + dir = os.path.dirname(project.worktree) + while dir != self.manifest.topdir: + try: + os.rmdir(dir) + except OSError: + break + dir = os.path.dirname(dir) new_project_paths.sort() fd = open(file_path, 'w') From 5732e47ebb7a096e3afad49687098c4181c4b300 Mon Sep 17 00:00:00 2001 From: Nico Sallembien Date: Mon, 26 Apr 2010 11:17:29 -0700 Subject: [PATCH 04/33] Strip refs/heads in the branch sent to the manifest server. The manifest server doesn't want to have refs/heads passed to it, so we need to strip that when the branch contains it. Change-Id: I044f8a9629220e886fd5e02e3c1ac4b4bb6020ba --- subcmds/sync.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/subcmds/sync.py b/subcmds/sync.py index 02dd82df..67213d3a 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -24,6 +24,7 @@ import time import xmlrpclib from git_command import GIT +from git_refs import R_HEADS from project import HEAD from project import Project from project import RemoteSpec @@ -205,6 +206,8 @@ uncommitted changes are present' % project.relpath p = self.manifest.manifestProject b = p.GetBranch(p.CurrentBranch) branch = b.merge + if branch.startswith(R_HEADS): + branch = branch[len(R_HEADS):] env = dict(os.environ) if (env.has_key('TARGET_PRODUCT') and From 719965af35a2fab96cb578c8a19a48a2cf9fe8e8 Mon Sep 17 00:00:00 2001 From: Nico Sallembien Date: Tue, 20 Apr 2010 15:28:19 -0700 Subject: [PATCH 05/33] Override manifest file only after it is fully written to disk. We called "Override()" before closing the file passed in argument. Change-Id: I15adb99deb14297ef72fcb1b0945eb246f172fb0 --- subcmds/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subcmds/sync.py b/subcmds/sync.py index 67213d3a..613cc81c 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -226,13 +226,13 @@ uncommitted changes are present' % project.relpath f = open(manifest_path, 'w') try: f.write(manifest_str) - self.manifest.Override(manifest_name) finally: f.close() except IOError: print >>sys.stderr, 'error: cannot write manifest to %s' % \ manifest_path sys.exit(1) + self.manifest.Override(manifest_name) else: print >>sys.stderr, 'error: %s' % manifest_str sys.exit(1) From 1c85f4e43ba2d337b1e8c49bb3c7814a5cb163ae Mon Sep 17 00:00:00 2001 From: Nico Sallembien Date: Tue, 27 Apr 2010 14:35:27 -0700 Subject: [PATCH 06/33] Rename _ssh_sock() to fix code style issue. Since _ssh_sock is imported out of the git_command module, the leading underscore should be removed from the function name. --- git_command.py | 4 ++-- git_config.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/git_command.py b/git_command.py index d56ad0a8..3309f378 100644 --- a/git_command.py +++ b/git_command.py @@ -30,7 +30,7 @@ LAST_CWD = None _ssh_proxy_path = None _ssh_sock_path = None -def _ssh_sock(create=True): +def ssh_sock(create=True): global _ssh_sock_path if _ssh_sock_path is None: if not create: @@ -119,7 +119,7 @@ class GitCommand(object): if disable_editor: env['GIT_EDITOR'] = ':' if ssh_proxy: - env['REPO_SSH_SOCK'] = _ssh_sock() + env['REPO_SSH_SOCK'] = ssh_sock() env['GIT_SSH'] = _ssh_proxy() if project: diff --git a/git_config.py b/git_config.py index 45a2d257..a7c82107 100644 --- a/git_config.py +++ b/git_config.py @@ -23,7 +23,7 @@ from signal import SIGTERM from urllib2 import urlopen, HTTPError from error import GitError, UploadError from trace import Trace -from git_command import GitCommand, _ssh_sock +from git_command import GitCommand, ssh_sock R_HEADS = 'refs/heads/' R_TAGS = 'refs/tags/' @@ -371,7 +371,7 @@ def _open_ssh(host, port): return False command = ['ssh', - '-o','ControlPath %s' % _ssh_sock(), + '-o','ControlPath %s' % ssh_sock(), '-p',str(port), '-M', '-N', @@ -399,7 +399,7 @@ def close_ssh(): pass _ssh_cache.clear() - d = _ssh_sock(create=False) + d = ssh_sock(create=False) if d: try: os.rmdir(os.path.dirname(d)) From ff6929dde8cae515f7221a60f21fff7c1297aade Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A4r=20=C3=85sf=C3=A4lt?= Date: Sat, 5 Sep 2009 23:10:56 +0200 Subject: [PATCH 07/33] branches: Enable output of multiple projects Fixes a bug introduced by 498a0e8a79ab76eeb6adc40f12b04d59820716f9 ("Make 'repo branches -a' the default behavior"). Change-Id: Ib739f82f4647890c46d7c9fb2f2e63a16a0481de --- subcmds/branches.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/subcmds/branches.py b/subcmds/branches.py index 0e3ab3c2..a4f8d360 100644 --- a/subcmds/branches.py +++ b/subcmds/branches.py @@ -136,7 +136,7 @@ is shown, then the branch appears in all projects. hdr('%c%c %-*s' % (current, published, width, name)) out.write(' |') - if in_cnt < project_cnt and (in_cnt == 1): + if in_cnt < project_cnt: fmt = out.write paths = [] if in_cnt < project_cnt - in_cnt: @@ -150,15 +150,17 @@ is shown, then the branch appears in all projects. for b in i.projects: have.add(b.project) for p in projects: - paths.append(p.relpath) + if not p in have: + paths.append(p.relpath) s = ' %s %s' % (type, ', '.join(paths)) if width + 7 + len(s) < 80: fmt(s) else: - out.nl() - fmt(' %s:' % type) + fmt(' %s:' % type) for p in paths: out.nl() - fmt(' %s' % p) + fmt(width*' ' + ' %s' % p) + else: + out.write(' in all projects') out.nl() From 879a9a5cf0f4ed61df6544949068babbee4f60e2 Mon Sep 17 00:00:00 2001 From: Dan Morrill Date: Tue, 4 May 2010 16:56:07 -0700 Subject: [PATCH 08/33] upload: Confirm unusually large number of uploaded commit Add a sentinel check to require a second explicit confirmation if the user is attempting to upload (or upload --replace) an unusually large number of commits. This may help the user to catch an accidentally incorrect rebase they had done previously. Change-Id: I12c4d102f90a631d6ad193486a70ffd520ef6ae0 --- subcmds/upload.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/subcmds/upload.py b/subcmds/upload.py index aea399b6..4dc11d28 100644 --- a/subcmds/upload.py +++ b/subcmds/upload.py @@ -20,6 +20,17 @@ from command import InteractiveCommand from editor import Editor from error import UploadError +UNUSUAL_COMMIT_THRESHOLD = 3 + +def _ConfirmManyUploads(multiple_branches=False): + if multiple_branches: + print "ATTENTION: One or more branches has an unusually high number of commits." + else: + print "ATTENTION: You are uploading an unusually high number of commits." + print "YOU PROBABLY DO NOT MEAN TO DO THIS. (Did you rebase across branches?)" + answer = raw_input("If you are sure you intend to do this, type 'yes': ").strip() + return answer == "yes" + def _die(fmt, *args): msg = fmt % args print >>sys.stderr, 'error: %s' % msg @@ -128,6 +139,10 @@ Gerrit Code Review: http://code.google.com/p/gerrit/ answer = sys.stdin.readline().strip() answer = answer in ('y', 'Y', 'yes', '1', 'true', 't') + if answer: + if len(branch.commits) > UNUSUAL_COMMIT_THRESHOLD: + answer = _ConfirmManyUploads() + if answer: self._UploadAndReport([branch], people) else: @@ -192,6 +207,16 @@ Gerrit Code Review: http://code.google.com/p/gerrit/ todo.append(branch) if not todo: _die("nothing uncommented for upload") + + many_commits = False + for branch in todo: + if len(branch.commits) > UNUSUAL_COMMIT_THRESHOLD: + many_commits = True + break + if many_commits: + if not _ConfirmManyUploads(multiple_branches=True): + _die("upload aborted by user") + self._UploadAndReport(todo, people) def _FindGerritChange(self, branch): @@ -258,6 +283,10 @@ Gerrit Code Review: http://code.google.com/p/gerrit/ print >>sys.stderr, " use 'repo upload' without --replace" sys.exit(1) + if len(branch.commits) > UNUSUAL_COMMIT_THRESHOLD: + if not _ConfirmManyUploads(multiple_branches=True): + _die("upload aborted by user") + branch.replace_changes = to_replace self._UploadAndReport([branch], people) From f0a9a1a30e60e92cec9bff4cae030478c276da4d Mon Sep 17 00:00:00 2001 From: Dan Morrill Date: Wed, 5 May 2010 08:18:35 -0700 Subject: [PATCH 09/33] upload: Move confirmation threshold from 3 to 5 commits Change-Id: I7275d195cf04f02694206b9f838540b0228ff5e1 --- subcmds/upload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subcmds/upload.py b/subcmds/upload.py index 4dc11d28..4fa5b432 100644 --- a/subcmds/upload.py +++ b/subcmds/upload.py @@ -20,7 +20,7 @@ from command import InteractiveCommand from editor import Editor from error import UploadError -UNUSUAL_COMMIT_THRESHOLD = 3 +UNUSUAL_COMMIT_THRESHOLD = 5 def _ConfirmManyUploads(multiple_branches=False): if multiple_branches: From ca8c32cd7ae7c3ae27bb6b649eafbfd54d77f916 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Tue, 11 May 2010 18:21:33 -0700 Subject: [PATCH 10/33] sync: kill git fetch process before SSH control master process If the SSH control master process is killed while an active git fetch is using its network socket, the underlying SSH client may not realize the connection was broken. This can lead to both the client and the server waiting indefinitely for network messages which will never be sent. Work around the problem by keeping track of any processes that use the tunnels we establish. If we are about to kill any of the SSH control masters that we started, ensure the clients using them are successfully killed first. Change-Id: Ida6c124dcb0c6a26bf7dd69cba2fbdc2ecd5b2fc Signed-off-by: Shawn O. Pearce --- git_command.py | 29 ++++++++++++++++++++++++++++- git_config.py | 7 ++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/git_command.py b/git_command.py index 3309f378..4ad908f6 100644 --- a/git_command.py +++ b/git_command.py @@ -17,6 +17,7 @@ import os import sys import subprocess import tempfile +from signal import SIGTERM from error import GitError from trace import REPO_TRACE, IsTrace, Trace @@ -29,6 +30,7 @@ LAST_CWD = None _ssh_proxy_path = None _ssh_sock_path = None +_ssh_clients = [] def ssh_sock(create=True): global _ssh_sock_path @@ -51,6 +53,24 @@ def _ssh_proxy(): 'git_ssh') return _ssh_proxy_path +def _add_ssh_client(p): + _ssh_clients.append(p) + +def _remove_ssh_client(p): + try: + _ssh_clients.remove(p) + except ValueError: + pass + +def terminate_ssh_clients(): + global _ssh_clients + for p in _ssh_clients: + try: + os.kill(p.pid, SIGTERM) + p.wait() + except OSError: + pass + _ssh_clients = [] class _GitCall(object): def version(self): @@ -188,6 +208,9 @@ class GitCommand(object): except Exception, e: raise GitError('%s: %s' % (command[1], e)) + if ssh_proxy: + _add_ssh_client(p) + self.process = p self.stdin = p.stdin @@ -210,4 +233,8 @@ class GitCommand(object): else: p.stderr = None - return self.process.wait() + try: + rc = p.wait() + finally: + _remove_ssh_client(p) + return rc diff --git a/git_config.py b/git_config.py index a7c82107..75936d40 100644 --- a/git_config.py +++ b/git_config.py @@ -23,7 +23,10 @@ from signal import SIGTERM from urllib2 import urlopen, HTTPError from error import GitError, UploadError from trace import Trace -from git_command import GitCommand, ssh_sock + +from git_command import GitCommand +from git_command import ssh_sock +from git_command import terminate_ssh_clients R_HEADS = 'refs/heads/' R_TAGS = 'refs/tags/' @@ -391,6 +394,8 @@ def _open_ssh(host, port): return True def close_ssh(): + terminate_ssh_clients() + for key,p in _ssh_cache.iteritems(): try: os.kill(p.pid, SIGTERM) From 6623b21e1073a70f7d5cc6eddd364bdab337b439 Mon Sep 17 00:00:00 2001 From: Nico Sallembien Date: Tue, 11 May 2010 12:57:01 -0700 Subject: [PATCH 11/33] Aliasing sync -s to 'smartsync' This alias will let people use this command without having to remember the option. Change-Id: I3256d9e8e884c5be9e77f70e9cfb73e0f0c544c6 --- subcmds/smartsync.py | 33 +++++++++++++++++++++++++++++++++ subcmds/sync.py | 9 +++++---- 2 files changed, 38 insertions(+), 4 deletions(-) create mode 100644 subcmds/smartsync.py diff --git a/subcmds/smartsync.py b/subcmds/smartsync.py new file mode 100644 index 00000000..1edbd35b --- /dev/null +++ b/subcmds/smartsync.py @@ -0,0 +1,33 @@ +# +# Copyright (C) 2010 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from sync import Sync + +class Smartsync(Sync): + common = True + helpSummary = "Update working tree to the latest known good revision" + helpUsage = """ +%prog [...] +""" + helpDescription = """ +The '%prog' command is a shortcut for sync -s. +""" + + def _Options(self, p): + Sync._Options(self, p, show_smart=False) + + def Execute(self, opt, args): + opt.smart_sync = True + Sync.Execute(self, opt, args) diff --git a/subcmds/sync.py b/subcmds/sync.py index 613cc81c..9b8a6122 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -94,7 +94,7 @@ later is required to fix a server side protocol bug. """ - def _Options(self, p): + def _Options(self, p, show_smart=True): p.add_option('-l','--local-only', dest='local_only', action='store_true', help="only update working tree, don't fetch") @@ -104,9 +104,10 @@ later is required to fix a server side protocol bug. p.add_option('-d','--detach', dest='detach_head', action='store_true', help='detach projects back to manifest revision') - p.add_option('-s', '--smart-sync', - dest='smart_sync', action='store_true', - help='smart sync using manifest from a known good build') + if show_smart: + p.add_option('-s', '--smart-sync', + dest='smart_sync', action='store_true', + help='smart sync using manifest from a known good build') g = p.add_option_group('repo Version options') g.add_option('--no-repo-verify', From 18afd7f679ab6271a34f4ec01e7755dd85c5dcf4 Mon Sep 17 00:00:00 2001 From: Roy Lee Date: Sun, 9 May 2010 04:32:08 +0800 Subject: [PATCH 12/33] sync: support --jobs to fetch projects simultaneously This patch does two things for being compatibile with those Python which are built without threading support: 1. As the Python document and Shawn suggested, import dummy_threading when the threading is not available. 2. Reserve the single threaded code and make it default. In cases the --jobs does not work properly with dummy_threading, we still have a safe fallback. Change-Id: I40909ef8e9b5c22f315c0a1da9be38eed8b0a2dc --- subcmds/sync.py | 53 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 46 insertions(+), 7 deletions(-) diff --git a/subcmds/sync.py b/subcmds/sync.py index 9b8a6122..6cac2e52 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -23,6 +23,11 @@ import sys import time import xmlrpclib +try: + import threading as _threading +except ImportError: + import dummy_threading as _threading + from git_command import GIT from git_refs import R_HEADS from project import HEAD @@ -35,6 +40,7 @@ from project import SyncBuffer from progress import Progress class Sync(Command, MirrorSafeCommand): + jobs = 1 common = True helpSummary = "Update working tree to the latest revision" helpUsage = """ @@ -104,6 +110,9 @@ later is required to fix a server side protocol bug. p.add_option('-d','--detach', dest='detach_head', action='store_true', help='detach projects back to manifest revision') + p.add_option('-j','--jobs', + dest='jobs', action='store', type='int', + help="number of projects to fetch simultaneously") if show_smart: p.add_option('-s', '--smart-sync', dest='smart_sync', action='store_true', @@ -117,16 +126,44 @@ later is required to fix a server side protocol bug. dest='repo_upgraded', action='store_true', help=SUPPRESS_HELP) + def _FetchHelper(self, project, lock, fetched, pm, sem): + if not project.Sync_NetworkHalf(): + print >>sys.stderr, 'error: Cannot fetch %s' % project.name + sem.release() + sys.exit(1) + + lock.acquire() + fetched.add(project.gitdir) + pm.update() + lock.release() + sem.release() + def _Fetch(self, projects): fetched = set() pm = Progress('Fetching projects', len(projects)) - for project in projects: - pm.update() - if project.Sync_NetworkHalf(): - fetched.add(project.gitdir) - else: - print >>sys.stderr, 'error: Cannot fetch %s' % project.name - sys.exit(1) + + if self.jobs == 1: + for project in projects: + pm.update() + if project.Sync_NetworkHalf(): + fetched.add(project.gitdir) + else: + print >>sys.stderr, 'error: Cannot fetch %s' % project.name + sys.exit(1) + else: + threads = set() + lock = _threading.Lock() + sem = _threading.Semaphore(self.jobs) + for project in projects: + sem.acquire() + t = _threading.Thread(target = self._FetchHelper, + args = (project, lock, fetched, pm, sem)) + threads.add(t) + t.start() + + for t in threads: + t.join() + pm.end() return fetched @@ -190,6 +227,8 @@ uncommitted changes are present' % project.relpath return 0 def Execute(self, opt, args): + if opt.jobs: + self.jobs = opt.jobs if opt.network_only and opt.detach_head: print >>sys.stderr, 'error: cannot combine -n and -d' sys.exit(1) From f4f04d9fa8fda995974c9214eee42c3ea68b1125 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Thu, 27 May 2010 16:48:36 -0700 Subject: [PATCH 13/33] Do not emit progress if stderr is not a tty Avoids logging progress data into cron logs, etc. Suggested-by: Michael Richardson Change-Id: I4eefa2c282f0ca0a95a0185612b52e2146669e4c Signed-off-by: Shawn O. Pearce --- progress.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/progress.py b/progress.py index b119b374..2ace7010 100644 --- a/progress.py +++ b/progress.py @@ -13,10 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import sys from time import time from trace import IsTrace +_NOT_TTY = not os.isatty(2) + class Progress(object): def __init__(self, title, total=0): self._title = title @@ -29,7 +32,7 @@ class Progress(object): def update(self, inc=1): self._done += inc - if IsTrace(): + if _NOT_TTY or IsTrace(): return if not self._show: @@ -56,7 +59,7 @@ class Progress(object): sys.stderr.flush() def end(self): - if IsTrace() or not self._show: + if _NOT_TTY or IsTrace() or not self._show: return if self._total <= 0: From 2daf66740bba0b2726462a547910d16cf0822db2 Mon Sep 17 00:00:00 2001 From: Matthew Buckett Date: Sat, 11 Jul 2009 09:43:47 -0400 Subject: [PATCH 14/33] Allow files to be copied into new folders Change-Id: I7f169e32be5a4328bb87ce7c2ff4b6529e925126 --- project.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/project.py b/project.py index ff896d01..1bcd9596 100644 --- a/project.py +++ b/project.py @@ -203,6 +203,10 @@ class _CopyFile: # remove existing file first, since it might be read-only if os.path.exists(dest): os.remove(dest) + else: + dir = os.path.dirname(dest) + if not os.path.isdir(dir): + os.makedirs(dir) shutil.copy(src, dest) # make the file read-only mode = os.stat(dest)[stat.ST_MODE] From 7198572dd7f5b9d95d83733a98691948a3eb9da3 Mon Sep 17 00:00:00 2001 From: Josh Guilfoyle Date: Sun, 16 Aug 2009 09:44:40 -0700 Subject: [PATCH 15/33] Do not invoke ssh with -p argument when no port has been specified. This change allows local SSH configuration to choose the port number to use when not explicitly set in the manifest. (cherry picked from commit 4c0f67046543c7c6ab24175e143001f14b76ea01) Change-Id: Ibea99cfe46b6a2cc27f754cc3944a2fe10f6fda4 --- git_config.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/git_config.py b/git_config.py index 75936d40..dc209ba8 100644 --- a/git_config.py +++ b/git_config.py @@ -359,10 +359,14 @@ class RefSpec(object): _ssh_cache = {} _ssh_master = True -def _open_ssh(host, port): +def _open_ssh(host, port=None): global _ssh_master - key = '%s:%s' % (host, port) + if port is not None: + key = '%s:%s' % (host, port) + else: + key = host + if key in _ssh_cache: return True @@ -375,10 +379,13 @@ def _open_ssh(host, port): command = ['ssh', '-o','ControlPath %s' % ssh_sock(), - '-p',str(port), '-M', '-N', host] + + if port is not None: + command[3:3] = ['-p',str(port)] + try: Trace(': %s', ' '.join(command)) p = subprocess.Popen(command) @@ -422,7 +429,7 @@ def _preconnect(url): if ':' in host: host, port = host.split(':') else: - port = 22 + port = None if scheme in ('ssh', 'git+ssh', 'ssh+git'): return _open_ssh(host, port) return False @@ -430,7 +437,7 @@ def _preconnect(url): m = URI_SCP.match(url) if m: host = m.group(1) - return _open_ssh(host, 22) + return _open_ssh(host) return False From feb39d61ef2de893b93153adc8f1f8140a54fc98 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Wed, 2 Jun 2010 17:18:13 +0200 Subject: [PATCH 16/33] Fix format string bugs in grep This fixes some format string bugs in grep which cause repo to with "TypeError: not enough arguments for format string" when grepping and the output contains a valid Python format string. Change-Id: Ice8968ea106148d409490e4f71a2833b0cc80816 --- subcmds/grep.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/subcmds/grep.py b/subcmds/grep.py index 4f714271..1cb5650b 100644 --- a/subcmds/grep.py +++ b/subcmds/grep.py @@ -204,7 +204,7 @@ contain a line that matches both expressions: else: out.project('--- project %s ---' % project.relpath) out.nl() - out.write(p.stderr) + out.write("%s", p.stderr) out.nl() continue have_match = True @@ -217,17 +217,17 @@ contain a line that matches both expressions: if have_rev and full_name: for line in r: rev, line = line.split(':', 1) - out.write(rev) + out.write("%s", rev) out.write(':') out.project(project.relpath) out.write('/') - out.write(line) + out.write("%s", line) out.nl() elif full_name: for line in r: out.project(project.relpath) out.write('/') - out.write(line) + out.write("%s", line) out.nl() else: for line in r: From 08a3f68d38eec81dfa66f9ea05080c37c863f322 Mon Sep 17 00:00:00 2001 From: Ben Komalo Date: Thu, 15 Jul 2010 16:03:02 -0700 Subject: [PATCH 17/33] upload: Automatically --cc folks in review.URL.autocopy The upload command will read review.URL.autocopy from the project's configuration and append the list of e-mails specified to the --cc argument of the upload command if a non-empty --re argument was provided. Change-Id: I2424517d17dd3444b20f0e6a003be6e70b8904f6 Signed-off-by: Shawn O. Pearce --- subcmds/upload.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/subcmds/upload.py b/subcmds/upload.py index 4fa5b432..ba532461 100644 --- a/subcmds/upload.py +++ b/subcmds/upload.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import copy import re import sys @@ -83,6 +84,14 @@ to "true" then repo will assume you always answer "y" at the prompt, and will not prompt you further. If it is set to "false" then repo will assume you always answer "n", and will abort. +review.URL.autocopy: + +To automatically copy a user or mailing list to all uploaded reviews, +you can set a per-project or global Git option to do so. Specifically, +review.URL.autocopy can be set to a comma separated list of reviewers +who you always want copied on all uploads with a non-empty --re +argument. + The URL must match the review URL listed in the manifest XML file, or in the .git/config within the project. For example: @@ -92,6 +101,7 @@ or in the .git/config within the project. For example: [review "http://review.example.com/"] autoupload = true + autocopy = johndoe@company.com,my-team-alias@company.com References ---------- @@ -219,6 +229,19 @@ Gerrit Code Review: http://code.google.com/p/gerrit/ self._UploadAndReport(todo, people) + def _AppendAutoCcList(self, branch, people): + """ + Appends the list of users in the CC list in the git project's config if a + non-empty reviewer list was found. + """ + + name = branch.name + project = branch.project + key = 'review.%s.autocopy' % project.GetBranch(name).remote.review + raw_list = project.config.GetString(key) + if not raw_list is None and len(people[0]) > 0: + people[1].extend([entry.strip() for entry in raw_list.split(',')]) + def _FindGerritChange(self, branch): last_pub = branch.project.WasPublished(branch.name) if last_pub is None: @@ -290,10 +313,13 @@ Gerrit Code Review: http://code.google.com/p/gerrit/ branch.replace_changes = to_replace self._UploadAndReport([branch], people) - def _UploadAndReport(self, todo, people): + def _UploadAndReport(self, todo, original_people): have_errors = False for branch in todo: try: + people = copy.deepcopy(original_people) + self._AppendAutoCcList(branch, people) + branch.UploadForReview(people) branch.uploaded = True except UploadError, e: From 9e426aa43231073c4a98dae3f6c16d67ab6f3b59 Mon Sep 17 00:00:00 2001 From: Daniel Sandler Date: Thu, 1 Apr 2010 10:42:33 -0400 Subject: [PATCH 18/33] rebase: Automatically rebase branch on upstrea Usage: repo rebase [[-i] ...] Rebases the current topic branch of the specified (or all) projects against the appropriate upstream. Note: Interactive rebase is currently only supported when exactly one project is specified on the command line. Change-Id: I7376e35f27a6585149def82938c1ca99f36db2c4 Signed-off-by: Shawn O. Pearce --- subcmds/rebase.py | 75 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 subcmds/rebase.py diff --git a/subcmds/rebase.py b/subcmds/rebase.py new file mode 100644 index 00000000..44cdd2eb --- /dev/null +++ b/subcmds/rebase.py @@ -0,0 +1,75 @@ +# +# Copyright (C) 2010 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +from command import Command +from git_command import GitCommand +from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M +from error import GitError + +class Rebase(Command): + common = True + helpSummary = "Rebase local branches on upstream branch" + helpUsage = """ +%prog {[...] | -i ...} +""" + helpDescription = """ +'%prog' uses git rebase to move local changes in the current topic branch to +the HEAD of the upstream history, useful when you have made commits in a topic +branch but need to incorporate new upstream changes "underneath" them. +""" + + def _Options(self, p): + p.add_option('-i', '--interactive', + dest="interactive", action="store_true", + help="interactive rebase (single project only)") + + def Execute(self, opt, args): + all = self.GetProjects(args) + one_project = len(all) == 1 + + if opt.interactive and not one_project: + print >>sys.stderr, 'error: interactive rebase not supported with multiple projects' + return -1 + + for project in all: + cb = project.CurrentBranch + if not cb: + if one_project: + print >>sys.stderr, "error: project %s has a detatched HEAD" % project.name + return -1 + # ignore branches with detatched HEADs + continue + + upbranch = project.GetBranch(cb) + if not upbranch.LocalMerge: + if one_project: + print >>sys.stderr, "error: project %s does not track any remote branches" % project.name + return -1 + # ignore branches without remotes + continue + + upstream = project.GetRevisionId() + + args = ["rebase"] + if opt.interactive: + args.append("-i") + args.append(upstream) + + print '# project %s: rebasing branch %s -> %s (%s)' % ( + project.relpath, cb, upbranch.LocalMerge, upstream[0:7]) + if GitCommand(project, args).Wait() != 0: + return -1 From 0cb1b3f687da4634e431953ef84fee59dd3f5d59 Mon Sep 17 00:00:00 2001 From: Julius Gustavsson Date: Thu, 17 Jun 2010 17:55:02 +0200 Subject: [PATCH 19/33] sync: Try fetching a tag as a last resort before giving up If a tagged commit is not reachable by the fetch refspec configured for the git (usually refs/heads/*) it will not be downloaded by 'git fetch'. The tag can however be downloaded with 'git fetch --tags' or 'git fetch tag '. This patch fixes the situation when a tag is not found after a 'git fetch'. Repo will issue 'git fetch tag ' before giving up completely. Change-Id: I87796a5e1d51fcf398f346a274b7a069df37599a Signed-off-by: Shawn O. Pearce --- project.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/project.py b/project.py index 1bcd9596..956f45bf 100644 --- a/project.py +++ b/project.py @@ -283,7 +283,7 @@ class Project(object): return os.path.exists(os.path.join(g, 'rebase-apply')) \ or os.path.exists(os.path.join(g, 'rebase-merge')) \ or os.path.exists(os.path.join(w, '.dotest')) - + def IsDirty(self, consider_untracked=True): """Is the working directory modified in some way? """ @@ -416,7 +416,7 @@ class Project(object): try: f = df[p] except KeyError: f = None - + if i: i_status = i.status.upper() else: i_status = '-' @@ -601,6 +601,18 @@ class Project(object): if not self._RemoteFetch(): return False + #Check that the requested ref was found after fetch + # + try: + self.GetRevisionId() + except ManifestInvalidRevisionError: + # if the ref is a tag. We can try fetching + # the tag manually as a last resort + # + rev = self.revisionExpr + if rev.startswith(R_TAGS): + self._RemoteFetch(None, rev[len(R_TAGS):]) + if self.worktree: self._InitMRef() else: @@ -982,7 +994,7 @@ class Project(object): ## Direct Git Commands ## - def _RemoteFetch(self, name=None): + def _RemoteFetch(self, name=None, tag=None): if not name: name = self.remote.name @@ -994,6 +1006,9 @@ class Project(object): if not self.worktree: cmd.append('--update-head-ok') cmd.append(name) + if tag is not None: + cmd.append('tag') + cmd.append(tag) return GitCommand(self, cmd, bare = True, From cc50bac8c7706082596d70756249d4964a67f281 Mon Sep 17 00:00:00 2001 From: Anthony Newnam Date: Thu, 8 Apr 2010 10:28:59 -0500 Subject: [PATCH 20/33] Warn users before uploading if there are local changes Change-Id: I231d7b6a3211e9f5ec71a542a0109b0c195d5e40 Signed-off-by: Shawn O. Pearce --- project.py | 21 +++++++++++++++++++++ subcmds/upload.py | 15 +++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/project.py b/project.py index 956f45bf..4e8fa0e0 100644 --- a/project.py +++ b/project.py @@ -368,6 +368,27 @@ class Project(object): ## Status Display ## + def HasChanges(self): + """Returns true if there are uncommitted changes. + """ + self.work_git.update_index('-q', + '--unmerged', + '--ignore-missing', + '--refresh') + if self.IsRebaseInProgress(): + return True + + if self.work_git.DiffZ('diff-index', '--cached', HEAD): + return True + + if self.work_git.DiffZ('diff-files'): + return True + + if self.work_git.LsOthers(): + return True + + return False + def PrintWorkTreeStatus(self): """Prints the status of the repository to stdout. """ diff --git a/subcmds/upload.py b/subcmds/upload.py index ba532461..5a426113 100644 --- a/subcmds/upload.py +++ b/subcmds/upload.py @@ -320,6 +320,21 @@ Gerrit Code Review: http://code.google.com/p/gerrit/ people = copy.deepcopy(original_people) self._AppendAutoCcList(branch, people) + # Check if there are local changes that may have been forgotten + if branch.project.HasChanges(): + key = 'review.%s.autoupload' % branch.project.remote.review + answer = branch.project.config.GetBoolean(key) + + # if they want to auto upload, let's not ask because it could be automated + if answer is None: + sys.stdout.write('Uncommitted changes in ' + branch.project.name + ' (did you forget to amend?). Continue uploading? (y/n) ') + a = sys.stdin.readline().strip().lower() + if a not in ('y', 'yes', 't', 'true', 'on'): + print >>sys.stderr, "skipping upload" + branch.uploaded = False + branch.error = 'User aborted' + continue + branch.UploadForReview(people) branch.uploaded = True except UploadError, e: From a5ece0e0505324218f38af02a1fe046ca2bcc278 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Thu, 15 Jul 2010 16:52:42 -0700 Subject: [PATCH 21/33] upload -t: Automatically include local branch name If the -t flag is given to upload, the local branch name is automatically sent to Gerrit Code Review as the topic branch name for the change(s). This requires the server to be Gerrit Code Review v2.1.3-53-gd50c94e or later, which isn't widely deployed right now, so the default is opt-out. Change-Id: I034fcacb405b7cb909147152db427fe69dd7bcbf Signed-off-by: Shawn O. Pearce --- project.py | 17 +++++++++++++---- subcmds/upload.py | 21 ++++++++++++--------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/project.py b/project.py index 4e8fa0e0..1b5d9a67 100644 --- a/project.py +++ b/project.py @@ -149,10 +149,11 @@ class ReviewableBranch(object): R_HEADS + self.name, '--') - def UploadForReview(self, people): + def UploadForReview(self, people, auto_topic=False): self.project.UploadForReview(self.name, self.replace_changes, - people) + people, + auto_topic=auto_topic) def GetPublishedRefs(self): refs = {} @@ -555,7 +556,10 @@ class Project(object): return rb return None - def UploadForReview(self, branch=None, replace_changes=None, people=([],[])): + def UploadForReview(self, branch=None, + replace_changes=None, + people=([],[]), + auto_topic=False): """Uploads the named branch for code review. """ if branch is None: @@ -587,10 +591,15 @@ class Project(object): for e in people[1]: rp.append('--cc=%s' % sq(e)) + ref_spec = '%s:refs/for/%s' % (R_HEADS + branch.name, dest_branch) + if auto_topic: + ref_spec = ref_spec + '/' + branch.name + cmd = ['push'] cmd.append('--receive-pack=%s' % " ".join(rp)) cmd.append(branch.remote.SshReviewUrl(self.UserEmail)) - cmd.append('%s:refs/for/%s' % (R_HEADS + branch.name, dest_branch)) + cmd.append(ref_spec) + if replace_changes: for change_id,commit_id in replace_changes.iteritems(): cmd.append('%s:refs/changes/%s/new' % (commit_id, change_id)) diff --git a/subcmds/upload.py b/subcmds/upload.py index 5a426113..569e31c1 100644 --- a/subcmds/upload.py +++ b/subcmds/upload.py @@ -111,6 +111,9 @@ Gerrit Code Review: http://code.google.com/p/gerrit/ """ def _Options(self, p): + p.add_option('-t', + dest='auto_topic', action='store_true', + help='Send local branch name to Gerrit Code Review') p.add_option('--replace', dest='replace', action='store_true', help='Upload replacement patchesets from this branch') @@ -121,7 +124,7 @@ Gerrit Code Review: http://code.google.com/p/gerrit/ type='string', action='append', dest='cc', help='Also send email to these email addresses.') - def _SingleBranch(self, branch, people): + def _SingleBranch(self, opt, branch, people): project = branch.project name = branch.name remote = project.GetBranch(name).remote @@ -154,11 +157,11 @@ Gerrit Code Review: http://code.google.com/p/gerrit/ answer = _ConfirmManyUploads() if answer: - self._UploadAndReport([branch], people) + self._UploadAndReport(opt, [branch], people) else: _die("upload aborted by user") - def _MultipleBranches(self, pending, people): + def _MultipleBranches(self, opt, pending, people): projects = {} branches = {} @@ -227,7 +230,7 @@ Gerrit Code Review: http://code.google.com/p/gerrit/ if not _ConfirmManyUploads(multiple_branches=True): _die("upload aborted by user") - self._UploadAndReport(todo, people) + self._UploadAndReport(opt, todo, people) def _AppendAutoCcList(self, branch, people): """ @@ -311,9 +314,9 @@ Gerrit Code Review: http://code.google.com/p/gerrit/ _die("upload aborted by user") branch.replace_changes = to_replace - self._UploadAndReport([branch], people) + self._UploadAndReport(opt, [branch], people) - def _UploadAndReport(self, todo, original_people): + def _UploadAndReport(self, opt, todo, original_people): have_errors = False for branch in todo: try: @@ -335,7 +338,7 @@ Gerrit Code Review: http://code.google.com/p/gerrit/ branch.error = 'User aborted' continue - branch.UploadForReview(people) + branch.UploadForReview(people, auto_topic=opt.auto_topic) branch.uploaded = True except UploadError, e: branch.error = e @@ -391,6 +394,6 @@ Gerrit Code Review: http://code.google.com/p/gerrit/ if not pending: print >>sys.stdout, "no branches ready for upload" elif len(pending) == 1 and len(pending[0][1]) == 1: - self._SingleBranch(pending[0][1][0], people) + self._SingleBranch(opt, pending[0][1][0], people) else: - self._MultipleBranches(pending, people) + self._MultipleBranches(opt, pending, people) From 3575b8f8bdc5f15de23db82499e0ce152f634a19 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Thu, 15 Jul 2010 17:00:14 -0700 Subject: [PATCH 22/33] upload: Allow review.HOST.username to override email Some users might need to use a different login name than the local part of their email address for their Gerrit Code Review user account. Allow it to be overridden with the review.HOST.username configuration variable. Change-Id: I714469142ac7feadf09fee9c26680c0e09076b75 Signed-off-by: Shawn O. Pearce --- git_config.py | 5 ++++- subcmds/upload.py | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/git_config.py b/git_config.py index dc209ba8..1382d5db 100644 --- a/git_config.py +++ b/git_config.py @@ -531,8 +531,11 @@ class Remote(object): def SshReviewUrl(self, userEmail): if self.ReviewProtocol != 'ssh': return None + username = self._config.GetString('review.%s.username' % self.review) + if username is None: + username = userEmail.split("@")[0] return 'ssh://%s@%s:%s/%s' % ( - userEmail.split("@")[0], + username, self._review_host, self._review_port, self.projectname) diff --git a/subcmds/upload.py b/subcmds/upload.py index 569e31c1..b47b37b8 100644 --- a/subcmds/upload.py +++ b/subcmds/upload.py @@ -92,6 +92,11 @@ review.URL.autocopy can be set to a comma separated list of reviewers who you always want copied on all uploads with a non-empty --re argument. +review.URL.username: + +Override the username used to connect to Gerrit Code Review. +By default the local part of the email address is used. + The URL must match the review URL listed in the manifest XML file, or in the .git/config within the project. For example: From a22f99ae41a9cdda2129c89678a6f581b0445c85 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Thu, 15 Jul 2010 17:40:41 -0700 Subject: [PATCH 23/33] rebase: Pass through more options Passing through --whitespace=fix to rebase can be useful to clean up a branch prior to uploading it for review. Change-Id: Id85f1912e5e11ff9602e3b342c2fd7441abe67d7 Signed-off-by: Shawn O. Pearce --- subcmds/rebase.py | 46 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/subcmds/rebase.py b/subcmds/rebase.py index 44cdd2eb..7c8e9389 100644 --- a/subcmds/rebase.py +++ b/subcmds/rebase.py @@ -37,6 +37,22 @@ branch but need to incorporate new upstream changes "underneath" them. dest="interactive", action="store_true", help="interactive rebase (single project only)") + p.add_option('-f', '--force-rebase', + dest='force_rebase', action='store_true', + help='Pass --force-rebase to git rebase') + p.add_option('--no-ff', + dest='no_ff', action='store_true', + help='Pass --no-ff to git rebase') + p.add_option('-q', '--quiet', + dest='quiet', action='store_true', + help='Pass --quiet to git rebase') + p.add_option('--autosquash', + dest='autosquash', action='store_true', + help='Pass --autosquash to git rebase') + p.add_option('--whitespace', + dest='whitespace', action='store', metavar='WS', + help='Pass --whitespace to git rebase') + def Execute(self, opt, args): all = self.GetProjects(args) one_project = len(all) == 1 @@ -49,7 +65,7 @@ branch but need to incorporate new upstream changes "underneath" them. cb = project.CurrentBranch if not cb: if one_project: - print >>sys.stderr, "error: project %s has a detatched HEAD" % project.name + print >>sys.stderr, "error: project %s has a detatched HEAD" % project.relpath return -1 # ignore branches with detatched HEADs continue @@ -57,19 +73,35 @@ branch but need to incorporate new upstream changes "underneath" them. upbranch = project.GetBranch(cb) if not upbranch.LocalMerge: if one_project: - print >>sys.stderr, "error: project %s does not track any remote branches" % project.name + print >>sys.stderr, "error: project %s does not track any remote branches" % project.relpath return -1 # ignore branches without remotes continue - upstream = project.GetRevisionId() - args = ["rebase"] + + if opt.whitespace: + args.append('--whitespace=%s' % opt.whitespace) + + if opt.quiet: + args.append('--quiet') + + if opt.force_rebase: + args.append('--force-rebase') + + if opt.no_ff: + args.append('--no-ff') + + if opt.autosquash: + args.append('--autosquash') + if opt.interactive: args.append("-i") - args.append(upstream) - print '# project %s: rebasing branch %s -> %s (%s)' % ( - project.relpath, cb, upbranch.LocalMerge, upstream[0:7]) + args.append(upbranch.LocalMerge) + + print >>sys.stderr, '# %s: rebasing %s -> %s' % \ + (project.relpath, cb, upbranch.LocalMerge) + if GitCommand(project, args).Wait() != 0: return -1 From 60829ba72fe81b1de1c1e9c6e0de486e9e90bddd Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 16 Jul 2010 07:42:45 -0700 Subject: [PATCH 24/33] upload: Fix --replace flag --replace started to fail due to a Python error, I forgot to pass through the opt structure to the replace function. Change-Id: Ifcd7a0c715c3fd9070a4c58208612a626382de35 Signed-off-by: Shawn O. Pearce --- subcmds/upload.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/subcmds/upload.py b/subcmds/upload.py index b47b37b8..153b3ebe 100644 --- a/subcmds/upload.py +++ b/subcmds/upload.py @@ -262,7 +262,7 @@ Gerrit Code Review: http://code.google.com/p/gerrit/ except: return "" - def _ReplaceBranch(self, project, people): + def _ReplaceBranch(self, opt, project, people): branch = project.CurrentBranch if not branch: print >>sys.stdout, "no branches ready for upload" @@ -388,7 +388,7 @@ Gerrit Code Review: http://code.google.com/p/gerrit/ print >>sys.stderr, \ 'error: --replace requires exactly one project' sys.exit(1) - self._ReplaceBranch(project_list[0], people) + self._ReplaceBranch(opt, project_list[0], people) return for project in project_list: From b715b148076ef589e806a98a183c8f4468f75cf6 Mon Sep 17 00:00:00 2001 From: Patrick Dubroy Date: Thu, 29 Jul 2010 17:10:47 -0700 Subject: [PATCH 25/33] Fix for handling values of EDITOR which contain a space. The shell swallows the 0th arg, which was the filename. Simple fix is to pass in an extra arg for the shell to swallow. Change-Id: Iad6304ba9ccea6e7262ee06ef87d3dac57dbde81 --- editor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/editor.py b/editor.py index 23aab542..62afbb91 100644 --- a/editor.py +++ b/editor.py @@ -82,7 +82,7 @@ least one of these before using this command.""" fd = None if re.compile("^.*[$ \t'].*$").match(editor): - args = [editor + ' "$@"'] + args = [editor + ' "$@"', 'sh'] shell = True else: args = [editor] From 06d029c1c8c06a6d446b7cc4d5e763ff0754b149 Mon Sep 17 00:00:00 2001 From: Doug Anderson Date: Wed, 27 Oct 2010 17:06:01 -0700 Subject: [PATCH 26/33] Check for existing SSH ControlMaster Be more thorough about checking for an existing ssh master by running a test command first, and only opening up a new master if the test fails to connect. Change-Id: I56fe8e7b4dbc123675b7f259e81d359ed0cd55cf Signed-off-by: Shawn O. Pearce --- git_config.py | 53 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/git_config.py b/git_config.py index 1382d5db..138470c5 100644 --- a/git_config.py +++ b/git_config.py @@ -356,18 +356,21 @@ class RefSpec(object): return s -_ssh_cache = {} +_master_processes = [] +_master_keys = set() _ssh_master = True def _open_ssh(host, port=None): global _ssh_master + # Check to see whether we already think that the master is running; if we + # think it's already running, return right away. if port is not None: key = '%s:%s' % (host, port) else: key = host - if key in _ssh_cache: + if key in _master_keys: return True if not _ssh_master \ @@ -377,15 +380,39 @@ def _open_ssh(host, port=None): # return False - command = ['ssh', - '-o','ControlPath %s' % ssh_sock(), - '-M', - '-N', - host] - + # We will make two calls to ssh; this is the common part of both calls. + command_base = ['ssh', + '-o','ControlPath %s' % ssh_sock(), + host] if port is not None: - command[3:3] = ['-p',str(port)] + command_base[1:1] = ['-p',str(port)] + # Since the key wasn't in _master_keys, we think that master isn't running. + # ...but before actually starting a master, we'll double-check. This can + # be important because we can't tell that that 'git@myhost.com' is the same + # as 'myhost.com' where "User git" is setup in the user's ~/.ssh/config file. + check_command = command_base + ['-O','check'] + try: + Trace(': %s', ' '.join(check_command)) + check_process = subprocess.Popen(check_command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + check_process.communicate() # read output, but ignore it... + isnt_running = check_process.wait() + + if not isnt_running: + # Our double-check found that the master _was_ infact running. Add to + # the list of keys. + _master_keys.add(key) + return True + except Exception: + # Ignore excpetions. We we will fall back to the normal command and print + # to the log there. + pass + + command = command_base[:1] + \ + ['-M', '-N'] + \ + command_base[1:] try: Trace(': %s', ' '.join(command)) p = subprocess.Popen(command) @@ -396,20 +423,22 @@ def _open_ssh(host, port=None): % (host,port, str(e)) return False - _ssh_cache[key] = p + _master_processes.append(p) + _master_keys.add(key) time.sleep(1) return True def close_ssh(): terminate_ssh_clients() - for key,p in _ssh_cache.iteritems(): + for p in _master_processes: try: os.kill(p.pid, SIGTERM) p.wait() except OSError: pass - _ssh_cache.clear() + del _master_processes[:] + _master_keys.clear() d = ssh_sock(create=False) if d: From ec1df9b7f6310ce5dd9432b727eb9f7bde9f2796 Mon Sep 17 00:00:00 2001 From: Doug Anderson Date: Wed, 27 Oct 2010 17:13:07 -0700 Subject: [PATCH 27/33] Don't allow git fetch to start ControlMaster To avoid connectivity problems, we don't want the ssh process that is started by git fetch to become a ControlMaster for the overall sync task. If it did, we would lose connectivity when git fetch was finished with the current project, causing later projects to not fetch efficiently. Change-Id: I8d0dcf9b361276ff8c8b5a6324cbd4a501e9c4dd Signed-off-by: Shawn O. Pearce --- git_ssh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git_ssh b/git_ssh index 63aa63c2..b1ab521e 100755 --- a/git_ssh +++ b/git_ssh @@ -1,2 +1,2 @@ #!/bin/sh -exec ssh -o "ControlPath $REPO_SSH_SOCK" "$@" +exec ssh -o "ControlMaster no" -o "ControlPath $REPO_SSH_SOCK" "$@" From 99482ae58a74e236fb40b65c267163a5690f39e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ulrik=20Sj=C3=B6lin?= Date: Fri, 29 Oct 2010 08:23:30 -0700 Subject: [PATCH 28/33] Only delete corrupt pickle config files if they exist os.remove() raises OSError if the file being removed doesn't exist. Check before calling to ensure we don't raise a useless exception on an already deleted file. Change-Id: I44c1c7dd97a47fcab8afb6c18fdf179158b6dab7 Signed-off-by: Shawn O. Pearce --- git_config.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/git_config.py b/git_config.py index 138470c5..8e3dfb1b 100644 --- a/git_config.py +++ b/git_config.py @@ -257,9 +257,11 @@ class GitConfig(object): finally: fd.close() except IOError: - os.remove(self._pickle) + if os.path.exists(self._pickle): + os.remove(self._pickle) except cPickle.PickleError: - os.remove(self._pickle) + if os.path.exists(self._pickle): + os.remove(self._pickle) def _ReadGit(self): """ From 88443387b1b0508f43b57e104821c6b375806fea Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 8 Oct 2010 10:02:09 +0200 Subject: [PATCH 29/33] sync: Enable use of git clone --reference Use git clone to initialize a new repository, and when possible allow callers to use --reference to reuse an existing checkout as the initial object storage area for the new checkout. Change-Id: Ie27f760247f311ce484c6d3e85a90d94da2febfc Signed-off-by: Shawn O. Pearce --- project.py | 101 ++++++++++++++++++++++++++++++++++++++++++++---- repo | 5 ++- subcmds/init.py | 14 ++++++- 3 files changed, 111 insertions(+), 9 deletions(-) diff --git a/project.py b/project.py index 1b5d9a67..8ffed842 100644 --- a/project.py +++ b/project.py @@ -622,13 +622,14 @@ class Project(object): """Perform only the network IO portion of the sync process. Local working directory/branch state is not affected. """ - if not self.Exists: + is_new = not self.Exists + if is_new: print >>sys.stderr print >>sys.stderr, 'Initializing project %s ...' % self.name self._InitGitDir() self._InitRemote() - if not self._RemoteFetch(): + if not self._RemoteFetch(initial = is_new): return False #Check that the requested ref was found after fetch @@ -1024,7 +1025,7 @@ class Project(object): ## Direct Git Commands ## - def _RemoteFetch(self, name=None, tag=None): + def _RemoteFetch(self, name=None, tag=None, initial=False): if not name: name = self.remote.name @@ -1032,6 +1033,60 @@ class Project(object): if self.GetRemote(name).PreConnectFetch(): ssh_proxy = True + if initial: + alt = os.path.join(self.gitdir, 'objects/info/alternates') + try: + fd = open(alt, 'rb') + try: + ref_dir = fd.readline() + if ref_dir and ref_dir.endswith('\n'): + ref_dir = ref_dir[:-1] + finally: + fd.close() + except IOError, e: + ref_dir = None + + if ref_dir and 'objects' == os.path.basename(ref_dir): + ref_dir = os.path.dirname(ref_dir) + packed_refs = os.path.join(self.gitdir, 'packed-refs') + remote = self.GetRemote(name) + + all = self.bare_ref.all + ids = set(all.values()) + tmp = set() + + for r, id in GitRefs(ref_dir).all.iteritems(): + if r not in all: + if r.startswith(R_TAGS) or remote.WritesTo(r): + all[r] = id + ids.add(id) + continue + + if id in ids: + continue + + r = 'refs/_alt/%s' % id + all[r] = id + ids.add(id) + tmp.add(r) + + ref_names = list(all.keys()) + ref_names.sort() + + tmp_packed = '' + old_packed = '' + + for r in ref_names: + line = '%s %s\n' % (all[r], r) + tmp_packed += line + if r not in tmp: + old_packed += line + + _lwrite(packed_refs, tmp_packed) + + else: + ref_dir = None + cmd = ['fetch'] if not self.worktree: cmd.append('--update-head-ok') @@ -1039,10 +1094,21 @@ class Project(object): if tag is not None: cmd.append('tag') cmd.append(tag) - return GitCommand(self, - cmd, - bare = True, - ssh_proxy = ssh_proxy).Wait() == 0 + + ok = GitCommand(self, + cmd, + bare = True, + ssh_proxy = ssh_proxy).Wait() == 0 + + if initial: + if ref_dir: + if old_packed != '': + _lwrite(packed_refs, old_packed) + else: + os.remove(packed_refs) + self.bare_git.pack_refs('--all', '--prune') + + return ok def _Checkout(self, rev, quiet=False): cmd = ['checkout'] @@ -1080,6 +1146,27 @@ class Project(object): os.makedirs(self.gitdir) self.bare_git.init() + mp = self.manifest.manifestProject + ref_dir = mp.config.GetString('repo.reference') + + if ref_dir: + mirror_git = os.path.join(ref_dir, self.name + '.git') + repo_git = os.path.join(ref_dir, '.repo', 'projects', + self.relpath + '.git') + + if os.path.exists(mirror_git): + ref_dir = mirror_git + + elif os.path.exists(repo_git): + ref_dir = repo_git + + else: + ref_dir = None + + if ref_dir: + _lwrite(os.path.join(self.gitdir, 'objects/info/alternates'), + os.path.join(ref_dir, 'objects') + '\n') + if self.manifest.IsMirror: self.config.SetString('core.bare', 'true') else: diff --git a/repo b/repo index 13742559..bdc05c3b 100755 --- a/repo +++ b/repo @@ -28,7 +28,7 @@ if __name__ == '__main__': del magic # increment this whenever we make important changes to this script -VERSION = (1, 8) +VERSION = (1, 9) # increment this if the MAINTAINER_KEYS block is modified KEYRING_VERSION = (1,0) @@ -118,6 +118,9 @@ group.add_option('-m', '--manifest-name', group.add_option('--mirror', dest='mirror', action='store_true', help='mirror the forrest') +group.add_option('--reference', + dest='reference', + help='location of mirror directory', metavar='DIR') # Tool group = init_optparse.add_option_group('repo Version options') diff --git a/subcmds/init.py b/subcmds/init.py index 4023ab6d..17edfa05 100644 --- a/subcmds/init.py +++ b/subcmds/init.py @@ -41,6 +41,13 @@ The optional -m argument can be used to specify an alternate manifest to be used. If no manifest is specified, the manifest default.xml will be used. +The --reference option can be used to point to a directory that +has the content of a --mirror sync. This will make the working +directory use as much data as possible from the local reference +directory when fetching from the server. This will make the sync +go a lot faster by reducing data traffic on the network. + + Switching Manifest Branches --------------------------- @@ -71,7 +78,9 @@ to update the working directory files. g.add_option('--mirror', dest='mirror', action='store_true', help='mirror the forrest') - + g.add_option('--reference', + dest='reference', + help='location of mirror directory', metavar='DIR') # Tool g = p.add_option_group('repo Version options') @@ -115,6 +124,9 @@ to update the working directory files. r.ResetFetch() r.Save() + if opt.reference: + m.config.SetString('repo.reference', opt.reference) + if opt.mirror: if is_new: m.config.SetString('repo.mirror', 'true') From 16614f86b3cc8d61ccae7197624fa93fc752767b Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 29 Oct 2010 12:05:43 -0700 Subject: [PATCH 30/33] sync --quiet: be more quiet Change-Id: I5e8363c7b32e4546d1236cfc5a32e01c3e5ea8e6 Signed-off-by: Shawn O. Pearce --- project.py | 17 +++++++++++------ subcmds/sync.py | 24 ++++++++++++++++-------- 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/project.py b/project.py index 8ffed842..ce85b863 100644 --- a/project.py +++ b/project.py @@ -618,18 +618,19 @@ class Project(object): ## Sync ## - def Sync_NetworkHalf(self): + def Sync_NetworkHalf(self, quiet=False): """Perform only the network IO portion of the sync process. Local working directory/branch state is not affected. """ is_new = not self.Exists if is_new: - print >>sys.stderr - print >>sys.stderr, 'Initializing project %s ...' % self.name + if not quiet: + print >>sys.stderr + print >>sys.stderr, 'Initializing project %s ...' % self.name self._InitGitDir() self._InitRemote() - if not self._RemoteFetch(initial = is_new): + if not self._RemoteFetch(initial=is_new, quiet=quiet): return False #Check that the requested ref was found after fetch @@ -642,7 +643,7 @@ class Project(object): # rev = self.revisionExpr if rev.startswith(R_TAGS): - self._RemoteFetch(None, rev[len(R_TAGS):]) + self._RemoteFetch(None, rev[len(R_TAGS):], quiet=quiet) if self.worktree: self._InitMRef() @@ -1025,7 +1026,9 @@ class Project(object): ## Direct Git Commands ## - def _RemoteFetch(self, name=None, tag=None, initial=False): + def _RemoteFetch(self, name=None, tag=None, + initial=False, + quiet=False): if not name: name = self.remote.name @@ -1088,6 +1091,8 @@ class Project(object): ref_dir = None cmd = ['fetch'] + if quiet: + cmd.append('--quiet') if not self.worktree: cmd.append('--update-head-ok') cmd.append(name) diff --git a/subcmds/sync.py b/subcmds/sync.py index 6cac2e52..1f4b137f 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -110,6 +110,9 @@ later is required to fix a server side protocol bug. p.add_option('-d','--detach', dest='detach_head', action='store_true', help='detach projects back to manifest revision') + p.add_option('-q','--quiet', + dest='quiet', action='store_true', + help='be more quiet') p.add_option('-j','--jobs', dest='jobs', action='store', type='int', help="number of projects to fetch simultaneously") @@ -126,8 +129,8 @@ later is required to fix a server side protocol bug. dest='repo_upgraded', action='store_true', help=SUPPRESS_HELP) - def _FetchHelper(self, project, lock, fetched, pm, sem): - if not project.Sync_NetworkHalf(): + def _FetchHelper(self, opt, project, lock, fetched, pm, sem): + if not project.Sync_NetworkHalf(quiet=opt.quiet): print >>sys.stderr, 'error: Cannot fetch %s' % project.name sem.release() sys.exit(1) @@ -138,14 +141,14 @@ later is required to fix a server side protocol bug. lock.release() sem.release() - def _Fetch(self, projects): + def _Fetch(self, projects, opt): fetched = set() pm = Progress('Fetching projects', len(projects)) if self.jobs == 1: for project in projects: pm.update() - if project.Sync_NetworkHalf(): + if project.Sync_NetworkHalf(quiet=opt.quiet): fetched.add(project.gitdir) else: print >>sys.stderr, 'error: Cannot fetch %s' % project.name @@ -157,7 +160,12 @@ later is required to fix a server side protocol bug. for project in projects: sem.acquire() t = _threading.Thread(target = self._FetchHelper, - args = (project, lock, fetched, pm, sem)) + args = (opt, + project, + lock, + fetched, + pm, + sem)) threads.add(t) t.start() @@ -291,7 +299,7 @@ uncommitted changes are present' % project.relpath _PostRepoUpgrade(self.manifest) if not opt.local_only: - mp.Sync_NetworkHalf() + mp.Sync_NetworkHalf(quiet=opt.quiet) if mp.HasChanges: syncbuf = SyncBuffer(mp.config) @@ -308,7 +316,7 @@ uncommitted changes are present' % project.relpath to_fetch.append(rp) to_fetch.extend(all) - fetched = self._Fetch(to_fetch) + fetched = self._Fetch(to_fetch, opt) _PostRepoFetch(rp, opt.no_repo_verify) if opt.network_only: # bail out now; the rest touches the working tree @@ -320,7 +328,7 @@ uncommitted changes are present' % project.relpath for project in all: if project.gitdir not in fetched: missing.append(project) - self._Fetch(missing) + self._Fetch(missing, opt) if self.manifest.IsMirror: # bail out now, we have no working tree From a0de6e8eab97f5dcdb2f51d4e09dd1568623ec58 Mon Sep 17 00:00:00 2001 From: Ficus Kirkpatrick Date: Fri, 22 Oct 2010 13:06:47 -0700 Subject: [PATCH 31/33] upload: Remove --replace option It hasn't been necessary for a long time, and its functionality can be accomplished with 'git push'. Change-Id: Ic00d3adbe4cee7be3955117489c69d6e90106559 --- project.py | 6 ---- subcmds/upload.py | 78 +---------------------------------------------- 2 files changed, 1 insertion(+), 83 deletions(-) diff --git a/project.py b/project.py index ce85b863..01dc8678 100644 --- a/project.py +++ b/project.py @@ -111,7 +111,6 @@ class ReviewableBranch(object): self.project = project self.branch = branch self.base = base - self.replace_changes = None @property def name(self): @@ -151,7 +150,6 @@ class ReviewableBranch(object): def UploadForReview(self, people, auto_topic=False): self.project.UploadForReview(self.name, - self.replace_changes, people, auto_topic=auto_topic) @@ -557,7 +555,6 @@ class Project(object): return None def UploadForReview(self, branch=None, - replace_changes=None, people=([],[]), auto_topic=False): """Uploads the named branch for code review. @@ -600,9 +597,6 @@ class Project(object): cmd.append(branch.remote.SshReviewUrl(self.UserEmail)) cmd.append(ref_spec) - if replace_changes: - for change_id,commit_id in replace_changes.iteritems(): - cmd.append('%s:refs/changes/%s/new' % (commit_id, change_id)) if GitCommand(self, cmd, bare = True).Wait() != 0: raise UploadError('Upload failed') diff --git a/subcmds/upload.py b/subcmds/upload.py index 153b3ebe..1964bffa 100644 --- a/subcmds/upload.py +++ b/subcmds/upload.py @@ -47,7 +47,7 @@ class Upload(InteractiveCommand): common = True helpSummary = "Upload changes for code review" helpUsage=""" -%prog [--re --cc] {[]... | --replace } +%prog [--re --cc] []... """ helpDescription = """ The '%prog' command is used to send changes to the Gerrit Code @@ -67,12 +67,6 @@ added to the respective list of users, and emails are sent to any new users. Users passed as --reviewers must already be registered with the code review system, or the upload will fail. -If the --replace option is passed the user can designate which -existing change(s) in Gerrit match up to the commits in the branch -being uploaded. For each matched pair of change,commit the commit -will be added as a new patch set, completely replacing the set of -files and description associated with the change in Gerrit. - Configuration ------------- @@ -119,9 +113,6 @@ Gerrit Code Review: http://code.google.com/p/gerrit/ p.add_option('-t', dest='auto_topic', action='store_true', help='Send local branch name to Gerrit Code Review') - p.add_option('--replace', - dest='replace', action='store_true', - help='Upload replacement patchesets from this branch') p.add_option('--re', '--reviewers', type='string', action='append', dest='reviewers', help='Request reviews from these people.') @@ -262,65 +253,6 @@ Gerrit Code Review: http://code.google.com/p/gerrit/ except: return "" - def _ReplaceBranch(self, opt, project, people): - branch = project.CurrentBranch - if not branch: - print >>sys.stdout, "no branches ready for upload" - return - branch = project.GetUploadableBranch(branch) - if not branch: - print >>sys.stdout, "no branches ready for upload" - return - - script = [] - script.append('# Replacing from branch %s' % branch.name) - - if len(branch.commits) == 1: - change = self._FindGerritChange(branch) - script.append('[%-6s] %s' % (change, branch.commits[0])) - else: - for commit in branch.commits: - script.append('[ ] %s' % commit) - - script.append('') - script.append('# Insert change numbers in the brackets to add a new patch set.') - script.append('# To create a new change record, leave the brackets empty.') - - script = Editor.EditString("\n".join(script)).split("\n") - - change_re = re.compile(r'^\[\s*(\d{1,})\s*\]\s*([0-9a-f]{1,}) .*$') - to_replace = dict() - full_hashes = branch.unabbrev_commits - - for line in script: - m = change_re.match(line) - if m: - c = m.group(1) - f = m.group(2) - try: - f = full_hashes[f] - except KeyError: - print 'fh = %s' % full_hashes - print >>sys.stderr, "error: commit %s not found" % f - sys.exit(1) - if c in to_replace: - print >>sys.stderr,\ - "error: change %s cannot accept multiple commits" % c - sys.exit(1) - to_replace[c] = f - - if not to_replace: - print >>sys.stderr, "error: no replacements specified" - print >>sys.stderr, " use 'repo upload' without --replace" - sys.exit(1) - - if len(branch.commits) > UNUSUAL_COMMIT_THRESHOLD: - if not _ConfirmManyUploads(multiple_branches=True): - _die("upload aborted by user") - - branch.replace_changes = to_replace - self._UploadAndReport(opt, [branch], people) - def _UploadAndReport(self, opt, todo, original_people): have_errors = False for branch in todo: @@ -383,14 +315,6 @@ Gerrit Code Review: http://code.google.com/p/gerrit/ cc = _SplitEmails(opt.cc) people = (reviewers,cc) - if opt.replace: - if len(project_list) != 1: - print >>sys.stderr, \ - 'error: --replace requires exactly one project' - sys.exit(1) - self._ReplaceBranch(opt, project_list[0], people) - return - for project in project_list: avail = project.GetUploadableBranches() if avail: From 5df6de075e5fb674368d38f858419425bc8d8d07 Mon Sep 17 00:00:00 2001 From: Andrei Warkentin Date: Fri, 2 Jul 2010 17:58:31 -0500 Subject: [PATCH 32/33] sync: Use --force-broken to continue other projects This adds a new flag -f/--force-broken that will allow the rest of the sync process to continue instead of bailing when a particular project fails to sync. Change-Id: I23680f2ee7927410f7ed930b1d469424c9aa246e Signed-off-by: Andrei Warkentin Signed-off-by: Shawn O. Pearce --- subcmds/sync.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/subcmds/sync.py b/subcmds/sync.py index 1f4b137f..ca78467b 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -70,6 +70,9 @@ The -s/--smart-sync option can be used to sync to a known good build as specified by the manifest-server element in the current manifest. +The -f/--force-broken option can be used to proceed with syncing +other projects if a project sync fails. + SSH Connections --------------- @@ -101,6 +104,9 @@ later is required to fix a server side protocol bug. """ def _Options(self, p, show_smart=True): + p.add_option('-f', '--force-broken', + dest='force_broken', action='store_true', + help="continue sync even if a project fails to sync") p.add_option('-l','--local-only', dest='local_only', action='store_true', help="only update working tree, don't fetch") @@ -132,8 +138,11 @@ later is required to fix a server side protocol bug. def _FetchHelper(self, opt, project, lock, fetched, pm, sem): if not project.Sync_NetworkHalf(quiet=opt.quiet): print >>sys.stderr, 'error: Cannot fetch %s' % project.name - sem.release() - sys.exit(1) + if opt.force_broken: + print >>sys.stderr, 'warn: --force-broken, continuing to sync' + else: + sem.release() + sys.exit(1) lock.acquire() fetched.add(project.gitdir) @@ -152,7 +161,10 @@ later is required to fix a server side protocol bug. fetched.add(project.gitdir) else: print >>sys.stderr, 'error: Cannot fetch %s' % project.name - sys.exit(1) + if opt.force_broken: + print >>sys.stderr, 'warn: --force-broken, continuing to sync' + else: + sys.exit(1) else: threads = set() lock = _threading.Lock() From 2b8db3ce3e7344b9f3b5216637c5af0d54be5656 Mon Sep 17 00:00:00 2001 From: Doug Anderson Date: Mon, 1 Nov 2010 15:08:06 -0700 Subject: [PATCH 33/33] Added feature to print a from manifest at the end of a sync. This feature is used to convey information on a when a branch has ceased development or if it is an experimental branch with a few gotchas, etc. You add it to your manifest XML by doing something like this: NOTE TO DEVELOPERS: If you checkin code, you have to pinky-swear that it contains no bugs. Anyone who breaks their promise will have tomatoes thrown at them in the team meeting. Be sure to bring an extra set of clothes. ... Carriage returns and indentation are relevant for the text in this tag. This feature was requested by Anush Elangovan on the ChromiumOS team. --- docs/manifest-format.txt | 5 +++- manifest_xml.py | 62 ++++++++++++++++++++++++++++++++++++++++ subcmds/sync.py | 5 ++++ 3 files changed, 71 insertions(+), 1 deletion(-) diff --git a/docs/manifest-format.txt b/docs/manifest-format.txt index 211344ee..2e1c8c35 100644 --- a/docs/manifest-format.txt +++ b/docs/manifest-format.txt @@ -20,12 +20,15 @@ A manifest XML file (e.g. 'default.xml') roughly conforms to the following DTD: + + diff --git a/manifest_xml.py b/manifest_xml.py index d0c9debe..9d68f09f 100644 --- a/manifest_xml.py +++ b/manifest_xml.py @@ -107,6 +107,15 @@ class XmlManifest(object): root = doc.createElement('manifest') doc.appendChild(root) + # Save out the notice. There's a little bit of work here to give it the + # right whitespace, which assumes that the notice is automatically indented + # by 4 by minidom. + if self.notice: + notice_element = root.appendChild(doc.createElement('notice')) + notice_lines = self.notice.splitlines() + indented_notice = ('\n'.join(" "*4 + line for line in notice_lines))[4:] + notice_element.appendChild(doc.createTextNode(indented_notice)) + d = self.default sort_remotes = list(self.remotes.keys()) sort_remotes.sort() @@ -179,6 +188,11 @@ class XmlManifest(object): self._Load() return self._default + @property + def notice(self): + self._Load() + return self._notice + @property def manifest_server(self): self._Load() @@ -193,6 +207,7 @@ class XmlManifest(object): self._projects = {} self._remotes = {} self._default = None + self._notice = None self.branch = None self._manifest_server = None @@ -263,6 +278,14 @@ class XmlManifest(object): if self._default is None: self._default = _Default() + for node in config.childNodes: + if node.nodeName == 'notice': + if self._notice is not None: + raise ManifestParseError, \ + 'duplicate notice in %s' % \ + (self.manifestFile) + self._notice = self._ParseNotice(node) + for node in config.childNodes: if node.nodeName == 'manifest-server': url = self._reqatt(node, 'url') @@ -338,6 +361,45 @@ class XmlManifest(object): d.revisionExpr = None return d + def _ParseNotice(self, node): + """ + reads a element from the manifest file + + The element is distinct from other tags in the XML in that the + data is conveyed between the start and end tag (it's not an empty-element + tag). + + The white space (carriage returns, indentation) for the notice element is + relevant and is parsed in a way that is based on how python docstrings work. + In fact, the code is remarkably similar to here: + http://www.python.org/dev/peps/pep-0257/ + """ + # Get the data out of the node... + notice = node.childNodes[0].data + + # Figure out minimum indentation, skipping the first line (the same line + # as the tag)... + minIndent = sys.maxint + lines = notice.splitlines() + for line in lines[1:]: + lstrippedLine = line.lstrip() + if lstrippedLine: + indent = len(line) - len(lstrippedLine) + minIndent = min(indent, minIndent) + + # Strip leading / trailing blank lines and also indentation. + cleanLines = [lines[0].strip()] + for line in lines[1:]: + cleanLines.append(line[minIndent:].rstrip()) + + # Clear completely blank lines from front and back... + while cleanLines and not cleanLines[0]: + del cleanLines[0] + while cleanLines and not cleanLines[-1]: + del cleanLines[-1] + + return '\n'.join(cleanLines) + def _ParseProject(self, node): """ reads a element from the manifest file diff --git a/subcmds/sync.py b/subcmds/sync.py index ca78467b..d6ea442a 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -361,6 +361,11 @@ uncommitted changes are present' % project.relpath if not syncbuf.Finish(): sys.exit(1) + # If there's a notice that's supposed to print at the end of the sync, print + # it now... + if self.manifest.notice: + print self.manifest.notice + def _PostRepoUpgrade(manifest): for project in manifest.projects.values(): if project.Exists: