diff --git a/docs/manifest_submodule.txt b/docs/manifest_submodule.txt index e7d1f643..1718284b 100644 --- a/docs/manifest_submodule.txt +++ b/docs/manifest_submodule.txt @@ -76,6 +76,12 @@ submodule..update This key is not supported by repo. If set, it will be ignored. +repo.notice +----------- + +A message displayed when repo sync uses this manifest. + + .review ======= diff --git a/docs/manifest_xml.txt b/docs/manifest_xml.txt index da0e69ff..37fbd5cd 100644 --- a/docs/manifest_xml.txt +++ b/docs/manifest_xml.txt @@ -20,11 +20,15 @@ A manifest XML file (e.g. 'default.xml') roughly conforms to the following DTD: + + @@ -34,6 +38,9 @@ following DTD: + + + @@ -89,6 +96,27 @@ Attribute `revision`: Name of a Git branch (e.g. `master` or revision attribute will use this revision. +Element manifest-server +----------------------- + +At most one manifest-server may be specified. The url attribute +is used to specify the URL of a manifest server, which is an +XML RPC service that will return a manifest in which each project +is pegged to a known good revision for the current branch and +target. + +The manifest server should implement: + + GetApprovedManifest(branch, target) + +The target to use is defined by environment variables TARGET_PRODUCT +and TARGET_BUILD_VARIANT. These variables are used to create a string +of the form $TARGET_PRODUCT-$TARGET_BUILD_VARIANT, e.g. passion-userdebug. +If one of those variables or both are not present, the program will call +GetApprovedManifest without the target paramater and the manifest server +should choose a reasonable default target. + + Element project --------------- diff --git a/editor.py b/editor.py index 23aab542..62afbb91 100644 --- a/editor.py +++ b/editor.py @@ -82,7 +82,7 @@ least one of these before using this command.""" fd = None if re.compile("^.*[$ \t'].*$").match(editor): - args = [editor + ' "$@"'] + args = [editor + ' "$@"', 'sh'] shell = True else: args = [editor] diff --git a/git_command.py b/git_command.py index 414c84a2..181e3724 100644 --- a/git_command.py +++ b/git_command.py @@ -17,6 +17,7 @@ import os import sys import subprocess import tempfile +from signal import SIGTERM from error import GitError from trace import REPO_TRACE, IsTrace, Trace @@ -29,8 +30,9 @@ LAST_CWD = None _ssh_proxy_path = None _ssh_sock_path = None +_ssh_clients = [] -def _ssh_sock(create=True): +def ssh_sock(create=True): global _ssh_sock_path if _ssh_sock_path is None: if not create: @@ -51,6 +53,24 @@ def _ssh_proxy(): 'git_ssh') return _ssh_proxy_path +def _add_ssh_client(p): + _ssh_clients.append(p) + +def _remove_ssh_client(p): + try: + _ssh_clients.remove(p) + except ValueError: + pass + +def terminate_ssh_clients(): + global _ssh_clients + for p in _ssh_clients: + try: + os.kill(p.pid, SIGTERM) + p.wait() + except OSError: + pass + _ssh_clients = [] class _GitCall(object): def version(self): @@ -119,7 +139,7 @@ class GitCommand(object): if disable_editor: env['GIT_EDITOR'] = ':' if ssh_proxy: - env['REPO_SSH_SOCK'] = _ssh_sock() + env['REPO_SSH_SOCK'] = ssh_sock() env['GIT_SSH'] = _ssh_proxy() if project: @@ -188,6 +208,9 @@ class GitCommand(object): except Exception, e: raise GitError('%s: %s' % (command[1], e)) + if ssh_proxy: + _add_ssh_client(p) + self.process = p self.stdin = p.stdin @@ -210,4 +233,8 @@ class GitCommand(object): else: p.stderr = None - return self.process.wait() + try: + rc = p.wait() + finally: + _remove_ssh_client(p) + return rc diff --git a/git_config.py b/git_config.py index 4a42c047..286e89ca 100644 --- a/git_config.py +++ b/git_config.py @@ -25,7 +25,10 @@ from signal import SIGTERM from urllib2 import urlopen, HTTPError from error import GitError, UploadError from trace import Trace -from git_command import GitCommand, _ssh_sock + +from git_command import GitCommand +from git_command import ssh_sock +from git_command import terminate_ssh_clients R_HEADS = 'refs/heads/' R_TAGS = 'refs/tags/' @@ -365,18 +368,21 @@ class RefSpec(object): return s -_ssh_cache = {} +_master_processes = [] +_master_keys = set() _ssh_master = True def _open_ssh(host, port=None): global _ssh_master + # Check to see whether we already think that the master is running; if we + # think it's already running, return right away. if port is not None: key = '%s:%s' % (host, port) else: key = host - if key in _ssh_cache: + if key in _master_keys: return True if not _ssh_master \ @@ -386,15 +392,39 @@ def _open_ssh(host, port=None): # return False - command = ['ssh', - '-o','ControlPath %s' % _ssh_sock(), - '-M', - '-N', - host] - + # We will make two calls to ssh; this is the common part of both calls. + command_base = ['ssh', + '-o','ControlPath %s' % ssh_sock(), + host] if port is not None: - command[3:3] = ['-p',str(port)] + command_base[1:1] = ['-p',str(port)] + # Since the key wasn't in _master_keys, we think that master isn't running. + # ...but before actually starting a master, we'll double-check. This can + # be important because we can't tell that that 'git@myhost.com' is the same + # as 'myhost.com' where "User git" is setup in the user's ~/.ssh/config file. + check_command = command_base + ['-O','check'] + try: + Trace(': %s', ' '.join(check_command)) + check_process = subprocess.Popen(check_command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + check_process.communicate() # read output, but ignore it... + isnt_running = check_process.wait() + + if not isnt_running: + # Our double-check found that the master _was_ infact running. Add to + # the list of keys. + _master_keys.add(key) + return True + except Exception: + # Ignore excpetions. We we will fall back to the normal command and print + # to the log there. + pass + + command = command_base[:1] + \ + ['-M', '-N'] + \ + command_base[1:] try: Trace(': %s', ' '.join(command)) p = subprocess.Popen(command) @@ -405,20 +435,24 @@ def _open_ssh(host, port=None): % (host,port, str(e)) return False - _ssh_cache[key] = p + _master_processes.append(p) + _master_keys.add(key) time.sleep(1) return True def close_ssh(): - for key,p in _ssh_cache.iteritems(): + terminate_ssh_clients() + + for p in _master_processes: try: os.kill(p.pid, SIGTERM) p.wait() except OSError: pass - _ssh_cache.clear() + del _master_processes[:] + _master_keys.clear() - d = _ssh_sock(create=False) + d = ssh_sock(create=False) if d: try: os.rmdir(os.path.dirname(d)) @@ -540,8 +574,11 @@ class Remote(object): def SshReviewUrl(self, userEmail): if self.ReviewProtocol != 'ssh': return None + username = self._config.GetString('review.%s.username' % self.review) + if username is None: + username = userEmail.split("@")[0] return 'ssh://%s@%s:%s/%s' % ( - userEmail.split("@")[0], + username, self._review_host, self._review_port, self.projectname) diff --git a/git_ssh b/git_ssh index 63aa63c2..b1ab521e 100755 --- a/git_ssh +++ b/git_ssh @@ -1,2 +1,2 @@ #!/bin/sh -exec ssh -o "ControlPath $REPO_SSH_SOCK" "$@" +exec ssh -o "ControlMaster no" -o "ControlPath $REPO_SSH_SOCK" "$@" diff --git a/manifest.py b/manifest.py index f737e866..c03cb4a7 100644 --- a/manifest.py +++ b/manifest.py @@ -41,6 +41,14 @@ class Manifest(object): def projects(self): return {} + @property + def notice(self): + return None + + @property + def manifest_server(self): + return None + def InitBranch(self): pass diff --git a/manifest_submodule.py b/manifest_submodule.py index 92f187a0..cac271cd 100644 --- a/manifest_submodule.py +++ b/manifest_submodule.py @@ -102,6 +102,10 @@ class SubmoduleManifest(Manifest): self._Load() return self._projects + @property + def notice(self): + return self._modules.GetString('repo.notice') + def InitBranch(self): m = self.manifestProject if m.CurrentBranch is None: @@ -266,6 +270,9 @@ class SubmoduleManifest(Manifest): if b.startswith(R_HEADS): b = b[len(R_HEADS):] + if old.notice: + gm.SetString('repo.notice', old.notice) + info = [] pm = Progress('Converting manifest', len(sort_projects)) for p in sort_projects: diff --git a/manifest_xml.py b/manifest_xml.py index 35318d0a..1d02f9d4 100644 --- a/manifest_xml.py +++ b/manifest_xml.py @@ -66,8 +66,8 @@ class XmlManifest(Manifest): self._Unload() - def Link(self, name): - """Update the repo metadata to use a different manifest. + def Override(self, name): + """Use a different manifest, just for the current instantiation. """ path = os.path.join(self.manifestProject.worktree, name) if not os.path.isfile(path): @@ -81,6 +81,11 @@ class XmlManifest(Manifest): finally: self._manifestFile = old + def Link(self, name): + """Update the repo metadata to use a different manifest. + """ + self.Override(name) + try: if os.path.exists(self._manifestFile): os.remove(self._manifestFile) @@ -103,6 +108,15 @@ class XmlManifest(Manifest): root = doc.createElement('manifest') doc.appendChild(root) + # Save out the notice. There's a little bit of work here to give it the + # right whitespace, which assumes that the notice is automatically indented + # by 4 by minidom. + if self.notice: + notice_element = root.appendChild(doc.createElement('notice')) + notice_lines = self.notice.splitlines() + indented_notice = ('\n'.join(" "*4 + line for line in notice_lines))[4:] + notice_element.appendChild(doc.createTextNode(indented_notice)) + d = self.default sort_remotes = list(self.remotes.keys()) sort_remotes.sort() @@ -124,6 +138,12 @@ class XmlManifest(Manifest): root.appendChild(e) root.appendChild(doc.createTextNode('')) + if self._manifest_server: + e = doc.createElement('manifest-server') + e.setAttribute('url', self._manifest_server) + root.appendChild(e) + root.appendChild(doc.createTextNode('')) + sort_projects = list(self.projects.keys()) sort_projects.sort() @@ -169,6 +189,16 @@ class XmlManifest(Manifest): self._Load() return self._default + @property + def notice(self): + self._Load() + return self._notice + + @property + def manifest_server(self): + self._Load() + return self._manifest_server + def InitBranch(self): m = self.manifestProject if m.CurrentBranch is None: @@ -184,7 +214,9 @@ class XmlManifest(Manifest): self._projects = {} self._remotes = {} self._default = None + self._notice = None self.branch = None + self._manifest_server = None def _Load(self): if not self._loaded: @@ -256,6 +288,23 @@ class XmlManifest(Manifest): if self._default is None: self._default = _Default() + for node in config.childNodes: + if node.nodeName == 'notice': + if self._notice is not None: + raise ManifestParseError, \ + 'duplicate notice in %s' % \ + (self.manifestFile) + self._notice = self._ParseNotice(node) + + for node in config.childNodes: + if node.nodeName == 'manifest-server': + url = self._reqatt(node, 'url') + if self._manifest_server is not None: + raise ManifestParseError, \ + 'duplicate manifest-server in %s' % \ + (self.manifestFile) + self._manifest_server = url + for node in config.childNodes: if node.nodeName == 'project': project = self._ParseProject(node) @@ -322,10 +371,49 @@ class XmlManifest(Manifest): d.revisionExpr = None return d + def _ParseNotice(self, node): + """ + reads a element from the manifest file + + The element is distinct from other tags in the XML in that the + data is conveyed between the start and end tag (it's not an empty-element + tag). + + The white space (carriage returns, indentation) for the notice element is + relevant and is parsed in a way that is based on how python docstrings work. + In fact, the code is remarkably similar to here: + http://www.python.org/dev/peps/pep-0257/ + """ + # Get the data out of the node... + notice = node.childNodes[0].data + + # Figure out minimum indentation, skipping the first line (the same line + # as the tag)... + minIndent = sys.maxint + lines = notice.splitlines() + for line in lines[1:]: + lstrippedLine = line.lstrip() + if lstrippedLine: + indent = len(line) - len(lstrippedLine) + minIndent = min(indent, minIndent) + + # Strip leading / trailing blank lines and also indentation. + cleanLines = [lines[0].strip()] + for line in lines[1:]: + cleanLines.append(line[minIndent:].rstrip()) + + # Clear completely blank lines from front and back... + while cleanLines and not cleanLines[0]: + del cleanLines[0] + while cleanLines and not cleanLines[-1]: + del cleanLines[-1] + + return '\n'.join(cleanLines) + def _ParseProject(self, node): """ reads a element from the manifest file - """ + """ name = self._reqatt(node, 'name') remote = self._get_remote(node) diff --git a/progress.py b/progress.py index b119b374..2ace7010 100644 --- a/progress.py +++ b/progress.py @@ -13,10 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import sys from time import time from trace import IsTrace +_NOT_TTY = not os.isatty(2) + class Progress(object): def __init__(self, title, total=0): self._title = title @@ -29,7 +32,7 @@ class Progress(object): def update(self, inc=1): self._done += inc - if IsTrace(): + if _NOT_TTY or IsTrace(): return if not self._show: @@ -56,7 +59,7 @@ class Progress(object): sys.stderr.flush() def end(self): - if IsTrace() or not self._show: + if _NOT_TTY or IsTrace() or not self._show: return if self._total <= 0: diff --git a/project.py b/project.py index 1cea959e..fde98ad7 100644 --- a/project.py +++ b/project.py @@ -111,7 +111,6 @@ class ReviewableBranch(object): self.project = project self.branch = branch self.base = base - self.replace_changes = None @property def name(self): @@ -149,10 +148,10 @@ class ReviewableBranch(object): R_HEADS + self.name, '--') - def UploadForReview(self, people): + def UploadForReview(self, people, auto_topic=False): self.project.UploadForReview(self.name, - self.replace_changes, - people) + people, + auto_topic=auto_topic) def GetPublishedRefs(self): refs = {} @@ -203,6 +202,10 @@ class _CopyFile: # remove existing file first, since it might be read-only if os.path.exists(dest): os.remove(dest) + else: + dir = os.path.dirname(dest) + if not os.path.isdir(dir): + os.makedirs(dir) shutil.copy(src, dest) # make the file read-only mode = os.stat(dest)[stat.ST_MODE] @@ -279,7 +282,7 @@ class Project(object): return os.path.exists(os.path.join(g, 'rebase-apply')) \ or os.path.exists(os.path.join(g, 'rebase-merge')) \ or os.path.exists(os.path.join(w, '.dotest')) - + def IsDirty(self, consider_untracked=True): """Is the working directory modified in some way? """ @@ -364,6 +367,27 @@ class Project(object): ## Status Display ## + def HasChanges(self): + """Returns true if there are uncommitted changes. + """ + self.work_git.update_index('-q', + '--unmerged', + '--ignore-missing', + '--refresh') + if self.IsRebaseInProgress(): + return True + + if self.work_git.DiffZ('diff-index', '--cached', HEAD): + return True + + if self.work_git.DiffZ('diff-files'): + return True + + if self.work_git.LsOthers(): + return True + + return False + def PrintWorkTreeStatus(self): """Prints the status of the repository to stdout. """ @@ -412,7 +436,7 @@ class Project(object): try: f = df[p] except KeyError: f = None - + if i: i_status = i.status.upper() else: i_status = '-' @@ -530,7 +554,9 @@ class Project(object): return rb return None - def UploadForReview(self, branch=None, replace_changes=None, people=([],[])): + def UploadForReview(self, branch=None, + people=([],[]), + auto_topic=False): """Uploads the named branch for code review. """ if branch is None: @@ -562,13 +588,15 @@ class Project(object): for e in people[1]: rp.append('--cc=%s' % sq(e)) + ref_spec = '%s:refs/for/%s' % (R_HEADS + branch.name, dest_branch) + if auto_topic: + ref_spec = ref_spec + '/' + branch.name + cmd = ['push'] cmd.append('--receive-pack=%s' % " ".join(rp)) cmd.append(branch.remote.SshReviewUrl(self.UserEmail)) - cmd.append('%s:refs/for/%s' % (R_HEADS + branch.name, dest_branch)) - if replace_changes: - for change_id,commit_id in replace_changes.iteritems(): - cmd.append('%s:refs/changes/%s/new' % (commit_id, change_id)) + cmd.append(ref_spec) + if GitCommand(self, cmd, bare = True).Wait() != 0: raise UploadError('Upload failed') @@ -584,19 +612,33 @@ class Project(object): ## Sync ## - def Sync_NetworkHalf(self): + def Sync_NetworkHalf(self, quiet=False): """Perform only the network IO portion of the sync process. Local working directory/branch state is not affected. """ - if not self.Exists: - print >>sys.stderr - print >>sys.stderr, 'Initializing project %s ...' % self.name + is_new = not self.Exists + if is_new: + if not quiet: + print >>sys.stderr + print >>sys.stderr, 'Initializing project %s ...' % self.name self._InitGitDir() self._InitRemote() - if not self._RemoteFetch(): + if not self._RemoteFetch(initial=is_new, quiet=quiet): return False + #Check that the requested ref was found after fetch + # + try: + self.GetRevisionId() + except ManifestInvalidRevisionError: + # if the ref is a tag. We can try fetching + # the tag manually as a last resort + # + rev = self.revisionExpr + if rev.startswith(R_TAGS): + self._RemoteFetch(None, rev[len(R_TAGS):], quiet=quiet) + if self.worktree: self.manifest.SetMRefs(self) else: @@ -978,7 +1020,9 @@ class Project(object): ## Direct Git Commands ## - def _RemoteFetch(self, name=None): + def _RemoteFetch(self, name=None, tag=None, + initial=False, + quiet=False): if not name: name = self.remote.name @@ -986,14 +1030,84 @@ class Project(object): if self.GetRemote(name).PreConnectFetch(): ssh_proxy = True + if initial: + alt = os.path.join(self.gitdir, 'objects/info/alternates') + try: + fd = open(alt, 'rb') + try: + ref_dir = fd.readline() + if ref_dir and ref_dir.endswith('\n'): + ref_dir = ref_dir[:-1] + finally: + fd.close() + except IOError, e: + ref_dir = None + + if ref_dir and 'objects' == os.path.basename(ref_dir): + ref_dir = os.path.dirname(ref_dir) + packed_refs = os.path.join(self.gitdir, 'packed-refs') + remote = self.GetRemote(name) + + all = self.bare_ref.all + ids = set(all.values()) + tmp = set() + + for r, id in GitRefs(ref_dir).all.iteritems(): + if r not in all: + if r.startswith(R_TAGS) or remote.WritesTo(r): + all[r] = id + ids.add(id) + continue + + if id in ids: + continue + + r = 'refs/_alt/%s' % id + all[r] = id + ids.add(id) + tmp.add(r) + + ref_names = list(all.keys()) + ref_names.sort() + + tmp_packed = '' + old_packed = '' + + for r in ref_names: + line = '%s %s\n' % (all[r], r) + tmp_packed += line + if r not in tmp: + old_packed += line + + _lwrite(packed_refs, tmp_packed) + + else: + ref_dir = None + cmd = ['fetch'] + if quiet: + cmd.append('--quiet') if not self.worktree: cmd.append('--update-head-ok') cmd.append(name) - return GitCommand(self, - cmd, - bare = True, - ssh_proxy = ssh_proxy).Wait() == 0 + if tag is not None: + cmd.append('tag') + cmd.append(tag) + + ok = GitCommand(self, + cmd, + bare = True, + ssh_proxy = ssh_proxy).Wait() == 0 + + if initial: + if ref_dir: + if old_packed != '': + _lwrite(packed_refs, old_packed) + else: + os.remove(packed_refs) + self.bare_git.pack_refs('--all', '--prune') + + return ok def _Checkout(self, rev, quiet=False): cmd = ['checkout'] @@ -1031,6 +1145,27 @@ class Project(object): os.makedirs(self.gitdir) self.bare_git.init() + mp = self.manifest.manifestProject + ref_dir = mp.config.GetString('repo.reference') + + if ref_dir: + mirror_git = os.path.join(ref_dir, self.name + '.git') + repo_git = os.path.join(ref_dir, '.repo', 'projects', + self.relpath + '.git') + + if os.path.exists(mirror_git): + ref_dir = mirror_git + + elif os.path.exists(repo_git): + ref_dir = repo_git + + else: + ref_dir = None + + if ref_dir: + _lwrite(os.path.join(self.gitdir, 'objects/info/alternates'), + os.path.join(ref_dir, 'objects') + '\n') + if self.manifest.IsMirror: self.config.SetString('core.bare', 'true') else: diff --git a/repo b/repo index 3a545cc6..cb6f6349 100755 --- a/repo +++ b/repo @@ -123,6 +123,9 @@ group.add_option('-m', '--manifest-name', group.add_option('--mirror', dest='mirror', action='store_true', help='mirror the forrest') +group.add_option('--reference', + dest='reference', + help='location of mirror directory', metavar='DIR') # Tool group = init_optparse.add_option_group('repo Version options') diff --git a/subcmds/branches.py b/subcmds/branches.py index 0e3ab3c2..a4f8d360 100644 --- a/subcmds/branches.py +++ b/subcmds/branches.py @@ -136,7 +136,7 @@ is shown, then the branch appears in all projects. hdr('%c%c %-*s' % (current, published, width, name)) out.write(' |') - if in_cnt < project_cnt and (in_cnt == 1): + if in_cnt < project_cnt: fmt = out.write paths = [] if in_cnt < project_cnt - in_cnt: @@ -150,15 +150,17 @@ is shown, then the branch appears in all projects. for b in i.projects: have.add(b.project) for p in projects: - paths.append(p.relpath) + if not p in have: + paths.append(p.relpath) s = ' %s %s' % (type, ', '.join(paths)) if width + 7 + len(s) < 80: fmt(s) else: - out.nl() - fmt(' %s:' % type) + fmt(' %s:' % type) for p in paths: out.nl() - fmt(' %s' % p) + fmt(width*' ' + ' %s' % p) + else: + out.write(' in all projects') out.nl() diff --git a/subcmds/grep.py b/subcmds/grep.py index 4f714271..1cb5650b 100644 --- a/subcmds/grep.py +++ b/subcmds/grep.py @@ -204,7 +204,7 @@ contain a line that matches both expressions: else: out.project('--- project %s ---' % project.relpath) out.nl() - out.write(p.stderr) + out.write("%s", p.stderr) out.nl() continue have_match = True @@ -217,17 +217,17 @@ contain a line that matches both expressions: if have_rev and full_name: for line in r: rev, line = line.split(':', 1) - out.write(rev) + out.write("%s", rev) out.write(':') out.project(project.relpath) out.write('/') - out.write(line) + out.write("%s", line) out.nl() elif full_name: for line in r: out.project(project.relpath) out.write('/') - out.write(line) + out.write("%s", line) out.nl() else: for line in r: diff --git a/subcmds/init.py b/subcmds/init.py index cdbbfdf7..2ca4e163 100644 --- a/subcmds/init.py +++ b/subcmds/init.py @@ -40,6 +40,17 @@ current working directory. The optional -b argument can be used to select the manifest branch to checkout and use. If no branch is specified, master is assumed. +The optional -m argument can be used to specify an alternate manifest +to be used. If no manifest is specified, the manifest default.xml +will be used. + +The --reference option can be used to point to a directory that +has the content of a --mirror sync. This will make the working +directory use as much data as possible from the local reference +directory when fetching from the server. This will make the sync +go a lot faster by reducing data traffic on the network. + + Switching Manifest Branches --------------------------- @@ -76,7 +87,9 @@ to update the working directory files. g.add_option('--mirror', dest='mirror', action='store_true', help='mirror the forrest') - + g.add_option('--reference', + dest='reference', + help='location of mirror directory', metavar='DIR') # Tool g = p.add_option_group('repo Version options') @@ -132,6 +145,9 @@ to update the working directory files. r.ResetFetch() r.Save() + if opt.reference: + m.config.SetString('repo.reference', opt.reference) + if opt.mirror: if is_new: m.config.SetString('repo.mirror', 'true') @@ -162,7 +178,11 @@ to update the working directory files. syncbuf = SyncBuffer(m.config) m.Sync_LocalHalf(syncbuf) syncbuf.Finish() + + if isinstance(self.manifest, XmlManifest): + self._LinkManifest(opt.manifest_name) _ReloadManifest(self) + self._ApplyOptions(opt, is_new) if not self.manifest.InitBranch(): @@ -200,8 +220,9 @@ to update the working directory files. print '' print 'Your identity is: %s <%s>' % (name, email) - sys.stdout.write('is this correct [yes/no]? ') - if 'yes' == sys.stdin.readline().strip(): + sys.stdout.write('is this correct [y/n]? ') + a = sys.stdin.readline().strip() + if a in ('yes', 'y', 't', 'true'): break if name != mp.UserName: @@ -249,8 +270,6 @@ to update the working directory files. def Execute(self, opt, args): git_require(MIN_GIT_VERSION, fail=True) self._SyncManifest(opt) - if isinstance(self.manifest, XmlManifest): - self._LinkManifest(opt.manifest_name) if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror: self._ConfigureUser() diff --git a/subcmds/rebase.py b/subcmds/rebase.py new file mode 100644 index 00000000..e341296d --- /dev/null +++ b/subcmds/rebase.py @@ -0,0 +1,107 @@ +# +# Copyright (C) 2010 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +from command import Command +from git_command import GitCommand +from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB +from error import GitError + +class Rebase(Command): + common = True + helpSummary = "Rebase local branches on upstream branch" + helpUsage = """ +%prog {[...] | -i ...} +""" + helpDescription = """ +'%prog' uses git rebase to move local changes in the current topic branch to +the HEAD of the upstream history, useful when you have made commits in a topic +branch but need to incorporate new upstream changes "underneath" them. +""" + + def _Options(self, p): + p.add_option('-i', '--interactive', + dest="interactive", action="store_true", + help="interactive rebase (single project only)") + + p.add_option('-f', '--force-rebase', + dest='force_rebase', action='store_true', + help='Pass --force-rebase to git rebase') + p.add_option('--no-ff', + dest='no_ff', action='store_true', + help='Pass --no-ff to git rebase') + p.add_option('-q', '--quiet', + dest='quiet', action='store_true', + help='Pass --quiet to git rebase') + p.add_option('--autosquash', + dest='autosquash', action='store_true', + help='Pass --autosquash to git rebase') + p.add_option('--whitespace', + dest='whitespace', action='store', metavar='WS', + help='Pass --whitespace to git rebase') + + def Execute(self, opt, args): + all = self.GetProjects(args) + one_project = len(all) == 1 + + if opt.interactive and not one_project: + print >>sys.stderr, 'error: interactive rebase not supported with multiple projects' + return -1 + + for project in all: + cb = project.CurrentBranch + if not cb: + if one_project: + print >>sys.stderr, "error: project %s has a detatched HEAD" % project.relpath + return -1 + # ignore branches with detatched HEADs + continue + + upbranch = project.GetBranch(cb) + if not upbranch.LocalMerge: + if one_project: + print >>sys.stderr, "error: project %s does not track any remote branches" % project.relpath + return -1 + # ignore branches without remotes + continue + + args = ["rebase"] + + if opt.whitespace: + args.append('--whitespace=%s' % opt.whitespace) + + if opt.quiet: + args.append('--quiet') + + if opt.force_rebase: + args.append('--force-rebase') + + if opt.no_ff: + args.append('--no-ff') + + if opt.autosquash: + args.append('--autosquash') + + if opt.interactive: + args.append("-i") + + args.append(upbranch.LocalMerge) + + print >>sys.stderr, '# %s: rebasing %s -> %s' % \ + (project.relpath, cb, upbranch.LocalMerge) + + if GitCommand(project, args).Wait() != 0: + return -1 diff --git a/subcmds/smartsync.py b/subcmds/smartsync.py new file mode 100644 index 00000000..1edbd35b --- /dev/null +++ b/subcmds/smartsync.py @@ -0,0 +1,33 @@ +# +# Copyright (C) 2010 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from sync import Sync + +class Smartsync(Sync): + common = True + helpSummary = "Update working tree to the latest known good revision" + helpUsage = """ +%prog [...] +""" + helpDescription = """ +The '%prog' command is a shortcut for sync -s. +""" + + def _Options(self, p): + Sync._Options(self, p, show_smart=False) + + def Execute(self, opt, args): + opt.smart_sync = True + Sync.Execute(self, opt, args) diff --git a/subcmds/sync.py b/subcmds/sync.py index d89c2b8c..7b77388b 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -17,11 +17,19 @@ from optparse import SUPPRESS_HELP import os import re import shutil +import socket import subprocess import sys import time +import xmlrpclib + +try: + import threading as _threading +except ImportError: + import dummy_threading as _threading from git_command import GIT +from git_refs import R_HEADS from project import HEAD from project import Project from project import RemoteSpec @@ -32,6 +40,7 @@ from project import SyncBuffer from progress import Progress class Sync(Command, MirrorSafeCommand): + jobs = 1 common = True helpSummary = "Update working tree to the latest revision" helpUsage = """ @@ -57,6 +66,13 @@ back to the manifest revision. This option is especially helpful if the project is currently on a topic branch, but the manifest revision is temporarily needed. +The -s/--smart-sync option can be used to sync to a known good +build as specified by the manifest-server element in the current +manifest. + +The -f/--force-broken option can be used to proceed with syncing +other projects if a project sync fails. + SSH Connections --------------- @@ -87,7 +103,10 @@ later is required to fix a server side protocol bug. """ - def _Options(self, p): + def _Options(self, p, show_smart=True): + p.add_option('-f', '--force-broken', + dest='force_broken', action='store_true', + help="continue sync even if a project fails to sync") p.add_option('-l','--local-only', dest='local_only', action='store_true', help="only update working tree, don't fetch") @@ -97,6 +116,16 @@ later is required to fix a server side protocol bug. p.add_option('-d','--detach', dest='detach_head', action='store_true', help='detach projects back to manifest revision') + p.add_option('-q','--quiet', + dest='quiet', action='store_true', + help='be more quiet') + p.add_option('-j','--jobs', + dest='jobs', action='store', type='int', + help="number of projects to fetch simultaneously") + if show_smart: + p.add_option('-s', '--smart-sync', + dest='smart_sync', action='store_true', + help='smart sync using manifest from a known good build') g = p.add_option_group('repo Version options') g.add_option('--no-repo-verify', @@ -106,16 +135,55 @@ later is required to fix a server side protocol bug. dest='repo_upgraded', action='store_true', help=SUPPRESS_HELP) - def _Fetch(self, projects): + def _FetchHelper(self, opt, project, lock, fetched, pm, sem): + if not project.Sync_NetworkHalf(quiet=opt.quiet): + print >>sys.stderr, 'error: Cannot fetch %s' % project.name + if opt.force_broken: + print >>sys.stderr, 'warn: --force-broken, continuing to sync' + else: + sem.release() + sys.exit(1) + + lock.acquire() + fetched.add(project.gitdir) + pm.update() + lock.release() + sem.release() + + def _Fetch(self, projects, opt): fetched = set() pm = Progress('Fetching projects', len(projects)) - for project in projects: - pm.update() - if project.Sync_NetworkHalf(): - fetched.add(project.gitdir) - else: - print >>sys.stderr, 'error: Cannot fetch %s' % project.name - sys.exit(1) + + if self.jobs == 1: + for project in projects: + pm.update() + if project.Sync_NetworkHalf(quiet=opt.quiet): + fetched.add(project.gitdir) + else: + print >>sys.stderr, 'error: Cannot fetch %s' % project.name + if opt.force_broken: + print >>sys.stderr, 'warn: --force-broken, continuing to sync' + else: + sys.exit(1) + else: + threads = set() + lock = _threading.Lock() + sem = _threading.Semaphore(self.jobs) + for project in projects: + sem.acquire() + t = _threading.Thread(target = self._FetchHelper, + args = (opt, + project, + lock, + fetched, + pm, + sem)) + threads.add(t) + t.start() + + for t in threads: + t.join() + pm.end() for project in projects: project.bare_git.gc('--auto') @@ -140,32 +208,36 @@ later is required to fix a server side protocol bug. if not path: continue if path not in new_project_paths: - project = Project( - manifest = self.manifest, - name = path, - remote = RemoteSpec('origin'), - gitdir = os.path.join(self.manifest.topdir, - path, '.git'), - worktree = os.path.join(self.manifest.topdir, path), - relpath = path, - revisionExpr = 'HEAD', - revisionId = None) - if project.IsDirty(): - print >>sys.stderr, 'error: Cannot remove project "%s": \ + """If the path has already been deleted, we don't need to do it + """ + if os.path.exists(self.manifest.topdir + '/' + path): + project = Project( + manifest = self.manifest, + name = path, + remote = RemoteSpec('origin'), + gitdir = os.path.join(self.manifest.topdir, + path, '.git'), + worktree = os.path.join(self.manifest.topdir, path), + relpath = path, + revisionExpr = 'HEAD', + revisionId = None) + + if project.IsDirty(): + print >>sys.stderr, 'error: Cannot remove project "%s": \ uncommitted changes are present' % project.relpath - print >>sys.stderr, ' commit changes, then run sync again' - return -1 - else: - print >>sys.stderr, 'Deleting obsolete path %s' % project.worktree - shutil.rmtree(project.worktree) - # Try deleting parent subdirs if they are empty - dir = os.path.dirname(project.worktree) - while dir != self.manifest.topdir: - try: - os.rmdir(dir) - except OSError: - break - dir = os.path.dirname(dir) + print >>sys.stderr, ' commit changes, then run sync again' + return -1 + else: + print >>sys.stderr, 'Deleting obsolete path %s' % project.worktree + shutil.rmtree(project.worktree) + # Try deleting parent subdirs if they are empty + dir = os.path.dirname(project.worktree) + while dir != self.manifest.topdir: + try: + os.rmdir(dir) + except OSError: + break + dir = os.path.dirname(dir) new_project_paths.sort() fd = open(file_path, 'w') @@ -177,6 +249,8 @@ uncommitted changes are present' % project.relpath return 0 def Execute(self, opt, args): + if opt.jobs: + self.jobs = opt.jobs if opt.network_only and opt.detach_head: print >>sys.stderr, 'error: cannot combine -n and -d' sys.exit(1) @@ -184,6 +258,51 @@ uncommitted changes are present' % project.relpath print >>sys.stderr, 'error: cannot combine -n and -l' sys.exit(1) + if opt.smart_sync: + if not self.manifest.manifest_server: + print >>sys.stderr, \ + 'error: cannot smart sync: no manifest server defined in manifest' + sys.exit(1) + try: + server = xmlrpclib.Server(self.manifest.manifest_server) + p = self.manifest.manifestProject + b = p.GetBranch(p.CurrentBranch) + branch = b.merge + if branch.startswith(R_HEADS): + branch = branch[len(R_HEADS):] + + env = dict(os.environ) + if (env.has_key('TARGET_PRODUCT') and + env.has_key('TARGET_BUILD_VARIANT')): + target = '%s-%s' % (env['TARGET_PRODUCT'], + env['TARGET_BUILD_VARIANT']) + [success, manifest_str] = server.GetApprovedManifest(branch, target) + else: + [success, manifest_str] = server.GetApprovedManifest(branch) + + if success: + manifest_name = "smart_sync_override.xml" + manifest_path = os.path.join(self.manifest.manifestProject.worktree, + manifest_name) + try: + f = open(manifest_path, 'w') + try: + f.write(manifest_str) + finally: + f.close() + except IOError: + print >>sys.stderr, 'error: cannot write manifest to %s' % \ + manifest_path + sys.exit(1) + self.manifest.Override(manifest_name) + else: + print >>sys.stderr, 'error: %s' % manifest_str + sys.exit(1) + except socket.error: + print >>sys.stderr, 'error: cannot connect to manifest server %s' % ( + self.manifest.manifest_server) + sys.exit(1) + rp = self.manifest.repoProject rp.PreSync() @@ -194,7 +313,7 @@ uncommitted changes are present' % project.relpath _PostRepoUpgrade(self.manifest) if not opt.local_only: - mp.Sync_NetworkHalf() + mp.Sync_NetworkHalf(quiet=opt.quiet) if mp.HasChanges: syncbuf = SyncBuffer(mp.config) @@ -211,7 +330,7 @@ uncommitted changes are present' % project.relpath to_fetch.append(rp) to_fetch.extend(all) - fetched = self._Fetch(to_fetch) + fetched = self._Fetch(to_fetch, opt) _PostRepoFetch(rp, opt.no_repo_verify) if opt.network_only: # bail out now; the rest touches the working tree @@ -230,7 +349,7 @@ uncommitted changes are present' % project.relpath for project in all: if project.gitdir not in fetched: missing.append(project) - self._Fetch(missing) + self._Fetch(missing, opt) if self.manifest.IsMirror: # bail out now, we have no working tree @@ -258,6 +377,9 @@ def _ReloadManifest(cmd): if old.__class__ != new.__class__: print >>sys.stderr, 'NOTICE: manifest format has changed ***' new.Upgrade_Local(old) + else: + if new.notice: + print new.notice def _PostRepoUpgrade(manifest): for project in manifest.projects.values(): diff --git a/subcmds/upload.py b/subcmds/upload.py index 2ab6a484..20822096 100644 --- a/subcmds/upload.py +++ b/subcmds/upload.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import copy import re import sys @@ -20,6 +21,17 @@ from command import InteractiveCommand from editor import Editor from error import UploadError +UNUSUAL_COMMIT_THRESHOLD = 5 + +def _ConfirmManyUploads(multiple_branches=False): + if multiple_branches: + print "ATTENTION: One or more branches has an unusually high number of commits." + else: + print "ATTENTION: You are uploading an unusually high number of commits." + print "YOU PROBABLY DO NOT MEAN TO DO THIS. (Did you rebase across branches?)" + answer = raw_input("If you are sure you intend to do this, type 'yes': ").strip() + return answer == "yes" + def _die(fmt, *args): msg = fmt % args print >>sys.stderr, 'error: %s' % msg @@ -35,7 +47,7 @@ class Upload(InteractiveCommand): common = True helpSummary = "Upload changes for code review" helpUsage=""" -%prog [--re --cc] {[]... | --replace } +%prog [--re --cc] []... """ helpDescription = """ The '%prog' command is used to send changes to the Gerrit Code @@ -55,12 +67,6 @@ added to the respective list of users, and emails are sent to any new users. Users passed as --reviewers must already be registered with the code review system, or the upload will fail. -If the --replace option (deprecated) is passed the user can designate -which existing change(s) in Gerrit match up to the commits in the -branch being uploaded. For each matched pair of change,commit the -commit will be added as a new patch set, completely replacing the -set of files and description associated with the change in Gerrit. - Configuration ------------- @@ -72,6 +78,19 @@ to "true" then repo will assume you always answer "y" at the prompt, and will not prompt you further. If it is set to "false" then repo will assume you always answer "n", and will abort. +review.URL.autocopy: + +To automatically copy a user or mailing list to all uploaded reviews, +you can set a per-project or global Git option to do so. Specifically, +review.URL.autocopy can be set to a comma separated list of reviewers +who you always want copied on all uploads with a non-empty --re +argument. + +review.URL.username: + +Override the username used to connect to Gerrit Code Review. +By default the local part of the email address is used. + The URL must match the review URL listed in the manifest XML file, or in the .git/config within the project. For example: @@ -81,6 +100,7 @@ or in the .git/config within the project. For example: [review "http://review.example.com/"] autoupload = true + autocopy = johndoe@company.com,my-team-alias@company.com References ---------- @@ -90,9 +110,9 @@ Gerrit Code Review: http://code.google.com/p/gerrit/ """ def _Options(self, p): - p.add_option('--replace', - dest='replace', action='store_true', - help='Upload replacement patchsets from this branch (deprecated)') + p.add_option('-t', + dest='auto_topic', action='store_true', + help='Send local branch name to Gerrit Code Review') p.add_option('--re', '--reviewers', type='string', action='append', dest='reviewers', help='Request reviews from these people.') @@ -100,7 +120,7 @@ Gerrit Code Review: http://code.google.com/p/gerrit/ type='string', action='append', dest='cc', help='Also send email to these email addresses.') - def _SingleBranch(self, branch, people): + def _SingleBranch(self, opt, branch, people): project = branch.project name = branch.name remote = project.GetBranch(name).remote @@ -129,11 +149,15 @@ Gerrit Code Review: http://code.google.com/p/gerrit/ answer = answer in ('y', 'Y', 'yes', '1', 'true', 't') if answer: - self._UploadAndReport([branch], people) + if len(branch.commits) > UNUSUAL_COMMIT_THRESHOLD: + answer = _ConfirmManyUploads() + + if answer: + self._UploadAndReport(opt, [branch], people) else: _die("upload aborted by user") - def _MultipleBranches(self, pending, people): + def _MultipleBranches(self, opt, pending, people): projects = {} branches = {} @@ -192,7 +216,30 @@ Gerrit Code Review: http://code.google.com/p/gerrit/ todo.append(branch) if not todo: _die("nothing uncommented for upload") - self._UploadAndReport(todo, people) + + many_commits = False + for branch in todo: + if len(branch.commits) > UNUSUAL_COMMIT_THRESHOLD: + many_commits = True + break + if many_commits: + if not _ConfirmManyUploads(multiple_branches=True): + _die("upload aborted by user") + + self._UploadAndReport(opt, todo, people) + + def _AppendAutoCcList(self, branch, people): + """ + Appends the list of users in the CC list in the git project's config if a + non-empty reviewer list was found. + """ + + name = branch.name + project = branch.project + key = 'review.%s.autocopy' % project.GetBranch(name).remote.review + raw_list = project.config.GetString(key) + if not raw_list is None and len(people[0]) > 0: + people[1].extend([entry.strip() for entry in raw_list.split(',')]) def _FindGerritChange(self, branch): last_pub = branch.project.WasPublished(branch.name) @@ -206,66 +253,29 @@ Gerrit Code Review: http://code.google.com/p/gerrit/ except: return "" - def _ReplaceBranch(self, project, people): - branch = project.CurrentBranch - if not branch: - print >>sys.stdout, "no branches ready for upload" - return - branch = project.GetUploadableBranch(branch) - if not branch: - print >>sys.stdout, "no branches ready for upload" - return - - script = [] - script.append('# Replacing from branch %s' % branch.name) - - if len(branch.commits) == 1: - change = self._FindGerritChange(branch) - script.append('[%-6s] %s' % (change, branch.commits[0])) - else: - for commit in branch.commits: - script.append('[ ] %s' % commit) - - script.append('') - script.append('# Insert change numbers in the brackets to add a new patch set.') - script.append('# To create a new change record, leave the brackets empty.') - - script = Editor.EditString("\n".join(script)).split("\n") - - change_re = re.compile(r'^\[\s*(\d{1,})\s*\]\s*([0-9a-f]{1,}) .*$') - to_replace = dict() - full_hashes = branch.unabbrev_commits - - for line in script: - m = change_re.match(line) - if m: - c = m.group(1) - f = m.group(2) - try: - f = full_hashes[f] - except KeyError: - print 'fh = %s' % full_hashes - print >>sys.stderr, "error: commit %s not found" % f - sys.exit(1) - if c in to_replace: - print >>sys.stderr,\ - "error: change %s cannot accept multiple commits" % c - sys.exit(1) - to_replace[c] = f - - if not to_replace: - print >>sys.stderr, "error: no replacements specified" - print >>sys.stderr, " use 'repo upload' without --replace" - sys.exit(1) - - branch.replace_changes = to_replace - self._UploadAndReport([branch], people) - - def _UploadAndReport(self, todo, people): + def _UploadAndReport(self, opt, todo, original_people): have_errors = False for branch in todo: try: - branch.UploadForReview(people) + people = copy.deepcopy(original_people) + self._AppendAutoCcList(branch, people) + + # Check if there are local changes that may have been forgotten + if branch.project.HasChanges(): + key = 'review.%s.autoupload' % branch.project.remote.review + answer = branch.project.config.GetBoolean(key) + + # if they want to auto upload, let's not ask because it could be automated + if answer is None: + sys.stdout.write('Uncommitted changes in ' + branch.project.name + ' (did you forget to amend?). Continue uploading? (y/n) ') + a = sys.stdin.readline().strip().lower() + if a not in ('y', 'yes', 't', 'true', 'on'): + print >>sys.stderr, "skipping upload" + branch.uploaded = False + branch.error = 'User aborted' + continue + + branch.UploadForReview(people, auto_topic=opt.auto_topic) branch.uploaded = True except UploadError, e: branch.error = e @@ -309,14 +319,6 @@ Gerrit Code Review: http://code.google.com/p/gerrit/ cc = _SplitEmails(opt.cc) people = (reviewers,cc) - if opt.replace: - if len(project_list) != 1: - print >>sys.stderr, \ - 'error: --replace requires exactly one project' - sys.exit(1) - self._ReplaceBranch(project_list[0], people) - return - for project in project_list: avail = project.GetUploadableBranches() if avail: @@ -325,6 +327,6 @@ Gerrit Code Review: http://code.google.com/p/gerrit/ if not pending: print >>sys.stdout, "no branches ready for upload" elif len(pending) == 1 and len(pending[0][1]) == 1: - self._SingleBranch(pending[0][1][0], people) + self._SingleBranch(opt, pending[0][1][0], people) else: - self._MultipleBranches(pending, people) + self._MultipleBranches(opt, pending, people)