Change repo sync to be more friendly when updating the tree

We now try to sync all projects that can be done safely first, before
we start rebasing user commits over the upstream.  This has the nice
effect of making the local tree as close to the upstream as possible
before the user has to start resolving merge conflicts, as that extra
information in other projects may aid in the conflict resolution.

Informational output is buffered and delayed until calculation for
all projects has been done, so that the user gets one concise list
of notice messages, rather than it interrupting the progress meter.

Fast-forward output is now prefixed with the project header, so the
user can see which project that update is taking place in, and make
some relation of the diffstat back to the project name.

Rebase output is now prefixed with the project header, so that if
the rebase fails, the user can see which project we were operating
on and can try to address the failure themselves.

Since rebase sits on a detached HEAD, we now look for an in-progress
rebase during sync, so we can alert the user that the given project
is in a state we cannot handle.

Signed-off-by: Shawn O. Pearce <sop@google.com>
This commit is contained in:
Shawn O. Pearce 2009-04-16 11:21:18 -07:00
parent 48244781c2
commit 350cde4c4b
4 changed files with 175 additions and 62 deletions

View File

@ -100,6 +100,9 @@ class Coloring(object):
else: else:
self._on = False self._on = False
def redirect(self, out):
self._out = out
@property @property
def is_on(self): def is_on(self):
return self._on return self._on

View File

@ -38,14 +38,6 @@ def _error(fmt, *args):
msg = fmt % args msg = fmt % args
print >>sys.stderr, 'error: %s' % msg print >>sys.stderr, 'error: %s' % msg
def _warn(fmt, *args):
msg = fmt % args
print >>sys.stderr, 'warn: %s' % msg
def _info(fmt, *args):
msg = fmt % args
print >>sys.stderr, 'info: %s' % msg
def not_rev(r): def not_rev(r):
return '^' + r return '^' + r
@ -576,13 +568,9 @@ class Project(object):
for file in self.copyfiles: for file in self.copyfiles:
file._Copy() file._Copy()
def Sync_LocalHalf(self, detach_head=False): def Sync_LocalHalf(self, syncbuf):
"""Perform only the local IO portion of the sync process. """Perform only the local IO portion of the sync process.
Network access is not required. Network access is not required.
Return:
True: the sync was successful
False: the sync requires user input
""" """
self._InitWorkTree() self._InitWorkTree()
self.CleanPublishedCache() self.CleanPublishedCache()
@ -597,19 +585,25 @@ class Project(object):
branch = self.CurrentBranch branch = self.CurrentBranch
if branch is None or detach_head: if branch is None or syncbuf.detach_head:
# Currently on a detached HEAD. The user is assumed to # Currently on a detached HEAD. The user is assumed to
# not have any local modifications worth worrying about. # not have any local modifications worth worrying about.
# #
if os.path.exists(os.path.join(self.worktree, '.dotest')) \
or os.path.exists(os.path.join(self.worktree, '.git', 'rebase-apply')):
syncbuf.fail(self, _PriorSyncFailedError())
return
lost = self._revlist(not_rev(rev), HEAD) lost = self._revlist(not_rev(rev), HEAD)
if lost: if lost:
_info("[%s] Discarding %d commits", self.name, len(lost)) syncbuf.info(self, "discarding %d commits", len(lost))
try: try:
self._Checkout(rev, quiet=True) self._Checkout(rev, quiet=True)
except GitError: except GitError, e:
return False syncbuf.fail(self, e)
return
self._CopyFiles() self._CopyFiles()
return True return
branch = self.GetBranch(branch) branch = self.GetBranch(branch)
merge = branch.LocalMerge merge = branch.LocalMerge
@ -618,16 +612,16 @@ class Project(object):
# The current branch has no tracking configuration. # The current branch has no tracking configuration.
# Jump off it to a deatched HEAD. # Jump off it to a deatched HEAD.
# #
_info("[%s] Leaving %s" syncbuf.info(self,
" (does not track any upstream)", "leaving %s; does not track upstream",
self.name,
branch.name) branch.name)
try: try:
self._Checkout(rev, quiet=True) self._Checkout(rev, quiet=True)
except GitError: except GitError, e:
return False syncbuf.fail(self, e)
return
self._CopyFiles() self._CopyFiles()
return True return
upstream_gain = self._revlist(not_rev(HEAD), rev) upstream_gain = self._revlist(not_rev(HEAD), rev)
pub = self.WasPublished(branch.name) pub = self.WasPublished(branch.name)
@ -639,25 +633,24 @@ class Project(object):
# commits are not yet merged upstream. We do not want # commits are not yet merged upstream. We do not want
# to rewrite the published commits so we punt. # to rewrite the published commits so we punt.
# #
_info("[%s] Branch %s is published," syncbuf.info(self,
" but is now %d commits behind.", "branch %s is published but is now %d commits behind",
self.name, branch.name, len(upstream_gain)) branch.name,
_info("[%s] Consider merging or rebasing the" len(upstream_gain))
" unpublished commits.", self.name) syncbuf.info(self, "consider merging or rebasing the unpublished commits")
return True return
elif upstream_gain: elif upstream_gain:
# We can fast-forward safely. # We can fast-forward safely.
# #
try: def _doff():
self._FastForward(rev) self._FastForward(rev)
except GitError:
return False
self._CopyFiles() self._CopyFiles()
return True syncbuf.later1(self, _doff)
return
else: else:
# Trivially no changes in the upstream. # Trivially no changes in the upstream.
# #
return True return
if merge == rev: if merge == rev:
try: try:
@ -672,8 +665,7 @@ class Project(object):
# and pray that the old upstream also wasn't in the habit # and pray that the old upstream also wasn't in the habit
# of rebasing itself. # of rebasing itself.
# #
_info("[%s] Manifest switched from %s to %s", syncbuf.info(self, "manifest switched %s...%s", merge, rev)
self.name, merge, rev)
old_merge = merge old_merge = merge
if rev == old_merge: if rev == old_merge:
@ -684,19 +676,19 @@ class Project(object):
if not upstream_lost and not upstream_gain: if not upstream_lost and not upstream_gain:
# Trivially no changes caused by the upstream. # Trivially no changes caused by the upstream.
# #
return True return
if self.IsDirty(consider_untracked=False): if self.IsDirty(consider_untracked=False):
_warn('[%s] commit (or discard) uncommitted changes' syncbuf.fail(self, _DirtyError())
' before sync', self.name) return
return False
if upstream_lost: if upstream_lost:
# Upstream rebased. Not everything in HEAD # Upstream rebased. Not everything in HEAD
# may have been caused by the user. # may have been caused by the user.
# #
_info("[%s] Discarding %d commits removed from upstream", syncbuf.info(self,
self.name, len(upstream_lost)) "discarding %d commits removed from upstream",
len(upstream_lost))
branch.remote = rem branch.remote = rem
branch.merge = self.revision branch.merge = self.revision
@ -704,23 +696,22 @@ class Project(object):
my_changes = self._revlist(not_rev(old_merge), HEAD) my_changes = self._revlist(not_rev(old_merge), HEAD)
if my_changes: if my_changes:
try: def _dorebase():
self._Rebase(upstream = old_merge, onto = rev) self._Rebase(upstream = old_merge, onto = rev)
except GitError: self._CopyFiles()
return False syncbuf.later2(self, _dorebase)
elif upstream_lost: elif upstream_lost:
try: try:
self._ResetHard(rev) self._ResetHard(rev)
except GitError:
return False
else:
try:
self._FastForward(rev)
except GitError:
return False
self._CopyFiles() self._CopyFiles()
return True except GitError, e:
syncbuf.fail(self, e)
return
else:
def _doff():
self._FastForward(rev)
self._CopyFiles()
syncbuf.later1(self, _doff)
def AddCopyFile(self, src, dest, absdest): def AddCopyFile(self, src, dest, absdest):
# dest should already be an absolute path, but src is project relative # dest should already be an absolute path, but src is project relative
@ -1212,6 +1203,113 @@ class Project(object):
return runner return runner
class _PriorSyncFailedError(Exception):
def __str__(self):
return 'prior sync failed; rebase still in progress'
class _DirtyError(Exception):
def __str__(self):
return 'contains uncommitted changes'
class _InfoMessage(object):
def __init__(self, project, text):
self.project = project
self.text = text
def Print(self, syncbuf):
syncbuf.out.info('%s/: %s', self.project.relpath, self.text)
syncbuf.out.nl()
class _Failure(object):
def __init__(self, project, why):
self.project = project
self.why = why
def Print(self, syncbuf):
syncbuf.out.fail('error: %s/: %s',
self.project.relpath,
str(self.why))
syncbuf.out.nl()
class _Later(object):
def __init__(self, project, action):
self.project = project
self.action = action
def Run(self, syncbuf):
out = syncbuf.out
out.project('project %s/', self.project.relpath)
out.nl()
try:
self.action()
out.nl()
return True
except GitError, e:
out.nl()
return False
class _SyncColoring(Coloring):
def __init__(self, config):
Coloring.__init__(self, config, 'reposync')
self.project = self.printer('header', attr = 'bold')
self.info = self.printer('info')
self.fail = self.printer('fail', fg='red')
class SyncBuffer(object):
def __init__(self, config, detach_head=False):
self._messages = []
self._failures = []
self._later_queue1 = []
self._later_queue2 = []
self.out = _SyncColoring(config)
self.out.redirect(sys.stderr)
self.detach_head = detach_head
self.clean = True
def info(self, project, fmt, *args):
self._messages.append(_InfoMessage(project, fmt % args))
def fail(self, project, err=None):
self._failures.append(_Failure(project, err))
self.clean = False
def later1(self, project, what):
self._later_queue1.append(_Later(project, what))
def later2(self, project, what):
self._later_queue2.append(_Later(project, what))
def Finish(self):
self._PrintMessages()
self._RunLater()
self._PrintMessages()
return self.clean
def _RunLater(self):
for q in ['_later_queue1', '_later_queue2']:
if not self._RunQueue(q):
return
def _RunQueue(self, queue):
for m in getattr(self, queue):
if not m.Run(self):
self.clean = False
return False
setattr(self, queue, [])
return True
def _PrintMessages(self):
for m in self._messages:
m.Print(self)
for m in self._failures:
m.Print(self)
self._messages = []
self._failures = []
class MetaProject(Project): class MetaProject(Project):
"""A special project housed under .repo. """A special project housed under .repo.
""" """

View File

@ -20,6 +20,7 @@ from color import Coloring
from command import InteractiveCommand, MirrorSafeCommand from command import InteractiveCommand, MirrorSafeCommand
from error import ManifestParseError from error import ManifestParseError
from remote import Remote from remote import Remote
from project import SyncBuffer
from git_command import git, MIN_GIT_VERSION from git_command import git, MIN_GIT_VERSION
class Init(InteractiveCommand, MirrorSafeCommand): class Init(InteractiveCommand, MirrorSafeCommand):
@ -129,7 +130,10 @@ default.xml will be used.
print >>sys.stderr, 'fatal: cannot obtain manifest %s' % r.url print >>sys.stderr, 'fatal: cannot obtain manifest %s' % r.url
sys.exit(1) sys.exit(1)
m.Sync_LocalHalf() syncbuf = SyncBuffer(m.config)
m.Sync_LocalHalf(syncbuf)
syncbuf.Finish()
if is_new or m.CurrentBranch is None: if is_new or m.CurrentBranch is None:
if not m.StartBranch('default'): if not m.StartBranch('default'):
print >>sys.stderr, 'fatal: cannot create default in manifest' print >>sys.stderr, 'fatal: cannot create default in manifest'

View File

@ -24,6 +24,7 @@ from project import HEAD
from command import Command, MirrorSafeCommand from command import Command, MirrorSafeCommand
from error import RepoChangedException, GitError from error import RepoChangedException, GitError
from project import R_HEADS from project import R_HEADS
from project import SyncBuffer
from progress import Progress from progress import Progress
class Sync(Command, MirrorSafeCommand): class Sync(Command, MirrorSafeCommand):
@ -112,7 +113,9 @@ revision is temporarily needed.
return return
if mp.HasChanges: if mp.HasChanges:
if not mp.Sync_LocalHalf(): syncbuf = SyncBuffer(mp.config)
mp.Sync_LocalHalf(syncbuf)
if not syncbuf.Finish():
sys.exit(1) sys.exit(1)
self.manifest._Unload() self.manifest._Unload()
@ -123,14 +126,17 @@ revision is temporarily needed.
missing.append(project) missing.append(project)
self._Fetch(*missing) self._Fetch(*missing)
syncbuf = SyncBuffer(mp.config,
detach_head = opt.detach_head)
pm = Progress('Syncing work tree', len(all)) pm = Progress('Syncing work tree', len(all))
for project in all: for project in all:
pm.update() pm.update()
if project.worktree: if project.worktree:
if not project.Sync_LocalHalf( project.Sync_LocalHalf(syncbuf)
detach_head=opt.detach_head):
sys.exit(1)
pm.end() pm.end()
print >>sys.stderr
if not syncbuf.Finish():
sys.exit(1)
def _PostRepoUpgrade(manifest): def _PostRepoUpgrade(manifest):
@ -143,7 +149,9 @@ def _PostRepoFetch(rp, no_repo_verify=False, verbose=False):
print >>sys.stderr, 'info: A new version of repo is available' print >>sys.stderr, 'info: A new version of repo is available'
print >>sys.stderr, '' print >>sys.stderr, ''
if no_repo_verify or _VerifyTag(rp): if no_repo_verify or _VerifyTag(rp):
if not rp.Sync_LocalHalf(): syncbuf = SyncBuffer(rp.config)
rp.Sync_LocalHalf(syncbuf)
if not syncbuf.Finish():
sys.exit(1) sys.exit(1)
print >>sys.stderr, 'info: Restarting repo with latest version' print >>sys.stderr, 'info: Restarting repo with latest version'
raise RepoChangedException(['--repo-upgraded']) raise RepoChangedException(['--repo-upgraded'])