Revert "Save cookies back to jar when fetching clone.bundle"

This reverts commit 4abf8e6ef8.

The curl process for updating the cookie file is not atomic.  When
fetching many bundles in parallel, we can sometimes corrupt the file
causing it to be cleared.  Since users should manage gitcookies on
their own, leave it read-only.

Bug: https://crbug.com/gerrit/12300
Change-Id: Id472c99b197bc4cf8533c649f8881509f38643c1
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/254092
Reviewed-by: David Pursehouse <dpursehouse@collab.net>
Tested-by: Mike Frysinger <vapier@google.com>

docs/release-process.md

@@ -161,7 +161,91 @@ You can create a short changelog using the command:
 $ git log --format="%h (%aN) %s" --no-merges origin/stable..$r
 ```
 
+## Project References
 
+Here's a table showing the relationship of major tools, their EOL dates, and
+their status in Ubuntu & Debian.
+Those distros tend to be good indicators of how long we need to support things.
+
+Things in bold indicate stuff to take note of, but does not guarantee that we
+still support them.
+Things in italics are things we used to care about but probably don't anymore.
+
+|   Date   |     EOL      | [Git][rel-g] | [Python][rel-p] | [Ubuntu][rel-u] / [Debian][rel-d] | Git | Python |
+|:--------:|:------------:|--------------|-----------------|-----------------------------------|-----|--------|
+| Oct 2008 | *Oct 2013*   |              | 2.6.0           | *10.04 Lucid* - 10.10 Maverick / *Squeeze* |
+| Dec 2008 | *Feb 2009*   |              | 3.0.0           |
+| Feb 2009 | *Mar 2012*   |              |                 | Debian 5 Lenny       | 1.5.6.5 | 2.5.2 |
+| Jun 2009 | *Jun 2016*   |              | 3.1.0           | *10.04 Lucid* - 10.10 Maverick / *Squeeze* |
+| Feb 2010 | *Oct 2012*   | 1.7.0        |                 | *10.04 Lucid* - *12.04 Precise* - 12.10 Quantal |
+| Apr 2010 | *Apr 2015*   |              |                 | *10.04 Lucid*        | 1.7.0.4  | 2.6.5 3.1.2  |
+| Jul 2010 | *Dec 2019*   |              | **2.7.0**       | 11.04 Natty - **<current>** |
+| Oct 2010 |              |              |                 | 10.10 Maverick       | 1.7.1    | 2.6.6 3.1.3  |
+| Feb 2011 | *Feb 2016*   |              |                 | Debian 6 Squeeze     | 1.7.2.5  | 2.6.6 3.1.3  |
+| Apr 2011 |              |              |                 | 11.04 Natty          | 1.7.4    | 2.7.1 3.2.0  |
+| Oct 2011 | *Feb 2016*   |              | 3.2.0           | 11.04 Natty - 12.10 Quantal |
+| Oct 2011 |              |              |                 | 11.10 Ocelot         | 1.7.5.4  | 2.7.2 3.2.2  |
+| Apr 2012 | *Apr 2019*   |              |                 | *12.04 Precise*      | 1.7.9.5  | 2.7.3 3.2.3  |
+| Sep 2012 | *Sep 2017*   |              | 3.3.0           | 13.04 Raring - 13.10 Saucy |
+| Oct 2012 | *Dec 2014*   | 1.8.0        |                 | 13.04 Raring - 13.10 Saucy |
+| Oct 2012 |              |              |                 | 12.10 Quantal        | 1.7.10.4 | 2.7.3 3.2.3  |
+| Apr 2013 |              |              |                 | 13.04 Raring         | 1.8.1.2  | 2.7.4 3.3.1  |
+| May 2013 | *May 2018*   |              |                 | Debian 7 Wheezy      | 1.7.10.4 | 2.7.3 3.2.3  |
+| Oct 2013 |              |              |                 | 13.10 Saucy          | 1.8.3.2  | 2.7.5 3.3.2  |
+| Feb 2014 | *Dec 2014*   | **1.9.0**    |                 | **14.04 Trusty** |
+| Mar 2014 | *Mar 2019*   |              | **3.4.0**       | **14.04 Trusty** - 15.10 Wily / **Jessie** |
+| Apr 2014 | **Apr 2022** |              |                 | **14.04 Trusty**     | 1.9.1    | 2.7.5 3.4.0  |
+| May 2014 | *Dec 2014*   | 2.0.0        |
+| Aug 2014 | *Dec 2014*   | **2.1.0**    |                 | 14.10 Utopic - 15.04 Vivid / **Jessie** |
+| Oct 2014 |              |              |                 | 14.10 Utopic         | 2.1.0    | 2.7.8 3.4.2  |
+| Nov 2014 | *Sep 2015*   | 2.2.0        |
+| Feb 2015 | *Sep 2015*   | 2.3.0        |
+| Apr 2015 | *May 2017*   | 2.4.0        |
+| Apr 2015 | **Jun 2020** |              |                 | **Debian 8 Jessie**  | 2.1.4    | 2.7.9 3.4.2  |
+| Apr 2015 |              |              |                 | 15.04 Vivid          | 2.1.4    | 2.7.9 3.4.3  |
+| Jul 2015 | *May 2017*   | 2.5.0        |                 | 15.10 Wily |
+| Sep 2015 | *May 2017*   | 2.6.0        |
+| Sep 2015 | **Sep 2020** |              | **3.5.0**       | **16.04 Xenial** - 17.04 Zesty / **Stretch** |
+| Oct 2015 |              |              |                 | 15.10 Wily           | 2.5.0    | 2.7.9 3.4.3  |
+| Jan 2016 | *Jul 2017*   | **2.7.0**    |                 | **16.04 Xenial** |
+| Mar 2016 | *Jul 2017*   | 2.8.0        |
+| Apr 2016 | **Apr 2024** |              |                 | **16.04 Xenial**     | 2.7.4    | 2.7.11 3.5.1 |
+| Jun 2016 | *Jul 2017*   | 2.9.0        |                 | 16.10 Yakkety |
+| Sep 2016 | *Sep 2017*   | 2.10.0       |
+| Oct 2016 |              |              |                 | 16.10 Yakkety        | 2.9.3    | 2.7.11 3.5.1 |
+| Nov 2016 | *Sep 2017*   | **2.11.0**   |                 | 17.04 Zesty / **Stretch** |
+| Dec 2016 | **Dec 2021** |              | **3.6.0**       | 17.10 Artful - **18.04 Bionic** - 18.10 Cosmic |
+| Feb 2017 | *Sep 2017*   | 2.12.0       |
+| Apr 2017 |              |              |                 | 17.04 Zesty          | 2.11.0   | 2.7.13 3.5.3 |
+| May 2017 | *May 2018*   | 2.13.0       |
+| Jun 2017 | **Jun 2022** |              |                 | **Debian 9 Stretch** | 2.11.0   | 2.7.13 3.5.3 |
+| Aug 2017 | *Dec 2019*   | 2.14.0       |                 | 17.10 Artful |
+| Oct 2017 | *Dec 2019*   | 2.15.0       |
+| Oct 2017 |              |              |                 | 17.10 Artful         | 2.14.1   | 2.7.14 3.6.3 |
+| Jan 2018 | *Dec 2019*   | 2.16.0       |
+| Apr 2018 | *Dec 2019*   | 2.17.0       |                 | **18.04 Bionic**     |
+| Apr 2018 | **Apr 2028** |              |                 | **18.04 Bionic**     | 2.17.0   | 2.7.15 3.6.5 |
+| Jun 2018 | *Dec 2019*   | 2.18.0       |
+| Jun 2018 | **Jun 2023** |              | 3.7.0           | 19.04 Disco - **20.04 Focal** / **Buster** |
+| Sep 2018 | *Dec 2019*   | 2.19.0       |                 | 18.10 Cosmic |
+| Oct 2018 |              |              |                 | 18.10 Cosmic         | 2.19.1   | 2.7.15 3.6.6 |
+| Dec 2018 | *Dec 2019*   | **2.20.0**   |                 | 19.04 Disco / **Buster** |
+| Feb 2019 | *Dec 2019*   | 2.21.0       |
+| Apr 2019 |              |              |                 | 19.04 Disco          | 2.20.1   | 2.7.16 3.7.3 |
+| Jun 2019 |              | 2.22.0       |
+| Jul 2019 | **Jul 2024** |              |                 | **Debian 10 Buster** | 2.20.1   | 2.7.16 3.7.3 |
+| Aug 2019 |              | 2.23.0       |
+| Oct 2019 | **Oct 2024** |              | 3.8.0           |
+| Oct 2019 |              |              |                 | 19.10 Eoan           | 2.20.1   | 2.7.17 3.7.5 |
+| Nov 2019 |              | 2.24.0       |
+| Jan 2020 |              | 2.25.0       |                 | **20.04 Focal** |
+| Apr 2020 | **Apr 2030** |              |                 | **20.04 Focal**      | 2.25.0   | 2.7.17 3.7.5 |
+
+
+[rel-d]: https://en.wikipedia.org/wiki/Debian_version_history
+[rel-g]: https://en.wikipedia.org/wiki/Git#Releases
+[rel-p]: https://en.wikipedia.org/wiki/History_of_Python#Table_of_versions
+[rel-u]: https://en.wikipedia.org/wiki/Ubuntu_version_history#Table_of_versions
 [example announcement]: https://groups.google.com/d/topic/repo-discuss/UGBNismWo1M/discussion
 [repo-discuss@googlegroups.com]: https://groups.google.com/forum/#!forum/repo-discuss
 [go/repo-release]: https://goto.google.com/repo-release

git_command.py

@@ -28,8 +28,17 @@ from repo_trace import REPO_TRACE, IsTrace, Trace
 from wrapper import Wrapper
 
 GIT = 'git'
-# Should keep in sync with the "repo" launcher file.
-MIN_GIT_VERSION = (2, 10, 2)
+# NB: These do not need to be kept in sync with the repo launcher script.
+# These may be much newer as it allows the repo launcher to roll between
+# different repo releases while source versions might require a newer git.
+#
+# The soft version is when we start warning users that the version is old and
+# we'll be dropping support for it.  We'll refuse to work with versions older
+# than the hard version.
+#
+# git-1.7 is in (EOL) Ubuntu Precise.  git-1.9 is in Ubuntu Trusty.
+MIN_GIT_VERSION_SOFT = (1, 9, 1)
+MIN_GIT_VERSION_HARD = (1, 7, 2)
 GIT_DIR = 'GIT_DIR'
 
 LAST_GITDIR = None

main.py

@@ -255,27 +255,41 @@ class _Repo(object):
     return result
 
 
-def _CheckWrapperVersion(ver, repo_path):
+def _CheckWrapperVersion(ver_str, repo_path):
+  """Verify the repo launcher is new enough for this checkout.
+
+  Args:
+    ver_str: The version string passed from the repo launcher when it ran us.
+    repo_path: The path to the repo launcher that loaded us.
+  """
+  # Refuse to work with really old wrapper versions.  We don't test these,
+  # so might as well require a somewhat recent sane version.
+  # v1.15 of the repo launcher was released in ~Mar 2012.
+  MIN_REPO_VERSION = (1, 15)
+  min_str = '.'.join(str(x) for x in MIN_REPO_VERSION)
+
   if not repo_path:
     repo_path = '~/bin/repo'
 
-  if not ver:
+  if not ver_str:
     print('no --wrapper-version argument', file=sys.stderr)
     sys.exit(1)
 
+  # Pull out the version of the repo launcher we know about to compare.
   exp = Wrapper().VERSION
-  ver = tuple(map(int, ver.split('.')))
-  if len(ver) == 1:
-    ver = (0, ver[0])
+  ver = tuple(map(int, ver_str.split('.')))
 
   exp_str = '.'.join(map(str, exp))
-  if exp[0] > ver[0] or ver < (0, 4):
+  if ver < MIN_REPO_VERSION:
     print("""
-!!! A new repo command (%5s) is available.    !!!
-!!! You must upgrade before you can continue:   !!!
+repo: error:
+!!! Your version of repo %s is too old.
+!!! We need at least version %s.
+!!! A new repo command (%s) is available.
+!!! You must upgrade before you can continue:
 
     cp %s %s
-""" % (exp_str, WrapperPath(), repo_path), file=sys.stderr)
+""" % (ver_str, min_str, exp_str, WrapperPath(), repo_path), file=sys.stderr)
     sys.exit(1)
 
   if exp > ver:

manifest_xml.py

@@ -985,11 +985,13 @@ class XmlManifest(object):
     # Assume paths might be used on case-insensitive filesystems.
     path = path.lower()
 
-    # We don't really need to reject '.' here, but there shouldn't really be a
-    # need to ever use it, so no need to accept it either.
-    for part in set(path.split(os.path.sep)):
-      if part in {'.', '..', '.git'} or part.startswith('.repo'):
-        return 'bad component: %s' % (part,)
+    # Some people use src="." to create stable links to projects.  Lets allow
+    # that but reject all other uses of "." to keep things simple.
+    parts = path.split(os.path.sep)
+    if parts != ['.']:
+      for part in set(parts):
+        if part in {'.', '..', '.git'} or part.startswith('.repo'):
+          return 'bad component: %s' % (part,)
 
     if not symlink and path.endswith(os.path.sep):
       return 'dirs not allowed'

project.py

@@ -382,7 +382,12 @@ class _LinkFile(object):
     Handles wild cards on the src linking all of the files in the source in to
     the destination directory.
     """
-    src = _SafeExpandPath(self.git_worktree, self.src)
+    # Some people use src="." to create stable links to projects.  Lets allow
+    # that but reject all other uses of "." to keep things simple.
+    if self.src == '.':
+      src = self.git_worktree
+    else:
+      src = _SafeExpandPath(self.git_worktree, self.src)
 
     if os.path.exists(src):
       # Entity exists so just a simple one to one link operation.
@@ -2068,7 +2073,7 @@ class Project(object):
       gitmodules_lines = []
       fd, temp_gitmodules_path = tempfile.mkstemp()
       try:
-        os.write(fd, p.stdout)
+        os.write(fd, p.stdout.encode('utf-8'))
         os.close(fd)
         cmd = ['config', '--file', temp_gitmodules_path, '--list']
         p = GitCommand(None, cmd, capture_stdout=True, capture_stderr=True,
@@ -2477,7 +2482,7 @@ class Project(object):
         platform_utils.remove(tmpPath)
     with GetUrlCookieFile(srcUrl, quiet) as (cookiefile, proxy):
       if cookiefile:
-        cmd += ['--cookie', cookiefile, '--cookie-jar', cookiefile]
+        cmd += ['--cookie', cookiefile]
       if proxy:
         cmd += ['--proxy', proxy]
       elif 'http_proxy' in os.environ and 'darwin' == sys.platform:

repo

@@ -166,7 +166,12 @@ TACbBS+Up3RpfYVfd63c1cDdlru13pQAn3NQy/SN858MkxN+zym86UBgOad2
 """
 
 GIT = 'git'                      # our git command
-MIN_GIT_VERSION = (2, 10, 2)     # minimum supported git version
+# NB: The version of git that the repo launcher requires may be much older than
+# the version of git that the main repo source tree requires.  Keeping this at
+# an older version also makes it easier for users to upgrade/rollback as needed.
+#
+# git-1.7 is in (EOL) Ubuntu Precise.
+MIN_GIT_VERSION = (1, 7, 2)      # minimum supported git version
 repodir = '.repo'                # name of repo's private directory
 S_repo = 'repo'                  # special repo repository
 S_manifests = 'manifests'        # special manifest repository

subcmds/init.py

@@ -34,7 +34,7 @@ from command import InteractiveCommand, MirrorSafeCommand
 from error import ManifestParseError
 from project import SyncBuffer
 from git_config import GitConfig
-from git_command import git_require, MIN_GIT_VERSION
+from git_command import git_require, MIN_GIT_VERSION_SOFT, MIN_GIT_VERSION_HARD
 import platform_utils
 
 class Init(InteractiveCommand, MirrorSafeCommand):
@@ -451,7 +451,12 @@ to update the working directory files.
       self.OptionParser.error('--mirror and --archive cannot be used together.')
 
   def Execute(self, opt, args):
-    git_require(MIN_GIT_VERSION, fail=True)
+    git_require(MIN_GIT_VERSION_HARD, fail=True)
+    if not git_require(MIN_GIT_VERSION_SOFT):
+      print('repo: warning: git-%s+ will soon be required; please upgrade your '
+            'version of git to maintain support.'
+            % ('.'.join(str(x) for x in MIN_GIT_VERSION_SOFT),),
+            file=sys.stderr)
 
     self._SyncManifest(opt)
     self._LinkManifest(opt.manifest_name)

subcmds/sync.py

@@ -560,9 +560,20 @@ later is required to fix a server side protocol bug.
   def _GCProjects(self, projects, opt, err_event):
     gc_gitdirs = {}
     for project in projects:
+      # Make sure pruning never kicks in with shared projects.
       if len(project.manifest.GetProjectsWithName(project.name)) > 1:
-        print('Shared project %s found, disabling pruning.' % project.name)
-        project.bare_git.config('--replace-all', 'gc.pruneExpire', 'never')
+        print('%s: Shared project %s found, disabling pruning.' %
+              (project.relpath, project.name))
+        if git_require((2, 7, 0)):
+          project.config.SetString('core.repositoryFormatVersion', '1')
+          project.config.SetString('extensions.preciousObjects', 'true')
+        else:
+          # This isn't perfect, but it's the best we can do with old git.
+          print('%s: WARNING: shared projects are unreliable when using old '
+                'versions of git; please upgrade to git-2.7.0+.'
+                % (project.relpath,),
+                file=sys.stderr)
+          project.config.SetString('gc.pruneExpire', 'never')
       gc_gitdirs[project.gitdir] = project.bare_git
 
     has_dash_c = git_require((1, 7, 2))

tests/test_manifest_xml.py

@@ -49,6 +49,8 @@ class ManifestValidateFilePaths(unittest.TestCase):
     # We allow symlinks to end in a slash since we allow them to point to dirs
     # in general.  Technically the slash isn't necessary.
     check('foo/', 'bar')
+    # We allow a single '.' to get a reference to the project itself.
+    check('.', 'bar')
 
   def test_bad_paths(self):
     """Make sure bad paths (src & dest) are rejected."""

tests/test_project.py

@@ -314,6 +314,14 @@ class LinkFile(CopyLinkTestCase):
     lf._Link()
     self.assertExists(os.path.join(self.topdir, 'foo'))
 
+  def test_src_self(self):
+    """Link to the project itself."""
+    dest = os.path.join(self.topdir, 'foo', 'bar')
+    lf = self.LinkFile('.', 'foo/bar')
+    lf._Link()
+    self.assertExists(dest)
+    self.assertEqual('../git-project', os.readlink(dest))
+
   def test_dest_subdir(self):
     """Link a file to a subdir of a checkout."""
     src = os.path.join(self.worktree, 'foo.txt')
@@ -323,6 +331,21 @@ class LinkFile(CopyLinkTestCase):
     lf._Link()
     self.assertExists(os.path.join(self.topdir, 'sub', 'dir', 'foo', 'bar'))
 
+  def test_src_block_relative(self):
+    """Do not allow relative symlinks."""
+    BAD_SOURCES = (
+        './',
+        '..',
+        '../',
+        'foo/.',
+        'foo/./bar',
+        'foo/..',
+        'foo/../foo',
+    )
+    for src in BAD_SOURCES:
+      lf = self.LinkFile(src, 'foo')
+      self.assertRaises(error.ManifestInvalidPathError, lf._Link)
+
   def test_update(self):
     """Make sure changed targets get updated."""
     dest = os.path.join(self.topdir, 'sym')