.github/workflows/test-ci.yml

@@ -14,7 +14,10 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [3.6, 3.7, 3.8]
+        python-version: [2.7, 3.6, 3.7, 3.8]
+        exclude:
+          - os: windows-latest
+            python-version: 2.7
     runs-on: ${{ matrix.os }}
 
     steps:

docs/internal-fs-layout.md

@@ -134,7 +134,6 @@ User controlled settings are initialized when running `repo init`.
 |-------------------|---------------------------|-------------|
 | manifest.groups   | `--groups` & `--platform` | The manifest groups to sync |
 | repo.archive      | `--archive`               | Use `git archive` for checkouts |
-| repo.clonebundle  | `--clone-bundle`          | Whether the initial sync used clone.bundle explicitly |
 | repo.clonefilter  | `--clone-filter`          | Filter setting when using [partial git clones] |
 | repo.depth        | `--depth`                 | Create shallow checkouts when cloning |
 | repo.dissociate   | `--dissociate`            | Dissociate from any reference/mirrors after initial clone |

docs/manifest-format.md

@@ -396,4 +396,10 @@ these extra projects.
 Manifest files stored in `$TOP_DIR/.repo/local_manifests/*.xml` will
 be loaded in alphabetical order.
 
-The legacy `$TOP_DIR/.repo/local_manifest.xml` path is no longer supported.
+Additional remotes and projects may also be added through a local
+manifest, stored in `$TOP_DIR/.repo/local_manifest.xml`. This method
+is deprecated in favor of using multiple manifest files as mentioned
+above.
+
+If `$TOP_DIR/.repo/local_manifest.xml` exists, it will be loaded before
+any manifest files stored in `$TOP_DIR/.repo/local_manifests/*.xml`.

docs/release-process.md

@@ -5,37 +5,6 @@ related topics and flows.
 
 [TOC]
 
-## Schedule
-
-There is no specific schedule for when releases are made.
-Usually it's more along the lines of "enough minor changes have been merged",
-or "there's a known issue the maintainers know should get fixed".
-If you find a fix has been merged for an issue important to you, but hasn't been
-released after a week or so, feel free to [contact] us to request a new release.
-
-### Release Freezes {#freeze}
-
-We try to observe a regular schedule for when **not** to release.
-If something goes wrong, staff need to be active in order to respond quickly &
-effectively.
-We also don't want to disrupt non-Google organizations if possible.
-
-We generally follow the rules:
-
-* Release during Mon - Thu, 9:00 - 14:00 [US PT]
-* Avoid holidays
-  * All regular [US holidays]
-  * Large international ones if possible
-  * All the various [New Years]
-    * Jan 1 in Gregorian calendar is the most obvious
-    * Check for large Lunar New Years too
-* Follow the normal [Google production freeze schedule]
-
-[US holidays]: https://en.wikipedia.org/wiki/Federal_holidays_in_the_United_States
-[US PT]: https://en.wikipedia.org/wiki/Pacific_Time_Zone
-[New Years]: https://en.wikipedia.org/wiki/New_Year
-[Google production freeze schedule]: http://goto.google.com/prod-freeze
-
 ## Launcher script
 
 The main repo script serves as a standalone program and is often referred to as
@@ -273,7 +242,6 @@ Things in italics are things we used to care about but probably don't anymore.
 | Apr 2020 | **Apr 2030** |              |                 | **20.04 Focal**      | 2.25.0   | 2.7.17 3.7.5 |
 
 
-[contact]: ../README.md#contact
 [rel-d]: https://en.wikipedia.org/wiki/Debian_version_history
 [rel-g]: https://en.wikipedia.org/wiki/Git#Releases
 [rel-p]: https://en.wikipedia.org/wiki/History_of_Python#Table_of_versions

git_config.py

@@ -362,7 +362,7 @@ class GitConfig(object):
     return c
 
   def _do(self, *args):
-    command = ['config', '--file', self.file, '--includes']
+    command = ['config', '--file', self.file]
     command.extend(args)
 
     p = GitCommand(None,

hooks.py

@@ -1,431 +0,0 @@
-# -*- coding:utf-8 -*-
-#
-# Copyright (C) 2008 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import json
-import os
-import re
-import sys
-import traceback
-
-from error import HookError
-from git_refs import HEAD
-
-from pyversion import is_python3
-if is_python3():
-  import urllib.parse
-else:
-  import imp
-  import urlparse
-  urllib = imp.new_module('urllib')
-  urllib.parse = urlparse
-  input = raw_input  # noqa: F821
-
-class RepoHook(object):
-  """A RepoHook contains information about a script to run as a hook.
-
-  Hooks are used to run a python script before running an upload (for instance,
-  to run presubmit checks).  Eventually, we may have hooks for other actions.
-
-  This shouldn't be confused with files in the 'repo/hooks' directory.  Those
-  files are copied into each '.git/hooks' folder for each project.  Repo-level
-  hooks are associated instead with repo actions.
-
-  Hooks are always python.  When a hook is run, we will load the hook into the
-  interpreter and execute its main() function.
-  """
-
-  def __init__(self,
-               hook_type,
-               hooks_project,
-               topdir,
-               manifest_url,
-               abort_if_user_denies=False):
-    """RepoHook constructor.
-
-    Params:
-      hook_type: A string representing the type of hook.  This is also used
-          to figure out the name of the file containing the hook.  For
-          example: 'pre-upload'.
-      hooks_project: The project containing the repo hooks.  If you have a
-          manifest, this is manifest.repo_hooks_project.  OK if this is None,
-          which will make the hook a no-op.
-      topdir: Repo's top directory (the one containing the .repo directory).
-          Scripts will run with CWD as this directory.  If you have a manifest,
-          this is manifest.topdir
-      manifest_url: The URL to the manifest git repo.
-      abort_if_user_denies: If True, we'll throw a HookError() if the user
-          doesn't allow us to run the hook.
-    """
-    self._hook_type = hook_type
-    self._hooks_project = hooks_project
-    self._manifest_url = manifest_url
-    self._topdir = topdir
-    self._abort_if_user_denies = abort_if_user_denies
-
-    # Store the full path to the script for convenience.
-    if self._hooks_project:
-      self._script_fullpath = os.path.join(self._hooks_project.worktree,
-                                           self._hook_type + '.py')
-    else:
-      self._script_fullpath = None
-
-  def _GetHash(self):
-    """Return a hash of the contents of the hooks directory.
-
-    We'll just use git to do this.  This hash has the property that if anything
-    changes in the directory we will return a different has.
-
-    SECURITY CONSIDERATION:
-      This hash only represents the contents of files in the hook directory, not
-      any other files imported or called by hooks.  Changes to imported files
-      can change the script behavior without affecting the hash.
-
-    Returns:
-      A string representing the hash.  This will always be ASCII so that it can
-      be printed to the user easily.
-    """
-    assert self._hooks_project, "Must have hooks to calculate their hash."
-
-    # We will use the work_git object rather than just calling GetRevisionId().
-    # That gives us a hash of the latest checked in version of the files that
-    # the user will actually be executing.  Specifically, GetRevisionId()
-    # doesn't appear to change even if a user checks out a different version
-    # of the hooks repo (via git checkout) nor if a user commits their own revs.
-    #
-    # NOTE: Local (non-committed) changes will not be factored into this hash.
-    # I think this is OK, since we're really only worried about warning the user
-    # about upstream changes.
-    return self._hooks_project.work_git.rev_parse('HEAD')
-
-  def _GetMustVerb(self):
-    """Return 'must' if the hook is required; 'should' if not."""
-    if self._abort_if_user_denies:
-      return 'must'
-    else:
-      return 'should'
-
-  def _CheckForHookApproval(self):
-    """Check to see whether this hook has been approved.
-
-    We'll accept approval of manifest URLs if they're using secure transports.
-    This way the user can say they trust the manifest hoster.  For insecure
-    hosts, we fall back to checking the hash of the hooks repo.
-
-    Note that we ask permission for each individual hook even though we use
-    the hash of all hooks when detecting changes.  We'd like the user to be
-    able to approve / deny each hook individually.  We only use the hash of all
-    hooks because there is no other easy way to detect changes to local imports.
-
-    Returns:
-      True if this hook is approved to run; False otherwise.
-
-    Raises:
-      HookError: Raised if the user doesn't approve and abort_if_user_denies
-          was passed to the consturctor.
-    """
-    if self._ManifestUrlHasSecureScheme():
-      return self._CheckForHookApprovalManifest()
-    else:
-      return self._CheckForHookApprovalHash()
-
-  def _CheckForHookApprovalHelper(self, subkey, new_val, main_prompt,
-                                  changed_prompt):
-    """Check for approval for a particular attribute and hook.
-
-    Args:
-      subkey: The git config key under [repo.hooks.<hook_type>] to store the
-          last approved string.
-      new_val: The new value to compare against the last approved one.
-      main_prompt: Message to display to the user to ask for approval.
-      changed_prompt: Message explaining why we're re-asking for approval.
-
-    Returns:
-      True if this hook is approved to run; False otherwise.
-
-    Raises:
-      HookError: Raised if the user doesn't approve and abort_if_user_denies
-          was passed to the consturctor.
-    """
-    hooks_config = self._hooks_project.config
-    git_approval_key = 'repo.hooks.%s.%s' % (self._hook_type, subkey)
-
-    # Get the last value that the user approved for this hook; may be None.
-    old_val = hooks_config.GetString(git_approval_key)
-
-    if old_val is not None:
-      # User previously approved hook and asked not to be prompted again.
-      if new_val == old_val:
-        # Approval matched.  We're done.
-        return True
-      else:
-        # Give the user a reason why we're prompting, since they last told
-        # us to "never ask again".
-        prompt = 'WARNING: %s\n\n' % (changed_prompt,)
-    else:
-      prompt = ''
-
-    # Prompt the user if we're not on a tty; on a tty we'll assume "no".
-    if sys.stdout.isatty():
-      prompt += main_prompt + ' (yes/always/NO)? '
-      response = input(prompt).lower()
-      print()
-
-      # User is doing a one-time approval.
-      if response in ('y', 'yes'):
-        return True
-      elif response == 'always':
-        hooks_config.SetString(git_approval_key, new_val)
-        return True
-
-    # For anything else, we'll assume no approval.
-    if self._abort_if_user_denies:
-      raise HookError('You must allow the %s hook or use --no-verify.' %
-                      self._hook_type)
-
-    return False
-
-  def _ManifestUrlHasSecureScheme(self):
-    """Check if the URI for the manifest is a secure transport."""
-    secure_schemes = ('file', 'https', 'ssh', 'persistent-https', 'sso', 'rpc')
-    parse_results = urllib.parse.urlparse(self._manifest_url)
-    return parse_results.scheme in secure_schemes
-
-  def _CheckForHookApprovalManifest(self):
-    """Check whether the user has approved this manifest host.
-
-    Returns:
-      True if this hook is approved to run; False otherwise.
-    """
-    return self._CheckForHookApprovalHelper(
-        'approvedmanifest',
-        self._manifest_url,
-        'Run hook scripts from %s' % (self._manifest_url,),
-        'Manifest URL has changed since %s was allowed.' % (self._hook_type,))
-
-  def _CheckForHookApprovalHash(self):
-    """Check whether the user has approved the hooks repo.
-
-    Returns:
-      True if this hook is approved to run; False otherwise.
-    """
-    prompt = ('Repo %s run the script:\n'
-              '  %s\n'
-              '\n'
-              'Do you want to allow this script to run')
-    return self._CheckForHookApprovalHelper(
-        'approvedhash',
-        self._GetHash(),
-        prompt % (self._GetMustVerb(), self._script_fullpath),
-        'Scripts have changed since %s was allowed.' % (self._hook_type,))
-
-  @staticmethod
-  def _ExtractInterpFromShebang(data):
-    """Extract the interpreter used in the shebang.
-
-    Try to locate the interpreter the script is using (ignoring `env`).
-
-    Args:
-      data: The file content of the script.
-
-    Returns:
-      The basename of the main script interpreter, or None if a shebang is not
-      used or could not be parsed out.
-    """
-    firstline = data.splitlines()[:1]
-    if not firstline:
-      return None
-
-    # The format here can be tricky.
-    shebang = firstline[0].strip()
-    m = re.match(r'^#!\s*([^\s]+)(?:\s+([^\s]+))?', shebang)
-    if not m:
-      return None
-
-    # If the using `env`, find the target program.
-    interp = m.group(1)
-    if os.path.basename(interp) == 'env':
-      interp = m.group(2)
-
-    return interp
-
-  def _ExecuteHookViaReexec(self, interp, context, **kwargs):
-    """Execute the hook script through |interp|.
-
-    Note: Support for this feature should be dropped ~Jun 2021.
-
-    Args:
-      interp: The Python program to run.
-      context: Basic Python context to execute the hook inside.
-      kwargs: Arbitrary arguments to pass to the hook script.
-
-    Raises:
-      HookError: When the hooks failed for any reason.
-    """
-    # This logic needs to be kept in sync with _ExecuteHookViaImport below.
-    script = """
-import json, os, sys
-path = '''%(path)s'''
-kwargs = json.loads('''%(kwargs)s''')
-context = json.loads('''%(context)s''')
-sys.path.insert(0, os.path.dirname(path))
-data = open(path).read()
-exec(compile(data, path, 'exec'), context)
-context['main'](**kwargs)
-""" % {
-        'path': self._script_fullpath,
-        'kwargs': json.dumps(kwargs),
-        'context': json.dumps(context),
-    }
-
-    # We pass the script via stdin to avoid OS argv limits.  It also makes
-    # unhandled exception tracebacks less verbose/confusing for users.
-    cmd = [interp, '-c', 'import sys; exec(sys.stdin.read())']
-    proc = subprocess.Popen(cmd, stdin=subprocess.PIPE)
-    proc.communicate(input=script.encode('utf-8'))
-    if proc.returncode:
-      raise HookError('Failed to run %s hook.' % (self._hook_type,))
-
-  def _ExecuteHookViaImport(self, data, context, **kwargs):
-    """Execute the hook code in |data| directly.
-
-    Args:
-      data: The code of the hook to execute.
-      context: Basic Python context to execute the hook inside.
-      kwargs: Arbitrary arguments to pass to the hook script.
-
-    Raises:
-      HookError: When the hooks failed for any reason.
-    """
-    # Exec, storing global context in the context dict.  We catch exceptions
-    # and convert to a HookError w/ just the failing traceback.
-    try:
-      exec(compile(data, self._script_fullpath, 'exec'), context)
-    except Exception:
-      raise HookError('%s\nFailed to import %s hook; see traceback above.' %
-                      (traceback.format_exc(), self._hook_type))
-
-    # Running the script should have defined a main() function.
-    if 'main' not in context:
-      raise HookError('Missing main() in: "%s"' % self._script_fullpath)
-
-    # Call the main function in the hook.  If the hook should cause the
-    # build to fail, it will raise an Exception.  We'll catch that convert
-    # to a HookError w/ just the failing traceback.
-    try:
-      context['main'](**kwargs)
-    except Exception:
-      raise HookError('%s\nFailed to run main() for %s hook; see traceback '
-                      'above.' % (traceback.format_exc(), self._hook_type))
-
-  def _ExecuteHook(self, **kwargs):
-    """Actually execute the given hook.
-
-    This will run the hook's 'main' function in our python interpreter.
-
-    Args:
-      kwargs: Keyword arguments to pass to the hook.  These are often specific
-          to the hook type.  For instance, pre-upload hooks will contain
-          a project_list.
-    """
-    # Keep sys.path and CWD stashed away so that we can always restore them
-    # upon function exit.
-    orig_path = os.getcwd()
-    orig_syspath = sys.path
-
-    try:
-      # Always run hooks with CWD as topdir.
-      os.chdir(self._topdir)
-
-      # Put the hook dir as the first item of sys.path so hooks can do
-      # relative imports.  We want to replace the repo dir as [0] so
-      # hooks can't import repo files.
-      sys.path = [os.path.dirname(self._script_fullpath)] + sys.path[1:]
-
-      # Initial global context for the hook to run within.
-      context = {'__file__': self._script_fullpath}
-
-      # Add 'hook_should_take_kwargs' to the arguments to be passed to main.
-      # We don't actually want hooks to define their main with this argument--
-      # it's there to remind them that their hook should always take **kwargs.
-      # For instance, a pre-upload hook should be defined like:
-      #   def main(project_list, **kwargs):
-      #
-      # This allows us to later expand the API without breaking old hooks.
-      kwargs = kwargs.copy()
-      kwargs['hook_should_take_kwargs'] = True
-
-      # See what version of python the hook has been written against.
-      data = open(self._script_fullpath).read()
-      interp = self._ExtractInterpFromShebang(data)
-      reexec = False
-      if interp:
-        prog = os.path.basename(interp)
-        if prog.startswith('python2') and sys.version_info.major != 2:
-          reexec = True
-        elif prog.startswith('python3') and sys.version_info.major == 2:
-          reexec = True
-
-      # Attempt to execute the hooks through the requested version of Python.
-      if reexec:
-        try:
-          self._ExecuteHookViaReexec(interp, context, **kwargs)
-        except OSError as e:
-          if e.errno == errno.ENOENT:
-            # We couldn't find the interpreter, so fallback to importing.
-            reexec = False
-          else:
-            raise
-
-      # Run the hook by importing directly.
-      if not reexec:
-        self._ExecuteHookViaImport(data, context, **kwargs)
-    finally:
-      # Restore sys.path and CWD.
-      sys.path = orig_syspath
-      os.chdir(orig_path)
-
-  def Run(self, user_allows_all_hooks, **kwargs):
-    """Run the hook.
-
-    If the hook doesn't exist (because there is no hooks project or because
-    this particular hook is not enabled), this is a no-op.
-
-    Args:
-      user_allows_all_hooks: If True, we will never prompt about running the
-          hook--we'll just assume it's OK to run it.
-      kwargs: Keyword arguments to pass to the hook.  These are often specific
-          to the hook type.  For instance, pre-upload hooks will contain
-          a project_list.
-
-    Raises:
-      HookError: If there was a problem finding the hook or the user declined
-          to run a required hook (from _CheckForHookApproval).
-    """
-    # No-op if there is no hooks project or if hook is disabled.
-    if ((not self._hooks_project) or (self._hook_type not in
-                                      self._hooks_project.enabled_repo_hooks)):
-      return
-
-    # Bail with a nice error if we can't find the hook.
-    if not os.path.isfile(self._script_fullpath):
-      raise HookError('Couldn\'t find repo hook: "%s"' % self._script_fullpath)
-
-    # Make sure the user is OK with running the hook.
-    if (not user_allows_all_hooks) and (not self._CheckForHookApproval()):
-      return
-
-    # Run the hook with the same version of python we're using.
-    self._ExecuteHook(**kwargs)

main.py

@@ -614,7 +614,7 @@ def _Main(argv):
     argv = list(sys.argv)
     argv.extend(rce.extra_args)
     try:
-      os.execv(sys.executable, [__file__] + argv)
+      os.execv(__file__, argv)
     except OSError as e:
       print('fatal: cannot restart repo after upgrade', file=sys.stderr)
       print('fatal: %s' % e, file=sys.stderr)

manifest_xml.py

@@ -31,7 +31,7 @@ else:
   urllib.parse = urlparse
 
 import gitc_utils
-from git_config import GitConfig, IsId
+from git_config import GitConfig
 from git_refs import R_HEADS, HEAD
 import platform_utils
 from project import RemoteSpec, Project, MetaProject
@@ -192,6 +192,7 @@ class XmlManifest(object):
     self.topdir = os.path.dirname(self.repodir)
     self.manifestFile = os.path.join(self.repodir, MANIFEST_FILE_NAME)
     self.globalConfig = GitConfig.ForUser()
+    self.localManifestWarning = False
     self.isGitcClient = False
     self._load_local_manifests = True
 
@@ -283,7 +284,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
   def _ParseGroups(self, groups):
     return [x for x in re.split(r'[,\s]+', groups) if x]
 
-  def Save(self, fd, peg_rev=False, peg_rev_upstream=True, peg_rev_dest_branch=True, groups=None):
+  def Save(self, fd, peg_rev=False, peg_rev_upstream=True, groups=None):
     """Write the current manifest out to the given file descriptor.
     """
     mp = self.manifestProject
@@ -388,13 +389,6 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
             # Only save the origin if the origin is not a sha1, and the default
             # isn't our value
             e.setAttribute('upstream', p.revisionExpr)
-
-        if peg_rev_dest_branch:
-          if p.dest_branch:
-            e.setAttribute('dest-branch', p.dest_branch)
-          elif value != p.revisionExpr:
-            e.setAttribute('dest-branch', p.revisionExpr)
-
       else:
         revision = self.remotes[p.remote.orig_name].revision or d.revisionExpr
         if not revision or revision != p.revisionExpr:
@@ -500,14 +494,6 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
     self._Load()
     return self._manifest_server
 
-  @property
-  def CloneBundle(self):
-    clone_bundle = self.manifestProject.config.GetBoolean('repo.clonebundle')
-    if clone_bundle is None:
-      return False if self.manifestProject.config.GetBoolean('repo.partialclone') else True
-    else:
-      return clone_bundle
-
   @property
   def CloneFilter(self):
     if self.manifestProject.config.GetBoolean('repo.partialclone'):
@@ -554,12 +540,15 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
                                           self.manifestProject.worktree))
 
       if self._load_local_manifests:
-        if os.path.exists(os.path.join(self.repodir, LOCAL_MANIFEST_NAME)):
+        local = os.path.join(self.repodir, LOCAL_MANIFEST_NAME)
-          print('error: %s is not supported; put local manifests in `%s`'
+        if os.path.exists(local):
-                'instead' % (LOCAL_MANIFEST_NAME,
+          if not self.localManifestWarning:
-                             os.path.join(self.repodir, LOCAL_MANIFESTS_DIR_NAME)),
+            self.localManifestWarning = True
-                file=sys.stderr)
+            print('warning: %s is deprecated; put local manifests '
-          sys.exit(1)
+                  'in `%s` instead' % (LOCAL_MANIFEST_NAME,
+                                       os.path.join(self.repodir, LOCAL_MANIFESTS_DIR_NAME)),
+                  file=sys.stderr)
+          nodes.append(self._ParseManifestXml(local, self.repodir))
 
         local_dir = os.path.abspath(os.path.join(self.repodir,
                                                  LOCAL_MANIFESTS_DIR_NAME))
@@ -705,10 +694,6 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
             p.groups.extend(groups)
           if revision:
             p.revisionExpr = revision
-            if IsId(revision):
-              p.revisionId = revision
-            else:
-              p.revisionId = None
           if remote:
             p.remote = remote.ToRemoteSpec(name)
       if node.nodeName == 'repo-hooks':
@@ -978,10 +963,6 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
     return project
 
   def GetProjectPaths(self, name, path):
-    # The manifest entries might have trailing slashes.  Normalize them to avoid
-    # unexpected filesystem behavior since we do string concatenation below.
-    path = path.rstrip('/')
-    name = name.rstrip('/')
     use_git_worktrees = False
     relpath = path
     if self.IsMirror:
@@ -1014,10 +995,6 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
     return os.path.relpath(relpath, parent_relpath)
 
   def GetSubprojectPaths(self, parent, name, path):
-    # The manifest entries might have trailing slashes.  Normalize them to avoid
-    # unexpected filesystem behavior since we do string concatenation below.
-    path = path.rstrip('/')
-    name = name.rstrip('/')
     relpath = self._JoinRelpath(parent.relpath, path)
     gitdir = os.path.join(parent.gitdir, 'subprojects', '%s.git' % path)
     objdir = os.path.join(parent.gitdir, 'subproject-objects', '%s.git' % name)

project.py

@@ -18,6 +18,7 @@ from __future__ import print_function
 import errno
 import filecmp
 import glob
+import json
 import os
 import random
 import re
@@ -28,12 +29,13 @@ import sys
 import tarfile
 import tempfile
 import time
+import traceback
 
 from color import Coloring
 from git_command import GitCommand, git_require
 from git_config import GitConfig, IsId, GetSchemeFromUrl, GetUrlCookieFile, \
     ID_RE
-from error import GitError, UploadError, DownloadError
+from error import GitError, HookError, UploadError, DownloadError
 from error import ManifestInvalidRevisionError, ManifestInvalidPathError
 from error import NoManifestException
 import platform_utils
@@ -403,8 +405,8 @@ class _LinkFile(object):
     else:
       src = _SafeExpandPath(self.git_worktree, self.src)
 
-    if not glob.has_magic(src):
+    if os.path.exists(src):
-      # Entity does not contain a wild card so just a simple one to one link operation.
+      # Entity exists so just a simple one to one link operation.
       dest = _SafeExpandPath(self.topdir, self.dest, skipfinal=True)
       # dest & src are absolute paths at this point.  Make sure the target of
       # the symlink is relative in the context of the repo client checkout.
@@ -412,7 +414,7 @@ class _LinkFile(object):
       self.__linkIt(relpath, dest)
     else:
       dest = _SafeExpandPath(self.topdir, self.dest)
-      # Entity contains a wild card.
+      # Entity doesn't exist assume there is a wild card
       if os.path.exists(dest) and not platform_utils.isdir(dest):
         _error('Link error: src with wildcard, %s must be a directory', dest)
       else:
@@ -449,6 +451,406 @@ class RemoteSpec(object):
     self.orig_name = orig_name
     self.fetchUrl = fetchUrl
 
+
+class RepoHook(object):
+
+  """A RepoHook contains information about a script to run as a hook.
+
+  Hooks are used to run a python script before running an upload (for instance,
+  to run presubmit checks).  Eventually, we may have hooks for other actions.
+
+  This shouldn't be confused with files in the 'repo/hooks' directory.  Those
+  files are copied into each '.git/hooks' folder for each project.  Repo-level
+  hooks are associated instead with repo actions.
+
+  Hooks are always python.  When a hook is run, we will load the hook into the
+  interpreter and execute its main() function.
+  """
+
+  def __init__(self,
+               hook_type,
+               hooks_project,
+               topdir,
+               manifest_url,
+               abort_if_user_denies=False):
+    """RepoHook constructor.
+
+    Params:
+      hook_type: A string representing the type of hook.  This is also used
+          to figure out the name of the file containing the hook.  For
+          example: 'pre-upload'.
+      hooks_project: The project containing the repo hooks.  If you have a
+          manifest, this is manifest.repo_hooks_project.  OK if this is None,
+          which will make the hook a no-op.
+      topdir: Repo's top directory (the one containing the .repo directory).
+          Scripts will run with CWD as this directory.  If you have a manifest,
+          this is manifest.topdir
+      manifest_url: The URL to the manifest git repo.
+      abort_if_user_denies: If True, we'll throw a HookError() if the user
+          doesn't allow us to run the hook.
+    """
+    self._hook_type = hook_type
+    self._hooks_project = hooks_project
+    self._manifest_url = manifest_url
+    self._topdir = topdir
+    self._abort_if_user_denies = abort_if_user_denies
+
+    # Store the full path to the script for convenience.
+    if self._hooks_project:
+      self._script_fullpath = os.path.join(self._hooks_project.worktree,
+                                           self._hook_type + '.py')
+    else:
+      self._script_fullpath = None
+
+  def _GetHash(self):
+    """Return a hash of the contents of the hooks directory.
+
+    We'll just use git to do this.  This hash has the property that if anything
+    changes in the directory we will return a different has.
+
+    SECURITY CONSIDERATION:
+      This hash only represents the contents of files in the hook directory, not
+      any other files imported or called by hooks.  Changes to imported files
+      can change the script behavior without affecting the hash.
+
+    Returns:
+      A string representing the hash.  This will always be ASCII so that it can
+      be printed to the user easily.
+    """
+    assert self._hooks_project, "Must have hooks to calculate their hash."
+
+    # We will use the work_git object rather than just calling GetRevisionId().
+    # That gives us a hash of the latest checked in version of the files that
+    # the user will actually be executing.  Specifically, GetRevisionId()
+    # doesn't appear to change even if a user checks out a different version
+    # of the hooks repo (via git checkout) nor if a user commits their own revs.
+    #
+    # NOTE: Local (non-committed) changes will not be factored into this hash.
+    # I think this is OK, since we're really only worried about warning the user
+    # about upstream changes.
+    return self._hooks_project.work_git.rev_parse('HEAD')
+
+  def _GetMustVerb(self):
+    """Return 'must' if the hook is required; 'should' if not."""
+    if self._abort_if_user_denies:
+      return 'must'
+    else:
+      return 'should'
+
+  def _CheckForHookApproval(self):
+    """Check to see whether this hook has been approved.
+
+    We'll accept approval of manifest URLs if they're using secure transports.
+    This way the user can say they trust the manifest hoster.  For insecure
+    hosts, we fall back to checking the hash of the hooks repo.
+
+    Note that we ask permission for each individual hook even though we use
+    the hash of all hooks when detecting changes.  We'd like the user to be
+    able to approve / deny each hook individually.  We only use the hash of all
+    hooks because there is no other easy way to detect changes to local imports.
+
+    Returns:
+      True if this hook is approved to run; False otherwise.
+
+    Raises:
+      HookError: Raised if the user doesn't approve and abort_if_user_denies
+          was passed to the consturctor.
+    """
+    if self._ManifestUrlHasSecureScheme():
+      return self._CheckForHookApprovalManifest()
+    else:
+      return self._CheckForHookApprovalHash()
+
+  def _CheckForHookApprovalHelper(self, subkey, new_val, main_prompt,
+                                  changed_prompt):
+    """Check for approval for a particular attribute and hook.
+
+    Args:
+      subkey: The git config key under [repo.hooks.<hook_type>] to store the
+          last approved string.
+      new_val: The new value to compare against the last approved one.
+      main_prompt: Message to display to the user to ask for approval.
+      changed_prompt: Message explaining why we're re-asking for approval.
+
+    Returns:
+      True if this hook is approved to run; False otherwise.
+
+    Raises:
+      HookError: Raised if the user doesn't approve and abort_if_user_denies
+          was passed to the consturctor.
+    """
+    hooks_config = self._hooks_project.config
+    git_approval_key = 'repo.hooks.%s.%s' % (self._hook_type, subkey)
+
+    # Get the last value that the user approved for this hook; may be None.
+    old_val = hooks_config.GetString(git_approval_key)
+
+    if old_val is not None:
+      # User previously approved hook and asked not to be prompted again.
+      if new_val == old_val:
+        # Approval matched.  We're done.
+        return True
+      else:
+        # Give the user a reason why we're prompting, since they last told
+        # us to "never ask again".
+        prompt = 'WARNING: %s\n\n' % (changed_prompt,)
+    else:
+      prompt = ''
+
+    # Prompt the user if we're not on a tty; on a tty we'll assume "no".
+    if sys.stdout.isatty():
+      prompt += main_prompt + ' (yes/always/NO)? '
+      response = input(prompt).lower()
+      print()
+
+      # User is doing a one-time approval.
+      if response in ('y', 'yes'):
+        return True
+      elif response == 'always':
+        hooks_config.SetString(git_approval_key, new_val)
+        return True
+
+    # For anything else, we'll assume no approval.
+    if self._abort_if_user_denies:
+      raise HookError('You must allow the %s hook or use --no-verify.' %
+                      self._hook_type)
+
+    return False
+
+  def _ManifestUrlHasSecureScheme(self):
+    """Check if the URI for the manifest is a secure transport."""
+    secure_schemes = ('file', 'https', 'ssh', 'persistent-https', 'sso', 'rpc')
+    parse_results = urllib.parse.urlparse(self._manifest_url)
+    return parse_results.scheme in secure_schemes
+
+  def _CheckForHookApprovalManifest(self):
+    """Check whether the user has approved this manifest host.
+
+    Returns:
+      True if this hook is approved to run; False otherwise.
+    """
+    return self._CheckForHookApprovalHelper(
+        'approvedmanifest',
+        self._manifest_url,
+        'Run hook scripts from %s' % (self._manifest_url,),
+        'Manifest URL has changed since %s was allowed.' % (self._hook_type,))
+
+  def _CheckForHookApprovalHash(self):
+    """Check whether the user has approved the hooks repo.
+
+    Returns:
+      True if this hook is approved to run; False otherwise.
+    """
+    prompt = ('Repo %s run the script:\n'
+              '  %s\n'
+              '\n'
+              'Do you want to allow this script to run')
+    return self._CheckForHookApprovalHelper(
+        'approvedhash',
+        self._GetHash(),
+        prompt % (self._GetMustVerb(), self._script_fullpath),
+        'Scripts have changed since %s was allowed.' % (self._hook_type,))
+
+  @staticmethod
+  def _ExtractInterpFromShebang(data):
+    """Extract the interpreter used in the shebang.
+
+    Try to locate the interpreter the script is using (ignoring `env`).
+
+    Args:
+      data: The file content of the script.
+
+    Returns:
+      The basename of the main script interpreter, or None if a shebang is not
+      used or could not be parsed out.
+    """
+    firstline = data.splitlines()[:1]
+    if not firstline:
+      return None
+
+    # The format here can be tricky.
+    shebang = firstline[0].strip()
+    m = re.match(r'^#!\s*([^\s]+)(?:\s+([^\s]+))?', shebang)
+    if not m:
+      return None
+
+    # If the using `env`, find the target program.
+    interp = m.group(1)
+    if os.path.basename(interp) == 'env':
+      interp = m.group(2)
+
+    return interp
+
+  def _ExecuteHookViaReexec(self, interp, context, **kwargs):
+    """Execute the hook script through |interp|.
+
+    Note: Support for this feature should be dropped ~Jun 2021.
+
+    Args:
+      interp: The Python program to run.
+      context: Basic Python context to execute the hook inside.
+      kwargs: Arbitrary arguments to pass to the hook script.
+
+    Raises:
+      HookError: When the hooks failed for any reason.
+    """
+    # This logic needs to be kept in sync with _ExecuteHookViaImport below.
+    script = """
+import json, os, sys
+path = '''%(path)s'''
+kwargs = json.loads('''%(kwargs)s''')
+context = json.loads('''%(context)s''')
+sys.path.insert(0, os.path.dirname(path))
+data = open(path).read()
+exec(compile(data, path, 'exec'), context)
+context['main'](**kwargs)
+""" % {
+        'path': self._script_fullpath,
+        'kwargs': json.dumps(kwargs),
+        'context': json.dumps(context),
+    }
+
+    # We pass the script via stdin to avoid OS argv limits.  It also makes
+    # unhandled exception tracebacks less verbose/confusing for users.
+    cmd = [interp, '-c', 'import sys; exec(sys.stdin.read())']
+    proc = subprocess.Popen(cmd, stdin=subprocess.PIPE)
+    proc.communicate(input=script.encode('utf-8'))
+    if proc.returncode:
+      raise HookError('Failed to run %s hook.' % (self._hook_type,))
+
+  def _ExecuteHookViaImport(self, data, context, **kwargs):
+    """Execute the hook code in |data| directly.
+
+    Args:
+      data: The code of the hook to execute.
+      context: Basic Python context to execute the hook inside.
+      kwargs: Arbitrary arguments to pass to the hook script.
+
+    Raises:
+      HookError: When the hooks failed for any reason.
+    """
+    # Exec, storing global context in the context dict.  We catch exceptions
+    # and convert to a HookError w/ just the failing traceback.
+    try:
+      exec(compile(data, self._script_fullpath, 'exec'), context)
+    except Exception:
+      raise HookError('%s\nFailed to import %s hook; see traceback above.' %
+                      (traceback.format_exc(), self._hook_type))
+
+    # Running the script should have defined a main() function.
+    if 'main' not in context:
+      raise HookError('Missing main() in: "%s"' % self._script_fullpath)
+
+    # Call the main function in the hook.  If the hook should cause the
+    # build to fail, it will raise an Exception.  We'll catch that convert
+    # to a HookError w/ just the failing traceback.
+    try:
+      context['main'](**kwargs)
+    except Exception:
+      raise HookError('%s\nFailed to run main() for %s hook; see traceback '
+                      'above.' % (traceback.format_exc(), self._hook_type))
+
+  def _ExecuteHook(self, **kwargs):
+    """Actually execute the given hook.
+
+    This will run the hook's 'main' function in our python interpreter.
+
+    Args:
+      kwargs: Keyword arguments to pass to the hook.  These are often specific
+          to the hook type.  For instance, pre-upload hooks will contain
+          a project_list.
+    """
+    # Keep sys.path and CWD stashed away so that we can always restore them
+    # upon function exit.
+    orig_path = os.getcwd()
+    orig_syspath = sys.path
+
+    try:
+      # Always run hooks with CWD as topdir.
+      os.chdir(self._topdir)
+
+      # Put the hook dir as the first item of sys.path so hooks can do
+      # relative imports.  We want to replace the repo dir as [0] so
+      # hooks can't import repo files.
+      sys.path = [os.path.dirname(self._script_fullpath)] + sys.path[1:]
+
+      # Initial global context for the hook to run within.
+      context = {'__file__': self._script_fullpath}
+
+      # Add 'hook_should_take_kwargs' to the arguments to be passed to main.
+      # We don't actually want hooks to define their main with this argument--
+      # it's there to remind them that their hook should always take **kwargs.
+      # For instance, a pre-upload hook should be defined like:
+      #   def main(project_list, **kwargs):
+      #
+      # This allows us to later expand the API without breaking old hooks.
+      kwargs = kwargs.copy()
+      kwargs['hook_should_take_kwargs'] = True
+
+      # See what version of python the hook has been written against.
+      data = open(self._script_fullpath).read()
+      interp = self._ExtractInterpFromShebang(data)
+      reexec = False
+      if interp:
+        prog = os.path.basename(interp)
+        if prog.startswith('python2') and sys.version_info.major != 2:
+          reexec = True
+        elif prog.startswith('python3') and sys.version_info.major == 2:
+          reexec = True
+
+      # Attempt to execute the hooks through the requested version of Python.
+      if reexec:
+        try:
+          self._ExecuteHookViaReexec(interp, context, **kwargs)
+        except OSError as e:
+          if e.errno == errno.ENOENT:
+            # We couldn't find the interpreter, so fallback to importing.
+            reexec = False
+          else:
+            raise
+
+      # Run the hook by importing directly.
+      if not reexec:
+        self._ExecuteHookViaImport(data, context, **kwargs)
+    finally:
+      # Restore sys.path and CWD.
+      sys.path = orig_syspath
+      os.chdir(orig_path)
+
+  def Run(self, user_allows_all_hooks, **kwargs):
+    """Run the hook.
+
+    If the hook doesn't exist (because there is no hooks project or because
+    this particular hook is not enabled), this is a no-op.
+
+    Args:
+      user_allows_all_hooks: If True, we will never prompt about running the
+          hook--we'll just assume it's OK to run it.
+      kwargs: Keyword arguments to pass to the hook.  These are often specific
+          to the hook type.  For instance, pre-upload hooks will contain
+          a project_list.
+
+    Raises:
+      HookError: If there was a problem finding the hook or the user declined
+          to run a required hook (from _CheckForHookApproval).
+    """
+    # No-op if there is no hooks project or if hook is disabled.
+    if ((not self._hooks_project) or (self._hook_type not in
+                                      self._hooks_project.enabled_repo_hooks)):
+      return
+
+    # Bail with a nice error if we can't find the hook.
+    if not os.path.isfile(self._script_fullpath):
+      raise HookError('Couldn\'t find repo hook: "%s"' % self._script_fullpath)
+
+    # Make sure the user is OK with running the hook.
+    if (not user_allows_all_hooks) and (not self._CheckForHookApproval()):
+      return
+
+    # Run the hook with the same version of python we're using.
+    self._ExecuteHook(**kwargs)
+
+
 class Project(object):
   # These objects can be shared between several working trees.
   shareable_files = ['description', 'info']
@@ -1909,27 +2311,6 @@ class Project(object):
     # Enable the extension!
     self.config.SetString('extensions.%s' % (key,), value)
 
-  def ResolveRemoteHead(self, name=None):
-    """Find out what the default branch (HEAD) points to.
-
-    Normally this points to refs/heads/master, but projects are moving to main.
-    Support whatever the server uses rather than hardcoding "master" ourselves.
-    """
-    if name is None:
-      name = self.remote.name
-
-    # The output will look like (NB: tabs are separators):
-    # ref: refs/heads/master	HEAD
-    # 5f6803b100bb3cd0f534e96e88c91373e8ed1c44	HEAD
-    output = self.bare_git.ls_remote('-q', '--symref', '--exit-code', name, 'HEAD')
-
-    for line in output.splitlines():
-      lhs, rhs = line.split('\t', 1)
-      if rhs == 'HEAD' and lhs.startswith('ref:'):
-        return lhs[4:].strip()
-
-    return None
-
   def _CheckForImmutableRevision(self):
     try:
       # if revision (sha or tag) is not present then following function
@@ -2412,7 +2793,7 @@ class Project(object):
 
           # Enable per-worktree config file support if possible.  This is more a
           # nice-to-have feature for users rather than a hard requirement.
-          if git_require((2, 20, 0)):
+          if git_require((2, 19, 0)):
             self.EnableRepositoryExtension('worktreeConfig')
 
       # If we have a separate directory to hold refs, initialize it as well.
@@ -2827,13 +3208,6 @@ class Project(object):
       self._bare = bare
       self._gitdir = gitdir
 
-    # __getstate__ and __setstate__ are required for pickling because __getattr__ exists.
-    def __getstate__(self):
-      return (self._project, self._bare, self._gitdir)
-
-    def __setstate__(self, state):
-      self._project, self._bare, self._gitdir = state
-
     def LsOthers(self):
       p = GitCommand(self._project,
                      ['ls-files',

release/sign-tag.py

@@ -18,7 +18,11 @@
 This is intended to be run only by the official Repo release managers, but it
 could be run by people maintaining their own fork of the project.
 
-NB: Check docs/release-process.md for production freeze information.
+NB: Avoid new releases on off-hours.  If something goes wrong, staff/oncall need
+to be active in order to respond quickly & effectively.  Recommend sticking to:
+* Mon - Thu, 9:00 - 14:00 PT (i.e. MTV time)
+* Avoid US holidays (and large international ones if possible)
+* Follow the normal Google production freeze schedule
 """
 
 import argparse

repo

@@ -133,7 +133,7 @@ if not REPO_REV:
   REPO_REV = 'stable'
 
 # increment this whenever we make important changes to this script
-VERSION = (2, 8)
+VERSION = (2, 5)
 
 # increment this if the MAINTAINER_KEYS block is modified
 KEYRING_VERSION = (2, 3)
@@ -317,11 +317,9 @@ def GetParser(gitc_init=False):
                    help='restrict manifest projects to ones with a specified '
                         'platform group [auto|all|none|linux|darwin|...]',
                    metavar='PLATFORM')
-  group.add_option('--clone-bundle', action='store_true',
-                   help='enable use of /clone.bundle on HTTP/HTTPS (default if not --partial-clone)')
   group.add_option('--no-clone-bundle',
-                   dest='clone_bundle', action='store_false',
+                   dest='clone_bundle', default=True, action='store_false',
-                   help='disable use of /clone.bundle on HTTP/HTTPS (default if --partial-clone)')
+                   help='disable use of /clone.bundle on HTTP/HTTPS')
   group.add_option('--no-tags',
                    dest='tags', default=True, action='store_false',
                    help="don't fetch tags in the manifest")
@@ -504,9 +502,6 @@ def _Init(args, gitc_init=False):
   opt.quiet = opt.output_mode is False
   opt.verbose = opt.output_mode is True
 
-  if opt.clone_bundle is None:
-    opt.clone_bundle = False if opt.partial_clone else True
-
   url = opt.repo_url or REPO_URL
   rev = opt.repo_rev or REPO_REV
 
@@ -966,7 +961,9 @@ def _FindRepo():
   repo = None
 
   olddir = None
-  while curdir != olddir and not repo:
+  while curdir != '/' \
+          and curdir != olddir \
+          and not repo:
     repo = os.path.join(curdir, repodir, REPO_MAIN)
     if not os.path.isfile(repo):
       repo = None
@@ -1172,10 +1169,6 @@ def main(orig_args):
   if my_main:
     repo_main = my_main
 
-  if not repo_main:
-    print("fatal: unable to find repo entry point", file=sys.stderr)
-    sys.exit(1)
-
   ver_str = '.'.join(map(str, VERSION))
   me = [sys.executable, repo_main,
         '--repo-dir=%s' % rel_repo_dir,

subcmds/forall.py

@@ -179,8 +179,6 @@ without iterating through the remaining projects.
         'annotations': dict((a.name, a.value) for a in project.annotations),
         'gitdir': project.gitdir,
         'worktree': project.worktree,
-        'upstream': project.upstream,
-        'dest_branch': project.dest_branch,
     }
 
   def ValidateOptions(self, opt, args):
@@ -319,8 +317,6 @@ def DoWork(project, mirror, opt, cmd, shell, cnt, config):
   setenv('REPO_REMOTE', project['remote_name'])
   setenv('REPO_LREV', project['lrev'])
   setenv('REPO_RREV', project['rrev'])
-  setenv('REPO_UPSTREAM', project['upstream'])
-  setenv('REPO_DEST_BRANCH', project['dest_branch'])
   setenv('REPO_I', str(cnt + 1))
   for name in project['annotations']:
     setenv("REPO__%s" % (name), project['annotations'][name])

subcmds/info.py

@@ -16,7 +16,7 @@
 
 from command import PagedCommand
 from color import Coloring
-from git_refs import R_M, R_HEADS
+from git_refs import R_M
 
 
 class _Coloring(Coloring):
@@ -127,10 +127,7 @@ class Info(PagedCommand):
     if not self.opt.local:
       project.Sync_NetworkHalf(quiet=True, current_branch_only=True)
 
-    branch = self.manifest.manifestProject.config.GetBranch('default').merge
+    logTarget = R_M + self.manifest.manifestProject.config.GetBranch("default").merge
-    if branch.startswith(R_HEADS):
-      branch = branch[len(R_HEADS):]
-    logTarget = R_M + branch
 
     bareTmp = project.bare_git._bare
     project.bare_git._bare = False

subcmds/init.py

@@ -54,8 +54,7 @@ from the server and is installed in the .repo/ directory in the
 current working directory.
 
 The optional -b argument can be used to select the manifest branch
-to checkout and use.  If no branch is specified, the remote's default
+to checkout and use.  If no branch is specified, master is assumed.
-branch is used.
 
 The optional -m argument can be used to specify an alternate manifest
 to be used. If no manifest is specified, the manifest default.xml
@@ -156,11 +155,9 @@ to update the working directory files.
                  help='restrict manifest projects to ones with a specified '
                       'platform group [auto|all|none|linux|darwin|...]',
                  metavar='PLATFORM')
-    g.add_option('--clone-bundle', action='store_true',
-                 help='force use of /clone.bundle on HTTP/HTTPS (default if not --partial-clone)')
     g.add_option('--no-clone-bundle',
-                 dest='clone_bundle', action='store_false',
+                 dest='clone_bundle', default=True, action='store_false',
-                 help='disable use of /clone.bundle on HTTP/HTTPS (default if --partial-clone)')
+                 help='disable use of /clone.bundle on HTTP/HTTPS')
     g.add_option('--no-tags',
                  dest='tags', default=True, action='store_false',
                  help="don't fetch tags in the manifest")
@@ -216,27 +213,24 @@ to update the working directory files.
 
       m._InitGitDir(mirror_git=mirrored_manifest_git)
 
+      if opt.manifest_branch:
+        m.revisionExpr = opt.manifest_branch
+      else:
+        m.revisionExpr = 'refs/heads/master'
+    else:
+      if opt.manifest_branch:
+        m.revisionExpr = opt.manifest_branch
+      else:
+        m.PreSync()
+
     self._ConfigureDepth(opt)
 
-    # Set the remote URL before the remote branch as we might need it below.
     if opt.manifest_url:
       r = m.GetRemote(m.remote.name)
       r.url = opt.manifest_url
       r.ResetFetch()
       r.Save()
 
-    if opt.manifest_branch:
-      m.revisionExpr = opt.manifest_branch
-    else:
-      if is_new:
-        default_branch = m.ResolveRemoteHead()
-        if default_branch is None:
-          # If the remote doesn't have HEAD configured, default to master.
-          default_branch = 'refs/heads/master'
-        m.revisionExpr = default_branch
-      else:
-        m.PreSync()
-
     groups = re.split(r'[,\s]+', opt.groups)
     all_platforms = ['linux', 'darwin', 'windows']
     platformize = lambda x: 'platform-' + x
@@ -309,11 +303,6 @@ to update the working directory files.
     else:
       opt.clone_filter = None
 
-    if opt.clone_bundle is None:
-      opt.clone_bundle = False if opt.partial_clone else True
-    else:
-      m.config.SetString('repo.clonebundle', 'true' if opt.clone_bundle else 'false')
-
     if opt.submodules:
       m.config.SetString('repo.submodules', 'true')
 
@@ -492,9 +481,6 @@ to update the working directory files.
     if opt.archive and opt.mirror:
       self.OptionParser.error('--mirror and --archive cannot be used together.')
 
-    if args:
-      self.OptionParser.error('init takes no arguments')
-
   def Execute(self, opt, args):
     git_require(MIN_GIT_VERSION_HARD, fail=True)
     if not git_require(MIN_GIT_VERSION_SOFT):

subcmds/manifest.py

@@ -30,16 +30,10 @@ class Manifest(PagedCommand):
   _helpDescription = """
 
 With the -o option, exports the current manifest for inspection.
-The manifest and (if present) local_manifests/ are combined
+The manifest and (if present) local_manifest.xml are combined
 together to produce a single manifest file.  This file can be stored
 in a Git repository for use during future 'repo init' invocations.
 
-The -r option can be used to generate a manifest file with project
-revisions set to the current commit hash.  These are known as
-"revision locked manifests", as they don't follow a particular branch.
-In this case, the 'upstream' attribute is set to the ref we were on
-when the manifest was generated.  The 'dest-branch' attribute is set
-to indicate the remote ref to push changes to via 'repo upload'.
 """
 
   @property
@@ -63,11 +57,6 @@ to indicate the remote ref to push changes to via 'repo upload'.
                  help='If in -r mode, do not write the upstream field.  '
                  'Only of use if the branch names for a sha1 manifest are '
                  'sensitive.')
-    p.add_option('--suppress-dest-branch', dest='peg_rev_dest_branch',
-                 default=True, action='store_false',
-                 help='If in -r mode, do not write the dest-branch field.  '
-                 'Only of use if the branch names for a sha1 manifest are '
-                 'sensitive.')
     p.add_option('-o', '--output-file',
                  dest='output_file',
                  default='-',
@@ -85,8 +74,7 @@ to indicate the remote ref to push changes to via 'repo upload'.
       fd = open(opt.output_file, 'w')
     self.manifest.Save(fd,
                        peg_rev=opt.peg_rev,
-                       peg_rev_upstream=opt.peg_rev_upstream,
+                       peg_rev_upstream=opt.peg_rev_upstream)
-                       peg_rev_dest_branch=opt.peg_rev_dest_branch)
     fd.close()
     if opt.output_file != '-':
       print('Saved manifest to %s' % opt.output_file, file=sys.stderr)

subcmds/status.py

@@ -16,13 +16,17 @@
 
 from __future__ import print_function
 
-import functools
 import glob
-import multiprocessing
+import itertools
 import os
 
 from command import PagedCommand
 
+try:
+  import threading as _threading
+except ImportError:
+  import dummy_threading as _threading
+
 from color import Coloring
 import platform_utils
 
@@ -91,20 +95,25 @@ the following meanings:
     p.add_option('-q', '--quiet', action='store_true',
                  help="only print the name of modified projects")
 
-  def _StatusHelper(self, quiet, project):
+  def _StatusHelper(self, project, clean_counter, sem, quiet):
     """Obtains the status for a specific project.
 
     Obtains the status for a project, redirecting the output to
-    the specified object.
+    the specified object. It will release the semaphore
+    when done.
 
     Args:
-      quiet: Where to output the status.
       project: Project to get status of.
-
+      clean_counter: Counter for clean projects.
-    Returns:
+      sem: Semaphore, will call release() when complete.
-      The status of the project.
+      output: Where to output the status.
     """
-    return project.PrintWorkTreeStatus(quiet=quiet)
+    try:
+      state = project.PrintWorkTreeStatus(quiet=quiet)
+      if state == 'CLEAN':
+        next(clean_counter)
+    finally:
+      sem.release()
 
   def _FindOrphans(self, dirs, proj_dirs, proj_dirs_parents, outstring):
     """find 'dirs' that are present in 'proj_dirs_parents' but not in 'proj_dirs'"""
@@ -124,18 +133,27 @@ the following meanings:
 
   def Execute(self, opt, args):
     all_projects = self.GetProjects(args)
-    counter = 0
+    counter = itertools.count()
 
     if opt.jobs == 1:
       for project in all_projects:
         state = project.PrintWorkTreeStatus(quiet=opt.quiet)
         if state == 'CLEAN':
-          counter += 1
+          next(counter)
     else:
-      with multiprocessing.Pool(opt.jobs) as pool:
+      sem = _threading.Semaphore(opt.jobs)
-        states = pool.map(functools.partial(self._StatusHelper, opt.quiet), all_projects)
+      threads = []
-        counter += states.count('CLEAN')
+      for project in all_projects:
-    if not opt.quiet and len(all_projects) == counter:
+        sem.acquire()
+
+        t = _threading.Thread(target=self._StatusHelper,
+                              args=(project, counter, sem, opt.quiet))
+        threads.append(t)
+        t.daemon = True
+        t.start()
+      for t in threads:
+        t.join()
+    if not opt.quiet and len(all_projects) == next(counter):
       print('nothing to commit (working directory clean)')
 
     if opt.orphans:

subcmds/sync.py

@@ -138,11 +138,11 @@ if the manifest server specified in the manifest file already includes
 credentials.
 
 By default, all projects will be synced. The --fail-fast option can be used
-to halt syncing as soon as possible when the first project fails to sync.
+to halt syncing as soon as possible when the the first project fails to sync.
 
 The --force-sync option can be used to overwrite existing git
 directories if they have previously been linked to a different
-object directory. WARNING: This may cause data to be lost since
+object direcotry. WARNING: This may cause data to be lost since
 refs may be removed when overwriting.
 
 The --force-remove-dirty option can be used to remove previously used
@@ -247,9 +247,8 @@ later is required to fix a server side protocol bug.
     p.add_option('-m', '--manifest-name',
                  dest='manifest_name',
                  help='temporary manifest to use for this sync', metavar='NAME.xml')
-    p.add_option('--clone-bundle', action='store_true',
+    p.add_option('--no-clone-bundle',
-                 help='enable use of /clone.bundle on HTTP/HTTPS')
+                 dest='clone_bundle', default=True, action='store_false',
-    p.add_option('--no-clone-bundle', dest='clone_bundle', action='store_false',
                  help='disable use of /clone.bundle on HTTP/HTTPS')
     p.add_option('-u', '--manifest-server-username', action='store',
                  dest='manifest_server_username',
@@ -837,9 +836,6 @@ later is required to fix a server side protocol bug.
     smart_sync_manifest_path = os.path.join(
         self.manifest.manifestProject.worktree, 'smart_sync_override.xml')
 
-    if opt.clone_bundle is None:
-      opt.clone_bundle = self.manifest.CloneBundle
-
     if opt.smart_sync or opt.smart_tag:
       manifest_name = self._SmartSyncSetup(opt, smart_sync_manifest_path)
     else:

subcmds/upload.py

@@ -23,8 +23,7 @@ from command import InteractiveCommand
 from editor import Editor
 from error import HookError, UploadError
 from git_command import GitCommand
-from git_refs import R_HEADS
+from project import RepoHook
-from hooks import RepoHook
 
 from pyversion import is_python3
 if not is_python3():
@@ -463,10 +462,7 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
         # Make sure our local branch is not setup to track a different remote branch
         merge_branch = self._GetMergeBranch(branch.project)
         if destination:
-          full_dest = destination
+          full_dest = 'refs/heads/%s' % destination
-          if not full_dest.startswith(R_HEADS):
-            full_dest = R_HEADS + full_dest
-
           if not opt.dest_branch and merge_branch and merge_branch != full_dest:
             print('merge branch %s does not match destination branch %s'
                   % (merge_branch, full_dest))
@@ -596,7 +592,7 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
           print('\nWARNING: pre-upload hooks failed, but uploading anyways.',
                 file=sys.stderr)
         else:
-          return 1
+          return
 
     if opt.reviewers:
       reviewers = _SplitEmails(opt.reviewers)

tests/test_hooks.py

@@ -1,60 +0,0 @@
-# -*- coding:utf-8 -*-
-#
-# Copyright (C) 2019 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Unittests for the hooks.py module."""
-
-from __future__ import print_function
-
-import hooks
-import unittest
-
-class RepoHookShebang(unittest.TestCase):
-  """Check shebang parsing in RepoHook."""
-
-  def test_no_shebang(self):
-    """Lines w/out shebangs should be rejected."""
-    DATA = (
-        '',
-        '# -*- coding:utf-8 -*-\n',
-        '#\n# foo\n',
-        '# Bad shebang in script\n#!/foo\n'
-    )
-    for data in DATA:
-      self.assertIsNone(hooks.RepoHook._ExtractInterpFromShebang(data))
-
-  def test_direct_interp(self):
-    """Lines whose shebang points directly to the interpreter."""
-    DATA = (
-        ('#!/foo', '/foo'),
-        ('#! /foo', '/foo'),
-        ('#!/bin/foo ', '/bin/foo'),
-        ('#! /usr/foo ', '/usr/foo'),
-        ('#! /usr/foo -args', '/usr/foo'),
-    )
-    for shebang, interp in DATA:
-      self.assertEqual(hooks.RepoHook._ExtractInterpFromShebang(shebang),
-                       interp)
-
-  def test_env_interp(self):
-    """Lines whose shebang launches through `env`."""
-    DATA = (
-        ('#!/usr/bin/env foo', 'foo'),
-        ('#!/bin/env foo', 'foo'),
-        ('#! /bin/env /bin/foo ', '/bin/foo'),
-    )
-    for shebang, interp in DATA:
-      self.assertEqual(hooks.RepoHook._ExtractInterpFromShebang(shebang),
-                       interp)

tests/test_project.py

@@ -44,6 +44,45 @@ def TempGitTree():
     platform_utils.rmtree(tempdir)
 
 
+class RepoHookShebang(unittest.TestCase):
+  """Check shebang parsing in RepoHook."""
+
+  def test_no_shebang(self):
+    """Lines w/out shebangs should be rejected."""
+    DATA = (
+        '',
+        '# -*- coding:utf-8 -*-\n',
+        '#\n# foo\n',
+        '# Bad shebang in script\n#!/foo\n'
+    )
+    for data in DATA:
+      self.assertIsNone(project.RepoHook._ExtractInterpFromShebang(data))
+
+  def test_direct_interp(self):
+    """Lines whose shebang points directly to the interpreter."""
+    DATA = (
+        ('#!/foo', '/foo'),
+        ('#! /foo', '/foo'),
+        ('#!/bin/foo ', '/bin/foo'),
+        ('#! /usr/foo ', '/usr/foo'),
+        ('#! /usr/foo -args', '/usr/foo'),
+    )
+    for shebang, interp in DATA:
+      self.assertEqual(project.RepoHook._ExtractInterpFromShebang(shebang),
+                       interp)
+
+  def test_env_interp(self):
+    """Lines whose shebang launches through `env`."""
+    DATA = (
+        ('#!/usr/bin/env foo', 'foo'),
+        ('#!/bin/env foo', 'foo'),
+        ('#! /bin/env /bin/foo ', '/bin/foo'),
+    )
+    for shebang, interp in DATA:
+      self.assertEqual(project.RepoHook._ExtractInterpFromShebang(shebang),
+                       interp)
+
+
 class FakeProject(object):
   """A fake for Project for basic functionality."""
 

tests/test_subcmds.py

@@ -1,43 +0,0 @@
-# Copyright (C) 2020 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Unittests for the subcmds module (mostly __init__.py than subcommands)."""
-
-import unittest
-
-import subcmds
-
-
-class AllCommands(unittest.TestCase):
-  """Check registered all_commands."""
-
-  def test_required_basic(self):
-    """Basic checking of registered commands."""
-    # NB: We don't test all subcommands as we want to avoid "change detection"
-    # tests, so we just look for the most common/important ones here that are
-    # unlikely to ever change.
-    for cmd in {'cherry-pick', 'help', 'init', 'start', 'sync', 'upload'}:
-      self.assertIn(cmd, subcmds.all_commands)
-
-  def test_naming(self):
-    """Verify we don't add things that we shouldn't."""
-    for cmd in subcmds.all_commands:
-      # Reject filename suffixes like "help.py".
-      self.assertNotIn('.', cmd)
-
-      # Make sure all '_' were converted to '-'.
-      self.assertNotIn('_', cmd)
-
-      # Reject internal python paths like "__init__".
-      self.assertFalse(cmd.startswith('__'))

tests/test_subcmds_init.py

@@ -1,49 +0,0 @@
-# Copyright (C) 2020 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Unittests for the subcmds/init.py module."""
-
-import unittest
-
-from subcmds import init
-
-
-class InitCommand(unittest.TestCase):
-  """Check registered all_commands."""
-
-  def setUp(self):
-    self.cmd = init.Init()
-
-  def test_cli_parser_good(self):
-    """Check valid command line options."""
-    ARGV = (
-        [],
-    )
-    for argv in ARGV:
-      opts, args = self.cmd.OptionParser.parse_args(argv)
-      self.cmd.ValidateOptions(opts, args)
-
-  def test_cli_parser_bad(self):
-    """Check invalid command line options."""
-    ARGV = (
-        # Too many arguments.
-        ['asdf'],
-
-        # Conflicting options.
-        ['--mirror', '--archive'],
-    )
-    for argv in ARGV:
-      opts, args = self.cmd.OptionParser.parse_args(argv)
-      with self.assertRaises(SystemExit):
-        self.cmd.ValidateOptions(opts, args)

tox.ini

@@ -15,10 +15,11 @@
 # https://tox.readthedocs.io/
 
 [tox]
-envlist = py36, py37, py38
+envlist = py27, py36, py37, py38
 
 [gh-actions]
 python =
+    2.7: py27
     3.6: py36
     3.7: py37
     3.8: py38
@@ -30,3 +31,8 @@ setenv =
     GIT_AUTHOR_NAME = Repo test author
     GIT_COMMITTER_NAME = Repo test committer
     EMAIL = repo@gerrit.nodomain
+
+[testenv:py27]
+deps =
+    mock
+    pytest