From 6a2400a4d097b6e510dc9b8ec06283517b9ca3ad Mon Sep 17 00:00:00 2001 From: Mike Frysinger Date: Tue, 16 Feb 2021 01:43:31 -0500 Subject: [PATCH] command: unify --job option & default values Extend the Command class to support adding the --jobs option to the parser if the command declares it supports running in parallel. Also pull the default value used for the number of local jobs into the command module so local commands can share it. Change-Id: I22b0f8d2cf69875013cec657b8e6c4385549ccac Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/297024 Tested-by: Mike Frysinger Reviewed-by: Chris Mcdonald --- command.py | 14 ++++++++++++++ subcmds/branches.py | 14 ++------------ subcmds/forall.py | 8 ++++---- subcmds/status.py | 7 +++---- subcmds/sync.py | 9 ++++----- 5 files changed, 27 insertions(+), 25 deletions(-) diff --git a/command.py b/command.py index ef2554d..7737ec7 100644 --- a/command.py +++ b/command.py @@ -23,6 +23,11 @@ from error import NoSuchProjectError from error import InvalidProjectGroupsError +# How many jobs to run in parallel by default? This assumes the jobs are +# largely I/O bound and do not hit the network. +DEFAULT_LOCAL_JOBS = min(os.cpu_count(), 8) + + class Command(object): """Base class for any command line action in repo. """ @@ -32,6 +37,10 @@ class Command(object): manifest = None _optparse = None + # Whether this command supports running in parallel. If greater than 0, + # it is the number of parallel jobs to default to. + PARALLEL_JOBS = None + def WantPager(self, _opt): return False @@ -72,6 +81,11 @@ class Command(object): def _Options(self, p): """Initialize the option parser. """ + if self.PARALLEL_JOBS is not None: + p.add_option( + '-j', '--jobs', + type=int, default=self.PARALLEL_JOBS, + help='number of jobs to run in parallel (default: %s)' % self.PARALLEL_JOBS) def _RegisteredEnvironmentOptions(self): """Get options that can be set from environment variables. diff --git a/subcmds/branches.py b/subcmds/branches.py index 20f5169..9665e85 100644 --- a/subcmds/branches.py +++ b/subcmds/branches.py @@ -16,7 +16,7 @@ import itertools import multiprocessing import sys from color import Coloring -from command import Command +from command import Command, DEFAULT_LOCAL_JOBS # Number of projects to submit to a single worker process at a time. # This number represents a tradeoff between the overhead of IPC and finer @@ -103,17 +103,7 @@ the branch appears in, or does not appear in. If no project list is shown, then the branch appears in all projects. """ - - def _Options(self, p): - """Add flags to CLI parser for this subcommand.""" - default_jobs = min(multiprocessing.cpu_count(), 8) - p.add_option( - '-j', - '--jobs', - type=int, - default=default_jobs, - help='Number of worker processes to spawn ' - '(default: %s)' % default_jobs) + PARALLEL_JOBS = DEFAULT_LOCAL_JOBS def Execute(self, opt, args): projects = self.GetProjects(args) diff --git a/subcmds/forall.py b/subcmds/forall.py index ef11851..d871b3e 100644 --- a/subcmds/forall.py +++ b/subcmds/forall.py @@ -21,7 +21,7 @@ import sys import subprocess from color import Coloring -from command import Command, MirrorSafeCommand +from command import DEFAULT_LOCAL_JOBS, Command, MirrorSafeCommand import platform_utils _CAN_COLOR = [ @@ -113,8 +113,11 @@ terminal and are not redirected. If -e is used, when a command exits unsuccessfully, '%prog' will abort without iterating through the remaining projects. """ + PARALLEL_JOBS = DEFAULT_LOCAL_JOBS def _Options(self, p): + super()._Options(p) + def cmd(option, opt_str, value, parser): setattr(parser.values, option.dest, list(parser.rargs)) while parser.rargs: @@ -148,9 +151,6 @@ without iterating through the remaining projects. g.add_option('-v', '--verbose', dest='verbose', action='store_true', help='Show command error messages') - g.add_option('-j', '--jobs', - dest='jobs', action='store', type='int', default=1, - help='number of commands to execute simultaneously') def WantPager(self, opt): return opt.project_header and opt.jobs == 1 diff --git a/subcmds/status.py b/subcmds/status.py index e293d75..f0f2e03 100644 --- a/subcmds/status.py +++ b/subcmds/status.py @@ -17,7 +17,7 @@ import glob import multiprocessing import os -from command import PagedCommand +from command import DEFAULT_LOCAL_JOBS, PagedCommand from color import Coloring import platform_utils @@ -76,11 +76,10 @@ the following meanings: d: deleted ( in index, not in work tree ) """ + PARALLEL_JOBS = DEFAULT_LOCAL_JOBS def _Options(self, p): - p.add_option('-j', '--jobs', - dest='jobs', action='store', type='int', default=2, - help="number of projects to check simultaneously") + super()._Options(p) p.add_option('-o', '--orphans', dest='orphans', action='store_true', help="include objects in working directory outside of repo projects") diff --git a/subcmds/sync.py b/subcmds/sync.py index 5b1024d..18d2256 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -177,12 +177,14 @@ If the remote SSH daemon is Gerrit Code Review, version 2.0.10 or later is required to fix a server side protocol bug. """ + PARALLEL_JOBS = 1 def _Options(self, p, show_smart=True): try: - self.jobs = self.manifest.default.sync_j + self.PARALLEL_JOBS = self.manifest.default.sync_j except ManifestParseError: - self.jobs = 1 + pass + super()._Options(p) p.add_option('-f', '--force-broken', dest='force_broken', action='store_true', @@ -222,9 +224,6 @@ later is required to fix a server side protocol bug. p.add_option('-q', '--quiet', dest='output_mode', action='store_false', help='only show errors') - p.add_option('-j', '--jobs', - dest='jobs', action='store', type='int', - help="projects to fetch simultaneously (default %d)" % self.jobs) p.add_option('-m', '--manifest-name', dest='manifest_name', help='temporary manifest to use for this sync', metavar='NAME.xml')