* [gentoo-commits] proj/portage:master commit in: repoman/man/, repoman/lib/repoman/tests/simple/, repoman/lib/repoman/, ...
@ 2019-11-20 10:30 Zac Medico
0 siblings, 0 replies; only message in thread
From: Zac Medico @ 2019-11-20 10:30 UTC (permalink / raw
To: gentoo-commits
commit: e9bb1e9681685f4e4d7174f51751356fd3f67d0c
Author: Sergei Trofimovich <slyfox <AT> gentoo <DOT> org>
AuthorDate: Tue Nov 19 00:21:13 2019 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Nov 20 10:29:14 2019 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=e9bb1e96
repoman: add --include-profiles=PROFILES
repoman slows down ~linearly with amount of profiles being scanned.
In case of amd64 we have 28 stable profiles.
To speed up processing and fit into time budged of various CIs we can
split the work across different processes that handle different profiles.
Example benchmark on ::haskell overlay:
$ ./repoman full --include-arches=amd64
~65 minutes
$ ./repoman full --include-profiles=default/linux/amd64/17.0
~4 minutes
This allows for a crude sharding of work across processes and allows for
cheap tree-wide scans for early failures.
Bug: https://bugs.gentoo.org/700456
Signed-off-by: Sergei Trofimovich <slyfox <AT> gentoo.org>
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
repoman/lib/repoman/actions.py | 4 ++++
repoman/lib/repoman/argparser.py | 7 +++++++
repoman/lib/repoman/modules/scan/depend/__init__.py | 3 ++-
repoman/lib/repoman/modules/scan/depend/profile.py | 9 +++++++--
repoman/lib/repoman/scanner.py | 5 +++++
repoman/lib/repoman/tests/simple/test_simple.py | 1 +
repoman/man/repoman.1 | 4 ++++
7 files changed, 30 insertions(+), 3 deletions(-)
diff --git a/repoman/lib/repoman/actions.py b/repoman/lib/repoman/actions.py
index 1c9989a72..92d4d4e94 100644
--- a/repoman/lib/repoman/actions.py
+++ b/repoman/lib/repoman/actions.py
@@ -412,6 +412,10 @@ the whole commit message to abort.
report_options.append(
"--include-arches=\"%s\"" %
" ".join(sorted(self.scanner.include_arches)))
+ if self.scanner.include_profiles is not None:
+ report_options.append(
+ "--include-profiles=\"%s\"" %
+ " ".join(sorted(self.scanner.include_profiles)))
if portage_version is None:
sys.stderr.write("Failed to insert portage version in message!\n")
diff --git a/repoman/lib/repoman/argparser.py b/repoman/lib/repoman/argparser.py
index fa0e6ff90..670a0e91d 100644
--- a/repoman/lib/repoman/argparser.py
+++ b/repoman/lib/repoman/argparser.py
@@ -164,6 +164,13 @@ def parse_args(argv, repoman_default_opts):
'A space separated list of arches used to '
'filter the selection of profiles for dependency checks'))
+ parser.add_argument(
+ '--include-profiles',
+ dest='include_profiles', metavar='PROFILES', action='append',
+ help=(
+ 'A space separated list of profiles used to '
+ 'define the selection of profiles for dependency checks'))
+
parser.add_argument(
'-d', '--include-dev', dest='include_dev', action='store_true',
default=False,
diff --git a/repoman/lib/repoman/modules/scan/depend/__init__.py b/repoman/lib/repoman/modules/scan/depend/__init__.py
index c3cc0ddeb..9068760bb 100644
--- a/repoman/lib/repoman/modules/scan/depend/__init__.py
+++ b/repoman/lib/repoman/modules/scan/depend/__init__.py
@@ -19,7 +19,8 @@ module_spec = {
'func_desc': {
},
'mod_kwargs': ['qatracker', 'portdb', 'profiles', 'options',
- 'repo_metadata', 'repo_settings', 'include_arches', 'caches',
+ 'repo_metadata', 'repo_settings', 'include_arches',
+ 'include_profiles', 'caches',
'repoman_incrementals', 'env', 'have', 'dev_keywords'
],
'func_kwargs': {
diff --git a/repoman/lib/repoman/modules/scan/depend/profile.py b/repoman/lib/repoman/modules/scan/depend/profile.py
index d980f4eca..39d8b550c 100644
--- a/repoman/lib/repoman/modules/scan/depend/profile.py
+++ b/repoman/lib/repoman/modules/scan/depend/profile.py
@@ -33,6 +33,7 @@ class ProfileDependsChecks(ScanBase):
@param options: cli options
@param repo_settings: repository settings instance
@param include_arches: set
+ @param include_profiles: set
@param caches: dictionary of our caches
@param repoman_incrementals: tuple
@param env: the environment
@@ -46,6 +47,7 @@ class ProfileDependsChecks(ScanBase):
self.options = kwargs.get('options')
self.repo_settings = kwargs.get('repo_settings')
self.include_arches = kwargs.get('include_arches')
+ self.include_profiles = kwargs.get('include_profiles')
self.caches = kwargs.get('caches')
self.repoman_incrementals = kwargs.get('repoman_incrementals')
self.env = kwargs.get('env')
@@ -81,8 +83,11 @@ class ProfileDependsChecks(ScanBase):
if arch not in self.include_arches:
continue
- relevant_profiles.extend(
- (keyword, groups, prof) for prof in self.profiles[arch])
+ for prof in self.profiles[arch]:
+ if self.include_profiles is not None:
+ if prof.sub_path not in self.include_profiles:
+ continue
+ relevant_profiles.append((keyword, groups, prof))
relevant_profiles.sort(key=sort_key)
diff --git a/repoman/lib/repoman/scanner.py b/repoman/lib/repoman/scanner.py
index 1b3242a51..06234b0ad 100644
--- a/repoman/lib/repoman/scanner.py
+++ b/repoman/lib/repoman/scanner.py
@@ -164,6 +164,10 @@ class Scanner(object):
if self.options.include_arches:
self.include_arches = set()
self.include_arches.update(*[x.split() for x in self.options.include_arches])
+ self.include_profiles = None
+ if self.options.include_profiles:
+ self.include_profiles = set()
+ self.include_profiles.update(*[x.split() for x in self.options.include_profiles])
# Disable the "self.modules['Ebuild'].notadded" check when not in commit mode and
# running `svn status` in every package dir will be too expensive.
@@ -190,6 +194,7 @@ class Scanner(object):
"repo_metadata": self.repo_metadata,
"profiles": profiles,
"include_arches": self.include_arches,
+ "include_profiles": self.include_profiles,
"caches": self.caches,
"repoman_incrementals": self.repoman_incrementals,
"env": self.env,
diff --git a/repoman/lib/repoman/tests/simple/test_simple.py b/repoman/lib/repoman/tests/simple/test_simple.py
index b0cc43297..c86a183f9 100644
--- a/repoman/lib/repoman/tests/simple/test_simple.py
+++ b/repoman/lib/repoman/tests/simple/test_simple.py
@@ -202,6 +202,7 @@ class SimpleRepomanTestCase(TestCase):
("", git_cmd + ("add", ".")),
("", git_cmd + ("commit", "-a", "-m", "add whole repo")),
("", repoman_cmd + ("full", "-d")),
+ ("", repoman_cmd + ("full", "--include-profiles", "default/linux/x86/test_profile")),
("", cp_cmd + (test_ebuild, test_ebuild[:-8] + "2.ebuild")),
("", git_cmd + ("add", test_ebuild[:-8] + "2.ebuild")),
("", repoman_cmd + ("commit", "-m", "cat/pkg: bump to version 2")),
diff --git a/repoman/man/repoman.1 b/repoman/man/repoman.1
index 766146f57..f33fb6098 100644
--- a/repoman/man/repoman.1
+++ b/repoman/man/repoman.1
@@ -120,6 +120,10 @@ Ignore masked packages (not allowed with commit mode)
A space separated list of arches used to filter the selection of
profiles for dependency checks.
.TP
+.BR "\-\-include\-profiles " PROFILES
+A space separated list of profiles used to
+define the selection of profiles for dependency checks.
+.TP
\fB\-d\fR, \fB\-\-include\-dev\fR
Include dev profiles in dependency checks.
.TP
^ permalink raw reply related [flat|nested] only message in thread
only message in thread, other threads:[~2019-11-20 10:30 UTC | newest]
Thread overview: (only message) (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2019-11-20 10:30 [gentoo-commits] proj/portage:master commit in: repoman/man/, repoman/lib/repoman/tests/simple/, repoman/lib/repoman/, Zac Medico
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox