* [gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/ebuild/, pym/repoman/modules/scan/keywords/, ...
@ 2016-04-24 1:00 Brian Dolbec
0 siblings, 0 replies; 3+ messages in thread
From: Brian Dolbec @ 2016-04-24 1:00 UTC (permalink / raw
To: gentoo-commits
commit: 8838c4317981315741df4334d5a1a172902f20a0
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Wed Apr 20 16:52:21 2016 +0000
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sun Apr 24 00:45:21 2016 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=8838c431
More new data structure change fixes
Fix arches not being cleared every ebuild.
pym/repoman/modules/scan/arches/arches.py | 1 +
pym/repoman/modules/scan/depend/depend.py | 1 +
pym/repoman/modules/scan/ebuild/__init__.py | 2 +-
pym/repoman/modules/scan/ebuild/ebuild.py | 2 +-
pym/repoman/modules/scan/ebuild/isebuild.py | 11 +++--
pym/repoman/modules/scan/eclasses/live.py | 5 ++-
pym/repoman/modules/scan/keywords/__init__.py | 4 +-
pym/repoman/modules/scan/metadata/__init__.py | 2 +-
pym/repoman/modules/scan/metadata/unused.py | 2 +-
pym/repoman/modules/scan/use/use_flags.py | 13 +++---
pym/repoman/scanner.py | 62 ++++++++++++++++++---------
11 files changed, 67 insertions(+), 38 deletions(-)
diff --git a/pym/repoman/modules/scan/arches/arches.py b/pym/repoman/modules/scan/arches/arches.py
index 8bc7a24..6e1c17d 100644
--- a/pym/repoman/modules/scan/arches/arches.py
+++ b/pym/repoman/modules/scan/arches/arches.py
@@ -69,6 +69,7 @@ class ArchChecks(ScanBase):
arches.add(('**', '**', ('**',)))
# update the dynamic data
dyn_arches = kwargs.get('arches')
+ dyn_arches.clear()
dyn_arches.update(arches)
return False
diff --git a/pym/repoman/modules/scan/depend/depend.py b/pym/repoman/modules/scan/depend/depend.py
index a5f91d3..cd1fb33 100644
--- a/pym/repoman/modules/scan/depend/depend.py
+++ b/pym/repoman/modules/scan/depend/depend.py
@@ -145,6 +145,7 @@ class DependChecks(ScanBase):
# update the dynamic data
dyn_unknown = kwargs.get('unknown_pkgs')
+ dyn_unknown.clear()
dyn_unknown.update(unknown_pkgs)
self.set_result_pass([
(kwargs.get('badlicsyntax'), badlicsyntax),
diff --git a/pym/repoman/modules/scan/ebuild/__init__.py b/pym/repoman/modules/scan/ebuild/__init__.py
index ea21ece..efb2feb 100644
--- a/pym/repoman/modules/scan/ebuild/__init__.py
+++ b/pym/repoman/modules/scan/ebuild/__init__.py
@@ -21,7 +21,7 @@ module_spec = {
'mod_kwargs': ['portdb', 'qatracker', 'repo_settings'
],
'func_kwargs': {'checkdirlist': 'list', 'checkdir': None, 'xpkg': None,
- 'validity_fuse': None, 'can_force': None, 'pkgs': 'dict',
+ 'validity_future': 'Future', 'can_force': None, 'pkgs': 'dict',
},
},
'ebuild-module': {
diff --git a/pym/repoman/modules/scan/ebuild/ebuild.py b/pym/repoman/modules/scan/ebuild/ebuild.py
index b015985..b02cdcd 100644
--- a/pym/repoman/modules/scan/ebuild/ebuild.py
+++ b/pym/repoman/modules/scan/ebuild/ebuild.py
@@ -86,7 +86,7 @@ class Ebuild(ScanBase):
self.qatracker.add_error(
"ebuild.notadded", self.xpkg + "/" + self.y_ebuild + ".ebuild")
# update the dynamic data
- self.set_result_raise([('ebuild', self)])
+ self.set_result_raise([(kwargs.get('ebuild'), self)])
return False
def set_pkg_data(self, **kwargs):
diff --git a/pym/repoman/modules/scan/ebuild/isebuild.py b/pym/repoman/modules/scan/ebuild/isebuild.py
index 3e288a3..24680e2 100644
--- a/pym/repoman/modules/scan/ebuild/isebuild.py
+++ b/pym/repoman/modules/scan/ebuild/isebuild.py
@@ -13,6 +13,7 @@ from portage import os
from repoman.qa_data import no_exec, allvars
from repoman.modules.scan.scanbase import ScanBase
+
class IsEbuild(ScanBase):
'''Performs basic tests to confirm it is an ebuild'''
@@ -88,9 +89,13 @@ class IsEbuild(ScanBase):
self.continue_ = True
self.set_result_pass([(can_force, False)])
# set our updated data
- self.set_result_raise([
- (kwargs.get('pkgs'), pkgs),
- ])
+ dyn_pkgs = kwargs.get('pkgs')
+ # clear() sets it to None,
+ # we don't want to kill the pointer reference
+ # just set it back to an empty dict()
+ for key in list(dyn_pkgs):
+ dyn_pkgs.pop(key)
+ dyn_pkgs.update(pkgs)
return self.continue_
@property
diff --git a/pym/repoman/modules/scan/eclasses/live.py b/pym/repoman/modules/scan/eclasses/live.py
index e5bb7dc..85e829c 100644
--- a/pym/repoman/modules/scan/eclasses/live.py
+++ b/pym/repoman/modules/scan/eclasses/live.py
@@ -25,10 +25,11 @@ class LiveEclassChecks(ScanBase):
@returns: dictionary, including {live_ebuild}
'''
+ ebuild = kwargs.get('ebuild').result()
# update the dynamic data
dyn_live = kwargs.get('live_ebuild')
- dyn_live.update(LIVE_ECLASSES.intersection(
- kwargs.get('ebuild').inherited))
+ #dyn_live.clear()
+ dyn_live.update(LIVE_ECLASSES.intersection(ebuild.inherited))
return False
def check(self, **kwargs):
diff --git a/pym/repoman/modules/scan/keywords/__init__.py b/pym/repoman/modules/scan/keywords/__init__.py
index 2d92dba..2b3f6ca 100644
--- a/pym/repoman/modules/scan/keywords/__init__.py
+++ b/pym/repoman/modules/scan/keywords/__init__.py
@@ -20,8 +20,8 @@ module_spec = {
},
'mod_kwargs': ['qatracker', 'options', 'repo_metadata', 'profiles',
],
- 'func_kwargs': {'pkg': None, 'xpkg': None, 'ebuild': None,
- 'y_ebuild': None, 'changed': None, 'live_ebuild': None,
+ 'func_kwargs': {'pkg': 'Future', 'xpkg': None, 'ebuild': 'Future',
+ 'y_ebuild': None, 'changed': None, 'live_ebuild': 'set',
},
},
}
diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py
index af9771b..9ccf377 100644
--- a/pym/repoman/modules/scan/metadata/__init__.py
+++ b/pym/repoman/modules/scan/metadata/__init__.py
@@ -48,7 +48,7 @@ module_spec = {
},
'mod_kwargs': ['qatracker',
],
- 'func_kwargs': {'ebuild': None, 'pkg': None,
+ 'func_kwargs': {'ebuild': None, 'pkg': 'Future',
},
},
'license-metadata': {
diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py
index 0184d78..fdca418 100644
--- a/pym/repoman/modules/scan/metadata/unused.py
+++ b/pym/repoman/modules/scan/metadata/unused.py
@@ -23,7 +23,7 @@ class UnusedCheck(ScanBase):
@param validity_future: Future instance
'''
xpkg = kwargs.get('xpkg')
- muselist = kwargs.get('muselist').result()
+ muselist = self.get_result(kwargs.get('muselist'), set())
used_useflags = kwargs.get('used_useflags')
valid_state = self.get_result(kwargs['validity_future'], True)
# check if there are unused local USE-descriptions in metadata.xml
diff --git a/pym/repoman/modules/scan/use/use_flags.py b/pym/repoman/modules/scan/use/use_flags.py
index 70f7583..b2b1cc0 100644
--- a/pym/repoman/modules/scan/use/use_flags.py
+++ b/pym/repoman/modules/scan/use/use_flags.py
@@ -42,20 +42,21 @@ class USEFlagChecks(ScanBase):
package = kwargs.get('xpkg')
ebuild = kwargs.get('ebuild').result()
y_ebuild = kwargs.get('y_ebuild')
- localUseFlags = kwargs.get('muselist').result()
+ localUseFlags = self.get_result(kwargs.get('muselist'), set())
+ dyn_ebuild_used = kwargs.get('ebuild_UsedUseFlags')
+ dyn_used = kwargs.get('used_useflags')
# reset state variables for the run
self.useFlags = []
self.defaultUseFlags = []
self.usedUseFlags = set()
+ dyn_ebuild_used.clear()
+ # perform the checks
self._checkGlobal(pkg)
self._checkMetadata(package, ebuild, y_ebuild, localUseFlags)
self._checkRequiredUSE(pkg, ebuild)
# update the dynamic data
- used_useflags = kwargs.get('used_useflags').union(self.usedUseFlags)
- dyn_ebuild_used = kwargs.get('ebuild_UsedUseFlags')
- dyn_ebuild_used.update(self.usedUseFlags)
- dyn_used = kwargs.get('used_useflags')
- dyn_used.update(used_useflags)
+ dyn_ebuild_used.union(self.usedUseFlags)
+ dyn_used.update(self.usedUseFlags)
return False
diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index 3d2c69c..299d118 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -203,11 +203,6 @@ class Scanner(object):
}
# initialize the plugin checks here
self.modules = {}
- for mod in ['manifests', 'isebuild', 'keywords', 'files', 'vcsstatus',
- 'fetches', 'pkgmetadata']:
- mod_class = MODULE_CONTROLLER.get_class(mod)
- logging.debug("Initializing class name: %s", mod_class.__name__)
- self.modules[mod_class.__name__] = mod_class(**self.set_kwargs(mod))
def set_kwargs(self, mod):
'''Creates a limited set of kwargs to pass to the module's __init__()
@@ -229,19 +224,32 @@ class Scanner(object):
@param dynamic_data: dictionary structure
@returns: dictionary
'''
+ #print("****** mod:", mod)
func_kwargs = MODULE_CONTROLLER.modules[mod]['func_kwargs']
+ #print(mod, "**** func_kwargs", func_kwargs)
# determine new keys
- required = set(func_kwargs.viewkeys())
- exist = set(dynamic_data.viewkeys())
- new = exist.difference(required)
+ required = set(list(func_kwargs))
+ #print(" required", required)
+ exist = set(list(dynamic_data))
+ #print(" exist", exist)
+ new = required.difference(exist)
+ #print(" new", new)
# update dynamic_data with initialized entries
for key in new:
- dynamic_data[key] = DATA_TYPES[func_kwargs['key']]()
+ #print(" adding", key, func_kwargs[key], DATA_TYPES[func_kwargs[key]])
+ dynamic_data[key] = DATA_TYPES[func_kwargs[key]]()
kwargs = {}
for key in required:
+ #print(" kwargs[key]", key, dynamic_data[key])
kwargs[key] = dynamic_data[key]
return kwargs
+ def reset_futures(self, dynamic_data):
+ for key in list(dynamic_data):
+ #if key in ['ebuild', 'pkg']: # and isinstance(dynamic_data[key], Future):
+ if isinstance(dynamic_data[key], Future) and key not in ['muselist']:
+ dynamic_data[key] = Future()
+
def scan_pkgs(self, can_force):
for xpkg in self.effective_scanlist:
xpkg_continue = False
@@ -270,16 +278,26 @@ class Scanner(object):
'repolevel': self.repolevel,
'catdir': catdir,
'pkgdir': pkgdir,
- 'validity_future': Future()
+ 'validity_future': Future(),
+ 'y_ebuild': None,
+ # this needs to be reset at the pkg level only,
+ # easiest is to just initialize it here
+ 'muselist': Future(),
}
# need to set it up for ==> self.modules or some other ordered list
- for mod in ['Manifests', 'IsEbuild', 'KeywordChecks', 'FileChecks',
- 'VCSStatus', 'FetchChecks', 'PkgMetadata']:
- logging.debug("scan_pkgs; module: %s", mod)
- do_it, functions = self.modules[mod].runInPkgs
+ for mod in [('manifests', 'Manifests'), ('isebuild', 'IsEbuild'),
+ ('keywords', 'KeywordChecks'), ('files', 'FileChecks'),
+ ('vcsstatus', 'VCSStatus'), ('fetches', 'FetchChecks'),
+ ('pkgmetadata', 'PkgMetadata'),
+ ]:
+ mod_class = MODULE_CONTROLLER.get_class(mod[0])
+ logging.debug("Initializing class name: %s", mod_class.__name__)
+ self.modules[mod_class.__name__] = mod_class(**self.set_kwargs(mod[0]))
+ logging.debug("scan_pkgs; module: %s", mod[1])
+ do_it, functions = self.modules[mod[1]].runInPkgs
if do_it:
for func in functions:
- _continue = func(**self.set_func_kwargs(mod, dynamic_data))
+ _continue = func(**self.set_func_kwargs(mod[0], dynamic_data))
if _continue:
# If we can't access all the metadata then it's totally unsafe to
# commit since there's no way to generate a correct Manifest.
@@ -312,6 +330,7 @@ class Scanner(object):
dynamic_data['used_useflags'] = set()
for y_ebuild in ebuildlist:
+ self.reset_futures(dynamic_data)
dynamic_data['y_ebuild'] = y_ebuild
y_ebuild_continue = False
@@ -320,7 +339,8 @@ class Scanner(object):
for mod in [('ebuild', 'Ebuild'), ('live', 'LiveEclassChecks'),
('eapi', 'EAPIChecks'), ('ebuild_metadata', 'EbuildMetadata'),
('thirdpartymirrors', 'ThirdPartyMirrors'),
- ('description', 'DescriptionChecks'), (None, 'KeywordChecks'),
+ ('description', 'DescriptionChecks'),
+ ('keywords', 'KeywordChecks'),
('arches', 'ArchChecks'), ('depend', 'DependChecks'),
('use_flags', 'USEFlagChecks'), ('ruby', 'RubyEclassChecks'),
('license', 'LicenseChecks'), ('restrict', 'RestrictChecks'),
@@ -329,17 +349,17 @@ class Scanner(object):
('options', 'Options'), ('profile', 'ProfileDependsChecks'),
('unknown', 'DependUnknown'),
]:
- if mod[0]:
+ if mod[0] and mod[1] not in self.modules:
mod_class = MODULE_CONTROLLER.get_class(mod[0])
logging.debug("Initializing class name: %s", mod_class.__name__)
- self.modules[mod[1]] = mod_class(**self.set_kwargs(mod))
+ self.modules[mod[1]] = mod_class(**self.set_kwargs(mod[0]))
logging.debug("scan_ebuilds: module: %s", mod[1])
do_it, functions = self.modules[mod[1]].runInEbuilds
logging.debug("do_it: %s, functions: %s", do_it, [x.__name__ for x in functions])
if do_it:
for func in functions:
logging.debug("\tRunning function: %s", func)
- _continue = func(**self.set_func_kwargs(mod, dynamic_data))
+ _continue = func(**self.set_func_kwargs(mod[0], dynamic_data))
if _continue:
# If we can't access all the metadata then it's totally unsafe to
# commit since there's no way to generate a correct Manifest.
@@ -363,14 +383,14 @@ class Scanner(object):
if mod[0]:
mod_class = MODULE_CONTROLLER.get_class(mod[0])
logging.debug("Initializing class name: %s", mod_class.__name__)
- self.modules[mod[1]] = mod_class(**self.set_kwargs(mod))
+ self.modules[mod[1]] = mod_class(**self.set_kwargs(mod[0]))
logging.debug("scan_ebuilds final checks: module: %s", mod[1])
do_it, functions = self.modules[mod[1]].runInFinal
logging.debug("do_it: %s, functions: %s", do_it, [x.__name__ for x in functions])
if do_it:
for func in functions:
logging.debug("\tRunning function: %s", func)
- _continue = func(**self.set_func_kwargs(mod, dynamic_data))
+ _continue = func(**self.set_func_kwargs(mod[0], dynamic_data))
if _continue:
xpkg_complete = True
# logging.debug("\t>>> Continuing")
^ permalink raw reply related [flat|nested] 3+ messages in thread
* [gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/ebuild/, pym/repoman/modules/scan/keywords/, ...
@ 2016-04-25 3:09 Brian Dolbec
0 siblings, 0 replies; 3+ messages in thread
From: Brian Dolbec @ 2016-04-25 3:09 UTC (permalink / raw
To: gentoo-commits
commit: 4a19fdd05e2aad4c0c97202c784f9d2f49538c67
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Apr 24 01:34:37 2016 +0000
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Mon Apr 25 03:03:04 2016 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=4a19fdd0
More new data structure change fixes
Fix arches not being cleared every ebuild.
pym/repoman/modules/scan/arches/arches.py | 1 +
pym/repoman/modules/scan/depend/depend.py | 1 +
pym/repoman/modules/scan/ebuild/__init__.py | 2 +-
pym/repoman/modules/scan/ebuild/ebuild.py | 2 +-
pym/repoman/modules/scan/ebuild/isebuild.py | 11 ++++--
pym/repoman/modules/scan/eclasses/live.py | 5 ++-
pym/repoman/modules/scan/keywords/__init__.py | 4 +-
pym/repoman/modules/scan/metadata/__init__.py | 2 +-
pym/repoman/modules/scan/metadata/unused.py | 2 +-
pym/repoman/modules/scan/use/use_flags.py | 13 +++---
pym/repoman/scanner.py | 57 +++++++++++++++++----------
11 files changed, 62 insertions(+), 38 deletions(-)
diff --git a/pym/repoman/modules/scan/arches/arches.py b/pym/repoman/modules/scan/arches/arches.py
index 8bc7a24..4df25a8 100644
--- a/pym/repoman/modules/scan/arches/arches.py
+++ b/pym/repoman/modules/scan/arches/arches.py
@@ -69,6 +69,7 @@ class ArchChecks(ScanBase):
arches.add(('**', '**', ('**',)))
# update the dynamic data
dyn_arches = kwargs.get('arches')
+ #dyn_arches.clear()
dyn_arches.update(arches)
return False
diff --git a/pym/repoman/modules/scan/depend/depend.py b/pym/repoman/modules/scan/depend/depend.py
index a5f91d3..cd1fb33 100644
--- a/pym/repoman/modules/scan/depend/depend.py
+++ b/pym/repoman/modules/scan/depend/depend.py
@@ -145,6 +145,7 @@ class DependChecks(ScanBase):
# update the dynamic data
dyn_unknown = kwargs.get('unknown_pkgs')
+ dyn_unknown.clear()
dyn_unknown.update(unknown_pkgs)
self.set_result_pass([
(kwargs.get('badlicsyntax'), badlicsyntax),
diff --git a/pym/repoman/modules/scan/ebuild/__init__.py b/pym/repoman/modules/scan/ebuild/__init__.py
index ea21ece..efb2feb 100644
--- a/pym/repoman/modules/scan/ebuild/__init__.py
+++ b/pym/repoman/modules/scan/ebuild/__init__.py
@@ -21,7 +21,7 @@ module_spec = {
'mod_kwargs': ['portdb', 'qatracker', 'repo_settings'
],
'func_kwargs': {'checkdirlist': 'list', 'checkdir': None, 'xpkg': None,
- 'validity_fuse': None, 'can_force': None, 'pkgs': 'dict',
+ 'validity_future': 'Future', 'can_force': None, 'pkgs': 'dict',
},
},
'ebuild-module': {
diff --git a/pym/repoman/modules/scan/ebuild/ebuild.py b/pym/repoman/modules/scan/ebuild/ebuild.py
index b015985..b02cdcd 100644
--- a/pym/repoman/modules/scan/ebuild/ebuild.py
+++ b/pym/repoman/modules/scan/ebuild/ebuild.py
@@ -86,7 +86,7 @@ class Ebuild(ScanBase):
self.qatracker.add_error(
"ebuild.notadded", self.xpkg + "/" + self.y_ebuild + ".ebuild")
# update the dynamic data
- self.set_result_raise([('ebuild', self)])
+ self.set_result_raise([(kwargs.get('ebuild'), self)])
return False
def set_pkg_data(self, **kwargs):
diff --git a/pym/repoman/modules/scan/ebuild/isebuild.py b/pym/repoman/modules/scan/ebuild/isebuild.py
index 3e288a3..24680e2 100644
--- a/pym/repoman/modules/scan/ebuild/isebuild.py
+++ b/pym/repoman/modules/scan/ebuild/isebuild.py
@@ -13,6 +13,7 @@ from portage import os
from repoman.qa_data import no_exec, allvars
from repoman.modules.scan.scanbase import ScanBase
+
class IsEbuild(ScanBase):
'''Performs basic tests to confirm it is an ebuild'''
@@ -88,9 +89,13 @@ class IsEbuild(ScanBase):
self.continue_ = True
self.set_result_pass([(can_force, False)])
# set our updated data
- self.set_result_raise([
- (kwargs.get('pkgs'), pkgs),
- ])
+ dyn_pkgs = kwargs.get('pkgs')
+ # clear() sets it to None,
+ # we don't want to kill the pointer reference
+ # just set it back to an empty dict()
+ for key in list(dyn_pkgs):
+ dyn_pkgs.pop(key)
+ dyn_pkgs.update(pkgs)
return self.continue_
@property
diff --git a/pym/repoman/modules/scan/eclasses/live.py b/pym/repoman/modules/scan/eclasses/live.py
index e5bb7dc..85e829c 100644
--- a/pym/repoman/modules/scan/eclasses/live.py
+++ b/pym/repoman/modules/scan/eclasses/live.py
@@ -25,10 +25,11 @@ class LiveEclassChecks(ScanBase):
@returns: dictionary, including {live_ebuild}
'''
+ ebuild = kwargs.get('ebuild').result()
# update the dynamic data
dyn_live = kwargs.get('live_ebuild')
- dyn_live.update(LIVE_ECLASSES.intersection(
- kwargs.get('ebuild').inherited))
+ #dyn_live.clear()
+ dyn_live.update(LIVE_ECLASSES.intersection(ebuild.inherited))
return False
def check(self, **kwargs):
diff --git a/pym/repoman/modules/scan/keywords/__init__.py b/pym/repoman/modules/scan/keywords/__init__.py
index 2d92dba..2b3f6ca 100644
--- a/pym/repoman/modules/scan/keywords/__init__.py
+++ b/pym/repoman/modules/scan/keywords/__init__.py
@@ -20,8 +20,8 @@ module_spec = {
},
'mod_kwargs': ['qatracker', 'options', 'repo_metadata', 'profiles',
],
- 'func_kwargs': {'pkg': None, 'xpkg': None, 'ebuild': None,
- 'y_ebuild': None, 'changed': None, 'live_ebuild': None,
+ 'func_kwargs': {'pkg': 'Future', 'xpkg': None, 'ebuild': 'Future',
+ 'y_ebuild': None, 'changed': None, 'live_ebuild': 'set',
},
},
}
diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py
index af9771b..9ccf377 100644
--- a/pym/repoman/modules/scan/metadata/__init__.py
+++ b/pym/repoman/modules/scan/metadata/__init__.py
@@ -48,7 +48,7 @@ module_spec = {
},
'mod_kwargs': ['qatracker',
],
- 'func_kwargs': {'ebuild': None, 'pkg': None,
+ 'func_kwargs': {'ebuild': None, 'pkg': 'Future',
},
},
'license-metadata': {
diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py
index 0184d78..fdca418 100644
--- a/pym/repoman/modules/scan/metadata/unused.py
+++ b/pym/repoman/modules/scan/metadata/unused.py
@@ -23,7 +23,7 @@ class UnusedCheck(ScanBase):
@param validity_future: Future instance
'''
xpkg = kwargs.get('xpkg')
- muselist = kwargs.get('muselist').result()
+ muselist = self.get_result(kwargs.get('muselist'), set())
used_useflags = kwargs.get('used_useflags')
valid_state = self.get_result(kwargs['validity_future'], True)
# check if there are unused local USE-descriptions in metadata.xml
diff --git a/pym/repoman/modules/scan/use/use_flags.py b/pym/repoman/modules/scan/use/use_flags.py
index 70f7583..b2b1cc0 100644
--- a/pym/repoman/modules/scan/use/use_flags.py
+++ b/pym/repoman/modules/scan/use/use_flags.py
@@ -42,20 +42,21 @@ class USEFlagChecks(ScanBase):
package = kwargs.get('xpkg')
ebuild = kwargs.get('ebuild').result()
y_ebuild = kwargs.get('y_ebuild')
- localUseFlags = kwargs.get('muselist').result()
+ localUseFlags = self.get_result(kwargs.get('muselist'), set())
+ dyn_ebuild_used = kwargs.get('ebuild_UsedUseFlags')
+ dyn_used = kwargs.get('used_useflags')
# reset state variables for the run
self.useFlags = []
self.defaultUseFlags = []
self.usedUseFlags = set()
+ dyn_ebuild_used.clear()
+ # perform the checks
self._checkGlobal(pkg)
self._checkMetadata(package, ebuild, y_ebuild, localUseFlags)
self._checkRequiredUSE(pkg, ebuild)
# update the dynamic data
- used_useflags = kwargs.get('used_useflags').union(self.usedUseFlags)
- dyn_ebuild_used = kwargs.get('ebuild_UsedUseFlags')
- dyn_ebuild_used.update(self.usedUseFlags)
- dyn_used = kwargs.get('used_useflags')
- dyn_used.update(used_useflags)
+ dyn_ebuild_used.union(self.usedUseFlags)
+ dyn_used.update(self.usedUseFlags)
return False
diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py
index 3d2c69c..eaf023b 100644
--- a/pym/repoman/scanner.py
+++ b/pym/repoman/scanner.py
@@ -203,11 +203,6 @@ class Scanner(object):
}
# initialize the plugin checks here
self.modules = {}
- for mod in ['manifests', 'isebuild', 'keywords', 'files', 'vcsstatus',
- 'fetches', 'pkgmetadata']:
- mod_class = MODULE_CONTROLLER.get_class(mod)
- logging.debug("Initializing class name: %s", mod_class.__name__)
- self.modules[mod_class.__name__] = mod_class(**self.set_kwargs(mod))
def set_kwargs(self, mod):
'''Creates a limited set of kwargs to pass to the module's __init__()
@@ -231,17 +226,25 @@ class Scanner(object):
'''
func_kwargs = MODULE_CONTROLLER.modules[mod]['func_kwargs']
# determine new keys
- required = set(func_kwargs.viewkeys())
- exist = set(dynamic_data.viewkeys())
- new = exist.difference(required)
+ required = set(list(func_kwargs))
+ exist = set(list(dynamic_data))
+ new = required.difference(exist)
# update dynamic_data with initialized entries
for key in new:
- dynamic_data[key] = DATA_TYPES[func_kwargs['key']]()
+ logging.debug("set_func_kwargs(); adding: %s, %s",
+ key, func_kwargs[key])
+ dynamic_data[key] = DATA_TYPES[func_kwargs[key]]()
kwargs = {}
for key in required:
kwargs[key] = dynamic_data[key]
return kwargs
+ def reset_futures(self, dynamic_data):
+ for key in list(dynamic_data):
+ #if key in ['ebuild', 'pkg']: # and isinstance(dynamic_data[key], Future):
+ if isinstance(dynamic_data[key], Future) and key not in ['muselist']:
+ dynamic_data[key] = Future()
+
def scan_pkgs(self, can_force):
for xpkg in self.effective_scanlist:
xpkg_continue = False
@@ -270,16 +273,26 @@ class Scanner(object):
'repolevel': self.repolevel,
'catdir': catdir,
'pkgdir': pkgdir,
- 'validity_future': Future()
+ 'validity_future': Future(),
+ 'y_ebuild': None,
+ # this needs to be reset at the pkg level only,
+ # easiest is to just initialize it here
+ 'muselist': Future(),
}
# need to set it up for ==> self.modules or some other ordered list
- for mod in ['Manifests', 'IsEbuild', 'KeywordChecks', 'FileChecks',
- 'VCSStatus', 'FetchChecks', 'PkgMetadata']:
- logging.debug("scan_pkgs; module: %s", mod)
- do_it, functions = self.modules[mod].runInPkgs
+ for mod in [('manifests', 'Manifests'), ('isebuild', 'IsEbuild'),
+ ('keywords', 'KeywordChecks'), ('files', 'FileChecks'),
+ ('vcsstatus', 'VCSStatus'), ('fetches', 'FetchChecks'),
+ ('pkgmetadata', 'PkgMetadata'),
+ ]:
+ mod_class = MODULE_CONTROLLER.get_class(mod[0])
+ logging.debug("Initializing class name: %s", mod_class.__name__)
+ self.modules[mod_class.__name__] = mod_class(**self.set_kwargs(mod[0]))
+ logging.debug("scan_pkgs; module: %s", mod[1])
+ do_it, functions = self.modules[mod[1]].runInPkgs
if do_it:
for func in functions:
- _continue = func(**self.set_func_kwargs(mod, dynamic_data))
+ _continue = func(**self.set_func_kwargs(mod[0], dynamic_data))
if _continue:
# If we can't access all the metadata then it's totally unsafe to
# commit since there's no way to generate a correct Manifest.
@@ -312,6 +325,7 @@ class Scanner(object):
dynamic_data['used_useflags'] = set()
for y_ebuild in ebuildlist:
+ self.reset_futures(dynamic_data)
dynamic_data['y_ebuild'] = y_ebuild
y_ebuild_continue = False
@@ -320,7 +334,8 @@ class Scanner(object):
for mod in [('ebuild', 'Ebuild'), ('live', 'LiveEclassChecks'),
('eapi', 'EAPIChecks'), ('ebuild_metadata', 'EbuildMetadata'),
('thirdpartymirrors', 'ThirdPartyMirrors'),
- ('description', 'DescriptionChecks'), (None, 'KeywordChecks'),
+ ('description', 'DescriptionChecks'),
+ ('keywords', 'KeywordChecks'),
('arches', 'ArchChecks'), ('depend', 'DependChecks'),
('use_flags', 'USEFlagChecks'), ('ruby', 'RubyEclassChecks'),
('license', 'LicenseChecks'), ('restrict', 'RestrictChecks'),
@@ -329,17 +344,17 @@ class Scanner(object):
('options', 'Options'), ('profile', 'ProfileDependsChecks'),
('unknown', 'DependUnknown'),
]:
- if mod[0]:
+ if mod[0] and mod[1] not in self.modules:
mod_class = MODULE_CONTROLLER.get_class(mod[0])
logging.debug("Initializing class name: %s", mod_class.__name__)
- self.modules[mod[1]] = mod_class(**self.set_kwargs(mod))
+ self.modules[mod[1]] = mod_class(**self.set_kwargs(mod[0]))
logging.debug("scan_ebuilds: module: %s", mod[1])
do_it, functions = self.modules[mod[1]].runInEbuilds
logging.debug("do_it: %s, functions: %s", do_it, [x.__name__ for x in functions])
if do_it:
for func in functions:
logging.debug("\tRunning function: %s", func)
- _continue = func(**self.set_func_kwargs(mod, dynamic_data))
+ _continue = func(**self.set_func_kwargs(mod[0], dynamic_data))
if _continue:
# If we can't access all the metadata then it's totally unsafe to
# commit since there's no way to generate a correct Manifest.
@@ -363,14 +378,14 @@ class Scanner(object):
if mod[0]:
mod_class = MODULE_CONTROLLER.get_class(mod[0])
logging.debug("Initializing class name: %s", mod_class.__name__)
- self.modules[mod[1]] = mod_class(**self.set_kwargs(mod))
+ self.modules[mod[1]] = mod_class(**self.set_kwargs(mod[0]))
logging.debug("scan_ebuilds final checks: module: %s", mod[1])
do_it, functions = self.modules[mod[1]].runInFinal
logging.debug("do_it: %s, functions: %s", do_it, [x.__name__ for x in functions])
if do_it:
for func in functions:
logging.debug("\tRunning function: %s", func)
- _continue = func(**self.set_func_kwargs(mod, dynamic_data))
+ _continue = func(**self.set_func_kwargs(mod[0], dynamic_data))
if _continue:
xpkg_complete = True
# logging.debug("\t>>> Continuing")
^ permalink raw reply related [flat|nested] 3+ messages in thread
* [gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/ebuild/, pym/repoman/modules/scan/keywords/, ...
@ 2016-04-25 15:32 Brian Dolbec
0 siblings, 0 replies; 3+ messages in thread
From: Brian Dolbec @ 2016-04-25 15:32 UTC (permalink / raw
To: gentoo-commits
commit: c24cac30a56bd7922d977999752b19ed3f64a300
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Apr 17 15:39:40 2016 +0000
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Mon Apr 25 15:28:53 2016 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=c24cac30
repoman: Populate module_spec's func_kwargs for all scan modules
First step towards a propoerly controlled module data exchange interface.
pym/repoman/modules/scan/arches/__init__.py | 2 +-
pym/repoman/modules/scan/depend/__init__.py | 10 +++++++---
pym/repoman/modules/scan/directories/__init__.py | 5 +++--
pym/repoman/modules/scan/eapi/__init__.py | 2 +-
pym/repoman/modules/scan/ebuild/__init__.py | 10 +++++++---
pym/repoman/modules/scan/eclasses/__init__.py | 5 +++--
pym/repoman/modules/scan/fetch/__init__.py | 4 +++-
pym/repoman/modules/scan/keywords/__init__.py | 3 ++-
pym/repoman/modules/scan/manifest/__init__.py | 2 +-
pym/repoman/modules/scan/metadata/__init__.py | 16 ++++++++++------
pym/repoman/modules/scan/mirrors/__init__.py | 2 +-
pym/repoman/modules/scan/status/__init__.py | 3 ++-
pym/repoman/modules/scan/use/__init__.py | 4 +++-
13 files changed, 44 insertions(+), 24 deletions(-)
diff --git a/pym/repoman/modules/scan/arches/__init__.py b/pym/repoman/modules/scan/arches/__init__.py
index b87e07c..e4a4806 100644
--- a/pym/repoman/modules/scan/arches/__init__.py
+++ b/pym/repoman/modules/scan/arches/__init__.py
@@ -20,7 +20,7 @@ module_spec = {
},
'mod_kwargs': ['options', 'repo_settings', 'profiles'
],
- 'func_kwargs': {'ebuild': None,
+ 'func_kwargs': {'ebuild': None, 'arches': 'set',
},
},
}
diff --git a/pym/repoman/modules/scan/depend/__init__.py b/pym/repoman/modules/scan/depend/__init__.py
index a0f2f9a..a2b8534 100644
--- a/pym/repoman/modules/scan/depend/__init__.py
+++ b/pym/repoman/modules/scan/depend/__init__.py
@@ -20,7 +20,9 @@ module_spec = {
},
'mod_kwargs': ['qatracker', 'portdb'
],
- 'func_kwargs': {
+ 'func_kwargs': {'ebuild': None, 'pkg': None, 'unknown_pkgs': 'set',
+ 'type_list': [], 'badlicsyntax': 'Future',
+ 'baddepsyntax': 'Future',
},
},
'profile-module': {
@@ -35,7 +37,8 @@ module_spec = {
'repo_settings', 'include_arches', 'caches',
'repoman_incrementals', 'env', 'have', 'dev_keywords'
],
- 'func_kwargs': {
+ 'func_kwargs': {'arches': 'set', 'ebuild': None, 'pkg': None,
+ 'unknown_pkgs': 'set', 'baddepsyntax': 'Future',
},
},
'unknown-module': {
@@ -48,7 +51,8 @@ module_spec = {
},
'mod_kwargs': ['qatracker',
],
- 'func_kwargs': {
+ 'func_kwargs': {'ebuild': None, 'unknown_pkgs': 'set',
+ 'baddepsyntax': 'Future',
},
},
}
diff --git a/pym/repoman/modules/scan/directories/__init__.py b/pym/repoman/modules/scan/directories/__init__.py
index d50ace5..0193843 100644
--- a/pym/repoman/modules/scan/directories/__init__.py
+++ b/pym/repoman/modules/scan/directories/__init__.py
@@ -20,7 +20,8 @@ module_spec = {
},
'mod_kwargs': ['portdb', 'qatracker', 'repo_settings', 'vcs_settings',
],
- 'func_kwargs': {
+ 'func_kwargs': {'checkdir': None, 'checkdirlist': None,
+ 'checkdir_relative': None, 'changed': None,
},
},
'mtime-module': {
@@ -33,7 +34,7 @@ module_spec = {
},
'mod_kwargs': ['vcs_settings',
],
- 'func_kwargs': {
+ 'func_kwargs': {'ebuild': None, 'changed': None, 'pkg': None,
},
},
}
diff --git a/pym/repoman/modules/scan/eapi/__init__.py b/pym/repoman/modules/scan/eapi/__init__.py
index 63618ab..376e79a 100644
--- a/pym/repoman/modules/scan/eapi/__init__.py
+++ b/pym/repoman/modules/scan/eapi/__init__.py
@@ -20,7 +20,7 @@ module_spec = {
},
'mod_kwargs': ['qatracker', 'repo_settings'
],
- 'func_kwargs': {
+ 'func_kwargs': {'ebuild': None,
},
},
}
diff --git a/pym/repoman/modules/scan/ebuild/__init__.py b/pym/repoman/modules/scan/ebuild/__init__.py
index 9cd49f8..eb9098c 100644
--- a/pym/repoman/modules/scan/ebuild/__init__.py
+++ b/pym/repoman/modules/scan/ebuild/__init__.py
@@ -20,7 +20,8 @@ module_spec = {
},
'mod_kwargs': ['portdb', 'qatracker', 'repo_settings'
],
- 'func_kwargs': {
+ 'func_kwargs': {'checkdirlist': 'list', 'checkdir': None, 'xpkg': None,
+ 'validity_fuse': None, 'can_force': 'Future', 'pkgs': 'dict',
},
},
'ebuild-module': {
@@ -33,7 +34,10 @@ module_spec = {
},
'mod_kwargs': ['qatracker', 'repo_settings', 'vcs_settings', 'checks',
],
- 'func_kwargs': {
+ 'func_kwargs': {'xpkg': None, 'y_ebuild': None, 'changed': None,
+ 'changelog_modified': None, 'pkg': 'Future', 'pkgs': None,
+ 'ebuild': 'Future', 'validity_future': None, 'pkgdir': None,
+ 'repolevel': None, 'catdir': None,
},
},
'multicheck-module': {
@@ -46,7 +50,7 @@ module_spec = {
},
'mod_kwargs': ['qatracker', 'options'
],
- 'func_kwargs': {
+ 'func_kwargs': {'ebuild': None, 'pkg': None,
},
},
}
diff --git a/pym/repoman/modules/scan/eclasses/__init__.py b/pym/repoman/modules/scan/eclasses/__init__.py
index c39589e..7be954c 100644
--- a/pym/repoman/modules/scan/eclasses/__init__.py
+++ b/pym/repoman/modules/scan/eclasses/__init__.py
@@ -20,7 +20,8 @@ module_spec = {
},
'mod_kwargs': ['qatracker', 'repo_metadata', 'repo_settings',
],
- 'func_kwargs': {
+ 'func_kwargs': {'live_ebuild': 'set', 'pkg': None, 'xpkg': None,
+ 'ebuild': None, 'y_ebuild': None,
},
},
'ruby-module': {
@@ -33,7 +34,7 @@ module_spec = {
},
'mod_kwargs': ['qatracker'
],
- 'func_kwargs': {
+ 'func_kwargs': {'pkg': None, 'ebuild': None,
},
},
}
diff --git a/pym/repoman/modules/scan/fetch/__init__.py b/pym/repoman/modules/scan/fetch/__init__.py
index 303eb19..795eabf 100644
--- a/pym/repoman/modules/scan/fetch/__init__.py
+++ b/pym/repoman/modules/scan/fetch/__init__.py
@@ -20,7 +20,9 @@ module_spec = {
},
'mod_kwargs': ['portdb', 'qatracker', 'repo_settings', 'vcs_settings',
],
- 'func_kwargs': {
+ 'func_kwargs': {'xpkg': None, 'checkdir': None,
+ 'checkdir_relative': None, 'changed': None,
+ 'src_uri_error': 'Future',
},
},
}
diff --git a/pym/repoman/modules/scan/keywords/__init__.py b/pym/repoman/modules/scan/keywords/__init__.py
index 471f447..2d92dba 100644
--- a/pym/repoman/modules/scan/keywords/__init__.py
+++ b/pym/repoman/modules/scan/keywords/__init__.py
@@ -20,7 +20,8 @@ module_spec = {
},
'mod_kwargs': ['qatracker', 'options', 'repo_metadata', 'profiles',
],
- 'func_kwargs': {
+ 'func_kwargs': {'pkg': None, 'xpkg': None, 'ebuild': None,
+ 'y_ebuild': None, 'changed': None, 'live_ebuild': None,
},
},
}
diff --git a/pym/repoman/modules/scan/manifest/__init__.py b/pym/repoman/modules/scan/manifest/__init__.py
index 4aa80cf..bd9f8da 100644
--- a/pym/repoman/modules/scan/manifest/__init__.py
+++ b/pym/repoman/modules/scan/manifest/__init__.py
@@ -20,7 +20,7 @@ module_spec = {
},
'mod_kwargs': ['options', 'portdb', 'qatracker', 'repo_settings',
],
- 'func_kwargs': {
+ 'func_kwargs': {'checkdir': None, 'xpkg': None,
},
},
}
diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py
index 90981d8..1e690bf 100644
--- a/pym/repoman/modules/scan/metadata/__init__.py
+++ b/pym/repoman/modules/scan/metadata/__init__.py
@@ -20,7 +20,8 @@ module_spec = {
},
'mod_kwargs': ['repo_settings', 'qatracker', 'options', 'metadata_dtd',
],
- 'func_kwargs': {
+ 'func_kwargs': {'xpkg': None, 'checkdir': None, 'checkdirlist': None,
+ 'repolevel': None, 'muselist': 'set',
},
},
'ebuild-metadata': {
@@ -33,7 +34,8 @@ module_spec = {
},
'mod_kwargs': ['qatracker',
],
- 'func_kwargs': {
+ 'func_kwargs': {'ebuild': None, 'catdir': None, 'live_ebuild': None,
+ 'xpkg': None, 'y_ebuild': None,
},
},
'description-metadata': {
@@ -46,7 +48,7 @@ module_spec = {
},
'mod_kwargs': ['qatracker',
],
- 'func_kwargs': {
+ 'func_kwargs': {'ebuild': None, 'pkg': None,
},
},
'license-metadata': {
@@ -59,7 +61,8 @@ module_spec = {
},
'mod_kwargs': ['qatracker', 'repo_metadata',
],
- 'func_kwargs': {
+ 'func_kwargs': {'xpkg': None, 'ebuild': None, 'y_ebuild': None,
+ 'badlicsyntax': None,
},
},
'restrict-metadata': {
@@ -72,7 +75,7 @@ module_spec = {
},
'mod_kwargs': ['qatracker',
],
- 'func_kwargs': {
+ 'func_kwargs': {'xpkg': None, 'ebuild': None, 'y_ebuild': None,
},
},
'unused-metadata': {
@@ -85,7 +88,8 @@ module_spec = {
},
'mod_kwargs': ['qatracker',
],
- 'func_kwargs': {
+ 'func_kwargs': {'xpkg': None, 'muselist': None, 'used_useflags': None,
+ 'validity_future': None,
},
},
}
diff --git a/pym/repoman/modules/scan/mirrors/__init__.py b/pym/repoman/modules/scan/mirrors/__init__.py
index 8ae58b5..1885dca 100644
--- a/pym/repoman/modules/scan/mirrors/__init__.py
+++ b/pym/repoman/modules/scan/mirrors/__init__.py
@@ -20,7 +20,7 @@ module_spec = {
},
'mod_kwargs': ['repo_settings', 'qatracker',
],
- 'func_kwargs': {
+ 'func_kwargs': {'ebuild': None, 'src_uri_error': None,
},
},
}
diff --git a/pym/repoman/modules/scan/status/__init__.py b/pym/repoman/modules/scan/status/__init__.py
index a5eeb3e..f4683bb 100644
--- a/pym/repoman/modules/scan/status/__init__.py
+++ b/pym/repoman/modules/scan/status/__init__.py
@@ -20,7 +20,8 @@ module_spec = {
},
'mod_kwargs': ['vcs_settings', 'checks',
],
- 'func_kwargs': {
+ 'func_kwargs': {'checkdir': None, 'checkdir_relative': None,
+ 'xpkg': None, 'eadded': 'Future',
},
},
}
diff --git a/pym/repoman/modules/scan/use/__init__.py b/pym/repoman/modules/scan/use/__init__.py
index 349980d..f157386 100644
--- a/pym/repoman/modules/scan/use/__init__.py
+++ b/pym/repoman/modules/scan/use/__init__.py
@@ -20,7 +20,9 @@ module_spec = {
},
'mod_kwargs': ['qatracker', 'uselist',
],
- 'func_kwargs': {
+ 'func_kwargs': {'pkg': None, 'xpkg': None, 'ebuild': None,
+ 'y_ebuild': None, 'muselist': None, 'ebuild_UsedUseFlags': 'set',
+ 'used_useflags': None,
},
},
}
^ permalink raw reply related [flat|nested] 3+ messages in thread
end of thread, other threads:[~2016-04-25 15:32 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2016-04-24 1:00 [gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/ebuild/, pym/repoman/modules/scan/keywords/, Brian Dolbec
-- strict thread matches above, loose matches on Subject: below --
2016-04-25 3:09 Brian Dolbec
2016-04-25 15:32 Brian Dolbec
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox