From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id 7E3E0138206 for ; Sun, 24 Apr 2016 01:00:16 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id C0DB7E0864; Sun, 24 Apr 2016 01:00:13 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 4B2F5E0864 for ; Sun, 24 Apr 2016 01:00:13 +0000 (UTC) Received: from oystercatcher.gentoo.org (unknown [IPv6:2a01:4f8:202:4333:225:90ff:fed9:fc84]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 9CDA2340C3F for ; Sun, 24 Apr 2016 01:00:11 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by oystercatcher.gentoo.org (Postfix) with ESMTP id 4EC717D for ; Sun, 24 Apr 2016 01:00:09 +0000 (UTC) From: "Brian Dolbec" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Brian Dolbec" Message-ID: <1461458721.8838c4317981315741df4334d5a1a172902f20a0.dolsen@gentoo> Subject: [gentoo-commits] proj/portage:repoman commit in: pym/repoman/modules/scan/ebuild/, pym/repoman/modules/scan/keywords/, ... X-VCS-Repository: proj/portage X-VCS-Files: pym/repoman/modules/scan/arches/arches.py pym/repoman/modules/scan/depend/depend.py pym/repoman/modules/scan/ebuild/__init__.py pym/repoman/modules/scan/ebuild/ebuild.py pym/repoman/modules/scan/ebuild/isebuild.py pym/repoman/modules/scan/eclasses/live.py pym/repoman/modules/scan/keywords/__init__.py pym/repoman/modules/scan/metadata/__init__.py pym/repoman/modules/scan/metadata/unused.py pym/repoman/modules/scan/use/use_flags.py pym/repoman/scanner.py X-VCS-Directories: pym/repoman/modules/scan/arches/ pym/repoman/ pym/repoman/modules/scan/eclasses/ pym/repoman/modules/scan/depend/ pym/repoman/modules/scan/use/ pym/repoman/modules/scan/ebuild/ pym/repoman/modules/scan/metadata/ pym/repoman/modules/scan/keywords/ X-VCS-Committer: dolsen X-VCS-Committer-Name: Brian Dolbec X-VCS-Revision: 8838c4317981315741df4334d5a1a172902f20a0 X-VCS-Branch: repoman Date: Sun, 24 Apr 2016 01:00:09 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Archives-Salt: faf76fdb-f014-4fc0-9354-de5dae03fc23 X-Archives-Hash: 15667cafa38b7383b631d3180dcdaf1e commit: 8838c4317981315741df4334d5a1a172902f20a0 Author: Brian Dolbec gentoo org> AuthorDate: Wed Apr 20 16:52:21 2016 +0000 Commit: Brian Dolbec gentoo org> CommitDate: Sun Apr 24 00:45:21 2016 +0000 URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=8838c431 More new data structure change fixes Fix arches not being cleared every ebuild. pym/repoman/modules/scan/arches/arches.py | 1 + pym/repoman/modules/scan/depend/depend.py | 1 + pym/repoman/modules/scan/ebuild/__init__.py | 2 +- pym/repoman/modules/scan/ebuild/ebuild.py | 2 +- pym/repoman/modules/scan/ebuild/isebuild.py | 11 +++-- pym/repoman/modules/scan/eclasses/live.py | 5 ++- pym/repoman/modules/scan/keywords/__init__.py | 4 +- pym/repoman/modules/scan/metadata/__init__.py | 2 +- pym/repoman/modules/scan/metadata/unused.py | 2 +- pym/repoman/modules/scan/use/use_flags.py | 13 +++--- pym/repoman/scanner.py | 62 ++++++++++++++++++--------- 11 files changed, 67 insertions(+), 38 deletions(-) diff --git a/pym/repoman/modules/scan/arches/arches.py b/pym/repoman/modules/scan/arches/arches.py index 8bc7a24..6e1c17d 100644 --- a/pym/repoman/modules/scan/arches/arches.py +++ b/pym/repoman/modules/scan/arches/arches.py @@ -69,6 +69,7 @@ class ArchChecks(ScanBase): arches.add(('**', '**', ('**',))) # update the dynamic data dyn_arches = kwargs.get('arches') + dyn_arches.clear() dyn_arches.update(arches) return False diff --git a/pym/repoman/modules/scan/depend/depend.py b/pym/repoman/modules/scan/depend/depend.py index a5f91d3..cd1fb33 100644 --- a/pym/repoman/modules/scan/depend/depend.py +++ b/pym/repoman/modules/scan/depend/depend.py @@ -145,6 +145,7 @@ class DependChecks(ScanBase): # update the dynamic data dyn_unknown = kwargs.get('unknown_pkgs') + dyn_unknown.clear() dyn_unknown.update(unknown_pkgs) self.set_result_pass([ (kwargs.get('badlicsyntax'), badlicsyntax), diff --git a/pym/repoman/modules/scan/ebuild/__init__.py b/pym/repoman/modules/scan/ebuild/__init__.py index ea21ece..efb2feb 100644 --- a/pym/repoman/modules/scan/ebuild/__init__.py +++ b/pym/repoman/modules/scan/ebuild/__init__.py @@ -21,7 +21,7 @@ module_spec = { 'mod_kwargs': ['portdb', 'qatracker', 'repo_settings' ], 'func_kwargs': {'checkdirlist': 'list', 'checkdir': None, 'xpkg': None, - 'validity_fuse': None, 'can_force': None, 'pkgs': 'dict', + 'validity_future': 'Future', 'can_force': None, 'pkgs': 'dict', }, }, 'ebuild-module': { diff --git a/pym/repoman/modules/scan/ebuild/ebuild.py b/pym/repoman/modules/scan/ebuild/ebuild.py index b015985..b02cdcd 100644 --- a/pym/repoman/modules/scan/ebuild/ebuild.py +++ b/pym/repoman/modules/scan/ebuild/ebuild.py @@ -86,7 +86,7 @@ class Ebuild(ScanBase): self.qatracker.add_error( "ebuild.notadded", self.xpkg + "/" + self.y_ebuild + ".ebuild") # update the dynamic data - self.set_result_raise([('ebuild', self)]) + self.set_result_raise([(kwargs.get('ebuild'), self)]) return False def set_pkg_data(self, **kwargs): diff --git a/pym/repoman/modules/scan/ebuild/isebuild.py b/pym/repoman/modules/scan/ebuild/isebuild.py index 3e288a3..24680e2 100644 --- a/pym/repoman/modules/scan/ebuild/isebuild.py +++ b/pym/repoman/modules/scan/ebuild/isebuild.py @@ -13,6 +13,7 @@ from portage import os from repoman.qa_data import no_exec, allvars from repoman.modules.scan.scanbase import ScanBase + class IsEbuild(ScanBase): '''Performs basic tests to confirm it is an ebuild''' @@ -88,9 +89,13 @@ class IsEbuild(ScanBase): self.continue_ = True self.set_result_pass([(can_force, False)]) # set our updated data - self.set_result_raise([ - (kwargs.get('pkgs'), pkgs), - ]) + dyn_pkgs = kwargs.get('pkgs') + # clear() sets it to None, + # we don't want to kill the pointer reference + # just set it back to an empty dict() + for key in list(dyn_pkgs): + dyn_pkgs.pop(key) + dyn_pkgs.update(pkgs) return self.continue_ @property diff --git a/pym/repoman/modules/scan/eclasses/live.py b/pym/repoman/modules/scan/eclasses/live.py index e5bb7dc..85e829c 100644 --- a/pym/repoman/modules/scan/eclasses/live.py +++ b/pym/repoman/modules/scan/eclasses/live.py @@ -25,10 +25,11 @@ class LiveEclassChecks(ScanBase): @returns: dictionary, including {live_ebuild} ''' + ebuild = kwargs.get('ebuild').result() # update the dynamic data dyn_live = kwargs.get('live_ebuild') - dyn_live.update(LIVE_ECLASSES.intersection( - kwargs.get('ebuild').inherited)) + #dyn_live.clear() + dyn_live.update(LIVE_ECLASSES.intersection(ebuild.inherited)) return False def check(self, **kwargs): diff --git a/pym/repoman/modules/scan/keywords/__init__.py b/pym/repoman/modules/scan/keywords/__init__.py index 2d92dba..2b3f6ca 100644 --- a/pym/repoman/modules/scan/keywords/__init__.py +++ b/pym/repoman/modules/scan/keywords/__init__.py @@ -20,8 +20,8 @@ module_spec = { }, 'mod_kwargs': ['qatracker', 'options', 'repo_metadata', 'profiles', ], - 'func_kwargs': {'pkg': None, 'xpkg': None, 'ebuild': None, - 'y_ebuild': None, 'changed': None, 'live_ebuild': None, + 'func_kwargs': {'pkg': 'Future', 'xpkg': None, 'ebuild': 'Future', + 'y_ebuild': None, 'changed': None, 'live_ebuild': 'set', }, }, } diff --git a/pym/repoman/modules/scan/metadata/__init__.py b/pym/repoman/modules/scan/metadata/__init__.py index af9771b..9ccf377 100644 --- a/pym/repoman/modules/scan/metadata/__init__.py +++ b/pym/repoman/modules/scan/metadata/__init__.py @@ -48,7 +48,7 @@ module_spec = { }, 'mod_kwargs': ['qatracker', ], - 'func_kwargs': {'ebuild': None, 'pkg': None, + 'func_kwargs': {'ebuild': None, 'pkg': 'Future', }, }, 'license-metadata': { diff --git a/pym/repoman/modules/scan/metadata/unused.py b/pym/repoman/modules/scan/metadata/unused.py index 0184d78..fdca418 100644 --- a/pym/repoman/modules/scan/metadata/unused.py +++ b/pym/repoman/modules/scan/metadata/unused.py @@ -23,7 +23,7 @@ class UnusedCheck(ScanBase): @param validity_future: Future instance ''' xpkg = kwargs.get('xpkg') - muselist = kwargs.get('muselist').result() + muselist = self.get_result(kwargs.get('muselist'), set()) used_useflags = kwargs.get('used_useflags') valid_state = self.get_result(kwargs['validity_future'], True) # check if there are unused local USE-descriptions in metadata.xml diff --git a/pym/repoman/modules/scan/use/use_flags.py b/pym/repoman/modules/scan/use/use_flags.py index 70f7583..b2b1cc0 100644 --- a/pym/repoman/modules/scan/use/use_flags.py +++ b/pym/repoman/modules/scan/use/use_flags.py @@ -42,20 +42,21 @@ class USEFlagChecks(ScanBase): package = kwargs.get('xpkg') ebuild = kwargs.get('ebuild').result() y_ebuild = kwargs.get('y_ebuild') - localUseFlags = kwargs.get('muselist').result() + localUseFlags = self.get_result(kwargs.get('muselist'), set()) + dyn_ebuild_used = kwargs.get('ebuild_UsedUseFlags') + dyn_used = kwargs.get('used_useflags') # reset state variables for the run self.useFlags = [] self.defaultUseFlags = [] self.usedUseFlags = set() + dyn_ebuild_used.clear() + # perform the checks self._checkGlobal(pkg) self._checkMetadata(package, ebuild, y_ebuild, localUseFlags) self._checkRequiredUSE(pkg, ebuild) # update the dynamic data - used_useflags = kwargs.get('used_useflags').union(self.usedUseFlags) - dyn_ebuild_used = kwargs.get('ebuild_UsedUseFlags') - dyn_ebuild_used.update(self.usedUseFlags) - dyn_used = kwargs.get('used_useflags') - dyn_used.update(used_useflags) + dyn_ebuild_used.union(self.usedUseFlags) + dyn_used.update(self.usedUseFlags) return False diff --git a/pym/repoman/scanner.py b/pym/repoman/scanner.py index 3d2c69c..299d118 100644 --- a/pym/repoman/scanner.py +++ b/pym/repoman/scanner.py @@ -203,11 +203,6 @@ class Scanner(object): } # initialize the plugin checks here self.modules = {} - for mod in ['manifests', 'isebuild', 'keywords', 'files', 'vcsstatus', - 'fetches', 'pkgmetadata']: - mod_class = MODULE_CONTROLLER.get_class(mod) - logging.debug("Initializing class name: %s", mod_class.__name__) - self.modules[mod_class.__name__] = mod_class(**self.set_kwargs(mod)) def set_kwargs(self, mod): '''Creates a limited set of kwargs to pass to the module's __init__() @@ -229,19 +224,32 @@ class Scanner(object): @param dynamic_data: dictionary structure @returns: dictionary ''' + #print("****** mod:", mod) func_kwargs = MODULE_CONTROLLER.modules[mod]['func_kwargs'] + #print(mod, "**** func_kwargs", func_kwargs) # determine new keys - required = set(func_kwargs.viewkeys()) - exist = set(dynamic_data.viewkeys()) - new = exist.difference(required) + required = set(list(func_kwargs)) + #print(" required", required) + exist = set(list(dynamic_data)) + #print(" exist", exist) + new = required.difference(exist) + #print(" new", new) # update dynamic_data with initialized entries for key in new: - dynamic_data[key] = DATA_TYPES[func_kwargs['key']]() + #print(" adding", key, func_kwargs[key], DATA_TYPES[func_kwargs[key]]) + dynamic_data[key] = DATA_TYPES[func_kwargs[key]]() kwargs = {} for key in required: + #print(" kwargs[key]", key, dynamic_data[key]) kwargs[key] = dynamic_data[key] return kwargs + def reset_futures(self, dynamic_data): + for key in list(dynamic_data): + #if key in ['ebuild', 'pkg']: # and isinstance(dynamic_data[key], Future): + if isinstance(dynamic_data[key], Future) and key not in ['muselist']: + dynamic_data[key] = Future() + def scan_pkgs(self, can_force): for xpkg in self.effective_scanlist: xpkg_continue = False @@ -270,16 +278,26 @@ class Scanner(object): 'repolevel': self.repolevel, 'catdir': catdir, 'pkgdir': pkgdir, - 'validity_future': Future() + 'validity_future': Future(), + 'y_ebuild': None, + # this needs to be reset at the pkg level only, + # easiest is to just initialize it here + 'muselist': Future(), } # need to set it up for ==> self.modules or some other ordered list - for mod in ['Manifests', 'IsEbuild', 'KeywordChecks', 'FileChecks', - 'VCSStatus', 'FetchChecks', 'PkgMetadata']: - logging.debug("scan_pkgs; module: %s", mod) - do_it, functions = self.modules[mod].runInPkgs + for mod in [('manifests', 'Manifests'), ('isebuild', 'IsEbuild'), + ('keywords', 'KeywordChecks'), ('files', 'FileChecks'), + ('vcsstatus', 'VCSStatus'), ('fetches', 'FetchChecks'), + ('pkgmetadata', 'PkgMetadata'), + ]: + mod_class = MODULE_CONTROLLER.get_class(mod[0]) + logging.debug("Initializing class name: %s", mod_class.__name__) + self.modules[mod_class.__name__] = mod_class(**self.set_kwargs(mod[0])) + logging.debug("scan_pkgs; module: %s", mod[1]) + do_it, functions = self.modules[mod[1]].runInPkgs if do_it: for func in functions: - _continue = func(**self.set_func_kwargs(mod, dynamic_data)) + _continue = func(**self.set_func_kwargs(mod[0], dynamic_data)) if _continue: # If we can't access all the metadata then it's totally unsafe to # commit since there's no way to generate a correct Manifest. @@ -312,6 +330,7 @@ class Scanner(object): dynamic_data['used_useflags'] = set() for y_ebuild in ebuildlist: + self.reset_futures(dynamic_data) dynamic_data['y_ebuild'] = y_ebuild y_ebuild_continue = False @@ -320,7 +339,8 @@ class Scanner(object): for mod in [('ebuild', 'Ebuild'), ('live', 'LiveEclassChecks'), ('eapi', 'EAPIChecks'), ('ebuild_metadata', 'EbuildMetadata'), ('thirdpartymirrors', 'ThirdPartyMirrors'), - ('description', 'DescriptionChecks'), (None, 'KeywordChecks'), + ('description', 'DescriptionChecks'), + ('keywords', 'KeywordChecks'), ('arches', 'ArchChecks'), ('depend', 'DependChecks'), ('use_flags', 'USEFlagChecks'), ('ruby', 'RubyEclassChecks'), ('license', 'LicenseChecks'), ('restrict', 'RestrictChecks'), @@ -329,17 +349,17 @@ class Scanner(object): ('options', 'Options'), ('profile', 'ProfileDependsChecks'), ('unknown', 'DependUnknown'), ]: - if mod[0]: + if mod[0] and mod[1] not in self.modules: mod_class = MODULE_CONTROLLER.get_class(mod[0]) logging.debug("Initializing class name: %s", mod_class.__name__) - self.modules[mod[1]] = mod_class(**self.set_kwargs(mod)) + self.modules[mod[1]] = mod_class(**self.set_kwargs(mod[0])) logging.debug("scan_ebuilds: module: %s", mod[1]) do_it, functions = self.modules[mod[1]].runInEbuilds logging.debug("do_it: %s, functions: %s", do_it, [x.__name__ for x in functions]) if do_it: for func in functions: logging.debug("\tRunning function: %s", func) - _continue = func(**self.set_func_kwargs(mod, dynamic_data)) + _continue = func(**self.set_func_kwargs(mod[0], dynamic_data)) if _continue: # If we can't access all the metadata then it's totally unsafe to # commit since there's no way to generate a correct Manifest. @@ -363,14 +383,14 @@ class Scanner(object): if mod[0]: mod_class = MODULE_CONTROLLER.get_class(mod[0]) logging.debug("Initializing class name: %s", mod_class.__name__) - self.modules[mod[1]] = mod_class(**self.set_kwargs(mod)) + self.modules[mod[1]] = mod_class(**self.set_kwargs(mod[0])) logging.debug("scan_ebuilds final checks: module: %s", mod[1]) do_it, functions = self.modules[mod[1]].runInFinal logging.debug("do_it: %s, functions: %s", do_it, [x.__name__ for x in functions]) if do_it: for func in functions: logging.debug("\tRunning function: %s", func) - _continue = func(**self.set_func_kwargs(mod, dynamic_data)) + _continue = func(**self.set_func_kwargs(mod[0], dynamic_data)) if _continue: xpkg_complete = True # logging.debug("\t>>> Continuing")