From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from pigeon.gentoo.org ([208.92.234.80] helo=lists.gentoo.org) by finch.gentoo.org with esmtp (Exim 4.60) (envelope-from ) id 1Qv69D-00051V-MD for garchives@archives.gentoo.org; Sun, 21 Aug 2011 11:25:01 +0000 Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id D55A621C186; Sun, 21 Aug 2011 11:24:52 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) by pigeon.gentoo.org (Postfix) with ESMTP id 4151C21C186 for ; Sun, 21 Aug 2011 11:24:50 +0000 (UTC) Received: from pelican.gentoo.org (unknown [66.219.59.40]) (using TLSv1 with cipher ADH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 1F7A91B4028 for ; Sun, 21 Aug 2011 11:24:50 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by pelican.gentoo.org (Postfix) with ESMTP id 71B8C80040 for ; Sun, 21 Aug 2011 11:24:49 +0000 (UTC) From: "Александр Берсенев" To: gentoo-commits@lists.gentoo.org Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Александр Берсенев" Message-ID: Subject: [gentoo-commits] proj/autodep:master commit in: / X-VCS-Repository: proj/autodep X-VCS-Files: integration_with_portage.patch X-VCS-Directories: / X-VCS-Committer: bay X-VCS-Committer-Name: Александр Берсенев X-VCS-Revision: cd034f40e6f1e6d8bcaac924083d7029a2429159 Date: Sun, 21 Aug 2011 11:24:49 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: quoted-printable X-Archives-Salt: X-Archives-Hash: 42a8b3de2f615609325f53ab981c123e commit: cd034f40e6f1e6d8bcaac924083d7029a2429159 Author: Alexander Bersenev hackerdom ru> AuthorDate: Sun Aug 21 17:23:29 2011 +0000 Commit: =D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B0=D0=BD=D0=B4=D1=80 =D0=91= =D0=B5=D1=80=D1=81=D0=B5=D0=BD=D0=B5=D0=B2 hackerdom ru> CommitDate: Sun Aug 21 17:23:29 2011 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=3Dproj/autodep.git;a= =3Dcommit;h=3Dcd034f40 portage integration patch is added --- integration_with_portage.patch | 899 ++++++++++++++++++++++++++++++++++= ++++++ 1 files changed, 899 insertions(+), 0 deletions(-) diff --git a/integration_with_portage.patch b/integration_with_portage.pa= tch new file mode 100644 index 0000000..a74beed --- /dev/null +++ b/integration_with_portage.patch @@ -0,0 +1,899 @@ +diff --git a/pym/_emerge/EbuildBuild.py b/pym/_emerge/EbuildBuild.py +index 0144cfc..1c423a3 100644 +--- a/pym/_emerge/EbuildBuild.py ++++ b/pym/_emerge/EbuildBuild.py +@@ -9,6 +9,8 @@ from _emerge.CompositeTask import CompositeTask + from _emerge.EbuildMerge import EbuildMerge + from _emerge.EbuildFetchonly import EbuildFetchonly + from _emerge.EbuildBuildDir import EbuildBuildDir ++from _emerge.EventsAnalyser import EventsAnalyser, FilterProcGenerator ++from _emerge.EventsLogger import EventsLogger + from _emerge.MiscFunctionsProcess import MiscFunctionsProcess + from portage.util import writemsg + import portage +@@ -21,7 +23,7 @@ from portage.package.ebuild._spawn_nofetch import spaw= n_nofetch + class EbuildBuild(CompositeTask): +=20 + __slots__ =3D ("args_set", "config_pool", "find_blockers", +- "ldpath_mtimes", "logger", "opts", "pkg", "pkg_count", ++ "ldpath_mtimes", "logger", "logserver", "opts", "pkg", "pkg_count", + "prefetcher", "settings", "world_atom") + \ + ("_build_dir", "_buildpkg", "_ebuild_path", "_issyspkg", "_tree") +=20 +@@ -244,8 +246,54 @@ class EbuildBuild(CompositeTask): +=20 + build =3D EbuildExecuter(background=3Dself.background, pkg=3Dpkg, + scheduler=3Dscheduler, settings=3Dsettings) ++ ++ build.addStartListener(self._build_start) ++ build.addExitListener(self._build_stop) ++ + self._start_task(build, self._build_exit) +=20 ++ def _build_start(self,phase): ++ if "depcheck" in self.settings["FEATURES"] or \ ++ "depcheckstrict" in self.settings["FEATURES"]: ++ # Lets start a log listening server ++ temp_path=3Dself.settings.get("T",self.settings["PORTAGE_TMPDIR"]) ++ =09 ++ if "depcheckstrict" not in self.settings["FEATURES"]: ++ # use default filter_proc ++ self.logserver=3DEventsLogger(socket_dir=3Dtemp_path) ++ else: ++ portage.util.writemsg("Getting list of allowed files..." + \ ++ "This may take some time\n") ++ filter_gen=3DFilterProcGenerator(self.pkg.cpv, self.settings) ++ filter_proc=3Dfilter_gen.get_filter_proc() ++ self.logserver=3DEventsLogger(socket_dir=3Dtemp_path,=20 ++ filter_proc=3Dfilter_proc) ++=09 ++ self.logserver.start() ++ =09 ++ # Copy socket path to LOG_SOCKET environment variable ++ env=3Dself.settings.configdict["pkg"] ++ env['LOG_SOCKET'] =3D self.logserver.socket_name ++ ++ #import pdb; pdb.set_trace() ++ ++ def _build_stop(self,phase): ++ if "depcheck" in self.settings["FEATURES"] or \ ++ "depcheckstrict" in self.settings["FEATURES"]: ++ # Delete LOG_SOCKET from environment ++ env=3Dself.settings.configdict["pkg"] ++ if 'LOG_SOCKET' in env: ++ del env['LOG_SOCKET'] ++ =09 ++ events=3Dself.logserver.stop() ++ self.logserver=3DNone ++ analyser=3DEventsAnalyser(self.pkg.cpv, events, self.settings) ++ analyser.display() # show the analyse ++ ++ #import pdb; pdb.set_trace() ++ =20 ++ ++ + def _fetch_failed(self): + # We only call the pkg_nofetch phase if either RESTRICT=3Dfetch + # is set or the package has explicitly overridden the default +diff --git a/pym/_emerge/EbuildPhase.py b/pym/_emerge/EbuildPhase.py +index f53570a..82c165d 100644 +--- a/pym/_emerge/EbuildPhase.py ++++ b/pym/_emerge/EbuildPhase.py +@@ -33,7 +33,8 @@ class EbuildPhase(CompositeTask): + ("_ebuild_lock",) +=20 + # FEATURES displayed prior to setup phase +- _features_display =3D ("ccache", "distcc", "distcc-pump", "fakeroot", ++ _features_display =3D ("ccache", "depcheck", "depcheckstrict" "distcc"= ,=20 ++ "distcc-pump", "fakeroot", + "installsources", "keeptemp", "keepwork", "nostrip", + "preserve-libs", "sandbox", "selinux", "sesandbox", + "splitdebug", "suidctl", "test", "userpriv", +diff --git a/pym/_emerge/EventsAnalyser.py b/pym/_emerge/EventsAnalyser.= py +new file mode 100644 +index 0000000..65ece7b +--- /dev/null ++++ b/pym/_emerge/EventsAnalyser.py +@@ -0,0 +1,511 @@ ++# Distributed under the terms of the GNU General Public License v2 ++ ++import portage ++from portage.dbapi._expand_new_virt import expand_new_virt ++from portage import os ++ ++import subprocess ++import re ++ ++class PortageUtils: ++ """ class for accessing the portage api """ ++ def __init__(self, settings): ++ """ test """ ++ self.settings=3Dsettings ++ self.vartree=3Dportage.vartree(settings=3Dsettings) ++ self.vardbapi=3Dportage.vardbapi(settings=3Dsettings, vartree=3Dself.= vartree) ++ self.portdbapi=3Dportage.portdbapi(mysettings=3Dsettings) ++ self.metadata_keys =3D [k for k in portage.auxdbkeys if not k.startsw= ith("UNUSED_")] ++ self.use=3Dself.settings["USE"] ++ ++ def get_best_visible_pkg(self,pkg): ++ """ ++ Gets best candidate on installing. Returns empty string if no found ++ ++ :param pkg: package name ++ ++ """ ++ try: ++ return self.portdbapi.xmatch("bestmatch-visible", pkg) ++ except: ++ return '' ++ ++ # non-recursive dependency getter ++ def get_dep(self,pkg,dep_type=3D["RDEPEND","DEPEND"]): ++ """ ++ Gets current dependencies of a package. Looks in portage db ++ =20 ++ :param pkg: name of package ++ :param dep_type: type of dependencies to recurse. Can be ["DEPEND"] = or=20 ++ ["RDEPEND", "DEPEND"] ++ :returns: **set** of packages names ++ """ ++ ret=3Dset() ++ =20 ++ pkg =3D self.get_best_visible_pkg(pkg)=09 ++ if not pkg: ++ return ret ++ =20 ++ # we found the best visible match in common tree ++ ++ ++ metadata =3D dict(zip(self.metadata_keys,=20 ++ self.portdbapi.aux_get(pkg, self.metadata_keys))) ++ dep_str =3D " ".join(metadata[k] for k in dep_type) ++ ++ # the IUSE default are very important for us ++ iuse_defaults=3D[ ++ u[1:] for u in metadata.get("IUSE",'').split() if u.startswith("+")] ++ =20 ++ use=3Dself.use.split() ++ =20 ++ for u in iuse_defaults: ++ if u not in use: ++ use.append(u) ++ ++ success, atoms =3D portage.dep_check(dep_str, None, self.settings,=20 ++ myuse=3Duse, myroot=3Dself.settings["ROOT"], ++ trees=3D{self.settings["ROOT"]:{"vartree":self.vartree, "porttree":= self.vartree}}) ++ if not success: ++ return ret ++ ++ for atom in atoms: ++ atomname =3D self.vartree.dep_bestmatch(atom) ++ ++ if not atomname: ++ continue ++ =09 ++ for unvirt_pkg in expand_new_virt(self.vardbapi,'=3D'+atomname): ++ for pkg in self.vartree.dep_match(unvirt_pkg): ++ ret.add(pkg) ++ ++ return ret ++ ++ # recursive dependency getter ++ def get_deps(self,pkg,dep_type=3D["RDEPEND","DEPEND"]): ++ """=20 ++ Gets current dependencies of a package on any depth=20 ++ All dependencies **must** be installed ++ =09 ++ :param pkg: name of package ++ :param dep_type: type of dependencies to recurse. Can be ["DEPEND"] o= r=20 ++ ["RDEPEND", "DEPEND"] ++ :returns: **set** of packages names ++ """ ++ ret=3Dset() ++ ++ ++ # get porttree dependencies on the first package ++ ++ pkg =3D self.portdbapi.xmatch("bestmatch-visible", pkg)=09 ++ if not pkg: ++ return ret ++ ++ known_packages=3Dset() ++ unknown_packages=3Dself.get_dep(pkg,dep_type) ++ ret=3Dret.union(unknown_packages) ++ =09 ++ while unknown_packages: ++ p=3Dunknown_packages.pop() ++ if p in known_packages: ++ continue ++ known_packages.add(p) ++ ++ metadata =3D dict(zip(self.metadata_keys, self.vardbapi.aux_get(p, s= elf.metadata_keys))) ++ ++ dep_str =3D " ".join(metadata[k] for k in dep_type) ++ =09 ++ # the IUSE default are very important for us ++ iuse_defaults=3D[ ++ u[1:] for u in metadata.get("IUSE",'').split() if u.startswith("+"= )] ++ =20 ++ use=3Dself.use.split() ++ =20 ++ for u in iuse_defaults: ++ if u not in use: ++ use.append(u) ++ =09 ++ success, atoms =3D portage.dep_check(dep_str, None, self.settings,=20 ++ myuse=3Duse, myroot=3Dself.settings["ROOT"], ++ trees=3D{self.settings["ROOT"]:{"vartree":self.vartree,"porttree": = self.vartree}}) ++ ++ if not success: ++ continue ++ ++ for atom in atoms: ++ atomname =3D self.vartree.dep_bestmatch(atom) ++ if not atomname: ++ continue ++ =09 ++ for unvirt_pkg in expand_new_virt(self.vardbapi,'=3D'+atomname): ++ for pkg in self.vartree.dep_match(unvirt_pkg): ++ ret.add(pkg) ++ unknown_packages.add(pkg) ++ return ret ++ ++ def get_deps_for_package_building(self, pkg): ++ """=20 ++ returns buildtime dependencies of current package and=20 ++ all runtime dependencies of that buildtime dependencies ++ """ ++ buildtime_deps=3Dself.get_dep(pkg, ["DEPEND"]) ++ runtime_deps=3Dset() ++ for dep in buildtime_deps: ++ runtime_deps=3Druntime_deps.union(self.get_deps(dep,["RDEPEND"])) ++ ++ ret=3Dbuildtime_deps.union(runtime_deps) ++ return ret ++ ++ def get_system_packages_list(self): ++ """=20 ++ returns all packages from system set. They are always implicit depend= encies ++ =09 ++ :returns: **list** of package names ++ """ ++ ret=3D[] ++ for atom in self.settings.packages: ++ for pre_pkg in self.vartree.dep_match(atom): ++ for unvirt_pkg in expand_new_virt(self.vardbapi,'=3D'+pre_pkg): ++ for pkg in self.vartree.dep_match(unvirt_pkg): ++ ret.append(pkg) ++ return ret ++ ++ ++class GentoolkitUtils: ++ """=20 ++ Interface with qfile and qlist utils. They are much faster than=20 ++ internals. ++ """ ++ ++ def getpackagesbyfiles(files): ++ """ ++ :param files: list of filenames ++ :returns: **dictionary** file->package, if file doesn't belong to any ++ package it not returned as key of this dictionary ++ """ ++ ret=3D{} ++ listtocheck=3D[] ++ for f in files: ++ if os.path.isdir(f): ++ ret[f]=3D"directory" ++ else: ++ listtocheck.append(f) ++ =09 ++ try: ++ proc=3Dsubprocess.Popen(['qfile']+['--nocolor','--exact','','--from'= ,'-'], ++ stdin=3Dsubprocess.PIPE, stdout=3Dsubprocess.PIPE,stderr=3Dsubproce= ss.PIPE,=20 ++ bufsize=3D4096) ++ =09 ++ out,err=3Dproc.communicate("\n".join(listtocheck).encode("utf8")) ++ =09 ++ lines=3Dout.decode("utf8").split("\n") ++ #print lines ++ line_re=3Dre.compile(r"^([^ ]+)\s+\(([^)]+)\)$") ++ for line in lines: ++ if len(line)=3D=3D0: ++ continue ++ match=3Dline_re.match(line) ++ if match: ++ ret[match.group(2)]=3Dmatch.group(1) ++ else: ++ portage.util.writemsg("Util qfile returned unparsable string: %s\n= " % line) ++ ++ except OSError as e: ++ portage.util.writemsg("Error while launching qfile: %s\n" % e) ++ =20 ++ =20 ++ return ret ++ =20 ++ def getfilesbypackages(packagenames): ++ """ ++ =09 ++ :param packagename: name of package ++ :returns: **list** of files in package with name *packagename* ++ """ ++ ret=3D[] ++ try: ++ proc=3Dsubprocess.Popen(['qlist']+['--nocolor',"--obj"]+packagenames= , ++ stdout=3Dsubprocess.PIPE,stderr=3Dsubprocess.PIPE,=20 ++ bufsize=3D4096) ++ =09 ++ out,err=3Dproc.communicate() ++ =09 ++ ret=3Dout.decode("utf8").split("\n") ++ if ret=3D=3D['']: ++ ret=3D[] ++ except OSError as e: ++ portage.util.writemsg("Error while launching qfile: %s\n" % e) ++ ++ return ret ++ =20 ++ def get_all_packages_files(): =20 ++ """ ++ Memory-hungry operation ++ =09 ++ :returns: **set** of all files that belongs to package ++ """ ++ ret=3D[] ++ try: ++ proc=3Dsubprocess.Popen(['qlist']+['--all',"--obj"], ++ stdout=3Dsubprocess.PIPE,stderr=3Dsubprocess.PIPE,=20 ++ bufsize=3D4096) ++ =20 ++ out,err=3Dproc.communicate() ++ =09 ++ ret=3Dout.decode("utf8").split("\n") ++ except OSError as e: ++ portage.util.writemsg("Error while launching qfile: %s\n" % e) ++ ++ return set(ret) ++ =20 ++class FilterProcGenerator: ++ def __init__(self, pkgname, settings): ++ portageutils=3DPortageUtils(settings=3Dsettings) ++ ++ deps_all=3Dportageutils.get_deps_for_package_building(pkgname) ++ deps_portage=3Dportageutils.get_dep('portage',["RDEPEND"]) ++ =09 ++ system_packages=3Dportageutils.get_system_packages_list() ++ ++ allfiles=3DGentoolkitUtils.get_all_packages_files() ++ portage.util.writemsg("All files list recieved, waiting for " \ ++ "a list of allowed files\n") ++ ++ ++ allowedpkgs=3Dsystem_packages+list(deps_portage)+list(deps_all)=09 ++ =09 ++ allowedfiles=3DGentoolkitUtils.getfilesbypackages(allowedpkgs) ++ #for pkg in allowedpkgs: ++ # allowedfiles+=3DGentoolkitUtils.getfilesbypackage(pkg) ++ ++ #import pdb; pdb.set_trace() ++ ++ # manually add all python interpreters to this list ++ allowedfiles+=3DGentoolkitUtils.getfilesbypackages(['python']) ++ allowedfiles=3Dset(allowedfiles) ++ =09 ++ deniedfiles=3Dallfiles-allowedfiles ++ ++ def filter_proc(eventname,filename,stage): ++ if filename in deniedfiles: ++ return False ++ return True ++ =09 ++ self.filter_proc=3Dfilter_proc ++ def get_filter_proc(self): ++ return self.filter_proc ++ ++class EventsAnalyser: ++ def __init__(self, pkgname, events, settings): ++ self.pkgname=3Dpkgname ++ self.events=3Devents ++ self.settings=3Dsettings ++ self.portageutils=3DPortageUtils(settings=3Dsettings) ++ ++ self.deps_all=3Dself.portageutils.get_deps_for_package_building(pkgna= me) ++ self.deps_direct=3Dself.portageutils.get_dep(pkgname,["DEPEND"]) ++ self.deps_portage=3Dself.portageutils.get_dep('portage',["RDEPEND"]) ++ =09 ++ self.system_packages=3Dself.portageutils.get_system_packages_list() ++ # All analyse work is here ++ =09 ++ # get unique filenames ++ filenames=3Dset() ++ for stage in events: ++ succ_events=3Dset(events[stage][0]) ++ fail_events=3Dset(events[stage][1]) ++ filenames=3Dfilenames.union(succ_events) ++ filenames=3Dfilenames.union(fail_events) ++ filenames=3Dlist(filenames) ++ ++ file_to_package=3DGentoolkitUtils.getpackagesbyfiles(filenames) ++ # This part is completly unreadable.=20 ++ # It converting one complex struct(returned by getfsevents) to anothe= r complex ++ # struct which good for generating output. ++ # ++ # Old struct is also used during output ++ ++ packagesinfo=3D{} ++ ++ for stage in sorted(events): ++ succ_events=3Devents[stage][0] ++ fail_events=3Devents[stage][1] ++ =20 ++ for filename in succ_events: ++ if filename in file_to_package: ++ package=3Dfile_to_package[filename] ++ else: ++ package=3D"unknown" ++ =09 ++ if not package in packagesinfo: ++ packagesinfo[package]=3D{} ++ stageinfo=3Dpackagesinfo[package] ++ if not stage in stageinfo: ++ stageinfo[stage]=3D{} ++ =09 ++ filesinfo=3Dstageinfo[stage] ++ if not filename in filesinfo: ++ filesinfo[filename]=3D{"found":[],"notfound":[]} ++ filesinfo[filename]["found"]=3Dsucc_events[filename] ++ =09 ++ for filename in fail_events: ++ if filename in file_to_package: ++ package=3Dfile_to_package[filename] ++ else: ++ package=3D"unknown" ++ if not package in packagesinfo: ++ packagesinfo[package]=3D{} ++ stageinfo=3Dpackagesinfo[package] ++ if not stage in stageinfo: ++ stageinfo[stage]=3D{} ++ =09 ++ filesinfo=3Dstageinfo[stage] ++ if not filename in filesinfo: ++ filesinfo[filename]=3D{"found":[],"notfound":[]} ++ filesinfo[filename]["notfound"]=3Dfail_events[filename] ++ self.packagesinfo=3Dpackagesinfo ++ =20 ++ def display(self): ++ portage.util.writemsg( ++ portage.output.colorize( ++ "WARN", "\nFile access report for %s:\n" % self.pkgname)) ++ ++ stagesorder=3D{"clean":1,"setup":2,"unpack":3,"prepare":4,"configure"= :5,"compile":6,"test":7, ++ "install":8,"preinst":9,"postinst":10,"prerm":11,"postrm":12,"unkno= wn":13} ++ packagesinfo=3Dself.packagesinfo ++ # print information grouped by package =20 ++ for package in sorted(packagesinfo): ++ # not showing special directory package ++ if package=3D=3D"directory": ++ continue ++ =09 ++ if package=3D=3D"unknown": ++ continue ++ ++ ++ is_pkg_in_dep=3Dpackage in self.deps_all ++ is_pkg_in_portage_dep=3Dpackage in self.deps_portage ++ is_pkg_in_system=3Dpackage in self.system_packages ++ is_pkg_python=3D"dev-lang/python" in package ++ ++ stages=3D[] ++ for stage in sorted(packagesinfo[package].keys(), key=3Dstagesorder.= get): ++ if stage!=3D"unknown": ++ stages.append(stage) ++ ++ if len(stages)=3D=3D0: ++ continue ++=09 ++ filenames=3D{} ++ for stage in stages: ++ for filename in packagesinfo[package][stage]: ++ if len(packagesinfo[package][stage][filename]["found"])!=3D0: ++ was_readed,was_writed=3Dpackagesinfo[package][stage][filename]["f= ound"] ++ if not filename in filenames: ++ filenames[filename]=3D['ok',was_readed,was_writed] ++ else: ++ status, old_was_readed, old_was_writed=3Dfilenames[filename] ++ filenames[filename]=3D[ ++ 'ok',old_was_readed | was_readed, old_was_writed | was_writed=20 ++ ] ++ if len(packagesinfo[package][stage][filename]["notfound"])!=3D0: ++ was_notfound,was_blocked=3Dpackagesinfo[package][stage][filename]= ["notfound"] ++ if not filename in filenames: ++ filenames[filename]=3D['err',was_notfound,was_blocked] ++ else: ++ status, old_was_notfound, old_was_blocked=3Dfilenames[filename] ++ filenames[filename]=3D[ ++ 'err',old_was_notfound | was_notfound, old_was_blocked | was_b= locked=20 ++ ] ++ =20 ++ ++ if is_pkg_in_dep: ++ portage.util.writemsg("[OK]") ++ elif is_pkg_in_system: ++ portage.util.writemsg("[SYSTEM]") ++ elif is_pkg_in_portage_dep: ++ portage.util.writemsg("[PORTAGE DEP]") ++ elif is_pkg_python: ++ portage.util.writemsg("[INTERPRETER]") ++ elif not self.is_package_useful(package,stages,filenames.keys()): ++ portage.util.writemsg("[LIKELY OK]") ++ else: ++ portage.util.writemsg(portage.output.colorize("BAD", "[NOT IN DEPS]= ")) ++ # show information about accessed files ++ ++ portage.util.writemsg(" %-40s: %s\n" % (package,stages)) ++ ++ # this is here for readability ++ action=3D{ ++ ('ok',False,False):"accessed", ++ ('ok',True,False):"readed", ++ ('ok',False,True):"writed", ++ ('ok',True,True):"readed and writed", ++ ('err',False,False):"other error", ++ ('err',True,False):"not found", ++ ('err',False,True):"blocked", ++ ('err',True,True):"not found and blocked" ++ } ++ =09 ++ filescounter=3D0 ++ =09 ++ for filename in filenames: ++ event_info=3Dtuple(filenames[filename]) ++ portage.util.writemsg(" %-56s %-21s\n" % (filename,action[event_in= fo])) ++ filescounter+=3D1 ++ if filescounter>10: ++ portage.util.writemsg(" ... and %d more ...\n" % (len(filenames)-= 10)) ++ break ++ # ... and one more check. Making sure that direct build time=20 ++ # dependencies were accessed ++ #import pdb; pdb.set_trace() ++ not_accessed_deps=3Dset(self.deps_direct)-set(self.packagesinfo.keys(= )) ++ if not_accessed_deps: ++ portage.util.writemsg(portage.output.colorize("WARN", "!!! ")) ++ portage.util.writemsg("Warning! Some build time dependencies " + \ ++ "of packages were not accessed: " + \ ++ " ".join(not_accessed_deps) + "\n") ++ =09 ++ def is_package_useful(self,pkg,stages,files): ++ """ some basic heuristics here to cut part of packages """ ++ ++ excluded_paths=3Dset( ++ ['/etc/sandbox.d/'] ++ ) ++ ++ excluded_packages=3Dset( ++ # autodep shows these two packages every time ++ ['net-zope/zope-fixers', 'net-zope/zope-interface'] ++ ) ++ ++ ++ def is_pkg_excluded(p): ++ for pkg in excluded_packages: ++ if p.startswith(pkg): # if package is excluded ++ return True ++ return False ++ ++ ++ def is_file_excluded(f): ++ for path in excluded_paths: ++ if f.startswith(path): # if path is excluded ++ return True ++ return False ++ ++ ++ if is_pkg_excluded(pkg): ++ return False ++ ++ for f in files: ++ if is_file_excluded(f): ++ continue ++ =09 ++ # test 1: package is not useful if all files are *.desktop or *.xml = or *.m4 =09 ++ if not (f.endswith(".desktop") or f.endswith(".xml") or f.endswith("= .m4") or f.endswith(".pc")): ++ break ++ else: ++ return False # we get here if cycle ends not with break ++ =20 ++ return True ++ =09 ++ =20 +\ No newline at end of file +diff --git a/pym/_emerge/EventsLogger.py b/pym/_emerge/EventsLogger.py +new file mode 100644 +index 0000000..68b3c67 +--- /dev/null ++++ b/pym/_emerge/EventsLogger.py +@@ -0,0 +1,180 @@ ++# Distributed under the terms of the GNU General Public License v2 ++ ++import io ++import sys ++import stat ++import socket ++import select ++import tempfile ++ ++import threading ++ ++from portage import os ++ ++class EventsLogger(threading.Thread): ++ def default_filter(eventname, filename, stage): ++ return True ++ =09 ++ def __init__(self, socket_dir=3D"/tmp/", filter_proc=3Ddefault_filter)= : ++ threading.Thread.__init__(self) # init the Thread ++ =09 ++ self.alive=3DFalse ++ =09 ++ self.main_thread=3Dthreading.currentThread() ++ =09 ++ self.socket_dir=3Dsocket_dir ++ self.filter_proc=3Dfilter_proc ++ =09 ++ self.socket_name=3DNone ++ self.socket_logger=3DNone ++ ++ self.events=3D{} ++ ++ try: ++ socket_dir_name =3D tempfile.mkdtemp(dir=3Dself.socket_dir, ++ prefix=3D"log_socket_") ++ =09 ++ socket_name =3D os.path.join(socket_dir_name, 'socket') ++ ++ except OSError as e: ++ return ++ =09 ++ self.socket_name=3Dsocket_name ++ =09 ++ #print(self.socket_name) ++ =09 ++ try: ++ socket_logger=3Dsocket.socket(socket.AF_UNIX, socket.SOCK_SEQPACKET) ++ socket_logger.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) ++ ++ socket_logger.bind(self.socket_name) ++ socket_logger.listen(64) ++ ++ except socket.error as e: ++ return ++ ++ self.socket_logger=3Dsocket_logger ++ ++ try: ++ # Allow connecting to socket for anyone ++ os.chmod(socket_dir_name, ++ stat.S_IRUSR|stat.S_IWUSR|stat.S_IXUSR| ++ stat.S_IROTH|stat.S_IWOTH|stat.S_IXOTH) ++ os.chmod(socket_name, ++ stat.S_IRUSR|stat.S_IWUSR|stat.S_IXUSR| ++ stat.S_IROTH|stat.S_IWOTH|stat.S_IXOTH) ++ except OSError as e: ++ return ++ =20 ++ def run(self): ++ """ Starts the log server """ ++ ++ self.alive=3DTrue ++ self.listen_thread=3Dthreading.currentThread() ++ clients=3D{} ++ =09 ++ epoll=3Dselect.epoll() ++ epoll.register(self.socket_logger.fileno(), select.EPOLLIN) ++ ++ while self.alive: ++ try: ++ sock_events =3D epoll.poll(3) ++ =09 ++ for fileno, sock_event in sock_events: ++ if fileno =3D=3D self.socket_logger.fileno(): ++ ret =3D self.socket_logger.accept() ++ if ret is None: ++ pass ++ else: ++ (client,addr)=3Dret ++ epoll.register(client.fileno(), select.EPOLLIN) ++ clients[client.fileno()]=3Dclient ++ elif sock_event & select.EPOLLIN: ++ s=3Dclients[fileno] ++ record=3Ds.recv(8192) ++ =09 ++ if not record: # if connection was closed ++ epoll.unregister(fileno) ++ clients[fileno].close() ++ del clients[fileno] ++ continue ++ =09 ++ #import pdb; pdb.set_trace() ++ try: ++ message=3Drecord.decode("utf8").split("\0") ++ except UnicodeDecodeError: ++ print("Bad message %s" % record) ++ continue ++ ++ # continue ++ ++ #print(message) ++ =09 ++ try: ++ if message[4]=3D=3D"ASKING": ++ if self.filter_proc(message[1],message[2],message[3]): ++ s.sendall(b"ALLOW\0") ++ else: ++ # TODO: log through portage infrastructure ++ #print("Blocking an access to %s" % message[2]) ++ s.sendall(b"DENY\0") ++ else: ++ eventname,filename,stage,result=3Dmessage[1:5] ++ ++ if not stage in self.events: ++ self.events[stage]=3D[{},{}] ++ =09 ++ hashofsucesses=3Dself.events[stage][0] ++ hashoffailures=3Dself.events[stage][1] ++ ++ if result=3D=3D"DENIED": ++ print("Blocking an access to %s" % filename) ++ ++ if result=3D=3D"OK": ++ if not filename in hashofsucesses: ++ hashofsucesses[filename]=3D[False,False] ++ =20 ++ readed_or_writed=3Dhashofsucesses[filename] ++ =20 ++ if eventname=3D=3D"read": ++ readed_or_writed[0]=3DTrue ++ elif eventname=3D=3D"write": ++ readed_or_writed[1]=3DTrue ++ =09 ++ elif result[0:3]=3D=3D"ERR" or result=3D=3D"DENIED": ++ if not filename in hashoffailures: ++ hashoffailures[filename]=3D[False,False] ++ notfound_or_blocked=3Dhashoffailures[filename] ++ =20 ++ if result=3D=3D"ERR/2": ++ notfound_or_blocked[0]=3DTrue ++ elif result=3D=3D"DENIED": ++ notfound_or_blocked[1]=3DTrue ++ ++ else: ++ print("Error in logger module<->analyser protocol") ++ =09 ++ except IndexError: ++ print("IndexError while parsing %s" % record) ++ except IOError as e: ++ if e.errno!=3D4: # handling "Interrupted system call" errors ++ raise ++ =20 ++ # if main thread doesnt exists then exit =20 ++ if not self.main_thread.is_alive(): ++ break ++ epoll.unregister(self.socket_logger.fileno()) ++ epoll.close() ++ self.socket_logger.close() ++ =09 ++ def stop(self): ++ """ Stops the log server. Returns all events """ ++ ++ self.alive=3DFalse ++ =09 ++ # Block the main thread until listener exists ++ self.listen_thread.join() ++ =09 ++ # We assume portage clears tmp folder, so no deleting a socket file ++ # We assume that no new socket data will arrive after this moment ++ return self.events +diff --git a/pym/portage/const.py b/pym/portage/const.py +index ecaa8f1..f34398d 100644 +--- a/pym/portage/const.py ++++ b/pym/portage/const.py +@@ -67,6 +67,8 @@ FAKEROOT_BINARY =3D "/usr/bin/fakeroot" + BASH_BINARY =3D "/bin/bash" + MOVE_BINARY =3D "/bin/mv" + PRELINK_BINARY =3D "/usr/sbin/prelink" ++AUTODEP_LIBRARY =3D "/usr/lib/file_hook.so" ++ +=20 + INVALID_ENV_FILE =3D "/etc/spork/is/not/valid/profile.env" + REPO_NAME_FILE =3D "repo_name" +@@ -88,7 +90,8 @@ EBUILD_PHASES =3D ("pretend", "setup", "unp= ack", "prepare", "configure" + SUPPORTED_FEATURES =3D frozenset([ + "allow-missing-manifests", + "assume-digests", "binpkg-logs", "buildpkg",= "buildsyspkg", "candy", +- "ccache", "chflags", "collision-protect", "c= ompress-build-logs", ++ "ccache", "chflags", "collision-protect", "c= ompress-build-logs",=20 ++ "depcheck", "depcheckstrict", + "digest", "distcc", "distcc-pump", "distlock= s", "ebuild-locks", "fakeroot", + "fail-clean", "fixpackages", "force-mirror",= "getbinpkg", + "installsources", "keeptemp", "keepwork", "f= ixlafiles", "lmirror", +diff --git a/pym/portage/package/ebuild/_config/special_env_vars.py b/py= m/portage/package/ebuild/_config/special_env_vars.py +index 87aa606..6d42809 100644 +--- a/pym/portage/package/ebuild/_config/special_env_vars.py ++++ b/pym/portage/package/ebuild/_config/special_env_vars.py +@@ -101,8 +101,8 @@ environ_whitelist +=3D [ + # other variables inherited from the calling environment + environ_whitelist +=3D [ + "CVS_RSH", "ECHANGELOG_USER", +- "GPG_AGENT_INFO", +- "SSH_AGENT_PID", "SSH_AUTH_SOCK", ++ "GPG_AGENT_INFO", "LOG_SOCKET", ++ "SSH_AGENT_PID", "SSH_AUTH_SOCK" + "STY", "WINDOW", "XAUTHORITY", + ] +=20 +diff --git a/pym/portage/package/ebuild/doebuild.py b/pym/portage/packag= e/ebuild/doebuild.py +index 49b67ac..c76c1ed 100644 +--- a/pym/portage/package/ebuild/doebuild.py ++++ b/pym/portage/package/ebuild/doebuild.py +@@ -1038,6 +1038,9 @@ def _spawn_actionmap(settings): + nosandbox =3D ("sandbox" not in features and \ + "usersandbox" not in features) +=20 ++ if "depcheck" in features or "depcheckstrict" in features: ++ nosandbox =3D True ++ + if not portage.process.sandbox_capable: + nosandbox =3D True +=20 +@@ -1215,7 +1218,10 @@ def spawn(mystring, mysettings, debug=3D0, free=3D= 0, droppriv=3D0, sesandbox=3D0, fakero + keywords["opt_name"] =3D "[%s/%s]" % \ + (mysettings.get("CATEGORY",""), mysettings.get("PF","")) +=20 +- if free or "SANDBOX_ACTIVE" in os.environ: ++ if "depcheck" in features or "depcheckstrict" in features: ++ keywords["opt_name"] +=3D " bash" ++ spawn_func =3D portage.process.spawn_autodep ++ elif free or "SANDBOX_ACTIVE" in os.environ: + keywords["opt_name"] +=3D " bash" + spawn_func =3D portage.process.spawn_bash + elif fakeroot: +diff --git a/pym/portage/process.py b/pym/portage/process.py +index 3c15370..6866a2f 100644 +--- a/pym/portage/process.py ++++ b/pym/portage/process.py +@@ -16,7 +16,7 @@ portage.proxy.lazyimport.lazyimport(globals(), + 'portage.util:dump_traceback', + ) +=20 +-from portage.const import BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY ++from portage.const import BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY,= AUTODEP_LIBRARY + from portage.exception import CommandNotFound +=20 + try: +@@ -39,6 +39,9 @@ else: + sandbox_capable =3D (os.path.isfile(SANDBOX_BINARY) and + os.access(SANDBOX_BINARY, os.X_OK)) +=20 ++autodep_capable =3D (os.path.isfile(AUTODEP_LIBRARY) and ++ os.access(AUTODEP_LIBRARY, os.X_OK)) ++ + fakeroot_capable =3D (os.path.isfile(FAKEROOT_BINARY) and + os.access(FAKEROOT_BINARY, os.X_OK)) +=20 +@@ -66,6 +69,16 @@ def spawn_bash(mycommand, debug=3DFalse, opt_name=3DN= one, **keywords): + args.append(mycommand) + return spawn(args, opt_name=3Dopt_name, **keywords) +=20 ++def spawn_autodep(mycommand, opt_name=3DNone, **keywords): ++ if not autodep_capable: ++ return spawn_bash(mycommand, opt_name=3Dopt_name, **keywords) ++ if "env" not in keywords or "LOG_SOCKET" not in keywords["env"]: ++ return spawn_bash(mycommand, opt_name=3Dopt_name, **keywords) ++ ++ # Core part: tell the loader to preload logging library ++ keywords["env"]["LD_PRELOAD"]=3DAUTODEP_LIBRARY ++ return spawn_bash(mycommand, opt_name=3Dopt_name, **keywords)=09 ++ + def spawn_sandbox(mycommand, opt_name=3DNone, **keywords): + if not sandbox_capable: + return spawn_bash(mycommand, opt_name=3Dopt_name, **keywords)