* [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/
@ 2011-03-25 4:34 Zac Medico
0 siblings, 0 replies; 29+ messages in thread
From: Zac Medico @ 2011-03-25 4:34 UTC (permalink / raw
To: gentoo-commits
commit: f823031ed33bda9579d265b62607380bb255dfdd
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Mar 25 03:03:28 2011 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Mar 25 03:03:28 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=f823031e
Preserve elog message continuity during updates.
This integrates the fix from commit
8209aeab647b1ab80a64d5931069b3533776ef75 with the asynchronous merge
changes from commit 7535cabdf2fab76fc55df83643157613dfd66be9.
---
pym/_emerge/Binpkg.py | 2 ++
pym/_emerge/EbuildBuild.py | 2 ++
pym/portage/dbapi/vartree.py | 6 +++---
3 files changed, 7 insertions(+), 3 deletions(-)
diff --git a/pym/_emerge/Binpkg.py b/pym/_emerge/Binpkg.py
index 62d44c4..fbd2a53 100644
--- a/pym/_emerge/Binpkg.py
+++ b/pym/_emerge/Binpkg.py
@@ -304,6 +304,8 @@ class Binpkg(CompositeTask):
def _unlock_builddir(self):
if self.opts.pretend or self.opts.fetchonly:
return
+ portage.elog.elog_process(self.pkg.cpv, self.settings,
+ phasefilter=("prerm", "postrm"))
portage.elog.elog_process(self.pkg.cpv, self.settings)
self._build_dir.unlock()
diff --git a/pym/_emerge/EbuildBuild.py b/pym/_emerge/EbuildBuild.py
index c7a5f5c..b72b3d5 100644
--- a/pym/_emerge/EbuildBuild.py
+++ b/pym/_emerge/EbuildBuild.py
@@ -243,6 +243,8 @@ class EbuildBuild(CompositeTask):
self.wait()
def _unlock_builddir(self):
+ portage.elog.elog_process(self.pkg.cpv, self.settings,
+ phasefilter=("prerm", "postrm"))
portage.elog.elog_process(self.pkg.cpv, self.settings)
self._build_dir.unlock()
diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 66e2955..a8888ae 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1668,7 +1668,7 @@ class dblink(object):
self._eerror(ebuild_phase, msg_lines)
- self._elog_process()
+ self._elog_process(phasefilter=("prerm", "postrm"))
if retval == os.EX_OK:
# myebuildpath might be None, so ensure
@@ -2784,10 +2784,10 @@ class dblink(object):
self._scheduler.dblinkElog(self,
phase, _eerror, lines)
- def _elog_process(self):
+ def _elog_process(self, phasefilter=None):
cpv = self.mycpv
if self._pipe is None:
- elog_process(cpv, self.settings)
+ elog_process(cpv, self.settings, phasefilter=phasefilter)
else:
logdir = os.path.join(self.settings["T"], "logging")
ebuild_logentries = collect_ebuild_messages(logdir)
^ permalink raw reply related [flat|nested] 29+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/
@ 2011-03-25 4:34 Zac Medico
0 siblings, 0 replies; 29+ messages in thread
From: Zac Medico @ 2011-03-25 4:34 UTC (permalink / raw
To: gentoo-commits
commit: 7535cabdf2fab76fc55df83643157613dfd66be9
Author: David James <davidjames <AT> google <DOT> com>
AuthorDate: Fri Mar 25 02:36:33 2011 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Mar 25 02:36:33 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=7535cabd
Merge packages asynchronously in Portage.
This allows for the scheduler to continue to run while packages are
being merged and installed, allowing for additional parallelism and
making better use of the CPUs.
Review URL: http://codereview.chromium.org/6713043
---
pym/_emerge/Binpkg.py | 21 ++++----
pym/_emerge/EbuildBuild.py | 13 +++--
pym/_emerge/EbuildMerge.py | 47 +++++++++++-------
pym/_emerge/MergeListItem.py | 21 ++++----
pym/_emerge/PackageMerge.py | 12 ++---
pym/portage/dbapi/_MergeProcess.py | 82 ++++++++++++++++++++++++++-----
pym/portage/dbapi/vartree.py | 94 ++++++++++++++++++++----------------
7 files changed, 182 insertions(+), 108 deletions(-)
diff --git a/pym/_emerge/Binpkg.py b/pym/_emerge/Binpkg.py
index 0058745..62d44c4 100644
--- a/pym/_emerge/Binpkg.py
+++ b/pym/_emerge/Binpkg.py
@@ -307,7 +307,7 @@ class Binpkg(CompositeTask):
portage.elog.elog_process(self.pkg.cpv, self.settings)
self._build_dir.unlock()
- def install(self):
+ def install(self, handler):
# This gives bashrc users an opportunity to do various things
# such as remove binary packages after they're installed.
@@ -320,19 +320,20 @@ class Binpkg(CompositeTask):
pkg=self.pkg, pkg_count=self.pkg_count,
pkg_path=self._pkg_path, scheduler=self.scheduler,
settings=settings, tree=self._tree, world_atom=self.world_atom)
+ task = merge.create_task()
+ task.addExitListener(self._install_exit)
+ self._start_task(task, handler)
- try:
- retval = merge.execute()
- finally:
- settings.pop("PORTAGE_BINPKG_FILE", None)
- self._unlock_builddir()
+ def _install_exit(self, task):
+ self.settings.pop("PORTAGE_BINPKG_FILE", None)
+ self._unlock_builddir()
- if retval == os.EX_OK and \
- 'binpkg-logs' not in self.settings.features and \
+ if self._default_final_exit(task) != os.EX_OK:
+ return
+
+ if 'binpkg-logs' not in self.settings.features and \
self.settings.get("PORTAGE_LOG_FILE"):
try:
os.unlink(self.settings["PORTAGE_LOG_FILE"])
except OSError:
pass
- return retval
-
diff --git a/pym/_emerge/EbuildBuild.py b/pym/_emerge/EbuildBuild.py
index 98ab245..c7a5f5c 100644
--- a/pym/_emerge/EbuildBuild.py
+++ b/pym/_emerge/EbuildBuild.py
@@ -314,7 +314,7 @@ class EbuildBuild(CompositeTask):
self._unlock_builddir()
self.wait()
- def install(self):
+ def install(self, exit_handler):
"""
Install the package and then clean up and release locks.
Only call this after the build has completed successfully
@@ -343,10 +343,11 @@ class EbuildBuild(CompositeTask):
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
- try:
- rval = merge.execute()
- finally:
- self._unlock_builddir()
+ task = merge.create_task()
+ task.addExitListener(self._install_exit)
+ self._start_task(task, exit_handler)
- return rval
+ def _install_exit(self, task):
+ self._unlock_builddir()
+ self._default_final_exit(task)
diff --git a/pym/_emerge/EbuildMerge.py b/pym/_emerge/EbuildMerge.py
index d73a262..6a58692 100644
--- a/pym/_emerge/EbuildMerge.py
+++ b/pym/_emerge/EbuildMerge.py
@@ -4,6 +4,8 @@
from _emerge.SlotObject import SlotObject
import portage
from portage import os
+from portage.dbapi._MergeProcess import MergeProcess
+from portage.dbapi.vartree import dblink
class EbuildMerge(SlotObject):
@@ -11,28 +13,35 @@ class EbuildMerge(SlotObject):
"pkg", "pkg_count", "pkg_path", "pretend",
"scheduler", "settings", "tree", "world_atom")
- def execute(self):
+ def create_task(self):
root_config = self.pkg.root_config
settings = self.settings
- retval = portage.merge(settings["CATEGORY"],
- settings["PF"], settings["D"],
- os.path.join(settings["PORTAGE_BUILDDIR"],
- "build-info"), root_config.root, settings,
- myebuild=settings["EBUILD"],
- mytree=self.tree, mydbapi=root_config.trees[self.tree].dbapi,
- vartree=root_config.trees["vartree"],
- prev_mtimes=self.ldpath_mtimes,
- scheduler=self.scheduler,
- blockers=self.find_blockers)
-
- if retval == os.EX_OK:
- self.world_atom(self.pkg)
- self._log_success()
-
- return retval
-
- def _log_success(self):
+ mycat = settings["CATEGORY"]
+ mypkg = settings["PF"]
+ pkgloc = settings["D"]
+ infloc = os.path.join(settings["PORTAGE_BUILDDIR"], "build-info")
+ myroot = root_config.root
+ myebuild = settings["EBUILD"]
+ mydbapi = root_config.trees[self.tree].dbapi
+ vartree = root_config.trees["vartree"]
+ background = (settings.get('PORTAGE_BACKGROUND') == '1')
+ logfile = settings.get('PORTAGE_LOG_FILE')
+
+ merge_task = MergeProcess(
+ dblink=dblink, mycat=mycat, mypkg=mypkg, settings=settings,
+ treetype=self.tree, vartree=vartree, scheduler=self.scheduler,
+ background=background, blockers=self.find_blockers, pkgloc=pkgloc,
+ infloc=infloc, myebuild=myebuild, mydbapi=mydbapi,
+ prev_mtimes=self.ldpath_mtimes, logfile=logfile)
+ merge_task.addExitListener(self._log_exit)
+ return merge_task
+
+ def _log_exit(self, task):
+ if task.returncode != os.EX_OK:
+ return
+
pkg = self.pkg
+ self.world_atom(pkg)
pkg_count = self.pkg_count
pkg_path = self.pkg_path
logger = self.logger
diff --git a/pym/_emerge/MergeListItem.py b/pym/_emerge/MergeListItem.py
index 1dcc178..768865e 100644
--- a/pym/_emerge/MergeListItem.py
+++ b/pym/_emerge/MergeListItem.py
@@ -111,7 +111,7 @@ class MergeListItem(CompositeTask):
self._install_task.wait()
return self.returncode
- def merge(self):
+ def merge(self, exit_handler):
pkg = self.pkg
build_opts = self.build_opts
@@ -135,15 +135,14 @@ class MergeListItem(CompositeTask):
world_atom=world_atom)
uninstall.start()
- retval = uninstall.wait()
- if retval != os.EX_OK:
- return retval
- return os.EX_OK
-
- if build_opts.fetchonly or \
+ self.returncode = uninstall.wait()
+ else:
+ self.returncode = os.EX_OK
+ exit_handler(self)
+ elif build_opts.fetchonly or \
build_opts.buildpkgonly:
- return self.returncode
-
- retval = self._install_task.install()
- return retval
+ exit_handler(self)
+ else:
+ self._current_task = self._install_task
+ self._install_task.install(exit_handler)
diff --git a/pym/_emerge/PackageMerge.py b/pym/_emerge/PackageMerge.py
index 4aecf8a..45d2e7d 100644
--- a/pym/_emerge/PackageMerge.py
+++ b/pym/_emerge/PackageMerge.py
@@ -4,11 +4,6 @@
from _emerge.AsynchronousTask import AsynchronousTask
from portage.output import colorize
class PackageMerge(AsynchronousTask):
- """
- TODO: Implement asynchronous merge so that the scheduler can
- run while a merge is executing.
- """
-
__slots__ = ("merge",)
def _start(self):
@@ -40,6 +35,9 @@ class PackageMerge(AsynchronousTask):
not self.merge.build_opts.buildpkgonly:
self.merge.statusMessage(msg)
- self.returncode = self.merge.merge()
- self.wait()
+ self.merge.merge(self.exit_handler)
+
+ def exit_handler(self, task):
+ self.returncode = task.returncode
+ self._wait_hook()
diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index f717d12..6e63f84 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -4,30 +4,72 @@
import signal
import traceback
+import errno
+import fcntl
import portage
-from portage import os
+from portage import os, StringIO
+import portage.elog.messages
+from _emerge.PollConstants import PollConstants
from _emerge.SpawnProcess import SpawnProcess
class MergeProcess(SpawnProcess):
"""
- Merge package files in a subprocess, so the Scheduler can run in the
- main thread while files are moved or copied asynchronously.
+ Merge packages in a subprocess, so the Scheduler can run in the main
+ thread while files are moved or copied asynchronously.
"""
- __slots__ = ('cfgfiledict', 'conf_mem_file', \
- 'destroot', 'dblink', 'srcroot',)
+ __slots__ = ('dblink', 'mycat', 'mypkg', 'settings', 'treetype',
+ 'vartree', 'scheduler', 'blockers', 'pkgloc', 'infloc', 'myebuild',
+ 'mydbapi', 'prev_mtimes', '_elog_reader_fd', '_elog_reg_id',
+ '_buf')
- def _spawn(self, args, fd_pipes=None, **kwargs):
+ def _elog_output_handler(self, fd, event):
+ output = None
+ if event & PollConstants.POLLIN:
+ try:
+ output = os.read(fd, self._bufsize)
+ except OSError as e:
+ if e.errno not in (errno.EAGAIN, errno.EINTR):
+ raise
+ if output:
+ lines = output.split('\n')
+ if len(lines) == 1:
+ self._buf += lines[0]
+ else:
+ lines[0] = self._buf + lines[0]
+ self._buf = lines.pop()
+ out = StringIO()
+ for line in lines:
+ funcname, phase, key, msg = line.split(' ', 3)
+ reporter = getattr(portage.elog.messages, funcname)
+ reporter(msg, phase=phase, key=key, out=out)
+
+ def _spawn(self, args, fd_pipes, **kwargs):
"""
Fork a subprocess, apply local settings, and call
- dblink._merge_process().
+ dblink.merge().
"""
+ files = self._files
+ elog_reader_fd, elog_writer_fd = os.pipe()
+ fcntl.fcntl(elog_reader_fd, fcntl.F_SETFL,
+ fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
+ mylink = self.dblink(self.mycat, self.mypkg, settings=self.settings,
+ treetype=self.treetype, vartree=self.vartree,
+ blockers=self.blockers, scheduler=self.scheduler,
+ pipe=elog_writer_fd)
+ fd_pipes[elog_writer_fd] = elog_writer_fd
+ self._elog_reg_id = self.scheduler.register(elog_reader_fd,
+ self._registered_events, self._elog_output_handler)
+
pid = os.fork()
if pid != 0:
+ self._elog_reader_fd = elog_reader_fd
+ self._buf = ""
portage.process.spawned_pids.append(pid)
return [pid]
+ os.close(elog_reader_fd)
portage.process._setup_pipes(fd_pipes)
# Use default signal handlers since the ones inherited
@@ -35,18 +77,19 @@ class MergeProcess(SpawnProcess):
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
- portage.output.havecolor = self.dblink.settings.get('NOCOLOR') \
+ portage.output.havecolor = self.settings.get('NOCOLOR') \
not in ('yes', 'true')
- # In this subprocess we want dblink._display_merge() to use
+ # In this subprocess we want mylink._display_merge() to use
# stdout/stderr directly since they are pipes. This behavior
- # is triggered when dblink._scheduler is None.
- self.dblink._scheduler = None
+ # is triggered when mylink._scheduler is None.
+ mylink._scheduler = None
rval = 1
try:
- rval = self.dblink._merge_process(self.srcroot, self.destroot,
- self.cfgfiledict, self.conf_mem_file)
+ rval = mylink.merge(self.pkgloc, self.infloc,
+ myebuild=self.myebuild, mydbapi=self.mydbapi,
+ prev_mtimes=self.prev_mtimes)
except SystemExit:
raise
except:
@@ -55,3 +98,16 @@ class MergeProcess(SpawnProcess):
# Call os._exit() from finally block, in order to suppress any
# finally blocks from earlier in the call stack. See bug #345289.
os._exit(rval)
+
+ def _unregister(self):
+ """
+ Unregister from the scheduler and close open files.
+ """
+ if self._elog_reg_id is not None:
+ self.scheduler.unregister(self._elog_reg_id)
+ self._elog_reg_id = None
+ if self._elog_reader_fd:
+ os.close(self._elog_reader_fd)
+ self._elog_reader_fd = None
+
+ super(MergeProcess, self)._unregister()
diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index bf48b15..66e2955 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -13,7 +13,8 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.dbapi._MergeProcess:MergeProcess',
'portage.dep:dep_getkey,isjustname,match_from_list,' + \
'use_reduce,_slot_re',
- 'portage.elog:elog_process,_preload_elog_modules',
+ 'portage.elog:collect_ebuild_messages,collect_messages,' + \
+ 'elog_process,_merge_logentries,_preload_elog_modules',
'portage.locks:lockdir,unlockdir',
'portage.output:bold,colorize',
'portage.package.ebuild.doebuild:doebuild_environment,' + \
@@ -1200,7 +1201,7 @@ class dblink(object):
_file_merge_yield_interval = 20
def __init__(self, cat, pkg, myroot=None, settings=None, treetype=None,
- vartree=None, blockers=None, scheduler=None):
+ vartree=None, blockers=None, scheduler=None, pipe=None):
"""
Creates a DBlink object for a given CPV.
The given CPV may not be present in the database already.
@@ -1259,6 +1260,7 @@ class dblink(object):
self._md5_merge_map = {}
self._hash_key = (self.myroot, self.mycpv)
self._protect_obj = None
+ self._pipe = pipe
def __hash__(self):
return hash(self._hash_key)
@@ -1502,7 +1504,7 @@ class dblink(object):
continue
others_in_slot.append(dblink(self.cat, catsplit(cur_cpv)[1],
settings=self.settings, vartree=self.vartree,
- treetype="vartree"))
+ treetype="vartree", pipe=self._pipe))
retval = self._security_check([self] + others_in_slot)
if retval:
@@ -1666,9 +1668,7 @@ class dblink(object):
self._eerror(ebuild_phase, msg_lines)
- # process logs created during pre/postrm
- elog_process(self.mycpv, self.settings,
- phasefilter=('prerm', 'postrm'))
+ self._elog_process()
if retval == os.EX_OK:
# myebuildpath might be None, so ensure
@@ -1764,7 +1764,7 @@ class dblink(object):
continue
others_in_slot.append(dblink(self.cat, catsplit(cur_cpv)[1],
settings=self.settings,
- vartree=self.vartree, treetype="vartree"))
+ vartree=self.vartree, treetype="vartree", pipe=self._pipe))
dest_root = self._eroot
dest_root_len = len(dest_root) - 1
@@ -2784,19 +2784,34 @@ class dblink(object):
self._scheduler.dblinkElog(self,
phase, _eerror, lines)
- def _elog_subprocess(self, funcname, phase, lines):
- """
- Subprocesses call this in order to create elog messages in
- $T, for collection by the main process.
- """
- cmd = "source %s/isolated-functions.sh ; " % \
- portage._shell_quote(self.settings["PORTAGE_BIN_PATH"])
- for line in lines:
- cmd += "%s %s ; " % (funcname, portage._shell_quote(line))
- env = self.settings.environ()
- env['EBUILD_PHASE'] = phase
- subprocess.call([portage.const.BASH_BINARY, "-c", cmd],
- env=env)
+ def _elog_process(self):
+ cpv = self.mycpv
+ if self._pipe is None:
+ elog_process(cpv, self.settings)
+ else:
+ logdir = os.path.join(self.settings["T"], "logging")
+ ebuild_logentries = collect_ebuild_messages(logdir)
+ py_logentries = collect_messages(key=cpv).get(cpv, {})
+ logentries = _merge_logentries(py_logentries, ebuild_logentries)
+ funcnames = {
+ "INFO": "einfo",
+ "LOG": "elog",
+ "WARN": "ewarn",
+ "QA": "eqawarn",
+ "ERROR": "eerror"
+ }
+ buffer = []
+ for phase, messages in logentries.items():
+ for key, lines in messages:
+ funcname = funcnames[key]
+ if isinstance(lines, basestring):
+ lines = [lines]
+ for line in lines:
+ fields = (funcname, phase, cpv, line.rstrip('\n'))
+ buffer.append(' '.join(fields))
+ buffer.append('\n')
+ if buffer:
+ os.write(self._pipe, ''.join(buffer))
def treewalk(self, srcroot, destroot, inforoot, myebuild, cleanup=0,
mydbapi=None, prev_mtimes=None):
@@ -2811,7 +2826,6 @@ class dblink(object):
unmerges old version (if required)
calls doebuild(mydo=pkg_postinst)
calls env_update
- calls elog_process
@param srcroot: Typically this is ${D}
@type srcroot: String (Path)
@@ -2921,7 +2935,7 @@ class dblink(object):
others_in_slot.append(dblink(self.cat, catsplit(cur_cpv)[1],
settings=config(clone=self.settings),
vartree=self.vartree, treetype="vartree",
- scheduler=self._scheduler))
+ scheduler=self._scheduler, pipe=self._pipe))
retval = self._security_check(others_in_slot)
if retval:
@@ -3069,8 +3083,6 @@ class dblink(object):
# check for package collisions
blockers = None
if self._blockers is not None:
- # This is only supposed to be called when
- # the vdb is locked, like it is here.
blockers = self._blockers()
if blockers is None:
blockers = []
@@ -3242,16 +3254,8 @@ class dblink(object):
cfgfiledict["IGNORE"] = 1
break
- merge_task = MergeProcess(
- background=(self.settings.get('PORTAGE_BACKGROUND') == '1'),
- cfgfiledict=cfgfiledict, conf_mem_file=conf_mem_file, dblink=self,
- destroot=destroot,
- logfile=self.settings.get('PORTAGE_LOG_FILE'),
- scheduler=(scheduler or PollScheduler().sched_iface),
- srcroot=srcroot)
-
- merge_task.start()
- rval = merge_task.wait()
+ rval = self._merge_contents(srcroot, destroot, cfgfiledict,
+ conf_mem_file)
if rval != os.EX_OK:
return rval
@@ -3438,7 +3442,7 @@ class dblink(object):
return backup_p
- def _merge_process(self, srcroot, destroot, cfgfiledict, conf_mem_file):
+ def _merge_contents(self, srcroot, destroot, cfgfiledict, conf_mem_file):
cfgfiledict_orig = cfgfiledict.copy()
@@ -3667,7 +3671,7 @@ class dblink(object):
msg.append(_("This file will be renamed to a different name:"))
msg.append(" '%s'" % backup_dest)
msg.append("")
- self._elog_subprocess("eerror", "preinst", msg)
+ self._eerror("preinst", msg)
if movefile(mydest, backup_dest,
mysettings=self.settings,
encoding=_encodings['merge']) is None:
@@ -3745,7 +3749,7 @@ class dblink(object):
msg.append(_("This file will be merged with a different name:"))
msg.append(" '%s'" % newdest)
msg.append("")
- self._elog_subprocess("eerror", "preinst", msg)
+ self._eerror("preinst", msg)
mydest = newdest
elif stat.S_ISREG(mydmode) or (stat.S_ISLNK(mydmode) and os.path.exists(mydest) and stat.S_ISREG(os.stat(mydest)[stat.ST_MODE])):
@@ -3929,7 +3933,7 @@ class dblink(object):
self._scheduler.dblinkEbuildPhase(
self, mydbapi, myebuild, phase)
- elog_process(self.mycpv, self.settings)
+ self._elog_process()
if 'noclean' not in self.settings.features and \
(retval == os.EX_OK or \
@@ -4029,10 +4033,16 @@ def merge(mycat, mypkg, pkgloc, infloc,
writemsg(_("Permission denied: access('%s', W_OK)\n") % settings['EROOT'],
noiselevel=-1)
return errno.EACCES
- mylink = dblink(mycat, mypkg, settings=settings, treetype=mytree,
- vartree=vartree, blockers=blockers, scheduler=scheduler)
- return mylink.merge(pkgloc, infloc, myebuild=myebuild,
- mydbapi=mydbapi, prev_mtimes=prev_mtimes)
+ background = (settings.get('PORTAGE_BACKGROUND') == '1')
+ merge_task = MergeProcess(
+ dblink=dblink, mycat=mycat, mypkg=mypkg, settings=settings,
+ treetype=mytree, vartree=vartree, scheduler=scheduler,
+ background=background, blockers=blockers, pkgloc=pkgloc,
+ infloc=infloc, myebuild=myebuild, mydbapi=mydbapi,
+ prev_mtimes=prev_mtimes, logfile=settings.get('PORTAGE_LOG_FILE'))
+ merge_task.start()
+ retcode = merge_task.wait()
+ return retcode
def unmerge(cat, pkg, myroot=None, settings=None,
mytrimworld=None, vartree=None,
^ permalink raw reply related [flat|nested] 29+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/
@ 2011-03-25 4:34 Zac Medico
0 siblings, 0 replies; 29+ messages in thread
From: Zac Medico @ 2011-03-25 4:34 UTC (permalink / raw
To: gentoo-commits
commit: 3081e651fc3cd3a0729bb1fbe2e93fbc58dcef0d
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Mar 25 04:32:38 2011 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Mar 25 04:32:38 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=3081e651
MergeProcess: Fix PORTAGE_BACKGROUND/LOG_FILE use
In this subprocess we don't want PORTAGE_BACKGROUND to suppress
stdout/stderr output since they are pipes. We also don't want to open
PORTAGE_LOG_FILE, since it will already be opened by the parent
process, so we set the PORTAGE_BACKGROUND="subprocess" value for use
in conditional logging code involving PORTAGE_LOG_FILE.
---
pym/_emerge/AbstractEbuildProcess.py | 6 ++++--
pym/_emerge/EbuildPhase.py | 29 ++++++++++++++++++-----------
pym/_emerge/MiscFunctionsProcess.py | 3 ++-
pym/portage/dbapi/_MergeProcess.py | 9 +++++++++
4 files changed, 33 insertions(+), 14 deletions(-)
diff --git a/pym/_emerge/AbstractEbuildProcess.py b/pym/_emerge/AbstractEbuildProcess.py
index d7f31be..39c613b 100644
--- a/pym/_emerge/AbstractEbuildProcess.py
+++ b/pym/_emerge/AbstractEbuildProcess.py
@@ -225,8 +225,10 @@ class AbstractEbuildProcess(SpawnProcess):
msg = _unicode_decode(out.getvalue(),
encoding=_encodings['content'], errors='replace')
if msg:
- self.scheduler.output(msg,
- log_path=self.settings.get("PORTAGE_LOG_FILE"))
+ log_path = None
+ if self.settings.get("PORTAGE_BACKGROUND") != "subprocess":
+ log_path = self.settings.get("PORTAGE_LOG_FILE")
+ self.scheduler.output(msg, log_path=log_path)
def _log_poll_exception(self, event):
self._elog("eerror",
diff --git a/pym/_emerge/EbuildPhase.py b/pym/_emerge/EbuildPhase.py
index e3270c8..a24608b 100644
--- a/pym/_emerge/EbuildPhase.py
+++ b/pym/_emerge/EbuildPhase.py
@@ -121,9 +121,10 @@ class EbuildPhase(CompositeTask):
# Don't open the log file during the clean phase since the
# open file can result in an nfs lock on $T/build.log which
# prevents the clean phase from removing $T.
- logfile = self.settings.get("PORTAGE_LOG_FILE")
- if self.phase in ("clean", "cleanrm"):
- logfile = None
+ logfile = None
+ if self.phase not in ("clean", "cleanrm") and \
+ self.settings.get("PORTAGE_BACKGROUND") != "subprocess":
+ logfile = self.settings.get("PORTAGE_LOG_FILE")
fd_pipes = None
if not self.background and self.phase == 'nofetch':
@@ -151,13 +152,16 @@ class EbuildPhase(CompositeTask):
if not fail:
self.returncode = None
+ logfile = None
+ if self.settings.get("PORTAGE_BACKGROUND") != "subprocess":
+ logfile = self.settings.get("PORTAGE_LOG_FILE")
+
if self.phase == "install":
out = portage.StringIO()
_check_build_log(self.settings, out=out)
msg = _unicode_decode(out.getvalue(),
encoding=_encodings['content'], errors='replace')
- self.scheduler.output(msg,
- log_path=self.settings.get("PORTAGE_LOG_FILE"))
+ self.scheduler.output(msg, log_path=logfile)
if fail:
self._die_hooks()
@@ -173,12 +177,10 @@ class EbuildPhase(CompositeTask):
msg = _unicode_decode(out.getvalue(),
encoding=_encodings['content'], errors='replace')
if msg:
- self.scheduler.output(msg,
- log_path=self.settings.get("PORTAGE_LOG_FILE"))
+ self.scheduler.output(msg, log_path=logfile)
post_phase_cmds = _post_phase_cmds.get(self.phase)
if post_phase_cmds is not None:
- logfile = settings.get("PORTAGE_LOG_FILE")
if logfile is not None and self.phase in ("install",):
# Log to a temporary file, since the code we are running
# reads PORTAGE_LOG_FILE for QA checks, and we want to
@@ -204,7 +206,10 @@ class EbuildPhase(CompositeTask):
self._assert_current(post_phase)
- log_path = self.settings.get("PORTAGE_LOG_FILE")
+ log_path = None
+ if self.settings.get("PORTAGE_BACKGROUND") != "subprocess":
+ log_path = self.settings.get("PORTAGE_LOG_FILE")
+
if post_phase.logfile is not None and \
post_phase.logfile != log_path:
# We were logging to a temp file (see above), so append
@@ -293,5 +298,7 @@ class EbuildPhase(CompositeTask):
msg = _unicode_decode(out.getvalue(),
encoding=_encodings['content'], errors='replace')
if msg:
- self.scheduler.output(msg,
- log_path=self.settings.get("PORTAGE_LOG_FILE"))
+ log_path = None
+ if self.settings.get("PORTAGE_BACKGROUND") != "subprocess":
+ log_path = self.settings.get("PORTAGE_LOG_FILE")
+ self.scheduler.output(msg, log_path=log_path)
diff --git a/pym/_emerge/MiscFunctionsProcess.py b/pym/_emerge/MiscFunctionsProcess.py
index ad8cefc..e6bb103 100644
--- a/pym/_emerge/MiscFunctionsProcess.py
+++ b/pym/_emerge/MiscFunctionsProcess.py
@@ -22,7 +22,8 @@ class MiscFunctionsProcess(AbstractEbuildProcess):
os.path.basename(portage.const.MISC_SH_BINARY))
self.args = [portage._shell_quote(misc_sh_binary)] + self.commands
- if self.logfile is None:
+ if self.logfile is None and \
+ self.settings.get("PORTAGE_BACKGROUND") != "subprocess":
self.logfile = settings.get("PORTAGE_LOG_FILE")
AbstractEbuildProcess._start(self)
diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index 6e63f84..a8c3c9d 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -85,6 +85,15 @@ class MergeProcess(SpawnProcess):
# is triggered when mylink._scheduler is None.
mylink._scheduler = None
+ # In this subprocess we don't want PORTAGE_BACKGROUND to
+ # suppress stdout/stderr output since they are pipes. We
+ # also don't want to open PORTAGE_LOG_FILE, since it will
+ # already be opened by the parent process, so we set the
+ # "subprocess" value for use in conditional logging code
+ # involving PORTAGE_LOG_FILE.
+ self.settings["PORTAGE_BACKGROUND"] = "subprocess"
+ self.settings.backup_changes("PORTAGE_BACKGROUND")
+
rval = 1
try:
rval = mylink.merge(self.pkgloc, self.infloc,
^ permalink raw reply related [flat|nested] 29+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/
@ 2011-03-26 3:24 Zac Medico
0 siblings, 0 replies; 29+ messages in thread
From: Zac Medico @ 2011-03-26 3:24 UTC (permalink / raw
To: gentoo-commits
commit: 86250c61c7e3fd227e9e2ccbb48358b1c80114d1
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Mar 26 03:24:10 2011 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Mar 26 03:24:10 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=86250c61
MergeProcess: call elog_process for replaced pkgs
---
pym/_emerge/Binpkg.py | 2 --
pym/_emerge/EbuildBuild.py | 2 --
pym/portage/dbapi/_MergeProcess.py | 9 ++++++++-
3 files changed, 8 insertions(+), 5 deletions(-)
diff --git a/pym/_emerge/Binpkg.py b/pym/_emerge/Binpkg.py
index fbd2a53..62d44c4 100644
--- a/pym/_emerge/Binpkg.py
+++ b/pym/_emerge/Binpkg.py
@@ -304,8 +304,6 @@ class Binpkg(CompositeTask):
def _unlock_builddir(self):
if self.opts.pretend or self.opts.fetchonly:
return
- portage.elog.elog_process(self.pkg.cpv, self.settings,
- phasefilter=("prerm", "postrm"))
portage.elog.elog_process(self.pkg.cpv, self.settings)
self._build_dir.unlock()
diff --git a/pym/_emerge/EbuildBuild.py b/pym/_emerge/EbuildBuild.py
index b72b3d5..c7a5f5c 100644
--- a/pym/_emerge/EbuildBuild.py
+++ b/pym/_emerge/EbuildBuild.py
@@ -243,8 +243,6 @@ class EbuildBuild(CompositeTask):
self.wait()
def _unlock_builddir(self):
- portage.elog.elog_process(self.pkg.cpv, self.settings,
- phasefilter=("prerm", "postrm"))
portage.elog.elog_process(self.pkg.cpv, self.settings)
self._build_dir.unlock()
diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index 809af37..10d8873 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -21,7 +21,7 @@ class MergeProcess(SpawnProcess):
__slots__ = ('dblink', 'mycat', 'mypkg', 'settings', 'treetype',
'vartree', 'scheduler', 'blockers', 'pkgloc', 'infloc', 'myebuild',
'mydbapi', 'prev_mtimes', '_elog_reader_fd', '_elog_reg_id',
- '_buf')
+ '_buf', '_elog_keys')
def _elog_output_handler(self, fd, event):
output = None
@@ -41,6 +41,7 @@ class MergeProcess(SpawnProcess):
out = StringIO()
for line in lines:
funcname, phase, key, msg = line.split(' ', 3)
+ self._elog_keys.add(key)
reporter = getattr(portage.elog.messages, funcname)
reporter(msg, phase=phase, key=key, out=out)
@@ -67,6 +68,7 @@ class MergeProcess(SpawnProcess):
os.close(elog_writer_fd)
self._elog_reader_fd = elog_reader_fd
self._buf = ""
+ self._elog_keys = set()
self.vartree.dbapi._pkgs_changed = True
portage.process.spawned_pids.append(pid)
return [pid]
@@ -124,5 +126,10 @@ class MergeProcess(SpawnProcess):
if self._elog_reader_fd:
os.close(self._elog_reader_fd)
self._elog_reader_fd = None
+ if self._elog_keys is not None:
+ for key in self._elog_keys:
+ portage.elog.elog_process(key, self.settings,
+ phasefilter=("prerm", "postrm"))
+ self._elog_keys = None
super(MergeProcess, self)._unregister()
^ permalink raw reply related [flat|nested] 29+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/
@ 2011-03-26 7:39 Zac Medico
0 siblings, 0 replies; 29+ messages in thread
From: Zac Medico @ 2011-03-26 7:39 UTC (permalink / raw
To: gentoo-commits
commit: a1a23d2bc25a324b24c180dc982d9442229dc9eb
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Mar 26 07:38:23 2011 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Mar 26 07:38:23 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=a1a23d2b
unmerge: fix PORTAGE_BACKGROUND logic
---
pym/_emerge/actions.py | 7 +++++++
pym/portage/dbapi/vartree.py | 13 ++++++++-----
2 files changed, 15 insertions(+), 5 deletions(-)
diff --git a/pym/_emerge/actions.py b/pym/_emerge/actions.py
index f79f927..53d1880 100644
--- a/pym/_emerge/actions.py
+++ b/pym/_emerge/actions.py
@@ -2623,6 +2623,13 @@ def action_uninstall(settings, trees, ldpath_mtimes,
sched._background = sched._background_mode()
sched._status_display.quiet = True
+ if sched._background:
+ sched.settings.unlock()
+ sched.settings["PORTAGE_BACKGROUND"] = "1"
+ sched.settings.backup_changes("PORTAGE_BACKGROUND")
+ sched.settings.lock()
+ sched.pkgsettings[root] = portage.config(clone=sched.settings)
+
if action in ('clean', 'unmerge') or \
(action == 'prune' and "--nodeps" in opts):
# When given a list of atoms, unmerge them in the order given.
diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index c90d86b..98f578f 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1477,12 +1477,15 @@ class dblink(object):
# We create a scheduler instance and use it to
# log unmerge output separately from merge output.
self._scheduler = PollScheduler().sched_iface
- if self.settings.get("PORTAGE_BACKGROUND_UNMERGE") == "1":
- self.settings["PORTAGE_BACKGROUND"] = "1"
- self.settings.backup_changes("PORTAGE_BACKGROUND")
+ if self.settings.get("PORTAGE_BACKGROUND") == "subprocess":
+ if self.settings.get("PORTAGE_BACKGROUND_UNMERGE") == "1":
+ self.settings["PORTAGE_BACKGROUND"] = "1"
+ self.settings.backup_changes("PORTAGE_BACKGROUND")
+ background = True
+ else:
+ self.settings.pop("PORTAGE_BACKGROUND", None)
+ elif self.settings.get("PORTAGE_BACKGROUND") == "1":
background = True
- else:
- self.settings.pop("PORTAGE_BACKGROUND", None)
self.vartree.dbapi._bump_mtime(self.mycpv)
showMessage = self._display_merge
^ permalink raw reply related [flat|nested] 29+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/
@ 2011-05-09 5:16 Zac Medico
0 siblings, 0 replies; 29+ messages in thread
From: Zac Medico @ 2011-05-09 5:16 UTC (permalink / raw
To: gentoo-commits
commit: b3d51db7521faa2f7c2dbc0d71894e46e74231d0
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon May 9 05:13:52 2011 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon May 9 05:13:52 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=b3d51db7
unmerge: use expand_new_virt for sys pkg warnings
---
pym/_emerge/actions.py | 64 +-------------------------------
pym/_emerge/unmerge.py | 9 ++++-
pym/portage/dbapi/_expand_new_virt.py | 66 +++++++++++++++++++++++++++++++++
3 files changed, 76 insertions(+), 63 deletions(-)
diff --git a/pym/_emerge/actions.py b/pym/_emerge/actions.py
index 29ecb38..9bc3aaf 100644
--- a/pym/_emerge/actions.py
+++ b/pym/_emerge/actions.py
@@ -31,7 +31,8 @@ from portage.cache.cache_errors import CacheError
from portage.const import GLOBAL_CONFIG_PATH, NEWS_LIB_PATH
from portage.const import _ENABLE_DYN_LINK_MAP, _ENABLE_SET_CONFIG
from portage.dbapi.dep_expand import dep_expand
-from portage.dep import Atom, extended_cp_match, _get_useflag_re
+from portage.dbapi._expand_new_virt import expand_new_virt
+from portage.dep import Atom, extended_cp_match
from portage.exception import InvalidAtom
from portage.output import blue, bold, colorize, create_color_func, darkgreen, \
red, yellow
@@ -1282,67 +1283,6 @@ def action_deselect(settings, trees, opts, atoms):
world_set.unlock()
return os.EX_OK
-def expand_new_virt(vardb, atom):
- """
- Iterate over the recursively expanded RDEPEND atoms of
- a new-style virtual. If atom is not a new-style virtual
- or it does not match an installed package then it is
- yielded without any expansion.
- """
- if not isinstance(atom, Atom):
- atom = Atom(atom)
- traversed = set()
- stack = [atom]
-
- while stack:
- atom = stack.pop()
- if atom.blocker:
- yield atom
- continue
-
- matches = vardb.match(atom)
- if not (matches and matches[-1].startswith("virtual/")):
- yield atom
- continue
-
- virt_cpv = matches[-1]
- if virt_cpv in traversed:
- continue
-
- traversed.add(virt_cpv)
- eapi, iuse, rdepend, use = vardb.aux_get(virt_cpv,
- ["EAPI", "IUSE", "RDEPEND", "USE"])
- if not portage.eapi_is_supported(eapi):
- yield atom
- continue
-
- # Validate IUSE and IUSE, for early detection of vardb corruption.
- useflag_re = _get_useflag_re(eapi)
- valid_iuse = []
- for x in iuse.split():
- if x[:1] in ("+", "-"):
- x = x[1:]
- if useflag_re.match(x) is not None:
- valid_iuse.append(x)
- valid_iuse = frozenset(valid_iuse)
-
- iuse_implicit_match = vardb.settings._iuse_implicit_match
- valid_use = []
- for x in use.split():
- if x in valid_iuse or iuse_implicit_match(x):
- valid_use.append(x)
- valid_use = frozenset(valid_use)
-
- success, atoms = portage.dep_check(rdepend,
- None, vardb.settings, myuse=valid_use,
- myroot=vardb.root, trees={vardb.root:{"porttree":vardb.vartree,
- "vartree":vardb.vartree}})
-
- if success:
- stack.extend(atoms)
- else:
- yield atom
-
class _info_pkgs_ver(object):
def __init__(self, ver, repo_suffix, provide_suffix):
self.ver = ver
diff --git a/pym/_emerge/unmerge.py b/pym/_emerge/unmerge.py
index 68b61ed..7e66ff9 100644
--- a/pym/_emerge/unmerge.py
+++ b/pym/_emerge/unmerge.py
@@ -8,6 +8,7 @@ import sys
import textwrap
import portage
from portage import os
+from portage.dbapi._expand_new_virt import expand_new_virt
from portage.output import bold, colorize, darkgreen, green
from portage._sets import SETPREFIX
from portage.util import cmp_sort_key
@@ -57,7 +58,13 @@ def unmerge(root_config, myopts, unmerge_action,
try:
if os.access(vdb_path, os.W_OK):
vdb_lock = portage.locks.lockdir(vdb_path)
- realsyslist = sets["system"].getAtoms()
+
+ realsyslist = []
+ for x in sets["system"].getAtoms():
+ for atom in expand_new_virt(vartree.dbapi, x):
+ if not atom.blocker:
+ realsyslist.append(atom)
+
syslist = []
for x in realsyslist:
mycp = portage.dep_getkey(x)
diff --git a/pym/portage/dbapi/_expand_new_virt.py b/pym/portage/dbapi/_expand_new_virt.py
new file mode 100644
index 0000000..7a233f1
--- /dev/null
+++ b/pym/portage/dbapi/_expand_new_virt.py
@@ -0,0 +1,66 @@
+# Copyright 2011 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import portage
+from portage.dep import Atom, _get_useflag_re
+
+def expand_new_virt(vardb, atom):
+ """
+ Iterate over the recursively expanded RDEPEND atoms of
+ a new-style virtual. If atom is not a new-style virtual
+ or it does not match an installed package then it is
+ yielded without any expansion.
+ """
+ if not isinstance(atom, Atom):
+ atom = Atom(atom)
+ traversed = set()
+ stack = [atom]
+
+ while stack:
+ atom = stack.pop()
+ if atom.blocker:
+ yield atom
+ continue
+
+ matches = vardb.match(atom)
+ if not (matches and matches[-1].startswith("virtual/")):
+ yield atom
+ continue
+
+ virt_cpv = matches[-1]
+ if virt_cpv in traversed:
+ continue
+
+ traversed.add(virt_cpv)
+ eapi, iuse, rdepend, use = vardb.aux_get(virt_cpv,
+ ["EAPI", "IUSE", "RDEPEND", "USE"])
+ if not portage.eapi_is_supported(eapi):
+ yield atom
+ continue
+
+ # Validate IUSE and IUSE, for early detection of vardb corruption.
+ useflag_re = _get_useflag_re(eapi)
+ valid_iuse = []
+ for x in iuse.split():
+ if x[:1] in ("+", "-"):
+ x = x[1:]
+ if useflag_re.match(x) is not None:
+ valid_iuse.append(x)
+ valid_iuse = frozenset(valid_iuse)
+
+ iuse_implicit_match = vardb.settings._iuse_implicit_match
+ valid_use = []
+ for x in use.split():
+ if x in valid_iuse or iuse_implicit_match(x):
+ valid_use.append(x)
+ valid_use = frozenset(valid_use)
+
+ success, atoms = portage.dep_check(rdepend,
+ None, vardb.settings, myuse=valid_use,
+ myroot=vardb.root, trees={vardb.root:{"porttree":vardb.vartree,
+ "vartree":vardb.vartree}})
+
+ if success:
+ stack.extend(atoms)
+ else:
+ yield atom
^ permalink raw reply related [flat|nested] 29+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/
@ 2011-05-24 0:33 Zac Medico
0 siblings, 0 replies; 29+ messages in thread
From: Zac Medico @ 2011-05-24 0:33 UTC (permalink / raw
To: gentoo-commits
commit: e451e90b148e58bc026b3a70f1c47f3c57ca4839
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue May 24 00:33:15 2011 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue May 24 00:33:15 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=e451e90b
MergeProcess: remove unnecessary dblink attribute
---
pym/_emerge/EbuildMerge.py | 3 +--
pym/portage/dbapi/_MergeProcess.py | 4 ++--
pym/portage/dbapi/vartree.py | 2 +-
3 files changed, 4 insertions(+), 5 deletions(-)
diff --git a/pym/_emerge/EbuildMerge.py b/pym/_emerge/EbuildMerge.py
index 28d17f9..9c35988 100644
--- a/pym/_emerge/EbuildMerge.py
+++ b/pym/_emerge/EbuildMerge.py
@@ -4,7 +4,6 @@
from _emerge.CompositeTask import CompositeTask
from portage import os
from portage.dbapi._MergeProcess import MergeProcess
-from portage.dbapi.vartree import dblink
class EbuildMerge(CompositeTask):
@@ -26,7 +25,7 @@ class EbuildMerge(CompositeTask):
logfile = settings.get('PORTAGE_LOG_FILE')
merge_task = MergeProcess(
- dblink=dblink, mycat=mycat, mypkg=mypkg, settings=settings,
+ mycat=mycat, mypkg=mypkg, settings=settings,
treetype=self.tree, vartree=vartree, scheduler=self.scheduler,
background=background, blockers=self.find_blockers, pkgloc=pkgloc,
infloc=infloc, myebuild=myebuild, mydbapi=mydbapi,
diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index 62f5bec..c228606 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -24,7 +24,7 @@ class MergeProcess(SpawnProcess):
thread while files are moved or copied asynchronously.
"""
- __slots__ = ('dblink', 'mycat', 'mypkg', 'settings', 'treetype',
+ __slots__ = ('mycat', 'mypkg', 'settings', 'treetype',
'vartree', 'scheduler', 'blockers', 'pkgloc', 'infloc', 'myebuild',
'mydbapi', 'prev_mtimes', '_elog_reader_fd', '_elog_reg_id',
'_buf', '_elog_keys', '_locked_vdb')
@@ -155,7 +155,7 @@ class MergeProcess(SpawnProcess):
# access to open database connections such as that
# used by the sqlite metadata cache module.
blockers = self.blockers()
- mylink = self.dblink(self.mycat, self.mypkg, settings=self.settings,
+ mylink = portage.dblink(self.mycat, self.mypkg, settings=self.settings,
treetype=self.treetype, vartree=self.vartree,
blockers=blockers, scheduler=self.scheduler,
pipe=elog_writer_fd)
diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index f53c209..1867cf2 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -4175,7 +4175,7 @@ def merge(mycat, mypkg, pkgloc, infloc,
return errno.EACCES
background = (settings.get('PORTAGE_BACKGROUND') == '1')
merge_task = MergeProcess(
- dblink=dblink, mycat=mycat, mypkg=mypkg, settings=settings,
+ mycat=mycat, mypkg=mypkg, settings=settings,
treetype=mytree, vartree=vartree,
scheduler=(scheduler or PollScheduler().sched_iface),
background=background, blockers=blockers, pkgloc=pkgloc,
^ permalink raw reply related [flat|nested] 29+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/
@ 2011-05-24 5:31 Zac Medico
0 siblings, 0 replies; 29+ messages in thread
From: Zac Medico @ 2011-05-24 5:31 UTC (permalink / raw
To: gentoo-commits
commit: f0f1bbe8fa9d3f698cbe529d2a11eec1ce437119
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue May 24 05:31:20 2011 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue May 24 05:31:20 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=f0f1bbe8
PackageUninstall: make async with MergeProcess
This fixes another ebuild-locks issue like the one fixed in commit
a81460175a441897282b0540cefff8060f2b92dc, but this time we use a
subprocess to ensure that the ebuild-locks for pkg_prerm and
pkg_postrm do not interfere with pkg_setup ebuild-locks held by
the main process.
---
pym/_emerge/PackageUninstall.py | 57 +++++++++++++++++++++++++++++++-----
pym/portage/dbapi/_MergeProcess.py | 30 ++++++++++++++----
pym/portage/dbapi/vartree.py | 36 +++++++++++++---------
3 files changed, 93 insertions(+), 30 deletions(-)
diff --git a/pym/_emerge/PackageUninstall.py b/pym/_emerge/PackageUninstall.py
index 0e91307..0829f50 100644
--- a/pym/_emerge/PackageUninstall.py
+++ b/pym/_emerge/PackageUninstall.py
@@ -4,17 +4,45 @@
import logging
import portage
from portage import os
+from portage.dbapi._MergeProcess import MergeProcess
+from portage.exception import UnsupportedAPIException
+from _emerge.EbuildBuildDir import EbuildBuildDir
from _emerge.emergelog import emergelog
from _emerge.CompositeTask import CompositeTask
from _emerge.unmerge import _unmerge_display
class PackageUninstall(CompositeTask):
+ """
+ Uninstall a package asynchronously in a subprocess. When
+ both parallel-install and ebuild-locks FEATURES are enabled,
+ it is essential for the ebuild-locks code to execute in a
+ subprocess, since the portage.locks module does not behave
+ as desired if we try to lock the same file multiple times
+ concurrently from the same process.
+ """
__slots__ = ("world_atom", "ldpath_mtimes", "opts",
- "pkg", "settings")
+ "pkg", "settings", "_builddir_lock")
def _start(self):
+ self.settings.setcpv(self.pkg)
+ vardb = self.pkg.root_config.trees["vartree"].dbapi
+ dbdir = vardb.getpath(self.pkg.cpv)
+ cat, pf = portage.catsplit(self.pkg.cpv)
+ myebuildpath = os.path.join(dbdir, pf + ".ebuild")
+
+ try:
+ portage.doebuild_environment(myebuildpath, "prerm",
+ settings=self.settings, db=vardb)
+ except UnsupportedAPIException:
+ # This is safe to ignore since this function is
+ # guaranteed to set PORTAGE_BUILDDIR even though
+ # it raises UnsupportedAPIException. The error
+ # will be logged when it prevents the pkg_prerm
+ # and pkg_postrm phases from executing.
+ pass
+
retval, pkgmap = _unmerge_display(self.pkg.root_config,
self.opts, "unmerge", [self.pkg.cpv], clean_delay=0,
writemsg_level=self._writemsg_level)
@@ -29,18 +57,31 @@ class PackageUninstall(CompositeTask):
self._emergelog("=== Unmerging... (%s)" % (self.pkg.cpv,))
cat, pf = portage.catsplit(self.pkg.cpv)
- retval = portage.unmerge(cat, pf, settings=self.settings,
- vartree=self.pkg.root_config.trees["vartree"],
- ldpath_mtimes=self.ldpath_mtimes,
- scheduler=self.scheduler)
- if retval != os.EX_OK:
+ self._builddir_lock = EbuildBuildDir(
+ scheduler=self.scheduler, settings=self.settings)
+ self._builddir_lock.lock()
+
+ portage.prepare_build_dirs(
+ settings=self.settings, cleanup=True)
+
+ unmerge_task = MergeProcess(
+ mycat=cat, mypkg=pf, settings=self.settings,
+ treetype="vartree", vartree=self.pkg.root_config.trees["vartree"],
+ scheduler=self.scheduler, background=self.background,
+ mydbapi=self.pkg.root_config.trees["vartree"].dbapi,
+ prev_mtimes=self.ldpath_mtimes,
+ logfile=self.settings.get("PORTAGE_LOG_FILE"), unmerge=True)
+
+ self._start_task(unmerge_task, self._unmerge_exit)
+
+ def _unmerge_exit(self, unmerge_task):
+ if self._final_exit(unmerge_task) != os.EX_OK:
self._emergelog(" !!! unmerge FAILURE: %s" % (self.pkg.cpv,))
else:
self._emergelog(" >>> unmerge success: %s" % (self.pkg.cpv,))
self.world_atom(self.pkg)
-
- self.returncode = retval
+ self._builddir_lock.unlock()
self.wait()
def _emergelog(self, msg):
diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index c228606..43bec72 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -3,6 +3,7 @@
import shutil
import signal
+import sys
import tempfile
import traceback
@@ -26,7 +27,7 @@ class MergeProcess(SpawnProcess):
__slots__ = ('mycat', 'mypkg', 'settings', 'treetype',
'vartree', 'scheduler', 'blockers', 'pkgloc', 'infloc', 'myebuild',
- 'mydbapi', 'prev_mtimes', '_elog_reader_fd', '_elog_reg_id',
+ 'mydbapi', 'prev_mtimes', 'unmerge', '_elog_reader_fd', '_elog_reg_id',
'_buf', '_elog_keys', '_locked_vdb')
def _start(self):
@@ -45,7 +46,8 @@ class MergeProcess(SpawnProcess):
settings.reset()
settings.setcpv(cpv, mydb=self.mydbapi)
- self._handle_self_reinstall()
+ if not self.unmerge:
+ self._handle_self_reinstall()
super(MergeProcess, self)._start()
def _lock_vdb(self):
@@ -170,7 +172,9 @@ class MergeProcess(SpawnProcess):
# FEATURES=parallel-install skips this lock in order to
# improve performance, and the risk is practically negligible.
self._lock_vdb()
- counter = self.vartree.dbapi.counter_tick()
+ counter = None
+ if not self.unmerge:
+ counter = self.vartree.dbapi.counter_tick()
pid = os.fork()
if pid != 0:
@@ -213,7 +217,7 @@ class MergeProcess(SpawnProcess):
# already be opened by the parent process, so we set the
# "subprocess" value for use in conditional logging code
# involving PORTAGE_LOG_FILE.
- if self.settings.get("PORTAGE_BACKGROUND") == "1":
+ if not self.unmerge and self.settings.get("PORTAGE_BACKGROUND") == "1":
# unmerge phases have separate logs
self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "1"
self.settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE")
@@ -222,9 +226,21 @@ class MergeProcess(SpawnProcess):
rval = 1
try:
- rval = mylink.merge(self.pkgloc, self.infloc,
- myebuild=self.myebuild, mydbapi=self.mydbapi,
- prev_mtimes=self.prev_mtimes, counter=counter)
+ if self.unmerge:
+ if not mylink.exists():
+ rval = os.EX_OK
+ elif mylink.unmerge(
+ ldpath_mtimes=self.prev_mtimes) == os.EX_OK:
+ mylink.lockdb()
+ try:
+ mylink.delete()
+ finally:
+ mylink.unlockdb()
+ rval = os.EX_OK
+ else:
+ rval = mylink.merge(self.pkgloc, self.infloc,
+ myebuild=self.myebuild, mydbapi=self.mydbapi,
+ prev_mtimes=self.prev_mtimes, counter=counter)
except SystemExit:
raise
except:
diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 1867cf2..a9d2d14 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1604,6 +1604,7 @@ class dblink(object):
DeprecationWarning, stacklevel=2)
background = False
+ log_path = None
if self._scheduler is None:
# We create a scheduler instance and use it to
# log unmerge output separately from merge output.
@@ -1614,7 +1615,8 @@ class dblink(object):
self.settings.backup_changes("PORTAGE_BACKGROUND")
background = True
else:
- self.settings.pop("PORTAGE_BACKGROUND", None)
+ # Our output is redirected and logged by the parent process.
+ log_path = self.settings.pop("PORTAGE_LOG_FILE", None)
elif self.settings.get("PORTAGE_BACKGROUND") == "1":
background = True
@@ -1647,7 +1649,6 @@ class dblink(object):
myebuildpath = None
failures = 0
ebuild_phase = "prerm"
- log_path = None
mystuff = os.listdir(self.dbdir)
for x in mystuff:
if x.endswith(".ebuild"):
@@ -1659,7 +1660,11 @@ class dblink(object):
write_atomic(os.path.join(self.dbdir, "PF"), self.pkg+"\n")
break
- self.settings.setcpv(self.mycpv, mydb=self.vartree.dbapi)
+ if self.mycpv != self.settings.mycpv or \
+ "SLOT" not in self.settings.configdict["pkg"]:
+ # We avoid a redundant setcpv call here when
+ # the caller has already taken care of it.
+ self.settings.setcpv(self.mycpv, mydb=self.vartree.dbapi)
if myebuildpath:
try:
doebuild_environment(myebuildpath, "prerm",
@@ -1677,21 +1682,22 @@ class dblink(object):
self._prune_plib_registry(unmerge=True, needed=needed,
preserve_paths=preserve_paths)
+ builddir_locked = "PORTAGE_BUILDIR_LOCKED" in self.settings
builddir_lock = None
- log_path = None
scheduler = self._scheduler
retval = os.EX_OK
try:
if myebuildpath:
- # Only create builddir_lock if doebuild_environment
- # succeeded, since that's needed to initialize
- # PORTAGE_BUILDDIR.
- builddir_lock = EbuildBuildDir(
- scheduler=scheduler,
- settings=self.settings)
- builddir_lock.lock()
- prepare_build_dirs(settings=self.settings, cleanup=True)
- log_path = self.settings.get("PORTAGE_LOG_FILE")
+ # Only create builddir_lock if the caller
+ # has not already acquired the lock.
+ if not builddir_locked:
+ builddir_lock = EbuildBuildDir(
+ scheduler=scheduler,
+ settings=self.settings)
+ builddir_lock.lock()
+ builddir_locked = True
+ prepare_build_dirs(settings=self.settings, cleanup=True)
+ log_path = self.settings.get("PORTAGE_LOG_FILE")
phase = EbuildPhase(background=background,
phase=ebuild_phase, scheduler=scheduler,
@@ -1728,7 +1734,7 @@ class dblink(object):
finally:
self.vartree.dbapi._bump_mtime(self.mycpv)
- if builddir_lock:
+ if builddir_locked:
try:
if myebuildpath:
if retval != os.EX_OK:
@@ -1771,7 +1777,7 @@ class dblink(object):
self._elog_process(phasefilter=("prerm", "postrm"))
- if retval == os.EX_OK and builddir_lock is not None:
+ if retval == os.EX_OK and builddir_locked:
# myebuildpath might be None, so ensure
# it has a sane value for the clean phase,
# even though it won't really be sourced.
^ permalink raw reply related [flat|nested] 29+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/
@ 2011-06-03 10:16 Zac Medico
0 siblings, 0 replies; 29+ messages in thread
From: Zac Medico @ 2011-06-03 10:16 UTC (permalink / raw
To: gentoo-commits
commit: 70c2a622e33e158066233da8c5ba3300d0c8fa61
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Jun 3 10:16:16 2011 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Jun 3 10:16:16 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=70c2a622
dblink.merge: remove unused scheduler callbacks
---
pym/_emerge/Scheduler.py | 68 +----------------------------------------
pym/portage/dbapi/vartree.py | 69 +++++++++++++++++------------------------
2 files changed, 30 insertions(+), 107 deletions(-)
diff --git a/pym/_emerge/Scheduler.py b/pym/_emerge/Scheduler.py
index b10439a..bdc13aa 100644
--- a/pym/_emerge/Scheduler.py
+++ b/pym/_emerge/Scheduler.py
@@ -86,8 +86,7 @@ class Scheduler(PollScheduler):
_fetch_log = os.path.join(_emerge_log_dir, 'emerge-fetch.log')
class _iface_class(SlotObject):
- __slots__ = ("dblinkEbuildPhase", "dblinkDisplayMerge",
- "dblinkElog", "dblinkEmergeLog", "fetch",
+ __slots__ = ("fetch",
"output", "register", "schedule",
"scheduleSetup", "scheduleUnpack", "scheduleYield",
"unregister")
@@ -220,10 +219,6 @@ class Scheduler(PollScheduler):
fetch_iface = self._fetch_iface_class(log_file=self._fetch_log,
schedule=self._schedule_fetch)
self._sched_iface = self._iface_class(
- dblinkEbuildPhase=self._dblink_ebuild_phase,
- dblinkDisplayMerge=self._dblink_display_merge,
- dblinkElog=self._dblink_elog,
- dblinkEmergeLog=self._dblink_emerge_log,
fetch=fetch_iface, output=self._task_output,
register=self._register,
schedule=self._schedule_wait,
@@ -613,67 +608,6 @@ class Scheduler(PollScheduler):
return blocker_dblinks
- def _dblink_pkg(self, pkg_dblink):
- cpv = pkg_dblink.mycpv
- type_name = RootConfig.tree_pkg_map[pkg_dblink.treetype]
- root_config = self.trees[pkg_dblink.myroot]["root_config"]
- installed = type_name == "installed"
- repo = pkg_dblink.settings.get("PORTAGE_REPO_NAME")
- return self._pkg(cpv, type_name, root_config,
- installed=installed, myrepo=repo)
-
- def _dblink_elog(self, pkg_dblink, phase, func, msgs):
-
- log_path = pkg_dblink.settings.get("PORTAGE_LOG_FILE")
- out = StringIO()
-
- for msg in msgs:
- func(msg, phase=phase, key=pkg_dblink.mycpv, out=out)
-
- out_str = out.getvalue()
-
- self._task_output(out_str, log_path=log_path)
-
- def _dblink_emerge_log(self, msg):
- self._logger.log(msg)
-
- def _dblink_display_merge(self, pkg_dblink, msg, level=0, noiselevel=0):
- log_path = pkg_dblink.settings.get("PORTAGE_LOG_FILE")
- background = self._background
-
- if log_path is None:
- if not (background and level < logging.WARN):
- portage.util.writemsg_level(msg,
- level=level, noiselevel=noiselevel)
- else:
- self._task_output(msg, log_path=log_path)
-
- def _dblink_ebuild_phase(self,
- pkg_dblink, pkg_dbapi, ebuild_path, phase):
- """
- Using this callback for merge phases allows the scheduler
- to run while these phases execute asynchronously, and allows
- the scheduler control output handling.
- """
-
- scheduler = self._sched_iface
- settings = pkg_dblink.settings
- background = self._background
- log_path = settings.get("PORTAGE_LOG_FILE")
-
- if phase in ('die_hooks', 'success_hooks'):
- ebuild_phase = MiscFunctionsProcess(background=background,
- commands=[phase], phase=phase,
- scheduler=scheduler, settings=settings)
- else:
- ebuild_phase = EbuildPhase(background=background,
- phase=phase, scheduler=scheduler,
- settings=settings)
- ebuild_phase.start()
- ebuild_phase.wait()
-
- return ebuild_phase.returncode
-
def _generate_digests(self):
"""
Generate digests if necessary for --digests or FEATURES=digest.
diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 889cb58..16cf48b 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -57,6 +57,7 @@ from portage import _unicode_encode
from _emerge.AsynchronousLock import AsynchronousLock
from _emerge.EbuildBuildDir import EbuildBuildDir
from _emerge.EbuildPhase import EbuildPhase
+from _emerge.emergelog import emergelog
from _emerge.PollScheduler import PollScheduler
from _emerge.MiscFunctionsProcess import MiscFunctionsProcess
@@ -2957,6 +2958,9 @@ class dblink(object):
if str_buffer:
os.write(self._pipe, _unicode_encode(''.join(str_buffer)))
+ def _emerge_log(self, msg):
+ emergelog(False, msg)
+
def treewalk(self, srcroot, destroot, inforoot, myebuild, cleanup=0,
mydbapi=None, prev_mtimes=None, counter=None):
"""
@@ -3003,8 +3007,6 @@ class dblink(object):
encoding=_encodings['content'], errors='strict')
showMessage = self._display_merge
- scheduler = self._scheduler
-
srcroot = normalize_path(srcroot).rstrip(os.path.sep) + os.path.sep
if not os.path.isdir(srcroot):
@@ -3359,12 +3361,12 @@ class dblink(object):
ensure_dirs(self.dbtmpdir)
# run preinst script
- if scheduler is None:
- showMessage(_(">>> Merging %(cpv)s to %(destroot)s\n") % {"cpv":self.mycpv, "destroot":destroot})
- a = _spawn_phase("preinst", self.settings)
- else:
- a = scheduler.dblinkEbuildPhase(
- self, mydbapi, myebuild, "preinst")
+ showMessage(_(">>> Merging %(cpv)s to %(destroot)s\n") % \
+ {"cpv":self.mycpv, "destroot":destroot})
+ phase = EbuildPhase(background=False, phase="preinst",
+ scheduler=self._scheduler, settings=self.settings)
+ phase.start()
+ a = phase.wait()
# XXX: Decide how to handle failures here.
if a != os.EX_OK:
@@ -3459,11 +3461,7 @@ class dblink(object):
match_from_list(PORTAGE_PACKAGE_ATOM, [self.mycpv]):
reinstall_self = True
- if scheduler is None:
- def emerge_log(msg):
- pass
- else:
- emerge_log = scheduler.dblinkEmergeLog
+ emerge_log = self._emerge_log
# If we have any preserved libraries then autoclean
# is forced so that preserve-libs logic doesn't have
@@ -3608,13 +3606,12 @@ class dblink(object):
os.path.join(self.dbpkgdir, "environment.bz2")
self.settings.backup_changes("PORTAGE_UPDATE_ENV")
try:
- if scheduler is None:
- a = _spawn_phase("postinst", self.settings)
- if a == os.EX_OK:
- showMessage(_(">>> %s merged.\n") % self.mycpv)
- else:
- a = scheduler.dblinkEbuildPhase(
- self, mydbapi, myebuild, "postinst")
+ phase = EbuildPhase(background=False, phase="postinst",
+ scheduler=self._scheduler, settings=self.settings)
+ phase.start()
+ a = phase.wait()
+ if a == os.EX_OK:
+ showMessage(_(">>> %s merged.\n") % self.mycpv)
finally:
self.settings.pop("PORTAGE_UPDATE_ENV", None)
@@ -4068,6 +4065,8 @@ class dblink(object):
if not parallel_install:
self.lockdb()
self.vartree.dbapi._bump_mtime(self.mycpv)
+ if self._scheduler is None:
+ self._scheduler = PollScheduler().sched_iface
try:
retval = self.treewalk(mergeroot, myroot, inforoot, myebuild,
cleanup=cleanup, mydbapi=mydbapi, prev_mtimes=prev_mtimes,
@@ -4075,8 +4074,7 @@ class dblink(object):
# If PORTAGE_BUILDDIR doesn't exist, then it probably means
# fail-clean is enabled, and the success/die hooks have
- # already been called by _emerge.EbuildPhase (via
- # self._scheduler.dblinkEbuildPhase) prior to cleaning.
+ # already been called by EbuildPhase.
if os.path.isdir(self.settings['PORTAGE_BUILDDIR']):
if retval == os.EX_OK:
@@ -4084,18 +4082,11 @@ class dblink(object):
else:
phase = 'die_hooks'
- if self._scheduler is None:
- ebuild_phase = MiscFunctionsProcess(
- background=False,
- commands=[phase],
- scheduler=PollScheduler().sched_iface,
- settings=self.settings)
- ebuild_phase.start()
- ebuild_phase.wait()
- else:
- self._scheduler.dblinkEbuildPhase(
- self, mydbapi, myebuild, phase)
-
+ ebuild_phase = MiscFunctionsProcess(
+ background=False, commands=[phase],
+ scheduler=self._scheduler, settings=self.settings)
+ ebuild_phase.start()
+ ebuild_phase.wait()
self._elog_process()
if 'noclean' not in self.settings.features and \
@@ -4106,12 +4097,10 @@ class dblink(object):
doebuild_environment(myebuild, "clean",
settings=self.settings, db=mydbapi)
- if self._scheduler is None:
- _spawn_phase("clean", self.settings)
- else:
- self._scheduler.dblinkEbuildPhase(
- self, mydbapi, myebuild, "clean")
-
+ phase = EbuildPhase(background=False, phase="clean",
+ scheduler=self._scheduler, settings=self.settings)
+ phase.start()
+ phase.wait()
finally:
self.settings.pop('REPLACING_VERSIONS', None)
if self.vartree.dbapi._linkmap is None:
^ permalink raw reply related [flat|nested] 29+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/
@ 2011-10-15 5:10 Zac Medico
0 siblings, 0 replies; 29+ messages in thread
From: Zac Medico @ 2011-10-15 5:10 UTC (permalink / raw
To: gentoo-commits
commit: 4580734caa18250bd0c2bc8241dc36f308cdef2b
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Oct 15 05:10:28 2011 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Oct 15 05:10:28 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=4580734c
emerge --metadata: fix breakage
The hunk that got removed from action_metadata in commit
d4ea29bf6a3ce35d49e0f54f9173e3a6e42da2d6 is actually needed when
FEATURES=metadata-transfer is enabled.
---
pym/_emerge/actions.py | 4 ++++
pym/portage/dbapi/porttree.py | 20 ++++++++++++++------
2 files changed, 18 insertions(+), 6 deletions(-)
diff --git a/pym/_emerge/actions.py b/pym/_emerge/actions.py
index c449b58..71a0902 100644
--- a/pym/_emerge/actions.py
+++ b/pym/_emerge/actions.py
@@ -1660,6 +1660,10 @@ def action_metadata(settings, portdb, myopts, porttrees=None):
porttrees_data = []
for path in porttrees:
src_db = portdb._pregen_auxdb.get(path)
+ if src_db is None:
+ # portdbapi does not populate _pregen_auxdb
+ # when FEATURES=metadata-transfer is enabled
+ src_db = portdb._create_pregen_cache(path)
if src_db is not None:
porttrees_data.append(TreeData(portdb.auxdb[path],
diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index bffae36..36b326c 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -182,6 +182,8 @@ class portdbapi(dbapi):
# ~harring
filtered_auxdbkeys = [x for x in auxdbkeys if not x.startswith("UNUSED_0")]
filtered_auxdbkeys.sort()
+ filtered_auxdbkeys = tuple(filtered_auxdbkeys)
+ self._filtered_auxdbkeys = filtered_auxdbkeys
# If secpass < 1, we don't want to write to the cache
# since then we won't be able to apply group permissions
# to the cache entries/directories.
@@ -212,14 +214,9 @@ class portdbapi(dbapi):
for x in self.porttrees:
if x in self._pregen_auxdb:
continue
- conf = self.repositories.get_repo_for_location(x)
- cache = conf.get_pregenerated_cache(filtered_auxdbkeys, readonly=True)
+ cache = self._create_pregen_cache(x)
if cache is not None:
self._pregen_auxdb[x] = cache
- try:
- cache.ec = self._repo_info[x].eclass_db
- except AttributeError:
- pass
# Selectively cache metadata in order to optimize dep matching.
self._aux_cache_keys = set(
["DEPEND", "EAPI", "INHERITED", "IUSE", "KEYWORDS", "LICENSE",
@@ -229,6 +226,17 @@ class portdbapi(dbapi):
self._aux_cache = {}
self._broken_ebuilds = set()
+ def _create_pregen_cache(self, tree):
+ conf = self.repositories.get_repo_for_location(tree)
+ cache = conf.get_pregenerated_cache(
+ self._filtered_auxdbkeys, readonly=True)
+ if cache is not None:
+ try:
+ cache.ec = self._repo_info[tree].eclass_db
+ except AttributeError:
+ pass
+ return cache
+
def _init_cache_dirs(self):
"""Create /var/cache/edb/dep and adjust permissions for the portage
group."""
^ permalink raw reply related [flat|nested] 29+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/
@ 2011-10-16 20:26 Zac Medico
0 siblings, 0 replies; 29+ messages in thread
From: Zac Medico @ 2011-10-16 20:26 UTC (permalink / raw
To: gentoo-commits
commit: 1272f5c55baa5c6d68d5ab3a1dceef69cc47e69b
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Oct 16 20:26:46 2011 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Oct 16 20:26:46 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=1272f5c5
EbuildMetadataPhase: avoid redundant EAPI parsing
---
pym/_emerge/EbuildMetadataPhase.py | 6 ++++--
pym/portage/dbapi/porttree.py | 2 +-
2 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/pym/_emerge/EbuildMetadataPhase.py b/pym/_emerge/EbuildMetadataPhase.py
index 2fbd29e..06cabe7 100644
--- a/pym/_emerge/EbuildMetadataPhase.py
+++ b/pym/_emerge/EbuildMetadataPhase.py
@@ -20,7 +20,7 @@ class EbuildMetadataPhase(SubProcess):
used to extract metadata from the ebuild.
"""
- __slots__ = ("cpv", "ebuild_hash", "fd_pipes", "metadata_callback",
+ __slots__ = ("cpv", "eapi", "ebuild_hash", "fd_pipes", "metadata_callback",
"metadata", "portdb", "repo_path", "settings") + \
("_raw_metadata",)
@@ -33,7 +33,9 @@ class EbuildMetadataPhase(SubProcess):
settings.setcpv(self.cpv)
ebuild_path = self.ebuild_hash.location
- eapi = None
+ # the caller can pass in eapi in order to avoid
+ # redundant _parse_eapi_ebuild_head calls
+ eapi = self.eapi
if eapi is None and \
'parse-eapi-ebuild-head' in settings.features:
eapi = portage._parse_eapi_ebuild_head(
diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 13c27e9..0ade59a 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -528,7 +528,7 @@ class portdbapi(dbapi):
mydata = self._metadata_callback(
mycpv, mylocation, {'EAPI':eapi}, ebuild_hash)
else:
- proc = EbuildMetadataPhase(cpv=mycpv,
+ proc = EbuildMetadataPhase(cpv=mycpv, eapi=eapi,
ebuild_hash=ebuild_hash,
metadata_callback=self._metadata_callback, portdb=self,
repo_path=mylocation,
^ permalink raw reply related [flat|nested] 29+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/
@ 2011-10-28 2:34 Zac Medico
0 siblings, 0 replies; 29+ messages in thread
From: Zac Medico @ 2011-10-28 2:34 UTC (permalink / raw
To: gentoo-commits
commit: df6078a76dc549b83de5a056a6a1dc52bdd69107
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Oct 28 02:32:49 2011 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Oct 28 02:32:49 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=df6078a7
stacklevel=3 for properties warnings
There's an extra level for the @property wrapper.
---
pym/_emerge/FakeVartree.py | 2 +-
pym/portage/dbapi/bintree.py | 2 +-
pym/portage/dbapi/porttree.py | 2 +-
pym/portage/dbapi/vartree.py | 4 ++--
4 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/pym/_emerge/FakeVartree.py b/pym/_emerge/FakeVartree.py
index a2b9232..626d471 100644
--- a/pym/_emerge/FakeVartree.py
+++ b/pym/_emerge/FakeVartree.py
@@ -77,7 +77,7 @@ class FakeVartree(vartree):
"_emerge.FakeVartree.FakeVartree"
" is deprecated. Use "
"settings['ROOT'] instead.",
- DeprecationWarning, stacklevel=2)
+ DeprecationWarning, stacklevel=3)
return self.settings['ROOT']
def _match_wrapper(self, cpv, use_cache=1):
diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 39bb030..0fb8be9 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -349,7 +349,7 @@ class binarytree(object):
"portage.dbapi.bintree.binarytree"
" is deprecated. Use "
"settings['ROOT'] instead.",
- DeprecationWarning, stacklevel=2)
+ DeprecationWarning, stacklevel=3)
return self.settings['ROOT']
def move_ent(self, mylist, repo_match=None):
diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 6f3d90f..fc7feb3 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -1097,7 +1097,7 @@ class portagetree(object):
"portage.dbapi.porttree.portagetree" + \
" is deprecated. Use " + \
"settings['ROOT'] instead.",
- DeprecationWarning, stacklevel=2)
+ DeprecationWarning, stacklevel=3)
return self.settings['ROOT']
def dep_bestmatch(self,mydep):
diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index f766453..7f94cc5 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -180,7 +180,7 @@ class vardbapi(dbapi):
"portage.dbapi.vartree.vardbapi"
" is deprecated. Use "
"settings['ROOT'] instead.",
- DeprecationWarning, stacklevel=2)
+ DeprecationWarning, stacklevel=3)
return self.settings['ROOT']
def getpath(self, mykey, filename=None):
@@ -1183,7 +1183,7 @@ class vartree(object):
"portage.dbapi.vartree.vartree"
" is deprecated. Use "
"settings['ROOT'] instead.",
- DeprecationWarning, stacklevel=2)
+ DeprecationWarning, stacklevel=3)
return self.settings['ROOT']
def getpath(self, mykey, filename=None):
^ permalink raw reply related [flat|nested] 29+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/
@ 2012-02-08 0:36 Zac Medico
0 siblings, 0 replies; 29+ messages in thread
From: Zac Medico @ 2012-02-08 0:36 UTC (permalink / raw
To: gentoo-commits
commit: 3f74dd21143044949e6344f3cbcc7308a6d75ef6
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Feb 8 00:36:32 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Feb 8 00:36:32 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=3f74dd21
PollScheduler: glib.io_add_watch() compatibility
---
pym/_emerge/AsynchronousLock.py | 4 +++
pym/_emerge/EbuildIpcDaemon.py | 2 +
pym/_emerge/EbuildMetadataPhase.py | 2 +
pym/_emerge/PipeReader.py | 4 +++
pym/_emerge/PollScheduler.py | 52 ++++++++++++++++++++++++-----------
pym/_emerge/Scheduler.py | 1 +
pym/_emerge/SpawnProcess.py | 4 +++
pym/portage/dbapi/_MergeProcess.py | 2 +
8 files changed, 54 insertions(+), 17 deletions(-)
diff --git a/pym/_emerge/AsynchronousLock.py b/pym/_emerge/AsynchronousLock.py
index 3593834..e166df3 100644
--- a/pym/_emerge/AsynchronousLock.py
+++ b/pym/_emerge/AsynchronousLock.py
@@ -143,6 +143,8 @@ class _LockThread(AbstractPollTask):
self.returncode = os.EX_OK
self.wait()
+ return True
+
def _cancel(self):
# There's currently no way to force thread termination.
pass
@@ -280,6 +282,8 @@ class _LockProcess(AbstractPollTask):
self.returncode = os.EX_OK
self.wait()
+ return True
+
def _unregister(self):
self._registered = False
diff --git a/pym/_emerge/EbuildIpcDaemon.py b/pym/_emerge/EbuildIpcDaemon.py
index 5dabe34..6a320cb 100644
--- a/pym/_emerge/EbuildIpcDaemon.py
+++ b/pym/_emerge/EbuildIpcDaemon.py
@@ -84,6 +84,8 @@ class EbuildIpcDaemon(FifoIpcDaemon):
if reply_hook is not None:
reply_hook()
+ return True
+
def _send_reply(self, reply):
# File streams are in unbuffered mode since we do atomic
# read and write of whole pickles. Use non-blocking mode so
diff --git a/pym/_emerge/EbuildMetadataPhase.py b/pym/_emerge/EbuildMetadataPhase.py
index d4f5bc0..f8da866 100644
--- a/pym/_emerge/EbuildMetadataPhase.py
+++ b/pym/_emerge/EbuildMetadataPhase.py
@@ -128,6 +128,8 @@ class EbuildMetadataPhase(SubProcess):
self._unregister_if_appropriate(event)
+ return True
+
def _set_returncode(self, wait_retval):
SubProcess._set_returncode(self, wait_retval)
# self._raw_metadata is None when _start returns
diff --git a/pym/_emerge/PipeReader.py b/pym/_emerge/PipeReader.py
index 0f784cf..a85d794 100644
--- a/pym/_emerge/PipeReader.py
+++ b/pym/_emerge/PipeReader.py
@@ -74,6 +74,8 @@ class PipeReader(AbstractPollTask):
self._unregister_if_appropriate(event)
+ return True
+
def _array_output_handler(self, fd, event):
for f in self.input_files.values():
@@ -93,6 +95,8 @@ class PipeReader(AbstractPollTask):
self._unregister_if_appropriate(event)
+ return True
+
def _unregister(self):
"""
Unregister from the scheduler and close open files.
diff --git a/pym/_emerge/PollScheduler.py b/pym/_emerge/PollScheduler.py
index 519a370..d6b971c 100644
--- a/pym/_emerge/PollScheduler.py
+++ b/pym/_emerge/PollScheduler.py
@@ -27,9 +27,12 @@ class PollScheduler(object):
__slots__ = ("output", "register", "schedule",
"source_remove", "timeout_add", "unregister")
+ class _io_handler_class(SlotObject):
+ __slots__ = ("args", "callback", "fd", "source_id")
+
class _timeout_handler_class(SlotObject):
- __slots__ = ("args", "function", "interval", "source_id",
- "timestamp")
+ __slots__ = ("args", "function", "interval",
+ "io_add_watch", "source_id", "timestamp")
def __init__(self):
self._terminated = threading.Event()
@@ -49,6 +52,7 @@ class PollScheduler(object):
self._scheduling = False
self._background = False
self.sched_iface = self._sched_iface_class(
+ io_add_watch=self._register,
output=self._task_output,
register=self._register,
schedule=self._schedule_wait,
@@ -248,8 +252,9 @@ class PollScheduler(object):
try:
while event_handlers:
f, event = self._next_poll_event()
- handler, reg_id = event_handlers[f]
- handler(f, event)
+ x = event_handlers[f]
+ if not x.callback(f, event, *x.args):
+ self._unregister(x.source_id)
event_handled = True
except StopIteration:
event_handled = True
@@ -277,8 +282,9 @@ class PollScheduler(object):
try:
while event_handlers and self._poll_event_queue:
f, event = self._next_poll_event()
- handler, reg_id = event_handlers[f]
- handler(f, event)
+ x = event_handlers[f]
+ if not x.callback(f, event, *x.args):
+ self._unregister(x.source_id)
events_handled += 1
except StopIteration:
events_handled += 1
@@ -332,20 +338,31 @@ class PollScheduler(object):
return bool(ready_timeouts)
- def _register(self, f, eventmask, handler):
+ def _register(self, f, condition, callback, *args):
"""
- @rtype: Integer
- @return: A unique registration id, for use in schedule() or
- unregister() calls.
+ Like glib.io_add_watch(), your function should return False to
+ stop being called, or True to continue being called. Any
+ additional positional arguments given here are passed to your
+ function when it's called.
+
+ @type f: int or object with fileno() method
+ @param f: a file descriptor to monitor
+ @type condition: int
+ @param condition: a condition mask
+ @type callback: callable
+ @param callback: a function to call
+ @rtype: int
+ @return: an integer ID of the event source
"""
if f in self._poll_event_handlers:
raise AssertionError("fd %d is already registered" % f)
self._event_handler_id += 1
- reg_id = self._event_handler_id
- self._poll_event_handler_ids[reg_id] = f
- self._poll_event_handlers[f] = (handler, reg_id)
- self._poll_obj.register(f, eventmask)
- return reg_id
+ source_id = self._event_handler_id
+ self._poll_event_handler_ids[source_id] = f
+ self._poll_event_handlers[f] = self._io_handler_class(
+ args=args, callback=callback, f=f, source_id=source_id)
+ self._poll_obj.register(f, condition)
+ return source_id
def _unregister(self, reg_id):
"""
@@ -409,8 +426,9 @@ class PollScheduler(object):
while (wait_ids is None and event_handlers) or \
(wait_ids is not None and wait_ids.intersection(handler_ids)):
f, event = self._next_poll_event(timeout=remaining_timeout)
- handler, reg_id = event_handlers[f]
- handler(f, event)
+ x = event_handlers[f]
+ if not x.callback(f, event, *x.args):
+ self._unregister(x.source_id)
event_handled = True
if condition is not None and condition():
break
diff --git a/pym/_emerge/Scheduler.py b/pym/_emerge/Scheduler.py
index e6f3e0e..55e327f 100644
--- a/pym/_emerge/Scheduler.py
+++ b/pym/_emerge/Scheduler.py
@@ -218,6 +218,7 @@ class Scheduler(PollScheduler):
schedule=self._schedule_fetch)
self._sched_iface = self._iface_class(
fetch=fetch_iface, output=self._task_output,
+ io_add_watch=self._register,
register=self._register,
schedule=self._schedule_wait,
scheduleSetup=self._schedule_setup,
diff --git a/pym/_emerge/SpawnProcess.py b/pym/_emerge/SpawnProcess.py
index ec5bf7d..411e7c6 100644
--- a/pym/_emerge/SpawnProcess.py
+++ b/pym/_emerge/SpawnProcess.py
@@ -218,6 +218,8 @@ class SpawnProcess(SubProcess):
self._unregister_if_appropriate(event)
+ return True
+
def _dummy_handler(self, fd, event):
"""
This method is mainly interested in detecting EOF, since
@@ -240,6 +242,8 @@ class SpawnProcess(SubProcess):
self._unregister_if_appropriate(event)
+ return True
+
def _unregister(self):
super(SpawnProcess, self)._unregister()
if self._log_file_real is not None:
diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index df501be..eed7bd4 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -83,6 +83,8 @@ class MergeProcess(SpawnProcess):
reporter = getattr(portage.elog.messages, funcname)
reporter(msg, phase=phase, key=key, out=out)
+ return True
+
def _spawn(self, args, fd_pipes, **kwargs):
"""
Fork a subprocess, apply local settings, and call
^ permalink raw reply related [flat|nested] 29+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/
@ 2012-02-08 1:16 Zac Medico
0 siblings, 0 replies; 29+ messages in thread
From: Zac Medico @ 2012-02-08 1:16 UTC (permalink / raw
To: gentoo-commits
commit: 610831146c27d46256df45f127474d509c9a7d31
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Feb 8 00:36:32 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Feb 8 01:15:47 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=61083114
PollScheduler: glib.io_add_watch() compatibility
---
pym/_emerge/AsynchronousLock.py | 4 +++
pym/_emerge/EbuildIpcDaemon.py | 2 +
pym/_emerge/EbuildMetadataPhase.py | 2 +
pym/_emerge/PipeReader.py | 4 +++
pym/_emerge/PollScheduler.py | 50 ++++++++++++++++++++++++-----------
pym/_emerge/Scheduler.py | 1 +
pym/_emerge/SpawnProcess.py | 4 +++
pym/portage/dbapi/_MergeProcess.py | 2 +
8 files changed, 53 insertions(+), 16 deletions(-)
diff --git a/pym/_emerge/AsynchronousLock.py b/pym/_emerge/AsynchronousLock.py
index 3593834..e166df3 100644
--- a/pym/_emerge/AsynchronousLock.py
+++ b/pym/_emerge/AsynchronousLock.py
@@ -143,6 +143,8 @@ class _LockThread(AbstractPollTask):
self.returncode = os.EX_OK
self.wait()
+ return True
+
def _cancel(self):
# There's currently no way to force thread termination.
pass
@@ -280,6 +282,8 @@ class _LockProcess(AbstractPollTask):
self.returncode = os.EX_OK
self.wait()
+ return True
+
def _unregister(self):
self._registered = False
diff --git a/pym/_emerge/EbuildIpcDaemon.py b/pym/_emerge/EbuildIpcDaemon.py
index 5dabe34..6a320cb 100644
--- a/pym/_emerge/EbuildIpcDaemon.py
+++ b/pym/_emerge/EbuildIpcDaemon.py
@@ -84,6 +84,8 @@ class EbuildIpcDaemon(FifoIpcDaemon):
if reply_hook is not None:
reply_hook()
+ return True
+
def _send_reply(self, reply):
# File streams are in unbuffered mode since we do atomic
# read and write of whole pickles. Use non-blocking mode so
diff --git a/pym/_emerge/EbuildMetadataPhase.py b/pym/_emerge/EbuildMetadataPhase.py
index d4f5bc0..f8da866 100644
--- a/pym/_emerge/EbuildMetadataPhase.py
+++ b/pym/_emerge/EbuildMetadataPhase.py
@@ -128,6 +128,8 @@ class EbuildMetadataPhase(SubProcess):
self._unregister_if_appropriate(event)
+ return True
+
def _set_returncode(self, wait_retval):
SubProcess._set_returncode(self, wait_retval)
# self._raw_metadata is None when _start returns
diff --git a/pym/_emerge/PipeReader.py b/pym/_emerge/PipeReader.py
index 0f784cf..a85d794 100644
--- a/pym/_emerge/PipeReader.py
+++ b/pym/_emerge/PipeReader.py
@@ -74,6 +74,8 @@ class PipeReader(AbstractPollTask):
self._unregister_if_appropriate(event)
+ return True
+
def _array_output_handler(self, fd, event):
for f in self.input_files.values():
@@ -93,6 +95,8 @@ class PipeReader(AbstractPollTask):
self._unregister_if_appropriate(event)
+ return True
+
def _unregister(self):
"""
Unregister from the scheduler and close open files.
diff --git a/pym/_emerge/PollScheduler.py b/pym/_emerge/PollScheduler.py
index 519a370..7cf8189 100644
--- a/pym/_emerge/PollScheduler.py
+++ b/pym/_emerge/PollScheduler.py
@@ -24,9 +24,12 @@ from _emerge.PollSelectAdapter import PollSelectAdapter
class PollScheduler(object):
class _sched_iface_class(SlotObject):
- __slots__ = ("output", "register", "schedule",
+ __slots__ = ("io_add_watch", "output", "register", "schedule",
"source_remove", "timeout_add", "unregister")
+ class _io_handler_class(SlotObject):
+ __slots__ = ("args", "callback", "fd", "source_id")
+
class _timeout_handler_class(SlotObject):
__slots__ = ("args", "function", "interval", "source_id",
"timestamp")
@@ -49,6 +52,7 @@ class PollScheduler(object):
self._scheduling = False
self._background = False
self.sched_iface = self._sched_iface_class(
+ io_add_watch=self._register,
output=self._task_output,
register=self._register,
schedule=self._schedule_wait,
@@ -248,8 +252,9 @@ class PollScheduler(object):
try:
while event_handlers:
f, event = self._next_poll_event()
- handler, reg_id = event_handlers[f]
- handler(f, event)
+ x = event_handlers[f]
+ if not x.callback(f, event, *x.args):
+ self._unregister(x.source_id)
event_handled = True
except StopIteration:
event_handled = True
@@ -277,8 +282,9 @@ class PollScheduler(object):
try:
while event_handlers and self._poll_event_queue:
f, event = self._next_poll_event()
- handler, reg_id = event_handlers[f]
- handler(f, event)
+ x = event_handlers[f]
+ if not x.callback(f, event, *x.args):
+ self._unregister(x.source_id)
events_handled += 1
except StopIteration:
events_handled += 1
@@ -332,20 +338,31 @@ class PollScheduler(object):
return bool(ready_timeouts)
- def _register(self, f, eventmask, handler):
+ def _register(self, f, condition, callback, *args):
"""
- @rtype: Integer
- @return: A unique registration id, for use in schedule() or
- unregister() calls.
+ Like glib.io_add_watch(), your function should return False to
+ stop being called, or True to continue being called. Any
+ additional positional arguments given here are passed to your
+ function when it's called.
+
+ @type f: int or object with fileno() method
+ @param f: a file descriptor to monitor
+ @type condition: int
+ @param condition: a condition mask
+ @type callback: callable
+ @param callback: a function to call
+ @rtype: int
+ @return: an integer ID of the event source
"""
if f in self._poll_event_handlers:
raise AssertionError("fd %d is already registered" % f)
self._event_handler_id += 1
- reg_id = self._event_handler_id
- self._poll_event_handler_ids[reg_id] = f
- self._poll_event_handlers[f] = (handler, reg_id)
- self._poll_obj.register(f, eventmask)
- return reg_id
+ source_id = self._event_handler_id
+ self._poll_event_handler_ids[source_id] = f
+ self._poll_event_handlers[f] = self._io_handler_class(
+ args=args, callback=callback, f=f, source_id=source_id)
+ self._poll_obj.register(f, condition)
+ return source_id
def _unregister(self, reg_id):
"""
@@ -409,8 +426,9 @@ class PollScheduler(object):
while (wait_ids is None and event_handlers) or \
(wait_ids is not None and wait_ids.intersection(handler_ids)):
f, event = self._next_poll_event(timeout=remaining_timeout)
- handler, reg_id = event_handlers[f]
- handler(f, event)
+ x = event_handlers[f]
+ if not x.callback(f, event, *x.args):
+ self._unregister(x.source_id)
event_handled = True
if condition is not None and condition():
break
diff --git a/pym/_emerge/Scheduler.py b/pym/_emerge/Scheduler.py
index e6f3e0e..55e327f 100644
--- a/pym/_emerge/Scheduler.py
+++ b/pym/_emerge/Scheduler.py
@@ -218,6 +218,7 @@ class Scheduler(PollScheduler):
schedule=self._schedule_fetch)
self._sched_iface = self._iface_class(
fetch=fetch_iface, output=self._task_output,
+ io_add_watch=self._register,
register=self._register,
schedule=self._schedule_wait,
scheduleSetup=self._schedule_setup,
diff --git a/pym/_emerge/SpawnProcess.py b/pym/_emerge/SpawnProcess.py
index ec5bf7d..411e7c6 100644
--- a/pym/_emerge/SpawnProcess.py
+++ b/pym/_emerge/SpawnProcess.py
@@ -218,6 +218,8 @@ class SpawnProcess(SubProcess):
self._unregister_if_appropriate(event)
+ return True
+
def _dummy_handler(self, fd, event):
"""
This method is mainly interested in detecting EOF, since
@@ -240,6 +242,8 @@ class SpawnProcess(SubProcess):
self._unregister_if_appropriate(event)
+ return True
+
def _unregister(self):
super(SpawnProcess, self)._unregister()
if self._log_file_real is not None:
diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index df501be..eed7bd4 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -83,6 +83,8 @@ class MergeProcess(SpawnProcess):
reporter = getattr(portage.elog.messages, funcname)
reporter(msg, phase=phase, key=key, out=out)
+ return True
+
def _spawn(self, args, fd_pipes, **kwargs):
"""
Fork a subprocess, apply local settings, and call
^ permalink raw reply related [flat|nested] 29+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/
@ 2012-02-10 1:28 Zac Medico
0 siblings, 0 replies; 29+ messages in thread
From: Zac Medico @ 2012-02-10 1:28 UTC (permalink / raw
To: gentoo-commits
commit: c85b1d7149f09e64b5ed4f5e45742b6e5a9df38b
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Feb 10 01:27:13 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Feb 10 01:27:13 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=c85b1d71
Use IO_* constants where appropriate.
---
pym/_emerge/AbstractPollTask.py | 20 +++++++++++++-------
pym/_emerge/AsynchronousLock.py | 9 ++++-----
pym/_emerge/EbuildIpcDaemon.py | 5 ++---
pym/_emerge/EbuildMetadataPhase.py | 5 ++---
pym/_emerge/PollScheduler.py | 9 ++++++++-
pym/_emerge/Scheduler.py | 6 ++++++
pym/portage/dbapi/_MergeProcess.py | 7 +++----
7 files changed, 38 insertions(+), 23 deletions(-)
diff --git a/pym/_emerge/AbstractPollTask.py b/pym/_emerge/AbstractPollTask.py
index 9229052..68282ef 100644
--- a/pym/_emerge/AbstractPollTask.py
+++ b/pym/_emerge/AbstractPollTask.py
@@ -8,16 +8,22 @@ import os
from portage.util import writemsg_level
from _emerge.AsynchronousTask import AsynchronousTask
-from _emerge.PollConstants import PollConstants
+
class AbstractPollTask(AsynchronousTask):
__slots__ = ("scheduler",) + \
("_registered",)
_bufsize = 4096
- _exceptional_events = PollConstants.POLLERR | PollConstants.POLLNVAL
- _registered_events = PollConstants.POLLIN | PollConstants.POLLHUP | \
- _exceptional_events
+
+ @property
+ def _exceptional_events(self):
+ return self.scheduler.IO_ERR | self.scheduler.IO_NVAL
+
+ @property
+ def _registered_events(self):
+ return self.scheduler.IO_IN | self.scheduler.IO_HUP | \
+ self._exceptional_events
def isAlive(self):
return bool(self._registered)
@@ -39,7 +45,7 @@ class AbstractPollTask(AsynchronousTask):
| 0 | None
"""
buf = None
- if event & PollConstants.POLLIN:
+ if event & self.scheduler.IO_IN:
buf = array.array('B')
try:
buf.fromfile(f, self._bufsize)
@@ -85,7 +91,7 @@ class AbstractPollTask(AsynchronousTask):
# bugs in all known versions of Python (including Python 2.7
# and Python 3.2).
buf = None
- if event & PollConstants.POLLIN:
+ if event & self.scheduler.IO_IN:
try:
buf = os.read(fd, self._bufsize)
except OSError as e:
@@ -117,7 +123,7 @@ class AbstractPollTask(AsynchronousTask):
self._log_poll_exception(event)
self._unregister()
self.cancel()
- elif event & PollConstants.POLLHUP:
+ elif event & self.scheduler.IO_HUP:
self._unregister()
self.wait()
diff --git a/pym/_emerge/AsynchronousLock.py b/pym/_emerge/AsynchronousLock.py
index 3cf0acb..c07df0b 100644
--- a/pym/_emerge/AsynchronousLock.py
+++ b/pym/_emerge/AsynchronousLock.py
@@ -20,7 +20,6 @@ from portage.locks import lockfile, unlockfile
from portage.util import writemsg_level
from _emerge.AbstractPollTask import AbstractPollTask
from _emerge.AsynchronousTask import AsynchronousTask
-from _emerge.PollConstants import PollConstants
from _emerge.SpawnProcess import SpawnProcess
class AsynchronousLock(AsynchronousTask):
@@ -118,7 +117,7 @@ class _LockThread(AbstractPollTask):
fcntl.fcntl(f, fcntl.F_SETFL,
fcntl.fcntl(f, fcntl.F_GETFL) | os.O_NONBLOCK)
self._reg_id = self.scheduler.register(self._files['pipe_read'],
- PollConstants.POLLIN, self._output_handler)
+ self.scheduler.IO_IN, self._output_handler)
self._registered = True
threading_mod = threading
if self._force_dummy:
@@ -132,7 +131,7 @@ class _LockThread(AbstractPollTask):
def _output_handler(self, f, event):
buf = None
- if event & PollConstants.POLLIN:
+ if event & self.scheduler.IO_IN:
try:
buf = os.read(self._files['pipe_read'], self._bufsize)
except OSError as e:
@@ -200,7 +199,7 @@ class _LockProcess(AbstractPollTask):
fcntl.fcntl(in_pr, fcntl.F_SETFL,
fcntl.fcntl(in_pr, fcntl.F_GETFL) | os.O_NONBLOCK)
self._reg_id = self.scheduler.register(in_pr,
- PollConstants.POLLIN, self._output_handler)
+ self.scheduler.IO_IN, self._output_handler)
self._registered = True
self._proc = SpawnProcess(
args=[portage._python_interpreter,
@@ -268,7 +267,7 @@ class _LockProcess(AbstractPollTask):
def _output_handler(self, f, event):
buf = None
- if event & PollConstants.POLLIN:
+ if event & self.scheduler.IO_IN:
try:
buf = os.read(self._files['pipe_in'], self._bufsize)
except OSError as e:
diff --git a/pym/_emerge/EbuildIpcDaemon.py b/pym/_emerge/EbuildIpcDaemon.py
index 6a320cb..5795bfb 100644
--- a/pym/_emerge/EbuildIpcDaemon.py
+++ b/pym/_emerge/EbuildIpcDaemon.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2011 Gentoo Foundation
+# Copyright 2010-2012 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
import errno
@@ -8,7 +8,6 @@ from portage import os
from portage.localization import _
from portage.util import writemsg_level
from _emerge.FifoIpcDaemon import FifoIpcDaemon
-from _emerge.PollConstants import PollConstants
class EbuildIpcDaemon(FifoIpcDaemon):
"""
@@ -34,7 +33,7 @@ class EbuildIpcDaemon(FifoIpcDaemon):
def _input_handler(self, fd, event):
# Read the whole pickle in a single atomic read() call.
data = None
- if event & PollConstants.POLLIN:
+ if event & self.scheduler.IO_IN:
# For maximum portability, use os.read() here since
# array.fromfile() and file.read() are both known to
# erroneously return an empty string from this
diff --git a/pym/_emerge/EbuildMetadataPhase.py b/pym/_emerge/EbuildMetadataPhase.py
index f8da866..a34542d 100644
--- a/pym/_emerge/EbuildMetadataPhase.py
+++ b/pym/_emerge/EbuildMetadataPhase.py
@@ -1,8 +1,7 @@
-# Copyright 1999-2011 Gentoo Foundation
+# Copyright 1999-2012 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
from _emerge.SubProcess import SubProcess
-from _emerge.PollConstants import PollConstants
import sys
from portage.cache.mappings import slot_dict_class
import portage
@@ -111,7 +110,7 @@ class EbuildMetadataPhase(SubProcess):
def _output_handler(self, fd, event):
- if event & PollConstants.POLLIN:
+ if event & self.scheduler.IO_IN:
while True:
try:
self._raw_metadata.append(
diff --git a/pym/_emerge/PollScheduler.py b/pym/_emerge/PollScheduler.py
index 3b86fa0..eafb8a2 100644
--- a/pym/_emerge/PollScheduler.py
+++ b/pym/_emerge/PollScheduler.py
@@ -20,7 +20,8 @@ from _emerge.getloadavg import getloadavg
class PollScheduler(object):
class _sched_iface_class(SlotObject):
- __slots__ = ("idle_add", "io_add_watch", "iteration",
+ __slots__ = ("IO_ERR", "IO_HUP", "IO_IN", "IO_NVAL", "IO_OUT",
+ "IO_PRI", "idle_add", "io_add_watch", "iteration",
"output", "register", "run",
"source_remove", "timeout_add", "unregister")
@@ -34,6 +35,12 @@ class PollScheduler(object):
self._background = False
self._event_loop = global_event_loop()
self.sched_iface = self._sched_iface_class(
+ IO_ERR=self._event_loop.IO_ERR,
+ IO_HUP=self._event_loop.IO_HUP,
+ IO_IN=self._event_loop.IO_IN,
+ IO_NVAL=self._event_loop.IO_NVAL,
+ IO_OUT=self._event_loop.IO_OUT,
+ IO_PRI=self._event_loop.IO_PRI,
idle_add=self._event_loop.idle_add,
io_add_watch=self._event_loop.io_add_watch,
iteration=self._event_loop.iteration,
diff --git a/pym/_emerge/Scheduler.py b/pym/_emerge/Scheduler.py
index 12f0871..674a11e 100644
--- a/pym/_emerge/Scheduler.py
+++ b/pym/_emerge/Scheduler.py
@@ -217,6 +217,12 @@ class Scheduler(PollScheduler):
fetch_iface = self._fetch_iface_class(log_file=self._fetch_log,
schedule=self._schedule_fetch)
self._sched_iface = self._iface_class(
+ IO_ERR=self._event_loop.IO_ERR,
+ IO_HUP=self._event_loop.IO_HUP,
+ IO_IN=self._event_loop.IO_IN,
+ IO_NVAL=self._event_loop.IO_NVAL,
+ IO_OUT=self._event_loop.IO_OUT,
+ IO_PRI=self._event_loop.IO_PRI,
fetch=fetch_iface, output=self._task_output,
idle_add=self._event_loop.idle_add,
io_add_watch=self._event_loop.io_add_watch,
diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index da278ea..9bb67c9 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2011 Gentoo Foundation
+# Copyright 2010-2012 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
import io
@@ -10,7 +10,6 @@ import fcntl
import portage
from portage import os, _unicode_decode
import portage.elog.messages
-from _emerge.PollConstants import PollConstants
from _emerge.SpawnProcess import SpawnProcess
class MergeProcess(SpawnProcess):
@@ -63,7 +62,7 @@ class MergeProcess(SpawnProcess):
def _elog_output_handler(self, fd, event):
output = None
- if event & PollConstants.POLLIN:
+ if event & self.scheduler.IO_IN:
try:
output = os.read(fd, self._bufsize)
except OSError as e:
@@ -83,7 +82,7 @@ class MergeProcess(SpawnProcess):
reporter = getattr(portage.elog.messages, funcname)
reporter(msg, phase=phase, key=key, out=out)
- if event & PollConstants.POLLHUP:
+ if event & self.scheduler.IO_HUP:
self.scheduler.unregister(self._elog_reg_id)
self._elog_reg_id = None
os.close(self._elog_reader_fd)
^ permalink raw reply related [flat|nested] 29+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/
@ 2012-02-14 23:40 Zac Medico
0 siblings, 0 replies; 29+ messages in thread
From: Zac Medico @ 2012-02-14 23:40 UTC (permalink / raw
To: gentoo-commits
commit: 11937db0fb2e25a30d855b084417f8d52547ff54
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Feb 14 23:39:57 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Feb 14 23:39:57 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=11937db0
Assert that fork returns int for bug 403697.
These cases should have been included with commit
6a94a074aa0475173a51f3f726377d4c407e986b.
---
pym/_emerge/EbuildFetcher.py | 3 +++
pym/portage/dbapi/_MergeProcess.py | 4 ++++
2 files changed, 7 insertions(+), 0 deletions(-)
diff --git a/pym/_emerge/EbuildFetcher.py b/pym/_emerge/EbuildFetcher.py
index 6ad4341..c0175b3 100644
--- a/pym/_emerge/EbuildFetcher.py
+++ b/pym/_emerge/EbuildFetcher.py
@@ -164,6 +164,9 @@ class EbuildFetcher(SpawnProcess):
pid = os.fork()
if pid != 0:
+ if not isinstance(pid, int):
+ raise AssertionError(
+ "fork returned non-integer: %s" % (repr(pid),))
portage.process.spawned_pids.append(pid)
return [pid]
diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index cf59265..1747a09 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -129,6 +129,10 @@ class MergeProcess(SpawnProcess):
pid = os.fork()
if pid != 0:
+ if not isinstance(pid, int):
+ raise AssertionError(
+ "fork returned non-integer: %s" % (repr(pid),))
+
os.close(elog_writer_fd)
self._elog_reader_fd = elog_reader_fd
self._buf = ""
^ permalink raw reply related [flat|nested] 29+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/
@ 2012-02-16 0:33 Zac Medico
0 siblings, 0 replies; 29+ messages in thread
From: Zac Medico @ 2012-02-16 0:33 UTC (permalink / raw
To: gentoo-commits
commit: 46107423f4a039af6528d14cb86123a05a67f85f
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Feb 16 00:33:10 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Feb 16 00:33:10 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=46107423
Comment about _setup_pipes / PyPy GC interaction.
---
pym/_emerge/EbuildFetcher.py | 6 +++++-
pym/portage/dbapi/_MergeProcess.py | 6 +++++-
2 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/pym/_emerge/EbuildFetcher.py b/pym/_emerge/EbuildFetcher.py
index c0175b3..f6dab54 100644
--- a/pym/_emerge/EbuildFetcher.py
+++ b/pym/_emerge/EbuildFetcher.py
@@ -171,7 +171,11 @@ class EbuildFetcher(SpawnProcess):
return [pid]
# TODO: Find out why PyPy 1.8 with close_fds=True triggers
- # "[Errno 9] Bad file descriptor" in subprocesses.
+ # "[Errno 9] Bad file descriptor" in subprocesses. It could
+ # be due to garbage collection of file objects that were not
+ # closed before going out of scope, since PyPy's garbage
+ # collector does not support the refcounting semantics that
+ # CPython does.
close_fds = platform.python_implementation() != 'PyPy'
portage.process._setup_pipes(fd_pipes, close_fds=close_fds)
diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index 21d43dd..9a185b4 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -157,7 +157,11 @@ class MergeProcess(SpawnProcess):
os.close(elog_reader_fd)
# TODO: Find out why PyPy 1.8 with close_fds=True triggers
- # "[Errno 9] Bad file descriptor" in subprocesses.
+ # "[Errno 9] Bad file descriptor" in subprocesses. It could
+ # be due to garbage collection of file objects that were not
+ # closed before going out of scope, since PyPy's garbage
+ # collector does not support the refcounting semantics that
+ # CPython does.
close_fds = platform.python_implementation() != 'PyPy'
portage.process._setup_pipes(fd_pipes, close_fds=close_fds)
^ permalink raw reply related [flat|nested] 29+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/
@ 2012-02-17 22:19 Zac Medico
0 siblings, 0 replies; 29+ messages in thread
From: Zac Medico @ 2012-02-17 22:19 UTC (permalink / raw
To: gentoo-commits
commit: 4d4e590947553d7d57cc7d303d5d6a9a2edfd948
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Feb 17 22:18:05 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Feb 17 22:18:05 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=4d4e5909
Remove redundant inherited __slots__ values.
---
pym/_emerge/EbuildExecuter.py | 2 +-
pym/portage/dbapi/_MergeProcess.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/pym/_emerge/EbuildExecuter.py b/pym/_emerge/EbuildExecuter.py
index 2b8f120..fd663a4 100644
--- a/pym/_emerge/EbuildExecuter.py
+++ b/pym/_emerge/EbuildExecuter.py
@@ -12,7 +12,7 @@ from portage.package.ebuild.doebuild import _prepare_fake_distdir
class EbuildExecuter(CompositeTask):
- __slots__ = ("pkg", "scheduler", "settings")
+ __slots__ = ("pkg", "settings")
_phases = ("prepare", "configure", "compile", "test", "install")
diff --git a/pym/portage/dbapi/_MergeProcess.py b/pym/portage/dbapi/_MergeProcess.py
index 9a185b4..301625c 100644
--- a/pym/portage/dbapi/_MergeProcess.py
+++ b/pym/portage/dbapi/_MergeProcess.py
@@ -21,7 +21,7 @@ class MergeProcess(SpawnProcess):
"""
__slots__ = ('mycat', 'mypkg', 'settings', 'treetype',
- 'vartree', 'scheduler', 'blockers', 'pkgloc', 'infloc', 'myebuild',
+ 'vartree', 'blockers', 'pkgloc', 'infloc', 'myebuild',
'mydbapi', 'prev_mtimes', 'unmerge', '_elog_reader_fd', '_elog_reg_id',
'_buf', '_elog_keys', '_locked_vdb')
^ permalink raw reply related [flat|nested] 29+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/
@ 2012-04-21 6:51 Zac Medico
0 siblings, 0 replies; 29+ messages in thread
From: Zac Medico @ 2012-04-21 6:51 UTC (permalink / raw
To: gentoo-commits
commit: d603f1440c814377fbc1965729fd9b6b008cf76d
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Apr 21 06:38:17 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Apr 21 06:45:31 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=d603f144
dbapi: account for unevaluated_atom in caches
This will fix bug 412391. This is analogous to the bug fixed in
commit 5438bb29c996d777b6343515995176912a7c137f.
---
pym/_emerge/PackageVirtualDbapi.py | 16 ++++++++++------
pym/_emerge/depgraph.py | 5 +++--
pym/portage/dbapi/porttree.py | 26 +++++++++++++-------------
pym/portage/dbapi/vartree.py | 5 +++--
pym/portage/dbapi/virtual.py | 16 ++++++++++------
5 files changed, 39 insertions(+), 29 deletions(-)
diff --git a/pym/_emerge/PackageVirtualDbapi.py b/pym/_emerge/PackageVirtualDbapi.py
index a692bb6..a34d21c 100644
--- a/pym/_emerge/PackageVirtualDbapi.py
+++ b/pym/_emerge/PackageVirtualDbapi.py
@@ -1,8 +1,9 @@
-# Copyright 1999-2011 Gentoo Foundation
+# Copyright 1999-2012 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
import sys
from portage.dbapi import dbapi
+from portage.dbapi.dep_expand import dep_expand
class PackageVirtualDbapi(dbapi):
"""
@@ -76,18 +77,21 @@ class PackageVirtualDbapi(dbapi):
self._match_cache = {}
def match(self, origdep, use_cache=1):
- result = self._match_cache.get(origdep)
+ atom = dep_expand(origdep, mydb=self, settings=self.settings)
+ cache_key = (atom, atom.unevaluated_atom)
+ result = self._match_cache.get(cache_key)
if result is not None:
return result[:]
- result = dbapi.match(self, origdep, use_cache=use_cache)
- self._match_cache[origdep] = result
+ result = list(self._iter_match(atom, self.cp_list(atom.cp)))
+ self._match_cache[cache_key] = result
return result[:]
def cpv_exists(self, cpv, myrepo=None):
return cpv in self._cpv_map
def cp_list(self, mycp, use_cache=1):
- cachelist = self._match_cache.get(mycp)
+ cache_key = (mycp, mycp)
+ cachelist = self._match_cache.get(cache_key)
# cp_list() doesn't expand old-style virtuals
if cachelist and cachelist[0].startswith(mycp):
return cachelist[:]
@@ -98,7 +102,7 @@ class PackageVirtualDbapi(dbapi):
cpv_list = [pkg.cpv for pkg in cpv_list]
self._cpv_sort_ascending(cpv_list)
if not (not cpv_list and mycp.startswith("virtual/")):
- self._match_cache[mycp] = cpv_list
+ self._match_cache[cache_key] = cpv_list
return cpv_list[:]
def cp_all(self):
diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py
index e77c0e8..e624559 100644
--- a/pym/_emerge/depgraph.py
+++ b/pym/_emerge/depgraph.py
@@ -6753,7 +6753,8 @@ class _dep_check_composite_db(dbapi):
return ret
def match(self, atom):
- ret = self._match_cache.get(atom)
+ cache_key = (atom, atom.unevaluated_atom)
+ ret = self._match_cache.get(cache_key)
if ret is not None:
return ret[:]
@@ -6801,7 +6802,7 @@ class _dep_check_composite_db(dbapi):
if len(ret) > 1:
self._cpv_sort_ascending(ret)
- self._match_cache[atom] = ret
+ self._match_cache[cache_key] = ret
return ret[:]
def _visible(self, pkg):
diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 382bcda..f9d78dc 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2011 Gentoo Foundation
+# Copyright 1998-2012 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
__all__ = [
@@ -731,7 +731,7 @@ class portdbapi(dbapi):
# profile (due to old-style virtuals). Do not propagate
# old-style virtuals since cp_list() doesn't expand them.
if not (not cachelist and mycp.startswith("virtual/")):
- self.xcache["match-all"][mycp] = cachelist
+ self.xcache["match-all"][(mycp, mycp)] = cachelist
return cachelist[:]
mysplit = mycp.split("/")
invalid_category = mysplit[0] not in self._categories
@@ -786,7 +786,7 @@ class portdbapi(dbapi):
# Do not propagate old-style virtuals since
# cp_list() doesn't expand them.
if not (not cachelist and mycp.startswith("virtual/")):
- self.xcache["match-all"][mycp] = cachelist
+ self.xcache["match-all"][(mycp, mycp)] = cachelist
return mylist
def freeze(self):
@@ -809,19 +809,21 @@ class portdbapi(dbapi):
"has been renamed to match-visible",
DeprecationWarning, stacklevel=2)
- #if no updates are being made to the tree, we can consult our xcache...
- if self.frozen:
- try:
- return self.xcache[level][origdep][:]
- except KeyError:
- pass
-
if mydep is None:
#this stuff only runs on first call of xmatch()
#create mydep, mykey from origdep
mydep = dep_expand(origdep, mydb=self, settings=self.settings)
mykey = mydep.cp
+ #if no updates are being made to the tree, we can consult our xcache...
+ cache_key = None
+ if self.frozen:
+ cache_key = (mydep, mydep.unevaluated_atom)
+ try:
+ return self.xcache[level][cache_key][:]
+ except KeyError:
+ pass
+
myval = None
mytree = None
if mydep.repo is not None:
@@ -930,9 +932,7 @@ class portdbapi(dbapi):
if self.frozen:
xcache_this_level = self.xcache.get(level)
if xcache_this_level is not None:
- xcache_this_level[mydep] = myval
- if origdep and origdep != mydep:
- xcache_this_level[origdep] = myval
+ xcache_this_level[cache_key] = myval
myval = myval[:]
return myval
diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index a3a6c76..ec9f87c 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -484,6 +484,7 @@ class vardbapi(dbapi):
"caching match function"
mydep = dep_expand(
origdep, mydb=self, use_cache=use_cache, settings=self.settings)
+ cache_key = (mydep, mydep.unevaluated_atom)
mykey = dep_getkey(mydep)
mycat = catsplit(mykey)[0]
if not use_cache:
@@ -505,8 +506,8 @@ class vardbapi(dbapi):
if mydep not in self.matchcache[mycat]:
mymatch = list(self._iter_match(mydep,
self.cp_list(mydep.cp, use_cache=use_cache)))
- self.matchcache[mycat][mydep] = mymatch
- return self.matchcache[mycat][mydep][:]
+ self.matchcache[mycat][cache_key] = mymatch
+ return self.matchcache[mycat][cache_key][:]
def findname(self, mycpv, myrepo=None):
return self.getpath(str(mycpv), filename=catsplit(mycpv)[1]+".ebuild")
diff --git a/pym/portage/dbapi/virtual.py b/pym/portage/dbapi/virtual.py
index ec97ffe..eed1407 100644
--- a/pym/portage/dbapi/virtual.py
+++ b/pym/portage/dbapi/virtual.py
@@ -1,8 +1,9 @@
-# Copyright 1998-2007 Gentoo Foundation
+# Copyright 1998-2012 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
from portage.dbapi import dbapi
+from portage.dbapi.dep_expand import dep_expand
from portage import cpv_getkey
class fakedbapi(dbapi):
@@ -31,18 +32,21 @@ class fakedbapi(dbapi):
self._match_cache = {}
def match(self, origdep, use_cache=1):
- result = self._match_cache.get(origdep, None)
+ atom = dep_expand(origdep, mydb=self, settings=self.settings)
+ cache_key = (atom, atom.unevaluated_atom)
+ result = self._match_cache.get(cache_key)
if result is not None:
return result[:]
- result = dbapi.match(self, origdep, use_cache=use_cache)
- self._match_cache[origdep] = result
+ result = list(self._iter_match(atom, self.cp_list(atom.cp)))
+ self._match_cache[cache_key] = result
return result[:]
def cpv_exists(self, mycpv, myrepo=None):
return mycpv in self.cpvdict
def cp_list(self, mycp, use_cache=1, myrepo=None):
- cachelist = self._match_cache.get(mycp)
+ cache_key = (mycp, mycp)
+ cachelist = self._match_cache.get(cache_key)
# cp_list() doesn't expand old-style virtuals
if cachelist and cachelist[0].startswith(mycp):
return cachelist[:]
@@ -51,7 +55,7 @@ class fakedbapi(dbapi):
cpv_list = []
self._cpv_sort_ascending(cpv_list)
if not (not cpv_list and mycp.startswith("virtual/")):
- self._match_cache[mycp] = cpv_list
+ self._match_cache[cache_key] = cpv_list
return cpv_list[:]
def cp_all(self):
^ permalink raw reply related [flat|nested] 29+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/
@ 2012-04-22 18:57 Zac Medico
0 siblings, 0 replies; 29+ messages in thread
From: Zac Medico @ 2012-04-22 18:57 UTC (permalink / raw
To: gentoo-commits
commit: d30db49bcaecbdeb20f6d4e180979b60b6260f69
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Apr 22 18:57:25 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Apr 22 18:57:25 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=d30db49b
dbapi.cp_list: remove special case for virtuals
Since commit d603f1440c814377fbc1965729fd9b6b008cf76d, the match caches
use the result from dep_expand for the cache_key, so the caches are
free of old-style virtual mappings. This allows the match caches to be
safely shared with cp_list.
---
pym/_emerge/PackageVirtualDbapi.py | 8 ++++----
pym/portage/dbapi/porttree.py | 14 +++++---------
pym/portage/dbapi/virtual.py | 5 +++--
3 files changed, 12 insertions(+), 15 deletions(-)
diff --git a/pym/_emerge/PackageVirtualDbapi.py b/pym/_emerge/PackageVirtualDbapi.py
index a34d21c..0f7be44 100644
--- a/pym/_emerge/PackageVirtualDbapi.py
+++ b/pym/_emerge/PackageVirtualDbapi.py
@@ -90,10 +90,11 @@ class PackageVirtualDbapi(dbapi):
return cpv in self._cpv_map
def cp_list(self, mycp, use_cache=1):
+ # NOTE: Cache can be safely shared with the match cache, since the
+ # match cache uses the result from dep_expand for the cache_key.
cache_key = (mycp, mycp)
cachelist = self._match_cache.get(cache_key)
- # cp_list() doesn't expand old-style virtuals
- if cachelist and cachelist[0].startswith(mycp):
+ if cachelist is not None:
return cachelist[:]
cpv_list = self._cp_map.get(mycp)
if cpv_list is None:
@@ -101,8 +102,7 @@ class PackageVirtualDbapi(dbapi):
else:
cpv_list = [pkg.cpv for pkg in cpv_list]
self._cpv_sort_ascending(cpv_list)
- if not (not cpv_list and mycp.startswith("virtual/")):
- self._match_cache[cache_key] = cpv_list
+ self._match_cache[cache_key] = cpv_list
return cpv_list[:]
def cp_all(self):
diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index f9d78dc..b060dd6 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -715,7 +715,8 @@ class portdbapi(dbapi):
return l
def cp_list(self, mycp, use_cache=1, mytree=None):
-
+ # NOTE: Cache can be safely shared with the match cache, since the
+ # match cache uses the result from dep_expand for the cache_key.
if self.frozen and mytree is not None \
and len(self.porttrees) == 1 \
and mytree == self.porttrees[0]:
@@ -728,10 +729,8 @@ class portdbapi(dbapi):
if cachelist is not None:
# Try to propagate this to the match-all cache here for
# repoman since he uses separate match-all caches for each
- # profile (due to old-style virtuals). Do not propagate
- # old-style virtuals since cp_list() doesn't expand them.
- if not (not cachelist and mycp.startswith("virtual/")):
- self.xcache["match-all"][(mycp, mycp)] = cachelist
+ # profile (due to old-style virtuals).
+ self.xcache["match-all"][(mycp, mycp)] = cachelist
return cachelist[:]
mysplit = mycp.split("/")
invalid_category = mysplit[0] not in self._categories
@@ -783,10 +782,7 @@ class portdbapi(dbapi):
if self.frozen and mytree is None:
cachelist = mylist[:]
self.xcache["cp-list"][mycp] = cachelist
- # Do not propagate old-style virtuals since
- # cp_list() doesn't expand them.
- if not (not cachelist and mycp.startswith("virtual/")):
- self.xcache["match-all"][(mycp, mycp)] = cachelist
+ self.xcache["match-all"][(mycp, mycp)] = cachelist
return mylist
def freeze(self):
diff --git a/pym/portage/dbapi/virtual.py b/pym/portage/dbapi/virtual.py
index eed1407..8a35d0c 100644
--- a/pym/portage/dbapi/virtual.py
+++ b/pym/portage/dbapi/virtual.py
@@ -45,6 +45,8 @@ class fakedbapi(dbapi):
return mycpv in self.cpvdict
def cp_list(self, mycp, use_cache=1, myrepo=None):
+ # NOTE: Cache can be safely shared with the match cache, since the
+ # match cache uses the result from dep_expand for the cache_key.
cache_key = (mycp, mycp)
cachelist = self._match_cache.get(cache_key)
# cp_list() doesn't expand old-style virtuals
@@ -54,8 +56,7 @@ class fakedbapi(dbapi):
if cpv_list is None:
cpv_list = []
self._cpv_sort_ascending(cpv_list)
- if not (not cpv_list and mycp.startswith("virtual/")):
- self._match_cache[cache_key] = cpv_list
+ self._match_cache[cache_key] = cpv_list
return cpv_list[:]
def cp_all(self):
^ permalink raw reply related [flat|nested] 29+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/
@ 2012-11-24 21:57 Zac Medico
0 siblings, 0 replies; 29+ messages in thread
From: Zac Medico @ 2012-11-24 21:57 UTC (permalink / raw
To: gentoo-commits
commit: abd18694835b9f8a5a515f9c6333a754703a0462
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Nov 24 21:56:51 2012 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Nov 24 21:56:51 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=abd18694
depgraph: split out similar_name_search func
This will allow the code to be re-used for bug #444596. Copyright
begins in 2011 since that's when the code for this feature was first
added in commit aa78cc8da18015b7d1e4eec277b5a7f940fe357c.
---
pym/_emerge/depgraph.py | 50 +++-----------------------
pym/portage/dbapi/_similar_name_search.py | 57 +++++++++++++++++++++++++++++
2 files changed, 62 insertions(+), 45 deletions(-)
diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py
index c97aa02..365fbf8 100644
--- a/pym/_emerge/depgraph.py
+++ b/pym/_emerge/depgraph.py
@@ -3,7 +3,6 @@
from __future__ import print_function
-import difflib
import errno
import io
import logging
@@ -19,6 +18,7 @@ from portage import _unicode_decode, _unicode_encode, _encodings
from portage.const import PORTAGE_PACKAGE_ATOM, USER_CONFIG_PATH
from portage.dbapi import dbapi
from portage.dbapi.dep_expand import dep_expand
+from portage.dbapi._similar_name_search import similar_name_search
from portage.dep import Atom, best_match_to_list, extract_affecting_use, \
check_required_use, human_readable_required_use, match_from_list, \
_repo_separator
@@ -3678,57 +3678,17 @@ class depgraph(object):
not cp_exists and \
self._frozen_config.myopts.get(
"--misspell-suggestions", "y") != "n":
- cp = myparent.atom.cp.lower()
- cat, pkg = portage.catsplit(cp)
- if cat == "null":
- cat = None
writemsg("\nemerge: searching for similar names..."
, noiselevel=-1)
- all_cp = set()
- all_cp.update(vardb.cp_all())
+ dbs = [vardb]
if "--usepkgonly" not in self._frozen_config.myopts:
- all_cp.update(portdb.cp_all())
+ dbs.append(portdb)
if "--usepkg" in self._frozen_config.myopts:
- all_cp.update(bindb.cp_all())
- # discard dir containing no ebuilds
- all_cp.discard(cp)
+ dbs.append(bindb)
- orig_cp_map = {}
- for cp_orig in all_cp:
- orig_cp_map.setdefault(cp_orig.lower(), []).append(cp_orig)
- all_cp = set(orig_cp_map)
-
- if cat:
- matches = difflib.get_close_matches(cp, all_cp)
- else:
- pkg_to_cp = {}
- for other_cp in list(all_cp):
- other_pkg = portage.catsplit(other_cp)[1]
- if other_pkg == pkg:
- # Check for non-identical package that
- # differs only by upper/lower case.
- identical = True
- for cp_orig in orig_cp_map[other_cp]:
- if portage.catsplit(cp_orig)[1] != \
- portage.catsplit(atom.cp)[1]:
- identical = False
- break
- if identical:
- # discard dir containing no ebuilds
- all_cp.discard(other_cp)
- continue
- pkg_to_cp.setdefault(other_pkg, set()).add(other_cp)
- pkg_matches = difflib.get_close_matches(pkg, pkg_to_cp)
- matches = []
- for pkg_match in pkg_matches:
- matches.extend(pkg_to_cp[pkg_match])
-
- matches_orig_case = []
- for cp in matches:
- matches_orig_case.extend(orig_cp_map[cp])
- matches = matches_orig_case
+ matches = similar_name_search(dbs, atom)
if len(matches) == 1:
writemsg("\nemerge: Maybe you meant " + matches[0] + "?\n"
diff --git a/pym/portage/dbapi/_similar_name_search.py b/pym/portage/dbapi/_similar_name_search.py
new file mode 100644
index 0000000..d569fbf
--- /dev/null
+++ b/pym/portage/dbapi/_similar_name_search.py
@@ -0,0 +1,57 @@
+# Copyright 2011-2012 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import difflib
+
+from portage.versions import catsplit
+
+def similar_name_search(dbs, atom):
+
+ cp = atom.cp.lower()
+ cat, pkg = catsplit(cp)
+ if cat == "null":
+ cat = None
+
+ all_cp = set()
+ for db in dbs:
+ all_cp.update(db.cp_all())
+
+ # discard dir containing no ebuilds
+ all_cp.discard(cp)
+
+ orig_cp_map = {}
+ for cp_orig in all_cp:
+ orig_cp_map.setdefault(cp_orig.lower(), []).append(cp_orig)
+ all_cp = set(orig_cp_map)
+
+ if cat:
+ matches = difflib.get_close_matches(cp, all_cp)
+ else:
+ pkg_to_cp = {}
+ for other_cp in list(all_cp):
+ other_pkg = catsplit(other_cp)[1]
+ if other_pkg == pkg:
+ # Check for non-identical package that
+ # differs only by upper/lower case.
+ identical = True
+ for cp_orig in orig_cp_map[other_cp]:
+ if catsplit(cp_orig)[1] != \
+ catsplit(atom.cp)[1]:
+ identical = False
+ break
+ if identical:
+ # discard dir containing no ebuilds
+ all_cp.discard(other_cp)
+ continue
+ pkg_to_cp.setdefault(other_pkg, set()).add(other_cp)
+
+ pkg_matches = difflib.get_close_matches(pkg, pkg_to_cp)
+ matches = []
+ for pkg_match in pkg_matches:
+ matches.extend(pkg_to_cp[pkg_match])
+
+ matches_orig_case = []
+ for cp in matches:
+ matches_orig_case.extend(orig_cp_map[cp])
+
+ return matches_orig_case
^ permalink raw reply related [flat|nested] 29+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/
@ 2012-12-23 5:33 Arfrever Frehtes Taifersar Arahesis
0 siblings, 0 replies; 29+ messages in thread
From: Arfrever Frehtes Taifersar Arahesis @ 2012-12-23 5:33 UTC (permalink / raw
To: gentoo-commits
commit: b4440ce6a87d7c6932132185697dd3c4888d752c
Author: Arfrever Frehtes Taifersar Arahesis <Arfrever <AT> Apache <DOT> Org>
AuthorDate: Sun Dec 23 05:32:24 2012 +0000
Commit: Arfrever Frehtes Taifersar Arahesis <arfrever.fta <AT> gmail <DOT> com>
CommitDate: Sun Dec 23 05:32:24 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=b4440ce6
Support subslots in command line arguments.
---
pym/_emerge/depgraph.py | 2 +-
pym/_emerge/is_valid_package_atom.py | 2 +-
pym/portage/dbapi/dep_expand.py | 4 ++--
3 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py
index 3cd44dd..fab87d9 100644
--- a/pym/_emerge/depgraph.py
+++ b/pym/_emerge/depgraph.py
@@ -2405,7 +2405,7 @@ class depgraph(object):
# came from, if any.
# 2) It takes away freedom from the resolver to choose other
# possible expansions when necessary.
- if "/" in x:
+ if "/" in x.split(":")[0]:
args.append(AtomArg(arg=x, atom=Atom(x, allow_repo=True),
root_config=root_config))
continue
diff --git a/pym/_emerge/is_valid_package_atom.py b/pym/_emerge/is_valid_package_atom.py
index a1e4294..c4f6ffd 100644
--- a/pym/_emerge/is_valid_package_atom.py
+++ b/pym/_emerge/is_valid_package_atom.py
@@ -14,7 +14,7 @@ def insert_category_into_atom(atom, category):
return ret
def is_valid_package_atom(x, allow_repo=False):
- if "/" not in x:
+ if "/" not in x.split(":")[0]:
x2 = insert_category_into_atom(x, 'cat')
if x2 != None:
x = x2
diff --git a/pym/portage/dbapi/dep_expand.py b/pym/portage/dbapi/dep_expand.py
index ac8ccf4..bb211c7 100644
--- a/pym/portage/dbapi/dep_expand.py
+++ b/pym/portage/dbapi/dep_expand.py
@@ -1,4 +1,4 @@
-# Copyright 2010 Gentoo Foundation
+# Copyright 2010-2012 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
__all__ = ["dep_expand"]
@@ -23,7 +23,7 @@ def dep_expand(mydep, mydb=None, use_cache=1, settings=None):
if mydep[0] == "*":
mydep = mydep[1:]
orig_dep = mydep
- has_cat = '/' in orig_dep
+ has_cat = '/' in orig_dep.split(':')[0]
if not has_cat:
alphanum = re.search(r'\w', orig_dep)
if alphanum:
^ permalink raw reply related [flat|nested] 29+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/
@ 2013-01-05 15:20 Zac Medico
0 siblings, 0 replies; 29+ messages in thread
From: Zac Medico @ 2013-01-05 15:20 UTC (permalink / raw
To: gentoo-commits
commit: e893125da2f260c73ac728c5f027113ef9aa2f73
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Jan 5 15:18:29 2013 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Jan 5 15:18:29 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=e893125d
BinpkgVerifier: use async FileDigester
---
pym/_emerge/BinpkgVerifier.py | 144 +++++++++++++++++++++++++++--------------
pym/portage/dbapi/bintree.py | 38 +++++++----
2 files changed, 120 insertions(+), 62 deletions(-)
diff --git a/pym/_emerge/BinpkgVerifier.py b/pym/_emerge/BinpkgVerifier.py
index fd4f266..2c69792 100644
--- a/pym/_emerge/BinpkgVerifier.py
+++ b/pym/_emerge/BinpkgVerifier.py
@@ -1,74 +1,120 @@
-# Copyright 1999-2012 Gentoo Foundation
+# Copyright 1999-2013 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
-from _emerge.AsynchronousTask import AsynchronousTask
-from portage.util import writemsg
+import errno
import io
import sys
+
+from _emerge.CompositeTask import CompositeTask
import portage
from portage import os
+from portage.checksum import (_apply_hash_filter,
+ _filter_unaccelarated_hashes, _hash_filter)
+from portage.output import EOutput
+from portage.util._async.FileDigester import FileDigester
from portage.package.ebuild.fetch import _checksum_failure_temp_file
-class BinpkgVerifier(AsynchronousTask):
- __slots__ = ("logfile", "pkg", "scheduler")
+class BinpkgVerifier(CompositeTask):
+ __slots__ = ("logfile", "pkg", "_digests", "_pkg_path")
def _start(self):
- """
- Note: Unlike a normal AsynchronousTask.start() method,
- this one does all work is synchronously. The returncode
- attribute will be set before it returns.
- """
-
- pkg = self.pkg
- root_config = pkg.root_config
- bintree = root_config.trees["bintree"]
- rval = os.EX_OK
+
+ bintree = self.pkg.root_config.trees["bintree"]
+ digests = bintree._get_digests(self.pkg)
+ if "size" not in digests:
+ self.returncode = os.EX_OK
+ self._async_wait()
+ return
+
+ digests = _filter_unaccelarated_hashes(digests)
+ hash_filter = _hash_filter(
+ bintree.settings.get("PORTAGE_CHECKSUM_FILTER", ""))
+ if not hash_filter.transparent:
+ digests = _apply_hash_filter(digests, hash_filter)
+
+ self._digests = digests
+ self._pkg_path = bintree.getname(self.pkg.cpv)
+
+ try:
+ size = os.stat(self._pkg_path).st_size
+ except OSError as e:
+ if e.errno not in (errno.ENOENT, errno.ESTALE):
+ raise
+ self.scheduler.output(("!!! Fetching Binary failed "
+ "for '%s'\n") % self.pkg.cpv, log_path=self.logfile,
+ background=self.background)
+ self.returncode = 1
+ self._async_wait()
+ return
+ else:
+ if size != digests["size"]:
+ self._digest_exception("size", size, digests["size"])
+ self.returncode = 1
+ self._async_wait()
+ return
+
+ self._start_task(FileDigester(file_path=self._pkg_path,
+ hash_names=(k for k in digests if k != "size"),
+ background=self.background, logfile=self.logfile,
+ scheduler=self.scheduler),
+ self._digester_exit)
+
+ def _digester_exit(self, digester):
+
+ if self._default_exit(digester) != os.EX_OK:
+ self.wait()
+ return
+
+ for hash_name in digester.hash_names:
+ if digester.digests[hash_name] != self._digests[hash_name]:
+ self._digest_exception(hash_name,
+ digester.digests[hash_name], self._digests[hash_name])
+ self.returncode = 1
+ self.wait()
+ return
+
+ if self.pkg.root_config.settings.get("PORTAGE_QUIET") != "1":
+ self._display_success()
+
+ self.returncode = os.EX_OK
+ self.wait()
+
+ def _display_success(self):
stdout_orig = sys.stdout
stderr_orig = sys.stderr
global_havecolor = portage.output.havecolor
out = io.StringIO()
- file_exists = True
try:
sys.stdout = out
sys.stderr = out
if portage.output.havecolor:
portage.output.havecolor = not self.background
- try:
- bintree.digestCheck(pkg)
- except portage.exception.FileNotFound:
- writemsg("!!! Fetching Binary failed " + \
- "for '%s'\n" % pkg.cpv, noiselevel=-1)
- rval = 1
- file_exists = False
- except portage.exception.DigestException as e:
- writemsg("\n!!! Digest verification failed:\n",
- noiselevel=-1)
- writemsg("!!! %s\n" % e.value[0],
- noiselevel=-1)
- writemsg("!!! Reason: %s\n" % e.value[1],
- noiselevel=-1)
- writemsg("!!! Got: %s\n" % e.value[2],
- noiselevel=-1)
- writemsg("!!! Expected: %s\n" % e.value[3],
- noiselevel=-1)
- rval = 1
- if rval == os.EX_OK:
- pass
- elif file_exists:
- pkg_path = bintree.getname(pkg.cpv)
- head, tail = os.path.split(pkg_path)
- temp_filename = _checksum_failure_temp_file(head, tail)
- writemsg("File renamed to '%s'\n" % (temp_filename,),
- noiselevel=-1)
+
+ eout = EOutput()
+ eout.ebegin("%s %s ;-)" % (os.path.basename(self._pkg_path),
+ " ".join(sorted(self._digests))))
+ eout.eend(0)
+
finally:
sys.stdout = stdout_orig
sys.stderr = stderr_orig
portage.output.havecolor = global_havecolor
- msg = out.getvalue()
- if msg:
- self.scheduler.output(msg, log_path=self.logfile,
- background=self.background)
+ self.scheduler.output(out.getvalue(), log_path=self.logfile,
+ background=self.background)
+
+ def _digest_exception(self, name, value, expected):
+
+ head, tail = os.path.split(self._pkg_path)
+ temp_filename = _checksum_failure_temp_file(head, tail)
- self.returncode = rval
- self._async_wait()
+ self.scheduler.output((
+ "\n!!! Digest verification failed:\n"
+ "!!! %s\n"
+ "!!! Reason: Failed on %s verification\n"
+ "!!! Got: %s\n"
+ "!!! Expected: %s\n"
+ "File renamed to '%s'\n") %
+ (self._pkg_path, name, value, expected, temp_filename),
+ log_path=self.logfile,
+ background=self.background)
diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 890e8e1..f6fc114 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -1356,19 +1356,14 @@ class binarytree(object):
f.close()
return pkgindex
- def digestCheck(self, pkg):
- """
- Verify digests for the given package and raise DigestException
- if verification fails.
- @rtype: bool
- @return: True if digests could be located, False otherwise.
- """
- cpv = pkg
- if not isinstance(cpv, basestring):
+ def _get_digests(self, pkg):
+
+ try:
cpv = pkg.cpv
- pkg = None
+ except AttributeError:
+ cpv = pkg
- pkg_path = self.getname(cpv)
+ digests = {}
metadata = None
if self._remotepkgs is None or cpv not in self._remotepkgs:
for d in self._load_pkgindex().packages:
@@ -1378,9 +1373,8 @@ class binarytree(object):
else:
metadata = self._remotepkgs[cpv]
if metadata is None:
- return False
+ return digests
- digests = {}
for k in hashfunc_map:
v = metadata.get(k)
if not v:
@@ -1394,9 +1388,27 @@ class binarytree(object):
writemsg(_("!!! Malformed SIZE attribute in remote " \
"metadata for '%s'\n") % cpv)
+ return digests
+
+ def digestCheck(self, pkg):
+ """
+ Verify digests for the given package and raise DigestException
+ if verification fails.
+ @rtype: bool
+ @return: True if digests could be located, False otherwise.
+ """
+
+ digests = self._get_digests(pkg)
+
if not digests:
return False
+ try:
+ cpv = pkg.cpv
+ except AttributeError:
+ cpv = pkg
+
+ pkg_path = self.getname(cpv)
hash_filter = _hash_filter(
self.settings.get("PORTAGE_CHECKSUM_FILTER", ""))
if not hash_filter.transparent:
^ permalink raw reply related [flat|nested] 29+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/
@ 2013-01-23 16:19 Zac Medico
0 siblings, 0 replies; 29+ messages in thread
From: Zac Medico @ 2013-01-23 16:19 UTC (permalink / raw
To: gentoo-commits
commit: 9ae58b611e7a0d2080f0f48c207c7323dfa06dc3
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Jan 23 16:19:06 2013 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Jan 23 16:19:06 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=9ae58b61
Package: filter built USE for bug #453400
The enabled flags must be consistent with implicit IUSE, in order to
avoid potential inconsistencies in USE dep matching (see bug #453400).
---
pym/_emerge/Package.py | 65 ++++++++++++++++++++++++-----------------
pym/portage/dbapi/__init__.py | 4 ++-
2 files changed, 41 insertions(+), 28 deletions(-)
diff --git a/pym/_emerge/Package.py b/pym/_emerge/Package.py
index 09d8b93..eee41eb 100644
--- a/pym/_emerge/Package.py
+++ b/pym/_emerge/Package.py
@@ -561,7 +561,7 @@ class Package(Task):
@property
def use(self):
if self._use is None:
- self._metadata._init_use()
+ self._init_use()
return self._use
def _get_pkgsettings(self):
@@ -570,6 +570,42 @@ class Package(Task):
pkgsettings.setcpv(self)
return pkgsettings
+ def _init_use(self):
+ if self.built:
+ # Use IUSE to validate USE settings for built packages,
+ # in case the package manager that built this package
+ # failed to do that for some reason (or in case of
+ # data corruption). The enabled flags must be consistent
+ # with implicit IUSE, in order to avoid potential
+ # inconsistencies in USE dep matching (see bug #453400).
+ use_str = self._metadata['USE']
+ is_valid_flag = self.iuse.is_valid_flag
+ enabled_flags = [x for x in use_str.split() if is_valid_flag(x)]
+ use_str = " ".join(enabled_flags)
+ self._use = self._use_class(
+ self, use_str)
+ else:
+ try:
+ use_str = _PackageMetadataWrapperBase.__getitem__(
+ self._metadata, 'USE')
+ except KeyError:
+ use_str = None
+ calculated_use = False
+ if not use_str:
+ use_str = self._get_pkgsettings()["PORTAGE_USE"]
+ calculated_use = True
+ _PackageMetadataWrapperBase.__setitem__(
+ self._metadata, 'USE', use_str)
+ self._use = self._use_class(
+ self, use_str)
+ # Initialize these now, since USE access has just triggered
+ # setcpv, and we want to cache the result of the force/mask
+ # calculations that were done.
+ if calculated_use:
+ self._use._init_force_mask()
+
+ return use_str
+
class _iuse(object):
__slots__ = ("__weakref__", "_iuse_implicit_match", "_pkg", "alias_mapping",
@@ -718,31 +754,6 @@ class _PackageMetadataWrapper(_PackageMetadataWrapperBase):
self.update(metadata)
- def _init_use(self):
- if self._pkg.built:
- use_str = self['USE']
- self._pkg._use = self._pkg._use_class(
- self._pkg, use_str)
- else:
- try:
- use_str = _PackageMetadataWrapperBase.__getitem__(self, 'USE')
- except KeyError:
- use_str = None
- calculated_use = False
- if not use_str:
- use_str = self._pkg._get_pkgsettings()["PORTAGE_USE"]
- calculated_use = True
- _PackageMetadataWrapperBase.__setitem__(self, 'USE', use_str)
- self._pkg._use = self._pkg._use_class(
- self._pkg, use_str)
- # Initialize these now, since USE access has just triggered
- # setcpv, and we want to cache the result of the force/mask
- # calculations that were done.
- if calculated_use:
- self._pkg._use._init_force_mask()
-
- return use_str
-
def __getitem__(self, k):
v = _PackageMetadataWrapperBase.__getitem__(self, k)
if k in self._use_conditional_keys:
@@ -760,7 +771,7 @@ class _PackageMetadataWrapper(_PackageMetadataWrapperBase):
elif k == 'USE' and not self._pkg.built:
if not v:
# This is lazy because it's expensive.
- v = self._init_use()
+ v = self._pkg._init_use()
return v
diff --git a/pym/portage/dbapi/__init__.py b/pym/portage/dbapi/__init__.py
index 3540c6d..79d1e7a 100644
--- a/pym/portage/dbapi/__init__.py
+++ b/pym/portage/dbapi/__init__.py
@@ -217,7 +217,9 @@ class dbapi(object):
# Use IUSE to validate USE settings for built packages,
# in case the package manager that built this package
# failed to do that for some reason (or in case of
- # data corruption).
+ # data corruption). The enabled flags must be consistent
+ # with implicit IUSE, in order to avoid potential
+ # inconsistencies in USE dep matching (see bug #453400).
use = frozenset(x for x in metadata["USE"].split()
if x in iuse or iuse_implicit_match(x))
missing_enabled = frozenset(x for x in
^ permalink raw reply related [flat|nested] 29+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/
@ 2013-03-13 5:56 Zac Medico
0 siblings, 0 replies; 29+ messages in thread
From: Zac Medico @ 2013-03-13 5:56 UTC (permalink / raw
To: gentoo-commits
commit: 8e130a60c9f4d6ab7f1de547015c4ce0b0045041
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Mar 13 05:56:43 2013 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Mar 13 05:56:43 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=8e130a60
iter_owners: event loop for spinner, bug #461412
---
pym/_emerge/depgraph.py | 22 +++++++++++++++++++++-
pym/_emerge/stdout_spinner.py | 13 ++++++++-----
pym/portage/dbapi/vartree.py | 25 +++++++++++++++++++++----
3 files changed, 50 insertions(+), 10 deletions(-)
diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py
index 7e3a0eb..dfef6a7 100644
--- a/pym/_emerge/depgraph.py
+++ b/pym/_emerge/depgraph.py
@@ -41,6 +41,8 @@ from portage.util import ensure_dirs
from portage.util import writemsg_level, write_atomic
from portage.util.digraph import digraph
from portage.util._async.TaskScheduler import TaskScheduler
+from portage.util._eventloop.EventLoop import EventLoop
+from portage.util._eventloop.global_event_loop import global_event_loop
from portage.versions import catpkgsplit
from _emerge.AtomArg import AtomArg
@@ -518,6 +520,9 @@ class depgraph(object):
self._select_atoms = self._select_atoms_highest_available
self._select_package = self._select_pkg_highest_available
+ self._event_loop = (portage._internal_caller and
+ global_event_loop() or EventLoop(main=False))
+
def _load_vdb(self):
"""
Load installed package metadata if appropriate. This used to be called
@@ -2594,7 +2599,22 @@ class depgraph(object):
continue
yield arg, atom
- def select_files(self, myfiles):
+ def select_files(self, args):
+ # Use the global event loop for spinner progress
+ # indication during file owner lookups (bug #461412).
+ spinner_id = None
+ try:
+ spinner = self._frozen_config.spinner
+ if spinner is not None and \
+ spinner.update is not spinner.update_quiet:
+ spinner_id = self._event_loop.idle_add(
+ self._frozen_config.spinner.update)
+ return self._select_files(args)
+ finally:
+ if spinner_id is not None:
+ self._event_loop.source_remove(spinner_id)
+
+ def _select_files(self, myfiles):
"""Given a list of .tbz2s, .ebuilds sets, and deps, populate
self._dynamic_config._initial_arg_list and call self._resolve to create the
appropriate depgraph and return a favorite list."""
diff --git a/pym/_emerge/stdout_spinner.py b/pym/_emerge/stdout_spinner.py
index 5ad31f0..670686a 100644
--- a/pym/_emerge/stdout_spinner.py
+++ b/pym/_emerge/stdout_spinner.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2009 Gentoo Foundation
+# Copyright 1999-2013 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
import platform
@@ -53,17 +53,18 @@ class stdout_spinner(object):
def update_basic(self):
self.spinpos = (self.spinpos + 1) % 500
if self._return_early():
- return
+ return True
if (self.spinpos % 100) == 0:
if self.spinpos == 0:
sys.stdout.write(". ")
else:
sys.stdout.write(".")
sys.stdout.flush()
+ return True
def update_scroll(self):
if self._return_early():
- return
+ return True
if(self.spinpos >= len(self.scroll_sequence)):
sys.stdout.write(darkgreen(" \b\b\b" + self.scroll_sequence[
len(self.scroll_sequence) - 1 - (self.spinpos % len(self.scroll_sequence))]))
@@ -71,13 +72,15 @@ class stdout_spinner(object):
sys.stdout.write(green("\b " + self.scroll_sequence[self.spinpos]))
sys.stdout.flush()
self.spinpos = (self.spinpos + 1) % (2 * len(self.scroll_sequence))
+ return True
def update_twirl(self):
self.spinpos = (self.spinpos + 1) % len(self.twirl_sequence)
if self._return_early():
- return
+ return True
sys.stdout.write("\b\b " + self.twirl_sequence[self.spinpos])
sys.stdout.flush()
+ return True
def update_quiet(self):
- return
+ return True
diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index a2fbf86..f918c2c 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1260,18 +1260,35 @@ class vardbapi(dbapi):
name = os.path.basename(path.rstrip(os.path.sep))
path_info_list.append((path, name, is_basename))
+ # Do work via the global event loop, so that it can be used
+ # for indication of progress during the search (bug #461412).
+ event_loop = (portage._internal_caller and
+ global_event_loop() or EventLoop(main=False))
root = self._vardb._eroot
- for cpv in self._vardb.cpv_all():
- dblnk = self._vardb._dblink(cpv)
+ def search_pkg(cpv):
+ dblnk = self._vardb._dblink(cpv)
for path, name, is_basename in path_info_list:
if is_basename:
for p in dblnk.getcontents():
if os.path.basename(p) == name:
- yield dblnk, p[len(root):]
+ search_pkg.results.append((dblnk, p[len(root):]))
else:
if dblnk.isowner(path):
- yield dblnk, path
+ search_pkg.results.append((dblnk, path))
+ search_pkg.complete = True
+ return False
+
+ search_pkg.results = []
+
+ for cpv in self._vardb.cpv_all():
+ del search_pkg.results[:]
+ search_pkg.complete = False
+ event_loop.idle_add(search_pkg, cpv)
+ while not search_pkg.complete:
+ event_loop.iteration()
+ for result in search_pkg.results:
+ yield result
class vartree(object):
"this tree will scan a var/db/pkg database located at root (passed to init)"
^ permalink raw reply related [flat|nested] 29+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/
@ 2014-02-24 9:23 Alexander Berntsen
0 siblings, 0 replies; 29+ messages in thread
From: Alexander Berntsen @ 2014-02-24 9:23 UTC (permalink / raw
To: gentoo-commits
commit: 34c0ea135617ee9bc1c3a684c109b3c0b27a31dd
Author: Alexander Berntsen <alexander <AT> plaimi <DOT> net>
AuthorDate: Mon Feb 24 09:09:13 2014 +0000
Commit: Alexander Berntsen <bernalex <AT> gentoo <DOT> org>
CommitDate: Mon Feb 24 09:19:47 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=34c0ea13
Clean up bad PORTAGE_BINHOST handling
Suggested-by: Arfrever Frehtes Taifersar Arahesis <Arfrever <AT> Apache.Org>
Reviewed-by: Arfrever Frehtes Taifersar Arahesis <Arfrever <AT> Apache.Org>
---
pym/_emerge/actions.py | 11 ++++++++---
pym/portage/dbapi/bintree.py | 10 ++++------
2 files changed, 12 insertions(+), 9 deletions(-)
diff --git a/pym/_emerge/actions.py b/pym/_emerge/actions.py
index 95c5c14..19659fd 100644
--- a/pym/_emerge/actions.py
+++ b/pym/_emerge/actions.py
@@ -44,7 +44,7 @@ from portage.dbapi.dep_expand import dep_expand
from portage.dbapi._expand_new_virt import expand_new_virt
from portage.dep import Atom
from portage.eclass_cache import hashed_path
-from portage.exception import InvalidAtom, InvalidData
+from portage.exception import InvalidAtom, InvalidData, ParseError
from portage.output import blue, bold, colorize, create_color_func, darkgreen, \
red, xtermTitle, xtermTitleReset, yellow
good = create_color_func("GOOD")
@@ -3701,8 +3701,13 @@ def run_action(emerge_config):
# Populate the bintree with current --getbinpkg setting.
# This needs to happen before expand_set_arguments(), in case
# any sets use the bintree.
- mytrees["bintree"].populate(
- getbinpkgs="--getbinpkg" in emerge_config.opts)
+ try:
+ mytrees["bintree"].populate(
+ getbinpkgs="--getbinpkg" in emerge_config.opts)
+ except ParseError as e:
+ writemsg("\n\n!!!%s.\nSee make.conf(5) for more info.\n"
+ % e, noiselevel=-1)
+ return 1
del mytrees, mydb
diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 7b61c1f..229ce3b 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -27,7 +27,7 @@ from portage.const import CACHE_PATH
from portage.dbapi.virtual import fakedbapi
from portage.dep import Atom, use_reduce, paren_enclose
from portage.exception import AlarmSignal, InvalidData, InvalidPackageName, \
- PermissionDenied, PortageException
+ ParseError, PermissionDenied, PortageException
from portage.localization import _
from portage import _movefile
from portage import os
@@ -909,11 +909,9 @@ class binarytree(object):
traceback.print_exc()
raise
- except ValueError as _:
- writemsg("\n\n!!! Invalid PORTAGE_BINHOST value "
- "'%s'.\nSee man make.conf(5) for more info."
- "\n" % url.lstrip(), noiselevel=-1)
- exit(1)
+ except ValueError:
+ raise ParseError("Invalid Portage BINHOST value '%s'"
+ % url.lstrip())
if f is None:
^ permalink raw reply related [flat|nested] 29+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/
@ 2014-11-20 4:08 Zac Medico
0 siblings, 0 replies; 29+ messages in thread
From: Zac Medico @ 2014-11-20 4:08 UTC (permalink / raw
To: gentoo-commits
commit: 103d27694197893badb3b5b5fb2e476e04f88033
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Nov 18 09:13:33 2014 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Nov 20 04:02:34 2014 +0000
URL: http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=103d2769
emerge: check for writable /var/db/pkg (490732)
If there are packages to be merged or unmerge, then bail out early
if /var/db/pkg is not writable (in order to avoid a fatal EROFS error
which would otherwise occur later on). Behavior remains unchanged for
--pretend mode. For --ask mode, it will bail out just after the last
relevant --ask prompt.
In contrast to the writeable_check module, which operates on files just
before they are merged, the new vardbapi.writable check is performed
before anything has been built (much earlier).
X-Gentoo-Bug: 490732
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=490732
Acked-by: Brian Dolbec <dolsen <AT> gentoo.org>
---
pym/_emerge/actions.py | 15 +++++++++++++++
pym/_emerge/unmerge.py | 8 ++++++++
pym/portage/dbapi/vartree.py | 12 ++++++++++++
3 files changed, 35 insertions(+)
diff --git a/pym/_emerge/actions.py b/pym/_emerge/actions.py
index 48b0826..6f7dfe0 100644
--- a/pym/_emerge/actions.py
+++ b/pym/_emerge/actions.py
@@ -428,6 +428,21 @@ def action_build(settings, trees, mtimedb,
# least show warnings about missed updates and such.
mydepgraph.display_problems()
+ if not Scheduler._opts_no_self_update.intersection(myopts):
+
+ eroots = set()
+ for x in mydepgraph.altlist():
+ if isinstance(x, Package) and x.operation == "merge":
+ eroots.add(x.root)
+
+ for eroot in eroots:
+ if not trees[eroot]["vartree"].dbapi.writable:
+ writemsg_level("!!! %s\n" %
+ _("Read-only file system: %s") %
+ trees[eroot]["vartree"].dbapi._dbroot,
+ level=logging.ERROR, noiselevel=-1)
+ return 1
+
if ("--resume" in myopts):
favorites=mtimedb["resume"]["favorites"]
diff --git a/pym/_emerge/unmerge.py b/pym/_emerge/unmerge.py
index df29000..03ce0a2 100644
--- a/pym/_emerge/unmerge.py
+++ b/pym/_emerge/unmerge.py
@@ -10,6 +10,7 @@ import textwrap
import portage
from portage import os
from portage.dbapi._expand_new_virt import expand_new_virt
+from portage.localization import _
from portage.output import bold, colorize, darkgreen, green
from portage._sets import SETPREFIX
from portage._sets.base import EditablePackageSet
@@ -546,6 +547,13 @@ def unmerge(root_config, myopts, unmerge_action,
print("Quitting.")
print()
return 128 + signal.SIGINT
+
+ if not vartree.dbapi.writable:
+ writemsg_level("!!! %s\n" %
+ _("Read-only file system: %s") % vartree.dbapi._dbroot,
+ level=logging.ERROR, noiselevel=-1)
+ return 1
+
#the real unmerging begins, after a short delay unless we're raging....
if not unmerge_action == "rage-clean" and clean_delay and not autoclean:
countdown(int(settings["CLEAN_DELAY"]), ">>> Unmerging")
diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 0fd1bd9..df031cd 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -33,6 +33,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.util.env_update:env_update',
'portage.util.listdir:dircache,listdir',
'portage.util.movefile:movefile',
+ 'portage.util.path:first_existing',
'portage.util.writeable_check:get_ro_checker',
'portage.util._dyn_libs.PreservedLibsRegistry:PreservedLibsRegistry',
'portage.util._dyn_libs.LinkageMapELF:LinkageMapELF@LinkageMap',
@@ -189,6 +190,17 @@ class vardbapi(dbapi):
self._cached_counter = None
@property
+ def writable(self):
+ """
+ Check if var/db/pkg is writable, or permissions are sufficient
+ to create it if it does not exist yet.
+ @rtype: bool
+ @return: True if var/db/pkg is writable or can be created,
+ False otherwise
+ """
+ return os.access(first_existing(self._dbroot), os.W_OK)
+
+ @property
def root(self):
warnings.warn("The root attribute of "
"portage.dbapi.vartree.vardbapi"
^ permalink raw reply related [flat|nested] 29+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/
@ 2015-08-30 23:44 Zac Medico
0 siblings, 0 replies; 29+ messages in thread
From: Zac Medico @ 2015-08-30 23:44 UTC (permalink / raw
To: gentoo-commits
commit: d87eeb7b8edf846330e3c2031eea2f326f149506
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Aug 29 05:10:45 2015 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Aug 30 23:42:42 2015 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=d87eeb7b
emerge --search: fix duplication of results (bug 559044)
Fix search._cp_all() to ensure sorted input for MultiIterGroupBy.
Add a boolean "sort" keyword argument to all dbapi.cp_all()
implementations, and set defaults for backward compatible behavior.
X-Gentoo-Bug: 559044
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=559044
Acked-by: Brian Dolbec <dolsen <AT> gentoo.org>
pym/_emerge/PackageVirtualDbapi.py | 4 ++--
pym/_emerge/search.py | 3 ++-
pym/portage/dbapi/IndexedPortdb.py | 4 ++--
pym/portage/dbapi/IndexedVardb.py | 6 +++---
pym/portage/dbapi/__init__.py | 4 ++--
pym/portage/dbapi/bintree.py | 4 ++--
pym/portage/dbapi/porttree.py | 6 ++++--
pym/portage/dbapi/vartree.py | 4 ++--
pym/portage/dbapi/virtual.py | 4 ++--
9 files changed, 21 insertions(+), 18 deletions(-)
diff --git a/pym/_emerge/PackageVirtualDbapi.py b/pym/_emerge/PackageVirtualDbapi.py
index 56a5576..26293dd 100644
--- a/pym/_emerge/PackageVirtualDbapi.py
+++ b/pym/_emerge/PackageVirtualDbapi.py
@@ -105,8 +105,8 @@ class PackageVirtualDbapi(dbapi):
self._match_cache[cache_key] = cpv_list
return cpv_list[:]
- def cp_all(self):
- return list(self._cp_map)
+ def cp_all(self, sort=False):
+ return sorted(self._cp_map) if sort else list(self._cp_map)
def cpv_all(self):
return list(self._cpv_map)
diff --git a/pym/_emerge/search.py b/pym/_emerge/search.py
index 41c182d..32d326e 100644
--- a/pym/_emerge/search.py
+++ b/pym/_emerge/search.py
@@ -72,7 +72,8 @@ class search(object):
def _cp_all(self):
iterators = []
for db in self._dbs:
- i = db.cp_all()
+ # MultiIterGroupBy requires sorted input
+ i = db.cp_all(sort=True)
try:
i = iter(i)
except TypeError:
diff --git a/pym/portage/dbapi/IndexedPortdb.py b/pym/portage/dbapi/IndexedPortdb.py
index e96f83c..510e027 100644
--- a/pym/portage/dbapi/IndexedPortdb.py
+++ b/pym/portage/dbapi/IndexedPortdb.py
@@ -127,14 +127,14 @@ class IndexedPortdb(object):
if new_cp is not None:
yield cp_group[0].cp
- def cp_all(self):
+ def cp_all(self, sort=True):
"""
Returns an ordered iterator instead of a list, so that search
results can be displayed incrementally.
"""
if self._cp_map is None:
return self._init_index()
- return iter(sorted(self._cp_map))
+ return iter(sorted(self._cp_map)) if sort else iter(self._cp_map)
def match(self, atom):
"""
diff --git a/pym/portage/dbapi/IndexedVardb.py b/pym/portage/dbapi/IndexedVardb.py
index 38bfeed..b946670 100644
--- a/pym/portage/dbapi/IndexedVardb.py
+++ b/pym/portage/dbapi/IndexedVardb.py
@@ -35,13 +35,13 @@ class IndexedVardb(object):
self._cp_map = None
- def cp_all(self):
+ def cp_all(self, sort=True):
"""
Returns an ordered iterator instead of a list, so that search
results can be displayed incrementally.
"""
if self._cp_map is not None:
- return iter(sorted(self._cp_map))
+ return iter(sorted(self._cp_map)) if sort else iter(self._cp_map)
delta_data = self._vardb._cache_delta.loadRace()
if delta_data is None:
@@ -62,7 +62,7 @@ class IndexedVardb(object):
cp_map[cpv.cp] = cp_list
cp_list.append(cpv)
- return iter(sorted(self._cp_map))
+ return iter(sorted(self._cp_map)) if sort else iter(self._cp_map)
def _iter_cp_all(self):
self._cp_map = cp_map = {}
diff --git a/pym/portage/dbapi/__init__.py b/pym/portage/dbapi/__init__.py
index 044faec..b6745fa 100644
--- a/pym/portage/dbapi/__init__.py
+++ b/pym/portage/dbapi/__init__.py
@@ -94,10 +94,10 @@ class dbapi(object):
cpv_list.extend(self.cp_list(cp))
return cpv_list
- def cp_all(self):
+ def cp_all(self, sort=False):
""" Implement this in a child class
Args
- None
+ sort - return sorted results
Returns:
A list of strings 1 per CP in the datastore
"""
diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index f415a63..9f47436 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -223,10 +223,10 @@ class bindbapi(fakedbapi):
self.bintree.populate()
return fakedbapi.cp_list(self, *pargs, **kwargs)
- def cp_all(self):
+ def cp_all(self, sort=False):
if not self.bintree.populated:
self.bintree.populate()
- return fakedbapi.cp_all(self)
+ return fakedbapi.cp_all(self, sort=sort)
def cpv_all(self):
if not self.bintree.populated:
diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index 590e3c5..d13fdee 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -706,7 +706,7 @@ class portdbapi(dbapi):
else:
return 0
- def cp_all(self, categories=None, trees=None, reverse=False):
+ def cp_all(self, categories=None, trees=None, reverse=False, sort=True):
"""
This returns a list of all keys in our tree or trees
@param categories: optional list of categories to search or
@@ -714,6 +714,7 @@ class portdbapi(dbapi):
@param trees: optional list of trees to search the categories in or
defaults to self.porttrees
@param reverse: reverse sort order (default is False)
+ @param sort: return sorted results (default is True)
@rtype list of [cat/pkg,...]
"""
d = {}
@@ -732,7 +733,8 @@ class portdbapi(dbapi):
continue
d[atom.cp] = None
l = list(d)
- l.sort(reverse=reverse)
+ if sort:
+ l.sort(reverse=reverse)
return l
def cp_list(self, mycp, use_cache=1, mytree=None):
diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 62d880e..927c645 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -491,7 +491,7 @@ class vardbapi(dbapi):
yield subpath
- def cp_all(self, use_cache=1):
+ def cp_all(self, use_cache=1, sort=False):
mylist = self.cpv_all(use_cache=use_cache)
d={}
for y in mylist:
@@ -506,7 +506,7 @@ class vardbapi(dbapi):
self.invalidentry(self.getpath(y))
continue
d[mysplit[0]+"/"+mysplit[1]] = None
- return list(d)
+ return sorted(d) if sort else list(d)
def checkblockers(self, origdep):
pass
diff --git a/pym/portage/dbapi/virtual.py b/pym/portage/dbapi/virtual.py
index 3b7d10e..a59c3b5 100644
--- a/pym/portage/dbapi/virtual.py
+++ b/pym/portage/dbapi/virtual.py
@@ -115,8 +115,8 @@ class fakedbapi(dbapi):
self._match_cache[cache_key] = cpv_list
return cpv_list[:]
- def cp_all(self):
- return list(self.cpdict)
+ def cp_all(self, sort=False):
+ return sorted(self.cpdict) if sort else list(self.cpdict)
def cpv_all(self):
if self._multi_instance:
^ permalink raw reply related [flat|nested] 29+ messages in thread
* [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/
@ 2017-04-03 20:03 Zac Medico
0 siblings, 0 replies; 29+ messages in thread
From: Zac Medico @ 2017-04-03 20:03 UTC (permalink / raw
To: gentoo-commits
commit: f479250c9cb9d82af4d621aa008d4d1e37a28a39
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Apr 2 00:16:53 2017 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Apr 3 20:01:57 2017 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=f479250c
emerge: fix --autounmask-continue to work with --getbinpkg (bug 614474)
Fix action_build to populate binarytree instances with remote package
metadata following config reload, using a new getbinpkg_refresh=False
parameter to prevent redundant fetching of remote package metadata.
X-Gentoo-bug: 614474
X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=614474
Fixes: e2d88ef3ff59 ("Add emerge --autounmask-continue option (bug 582624)")
Acked-by: Brian Dolbec <dolsen <AT> gentoo.org>
pym/_emerge/actions.py | 15 +++++++++++++++
pym/portage/dbapi/bintree.py | 19 +++++++++++++++----
2 files changed, 30 insertions(+), 4 deletions(-)
diff --git a/pym/_emerge/actions.py b/pym/_emerge/actions.py
index cc0269d5d..818fab90a 100644
--- a/pym/_emerge/actions.py
+++ b/pym/_emerge/actions.py
@@ -347,6 +347,21 @@ def action_build(emerge_config, trees=DeprecationWarning,
adjust_configs(emerge_config.opts, emerge_config.trees)
settings, trees, mtimedb = emerge_config
+ # After config reload, the freshly instantiated binarytree
+ # instances need to load remote metadata if --getbinpkg
+ # is enabled. Use getbinpkg_refresh=False to use cached
+ # metadata, since the cache is already fresh.
+ if "--getbinpkg" in emerge_config.opts:
+ for root_trees in emerge_config.trees.values():
+ try:
+ root_trees["bintree"].populate(
+ getbinpkgs=True,
+ getbinpkg_refresh=False)
+ except ParseError as e:
+ writemsg("\n\n!!!%s.\nSee make.conf(5) for more info.\n"
+ % e, noiselevel=-1)
+ return 1
+
if "--autounmask-only" in myopts:
mydepgraph.display_problems()
return 0
diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 12c3d3e51..ca90ba8f9 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -510,8 +510,16 @@ class binarytree(object):
except PortageException:
pass
- def populate(self, getbinpkgs=0):
- "populates the binarytree"
+ def populate(self, getbinpkgs=False, getbinpkg_refresh=True):
+ """
+ Populates the binarytree with package metadata.
+
+ @param getbinpkgs: include remote packages
+ @type getbinpkgs: bool
+ @param getbinpkg_refresh: attempt to refresh the cache
+ of remote package metadata if getbinpkgs is also True
+ @type getbinpkg_refresh: bool
+ """
if self._populating:
return
@@ -522,13 +530,13 @@ class binarytree(object):
pkgindex_lock = lockfile(self._pkgindex_file,
wantnewlockfile=1)
self._populating = True
- self._populate(getbinpkgs)
+ self._populate(getbinpkgs, getbinpkg_refresh=getbinpkg_refresh)
finally:
if pkgindex_lock:
unlockfile(pkgindex_lock)
self._populating = False
- def _populate(self, getbinpkgs=0):
+ def _populate(self, getbinpkgs=False, getbinpkg_refresh=True):
if (not os.path.isdir(self.pkgdir) and not getbinpkgs):
return 0
@@ -832,6 +840,9 @@ class binarytree(object):
url = base_url.rstrip("/") + "/Packages"
f = None
+ if not getbinpkg_refresh and local_timestamp:
+ raise UseCachedCopyOfRemoteIndex()
+
try:
ttl = float(pkgindex.header.get("TTL", 0))
except ValueError:
^ permalink raw reply related [flat|nested] 29+ messages in thread
end of thread, other threads:[~2017-04-03 20:03 UTC | newest]
Thread overview: 29+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2011-05-24 0:33 [gentoo-commits] proj/portage:master commit in: pym/portage/dbapi/, pym/_emerge/ Zac Medico
-- strict thread matches above, loose matches on Subject: below --
2017-04-03 20:03 Zac Medico
2015-08-30 23:44 Zac Medico
2014-11-20 4:08 Zac Medico
2014-02-24 9:23 Alexander Berntsen
2013-03-13 5:56 Zac Medico
2013-01-23 16:19 Zac Medico
2013-01-05 15:20 Zac Medico
2012-12-23 5:33 Arfrever Frehtes Taifersar Arahesis
2012-11-24 21:57 Zac Medico
2012-04-22 18:57 Zac Medico
2012-04-21 6:51 Zac Medico
2012-02-17 22:19 Zac Medico
2012-02-16 0:33 Zac Medico
2012-02-14 23:40 Zac Medico
2012-02-10 1:28 Zac Medico
2012-02-08 1:16 Zac Medico
2012-02-08 0:36 Zac Medico
2011-10-28 2:34 Zac Medico
2011-10-16 20:26 Zac Medico
2011-10-15 5:10 Zac Medico
2011-06-03 10:16 Zac Medico
2011-05-24 5:31 Zac Medico
2011-05-09 5:16 Zac Medico
2011-03-26 7:39 Zac Medico
2011-03-26 3:24 Zac Medico
2011-03-25 4:34 Zac Medico
2011-03-25 4:34 Zac Medico
2011-03-25 4:34 Zac Medico
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox