From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id C84F4138E35 for ; Tue, 18 Feb 2014 21:36:08 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id AA5F8E09D7; Tue, 18 Feb 2014 21:36:07 +0000 (UTC) Received: from mail-vc0-f171.google.com (mail-vc0-f171.google.com [209.85.220.171]) (using TLSv1 with cipher ECDHE-RSA-RC4-SHA (128/128 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 067AFE099F for ; Tue, 18 Feb 2014 21:36:06 +0000 (UTC) Received: by mail-vc0-f171.google.com with SMTP id le5so13976411vcb.30 for ; Tue, 18 Feb 2014 13:36:06 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20120113; h=mime-version:in-reply-to:references:from:date:message-id:subject:to :content-type; bh=eVVXzX0nnXzMj7rg7K8PJXU5VRJwBpb6erZgGgUb29s=; b=m7XnwgAxxYrgQe35ukya814F3x/QqJ58gZb6ha/ol7w0lqOP0lBgBwDlThiiPFaDvQ XLzG4mqjZwbuYJTyTHpW7PeX8oL339OQJApcCY+yRPd9Nr9VQ956Awz0bCY80uJRnZyO nlKAYeMjOoJRPfGIPIuZQ8mzE0HQQY0cHphLghf8B4WcNiIvOqZ+xZmdzeUoquAxhSQE bCLCX8D+41iADpdnqekJuaZ9SgeOsFTYOBoi7fMiq48Mynt1vS4Aebsn+W9NPOe08mdy cJhnchxIvhdK0+EZbevbKWxrATOUdkAIpIn9zJsd2M5Ep+WtzTNDI+F5Cd2/RH0bjuQb vNLw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20130820; h=x-gm-message-state:mime-version:in-reply-to:references:from:date :message-id:subject:to:content-type; bh=eVVXzX0nnXzMj7rg7K8PJXU5VRJwBpb6erZgGgUb29s=; b=X3f1YnHwpc3e/xU2sjdFIWKFyrqFuRRipVBp2zpxopjZGFgm/Gsc7Y7Dk1jNQ7y2c1 tZm7pA834xFpnmLD2yHH5mbbWE7HFgqDo7g7ru3R4o1eMwxCdas8SzBbOsA5wWRXriHC 2mNp6hPXpDBugKgf/1c+eJLXDAuOS3YgFknTULBc3sgPtaXICccM7bJ67uVAFXElJm6k E2+Y64JpIGW4RaqZ9kZ984Brfkao9CRYrKlL7Dr31ttGuRdEXD8Cn6qD1KmB/MaUYpP2 3UYiGd+7tvzNYPuQ/cTSh6Tf65y4/I3jufvRHQxC5Dhunw4lNCs5iPlIbhewknU47JTW 0waw== X-Gm-Message-State: ALoCoQk35F9x6fSfureSe3u+bLICM7O51qPc0xfWpJo8Tj9/faheihDpn2cOl8ccrbl8xxgmsMtMTD+Z+y/0w3NYaykyLEqUU1Y2LQocILzPLtIvRKT9mmQFaUb5jqRHKG9wVfdmKd9NWIz2NowqYHFySpjGtP+sebSEnhjgpUaTq5oe1hnYxSot2+K4HaL+APvY7wfmR2Mj0S67F+s2K9Fu4VfkG67LtA== X-Received: by 10.220.188.10 with SMTP id cy10mr7025305vcb.36.1392759366096; Tue, 18 Feb 2014 13:36:06 -0800 (PST) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-portage-dev@lists.gentoo.org Reply-to: gentoo-portage-dev@lists.gentoo.org MIME-Version: 1.0 Received: by 10.52.249.17 with HTTP; Tue, 18 Feb 2014 13:35:46 -0800 (PST) In-Reply-To: <1392757337-26882-2-git-send-email-SebastianLuther@gmx.de> References: <1392757337-26882-1-git-send-email-SebastianLuther@gmx.de> <1392757337-26882-2-git-send-email-SebastianLuther@gmx.de> From: David James Date: Tue, 18 Feb 2014 13:35:46 -0800 Message-ID: Subject: Re: [gentoo-portage-dev] [PATCH] portdbapi: Add cache to avoid repeated failing os.access calls To: gentoo-portage-dev@lists.gentoo.org Content-Type: multipart/alternative; boundary=047d7bfea07c46312b04f2b50f60 X-Archives-Salt: 06a9275e-ea86-445d-b3e7-d3741d90a8a2 X-Archives-Hash: ae43ba1d78b943ae397d23801e1bf574 --047d7bfea07c46312b04f2b50f60 Content-Type: text/plain; charset=UTF-8 On Tue, Feb 18, 2014 at 1:02 PM, Sebastian Luther wrote: > --- > pym/portage/dbapi/porttree.py | 13 +++++++++++++ > 1 file changed, 13 insertions(+) > > diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py > index 590e3c5..2dc406b 100644 > --- a/pym/portage/dbapi/porttree.py > +++ b/pym/portage/dbapi/porttree.py > @@ -254,6 +254,7 @@ class portdbapi(dbapi): > > self._aux_cache = {} > self._broken_ebuilds = set() > + self._findname2_cache = {} > > @property > def _event_loop(self): > @@ -372,6 +373,14 @@ class portdbapi(dbapi): > the file we wanted. > If myrepo is not None it will find packages from this > repository(overlay) > """ > + cache_key = (mycpv, mytree, myrepo) > + try: > + return self._findname2_cache[cache_key] > + except KeyError: > + self._findname2_cache[cache_key] = (None, 0) > To me, it seems potentially error-prone to cache a (potentially) incorrect value and then correct it later. Would it be possible to refactor your patch so that we only cache the value when we know the final answer? > + except TypeError: > In what circumstances does it happen that mytree / myrepo are unhashable types? Can you add a comment to explain this? + cache_key = None > + > if not mycpv: > return (None, 0) > > @@ -383,6 +392,8 @@ class portdbapi(dbapi): > mysplit = mycpv.split("/") > psplit = pkgsplit(mysplit[1]) > if psplit is None or len(mysplit) != 2: > + if cache_key: > + del self._findname2_cache[cache_key] raise InvalidPackageName(mycpv) > > # For optimal performace in this hot spot, we do manual > unicode > @@ -402,6 +413,8 @@ class portdbapi(dbapi): > filename = x + _os.sep + relative_path > if _os.access(_unicode_encode(filename, > encoding=encoding, errors=errors), > _os.R_OK): > + if cache_key: > + self._findname2_cache[cache_key] = > (filename, x) > return (filename, x) > return (None, 0) > > -- > 1.8.3.2 > > > --047d7bfea07c46312b04f2b50f60 Content-Type: text/html; charset=UTF-8 Content-Transfer-Encoding: quoted-printable
On T= ue, Feb 18, 2014 at 1:02 PM, Sebastian Luther <SebastianLuther@gmx.de= > wrote:
---
=C2=A0pym/portage/dbapi/porttree.py | 13 +++++++++++++
=C2=A01 file changed, 13 insertions(+)

diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py<= br> index 590e3c5..2dc406b 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -254,6 +254,7 @@ class portdbapi(dbapi):

=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self._aux_cache =3D= {}
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self._broken_ebuild= s =3D set()
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self._findname2_cache = =3D {}

=C2=A0 =C2=A0 =C2=A0 =C2=A0 @property
=C2=A0 =C2=A0 =C2=A0 =C2=A0 def _event_loop(self):
@@ -372,6 +373,14 @@ class portdbapi(dbapi):
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 the file we wanted.=
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 If myrepo is not No= ne it will find packages from this repository(overlay)
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 """<= br> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 cache_key =3D (mycpv, my= tree, myrepo)
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 try:
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 return self._findname2_cache[cache_key]
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 except KeyError:
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 self._findname2_cache[cache_key] =3D (None, 0)
To me, it seems potentially error-prone to cache a (potentially= ) incorrect value and then correct it later. Would it be possible to refact= or your patch so that we only cache the value when we know the final answer= ?
=C2=A0
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 except TypeError:

In what circumstances does it happen that myt= ree / myrepo are unhashable types? Can you add a comment to explain this?

+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 cache_key =3D None
+
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 if not mycpv:
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 return (None, 0)

@@ -383,6 +392,8 @@ class portdbapi(dbapi):
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 mysplit =3D mycpv.s= plit("/")
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 psplit =3D pkgsplit= (mysplit[1])
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 if psplit is None o= r len(mysplit) !=3D 2:
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 if cache_key:
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 del self._findname2_cache[cache_key]=C2=A0<= /blockquote>
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 raise InvalidPackageName(mycpv)

=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 # For optimal perfo= rmace in this hot spot, we do manual unicode
@@ -402,6 +413,8 @@ class portdbapi(dbapi):
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 filename =3D x + _os.sep + relative_path
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 if _os.access(_unicode_encode(filename,
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 encoding=3Dencoding, errors=3Derrors= ), _os.R_OK):
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 if cache_key:
+ =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self._findname2= _cache[cache_key] =3D (filename, x)
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 return (filename, x)
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 return (None, 0)
--
1.8.3.2



--047d7bfea07c46312b04f2b50f60--