From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from pigeon.gentoo.org ([208.92.234.80] helo=lists.gentoo.org) by finch.gentoo.org with esmtp (Exim 4.60) (envelope-from ) id 1SkOd2-0007iE-3M for garchives@archives.gentoo.org; Fri, 29 Jun 2012 00:00:04 +0000 Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id E5680E058F; Thu, 28 Jun 2012 23:59:56 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) by pigeon.gentoo.org (Postfix) with ESMTP id A42D8E058F for ; Thu, 28 Jun 2012 23:59:56 +0000 (UTC) Received: from hornbill.gentoo.org (hornbill.gentoo.org [94.100.119.163]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id AC6811B4084 for ; Thu, 28 Jun 2012 23:59:55 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by hornbill.gentoo.org (Postfix) with ESMTP id 4856EE543A for ; Thu, 28 Jun 2012 23:59:53 +0000 (UTC) From: "Slava Bacherikov" To: gentoo-commits@lists.gentoo.org Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Slava Bacherikov" Message-ID: <1340916315.ecd1dd1c2c3cc2a556bd7dd7d1367eab669bd993.bacher09@gentoo> Subject: [gentoo-commits] proj/gentoo-packages:master commit in: gpackages/apps/packages/ X-VCS-Repository: proj/gentoo-packages X-VCS-Files: gpackages/apps/packages/managers.py gpackages/apps/packages/views.py X-VCS-Directories: gpackages/apps/packages/ X-VCS-Committer: bacher09 X-VCS-Committer-Name: Slava Bacherikov X-VCS-Revision: ecd1dd1c2c3cc2a556bd7dd7d1367eab669bd993 X-VCS-Branch: master Date: Thu, 28 Jun 2012 23:59:53 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: quoted-printable X-Archives-Salt: 90839471-6f94-4d55-bd42-67b47c33480b X-Archives-Hash: 3f5b47ff73c932363bafcb3856e47ee6 commit: ecd1dd1c2c3cc2a556bd7dd7d1367eab669bd993 Author: Slava Bacherikov bacher09 org> AuthorDate: Thu Jun 28 20:20:03 2012 +0000 Commit: Slava Bacherikov bacherikov org ua> CommitDate: Thu Jun 28 20:45:15 2012 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=3Dproj/gentoo-packag= es.git;a=3Dcommit;h=3Decd1dd1c Use prefetcher for keywords prefetch in ebuilds --- gpackages/apps/packages/managers.py | 81 ++++++++++++-----------------= ------ gpackages/apps/packages/views.py | 2 +- 2 files changed, 29 insertions(+), 54 deletions(-) diff --git a/gpackages/apps/packages/managers.py b/gpackages/apps/package= s/managers.py index cfe0166..c1fd5e9 100644 --- a/gpackages/apps/packages/managers.py +++ b/gpackages/apps/packages/managers.py @@ -30,7 +30,9 @@ class EbuildsWithKeywrods(Prefetcher): self.keywords =3D keywords =20 def filter(self, ids): - return packages.models.EbuildModel.objects.filter(package__in =3D= ids).order_by('-version', '-revision').prefetch_with_keywords(self.keywo= rds) + return packages.models.EbuildModel.objects. \ + filter(package__in =3D ids).order_by('-version', '-revision'= ). \ + prefetch_keywords(self.keywords) =20 def reverse_mapper(self, ebuild): return [ebuild.package_id] @@ -38,6 +40,21 @@ class EbuildsWithKeywrods(Prefetcher): def decorator(self, package, ebuilds =3D ()): setattr(package, 'ebuilds', ebuilds) =20 +class KeywordsPrefetch(Prefetcher): + def __init__(self, arches): + self.arches =3D arches + + def filter(self, ids): + return packages.models.Keyword.objects. \ + filter(ebuild__in =3D ids, arch__name__in =3D self.arches). = \ + select_related('arch') + + def reverse_mapper(self, keyword): + return [keyword.ebuild_id] + + def decorator(self, ebuild, keywords =3D ()): + setattr(ebuild, '_prefetched_keywords', keywords) + =20 class PackageMixin(object): def get(self, package =3D None, *args, **kwargs): @@ -110,59 +127,17 @@ class EbuildMixin(object): def all_by_numbers(self): return super(EbuildMixin, self).order_by('version', 'revision') =20 -class EbuildQuerySet(EbuildMixin, models.query.QuerySet): - - def __init__(self, *args, **kwargs): - super(EbuildQuerySet, self).__init__(*args, **kwargs) - self._arches_set =3D None - self._cache_keywords =3D None - =20 - # Maybe use https://github.com/ionelmc/django-prefetch ? - def prefetch_with_keywords(self, arch_list): - arch_set =3D set(arch_list) - arch_set.add('*') - self._arches_set =3D arch_set - return self - - def __old_iter__(self): - return super(EbuildQuerySet, self).__iter__() - - def _clone(self, *args, **kwargs): - c =3D super(EbuildQuerySet, self)._clone(*args, **kwargs) - c._arches_set =3D self._arches_set - return c - - def _prefetch_keywords(self): - arch_set =3D self._arches_set - pk_keys =3D [ebuild.pk for ebuild in self.__old_iter__()] - query =3D packages.models.Keyword.objects.\ - filter(ebuild__in =3D pk_keys, arch__name__in =3D arch_set).= \ - select_related('arch') +class EbuildQuerySet(EbuildMixin, PrefetchQuerySet): + + def prefetch_keywords(self, arch_list): + return self.prefetch(P('keywords', arches =3D arch_list)) =20 - cache_query =3D defaultdict(list) - for keyword in query: - cache_query[keyword.ebuild_id].append(keyword) - - self._cache_keywords =3D cache_query - - def __iter__(self): - arch_set =3D self._arches_set - if arch_set is None: - for ebuild in self.__old_iter__(): - yield ebuild - else: - if self._cache_keywords is None: - self._prefetch_keywords() - - cache_query =3D self._cache_keywords - =20 - for ebuild in self.__old_iter__(): - ebuild._prefetched_keywords =3D cache_query[ebuild.pk] - yield ebuild - -class EbuildManager(EbuildMixin, models.Manager): - def get_query_set(self): - return EbuildQuerySet(self.model, using =3D self._db) +class EbuildManager(EbuildMixin, PrefetchManagerMixin): + prefetch_definitions =3D {'keywords': KeywordsPrefetch} + + @classmethod + def get_query_set_class(cls): + return EbuildQuerySet =20 class HerdsMixin(object):#{{{ def filter(self, *args, **kwargs): diff --git a/gpackages/apps/packages/views.py b/gpackages/apps/packages/v= iews.py index 21fa07c..4639555 100644 --- a/gpackages/apps/packages/views.py +++ b/gpackages/apps/packages/views.py @@ -43,7 +43,7 @@ class EbuildsListView(ContextListView): queryset =3D EbuildModel.objects.all(). \ select_related('package', 'package__virtual_package', - 'package__virtual_package__category').prefetch_wi= th_keywords(arches) + 'package__virtual_package__category').prefetch_ke= ywords(arches) =20 class PackagesListsView(ContextListView): arches =3D ['alpha', 'amd64', 'arm', 'hppa', 'ia64', 'ppc', 'ppc64',= 'sparc', 'x86']