From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from pigeon.gentoo.org ([208.92.234.80] helo=lists.gentoo.org) by finch.gentoo.org with esmtp (Exim 4.60) (envelope-from ) id 1SdQGz-0008A5-8Q for garchives@archives.gentoo.org; Sat, 09 Jun 2012 18:20:31 +0000 Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id A9FB3E0539; Sat, 9 Jun 2012 18:19:51 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) by pigeon.gentoo.org (Postfix) with ESMTP id 689A7E0539 for ; Sat, 9 Jun 2012 18:19:51 +0000 (UTC) Received: from hornbill.gentoo.org (hornbill.gentoo.org [94.100.119.163]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 7D19E1B4065 for ; Sat, 9 Jun 2012 18:19:50 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by hornbill.gentoo.org (Postfix) with ESMTP id 73E93E543E for ; Sat, 9 Jun 2012 18:19:48 +0000 (UTC) From: "Slava Bacherikov" To: gentoo-commits@lists.gentoo.org Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Slava Bacherikov" Message-ID: <1339265860.e0b3afa49d0b5b1e99c92e5d766a1a443dc5aa77.bacher09@gentoo> Subject: [gentoo-commits] proj/gentoo-packages:master commit in: gpackages/apps/packages/, gpackages/apps/packages/management/commands/ X-VCS-Repository: proj/gentoo-packages X-VCS-Files: gpackages/apps/packages/management/commands/scanpackages.py gpackages/apps/packages/models.py gpackages/apps/packages/scan.py X-VCS-Directories: gpackages/apps/packages/ gpackages/apps/packages/management/commands/ X-VCS-Committer: bacher09 X-VCS-Committer-Name: Slava Bacherikov X-VCS-Revision: e0b3afa49d0b5b1e99c92e5d766a1a443dc5aa77 X-VCS-Branch: master Date: Sat, 9 Jun 2012 18:19:48 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: quoted-printable X-Archives-Salt: 8d1c5a50-a024-41cb-9897-369000dbdd98 X-Archives-Hash: a5823cb2f7942e777c3bc05fa9c86d55 commit: e0b3afa49d0b5b1e99c92e5d766a1a443dc5aa77 Author: Slava Bacherikov bacher09 org> AuthorDate: Sat Jun 9 18:17:40 2012 +0000 Commit: Slava Bacherikov bacherikov org ua> CommitDate: Sat Jun 9 18:17:40 2012 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=3Dproj/gentoo-packag= es.git;a=3Dcommit;h=3De0b3afa4 Add support verbosity levels in scan --- .../packages/management/commands/scanpackages.py | 3 +- gpackages/apps/packages/models.py | 6 +- gpackages/apps/packages/scan.py | 47 ++++++++++++++= ++---- 3 files changed, 43 insertions(+), 13 deletions(-) diff --git a/gpackages/apps/packages/management/commands/scanpackages.py = b/gpackages/apps/packages/management/commands/scanpackages.py index 3f5010c..8f47848 100644 --- a/gpackages/apps/packages/management/commands/scanpackages.py +++ b/gpackages/apps/packages/management/commands/scanpackages.py @@ -12,9 +12,10 @@ class Command(BaseCommand): args =3D '' help =3D 'Will scan package tree and update info about it in databas= e' def handle(self, *args, **options): + verbosity =3D int(options['verbosity']) st =3D datetime.datetime.now() #scan.scanpackages() #scan.scan_all_repos() - Scanner().scan_all_repos() + Scanner(verbosity =3D verbosity).scan_all_repos() self.stdout.write(unicode((datetime.datetime.now() - st).total_s= econds())) self.stdout.write("\n") diff --git a/gpackages/apps/packages/models.py b/gpackages/apps/packages/= models.py index 713cb70..3486399 100644 --- a/gpackages/apps/packages/models.py +++ b/gpackages/apps/packages/models.py @@ -36,7 +36,7 @@ class CategoryModel(models.Model): category =3D models.CharField(unique =3D True, max_length =3D 70) =20 def __unicode__(self): - return self.category + return unicode(self.category) =20 class MaintainerModel(AbstractDateTimeModel): =20 @@ -126,11 +126,11 @@ class PackageModel(AbstractDateTimeModel): objects =3D managers.PackageManager() =20 def __unicode__(self): - return self.cp + return unicode(self.cp) =20 @property def cp(self): - return '%s/%s' % (self.category, self.name) + return "%s/%s" % (unicode(self.category), self.name) =20 def init_by_package(self, package, category =3D None): self.name =3D package.name diff --git a/gpackages/apps/packages/scan.py b/gpackages/apps/packages/sc= an.py index d0f476e..a7a77b9 100644 --- a/gpackages/apps/packages/scan.py +++ b/gpackages/apps/packages/scan.py @@ -1,4 +1,5 @@ from packages import models +import sys from django.db import IntegrityError from collections import defaultdict from generic import StrThatIgnoreCase @@ -87,7 +88,7 @@ def _get_items(items_list, Model, field_name, cache_var= ): =20 =20 class Scanner(object): - def __init__(self, verbosity =3D 0): + def __init__(self, verbosity =3D 1): # maintainers_cache: maintainer.email as key, and maintainer obj= ect as # value self.maintainers_cache =3D {} @@ -105,7 +106,16 @@ class Scanner(object): =20 self.arches_cache =3D {} =20 - self.verbosity =3D verbosity + self.verbosity =3D int(verbosity) + + def write(self, what, verbosity =3D 1): + if verbosity <=3D self.verbosity: + sys.stdout.write(what) + + def output(self, format_str, whats, verbosity =3D 1): + # Maybe implement lazy format string ? + if verbosity <=3D self.verbosity: + sys.stdout.write(format_str % whats) =20 def get_existent_maintainers(self): return models.MaintainerModel.objects.all() @@ -148,11 +158,15 @@ class Scanner(object): # updating maintainer_object.update_by_maintainer(maintainer_cm= p) maintainer_object.save(force_update =3D True) + + # Show info if need + self.output("update maintainer '%s'\n", maintainer_o= bject, 2) mo_dict[maintainer_object.email] =3D maintainer_object =20 to_create =3D [] for maintainer in maintainers_dict.iterkeys(): if maintainer.email not in mo_dict: + self.output("create maintainer '%s'\n", maintainer, 2) to_create.append(maintainer) =20 mobjects =3D _create_objects(models.MaintainerModel, 'maintainer= ', to_create) @@ -170,6 +184,7 @@ class Scanner(object): for herd_object in existent_herds: if herd_object.name not in herds_dict: to_del.append(herd_object.pk) + self.output("herd to del '%s'\n", herd_object, 2) else: herd_cmp =3D herds_dict[herd_object.name] # need update ? @@ -177,6 +192,7 @@ class Scanner(object): # updating=20 herd_object.update_by_herd(herd_cmp) herd_object.save(force_update =3D True) + self.output("update herd '%s'\n", herd_object, 2) herds_objects_dict[herd_object.name] =3D herd_object =20 models.HerdsModel.objects.filter(pk__in =3D to_del).delete() @@ -185,6 +201,7 @@ class Scanner(object): for herd in herds_dict.itervalues(): if herd.name not in herds_objects_dict: to_create.append(herd) + self.output("create herd '%s'\n", herd, 2) =20 cobjects =3D _create_objects(models.HerdsModel, 'herd', to_creat= e) _update_cache_by_queryset(herds_objects_dict, cobjects, 'name') @@ -213,6 +230,9 @@ class Scanner(object): herd_object.maintainers.clear() herd_object.maintainers.add(*res[herd_name]) =20 + self.output("add maintainers '%s' to mantainer '%s'\n",=20 + (res[herd_name], herd_object), 3) + def get_maintainers(self): if not self.maintainers_cache_loaded: self.load_maintainers_to_cache() @@ -221,8 +241,12 @@ class Scanner(object): def scan_all_repos(self, scan_herds =3D True): #cache_dict =3D anydbm.open('cache.db','c') if scan_herds: + self.write('Scaning herds\n', 3) self.scan_herds() + for repo in portage.iter_trees(): + self.output("Scaning repository '%s'\n", repo.name, 3) + repo_obj, repo_created =3D models.RepositoryModel.objects.ge= t_or_create(name =3D repo.name) self.scanpackages(repo, repo_obj) #cache_dict.close() @@ -265,7 +289,8 @@ class Scanner(object): if arch_name in self.arches_cache: arch =3D self.arches_cache[arch_name] else: - arch, created =3D models.ArchesModel.objects.get_or_create(n= ame =3D arch_name) + arch, created =3D models.ArchesModel.objects \ + .get_or_create(name =3D arch_name) self.arches_cache[arch_name] =3D arch return arch =20 @@ -309,6 +334,8 @@ class Scanner(object): ebuild_object.save(force_insert=3DTrue) self.add_related_to_ebuild(ebuild, ebuild_object) =20 + self.output("ebuild created '%s'\n", ebuild_object, 3) + def clear_related_to_package(self, package_object): package_object.herds.clear() package_object.maintainers.clear() @@ -325,15 +352,20 @@ class Scanner(object): def update_ebuilds(self, package, package_object, delete =3D True): not_del =3D [] for ebuild in package.iter_ebuilds(): - ebuild_object, ebuild_created =3D models.EbuildModel.objects= .get_or_create(ebuild =3D ebuild, package =3D package_object) + ebuild_object, ebuild_created =3D models.EbuildModel.objects= \ + .get_or_create(ebuild =3D ebuild, package =3D package_ob= ject) + not_del.append(ebuild_object.pk) if ebuild_created: self.add_related_to_ebuild(ebuild, ebuild_object) - continue + self.output("ebuild created '%s'\n", ebuild_object, 3) + if ebuild_object.check_or_need_update(ebuild): ebuild_object.update_by_ebuild(ebuild) self.update_related_to_ebuild(ebuild, ebuild_object) ebuild_object.save(force_update =3D True) + + self.output("ebuild updated '%s'\n", ebuild_object, 3) if delete: models.EbuildModel.objects.filter(package =3D package_object= ).exclude(pk__in =3D not_del).delete() =20 @@ -351,9 +383,6 @@ class Scanner(object): def scanpackages(self, porttree, porttree_obj, delete =3D True, force_update =3D False, update_cache =3D True, use_= cache =3D True): =20 - # Load homepages to cache - #for homepage in models.HomepageModel.objects.all(): - #homepages_cache[homepage.url] =3D homepage existend_categorys =3D [] for category in porttree.iter_categories(): existend_packages =3D [] @@ -367,7 +396,7 @@ class Scanner(object): #val =3D cache_dict[key] #if val is not None and val =3D=3D package.manifest_= sha1: #continue - print('%-44s [%s]' % (package, porttree)) + self.output('%-44s [%s]\n', (package, porttree)) package_object, package_created =3D models.PackageModel.= objects.only('changelog_hash', 'manifest_hash', 'metadata_hash') \ .get_or_create(package =3D package, category= =3D category_object, repository =3D porttree_obj) #if update_cache: