From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from pigeon.gentoo.org ([208.92.234.80] helo=lists.gentoo.org) by finch.gentoo.org with esmtp (Exim 4.60) (envelope-from ) id 1SdQGy-0008A2-EB for garchives@archives.gentoo.org; Sat, 09 Jun 2012 18:20:31 +0000 Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 8174CE055B; Sat, 9 Jun 2012 18:19:51 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) by pigeon.gentoo.org (Postfix) with ESMTP id 2D1E4E0531 for ; Sat, 9 Jun 2012 18:19:50 +0000 (UTC) Received: from hornbill.gentoo.org (hornbill.gentoo.org [94.100.119.163]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 370EC1B407D for ; Sat, 9 Jun 2012 18:19:50 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by hornbill.gentoo.org (Postfix) with ESMTP id 08B43E543A for ; Sat, 9 Jun 2012 18:19:48 +0000 (UTC) From: "Slava Bacherikov" To: gentoo-commits@lists.gentoo.org Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Slava Bacherikov" Message-ID: <1339256724.d482308972446f7649ff7fbf446512160e57cfb2.bacher09@gentoo> Subject: [gentoo-commits] proj/gentoo-packages:master commit in: gpackages/apps/packages/, gpackages/apps/packages/management/commands/ X-VCS-Repository: proj/gentoo-packages X-VCS-Files: gpackages/apps/packages/management/commands/scanpackages.py gpackages/apps/packages/managers.py gpackages/apps/packages/models.py gpackages/apps/packages/scan.py X-VCS-Directories: gpackages/apps/packages/ gpackages/apps/packages/management/commands/ X-VCS-Committer: bacher09 X-VCS-Committer-Name: Slava Bacherikov X-VCS-Revision: d482308972446f7649ff7fbf446512160e57cfb2 X-VCS-Branch: master Date: Sat, 9 Jun 2012 18:19:48 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: quoted-printable X-Archives-Salt: e978460c-8632-41c6-aa1a-4edc65edd4cf X-Archives-Hash: d2a70789ebafcc872b5e04bce88e2038 commit: d482308972446f7649ff7fbf446512160e57cfb2 Author: Slava Bacherikov bacher09 org> AuthorDate: Sat Jun 9 15:45:24 2012 +0000 Commit: Slava Bacherikov bacherikov org ua> CommitDate: Sat Jun 9 15:45:24 2012 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=3Dproj/gentoo-packag= es.git;a=3Dcommit;h=3Dd4823089 Refactoring scan.py --- .../packages/management/commands/scanpackages.py | 9 +- gpackages/apps/packages/managers.py | 5 +- gpackages/apps/packages/models.py | 1 - gpackages/apps/packages/scan.py | 539 +++++++++++---= ------ 4 files changed, 309 insertions(+), 245 deletions(-) diff --git a/gpackages/apps/packages/management/commands/scanpackages.py = b/gpackages/apps/packages/management/commands/scanpackages.py index 35a117b..3f5010c 100644 --- a/gpackages/apps/packages/management/commands/scanpackages.py +++ b/gpackages/apps/packages/management/commands/scanpackages.py @@ -1,7 +1,8 @@ from django.core.management.base import BaseCommand, CommandError import datetime import logging -from packages import scan +from packages.scan import Scanner +from optparse import make_option #l =3D logging.getLogger('django.db.backends') #l.setLevel(logging.DEBUG) #l.addHandler(logging.FileHandler('database.log')) @@ -13,5 +14,7 @@ class Command(BaseCommand): def handle(self, *args, **options): st =3D datetime.datetime.now() #scan.scanpackages() - scan.scan_all_repos() - print (datetime.datetime.now() - st).total_seconds() + #scan.scan_all_repos() + Scanner().scan_all_repos() + self.stdout.write(unicode((datetime.datetime.now() - st).total_s= econds())) + self.stdout.write("\n") diff --git a/gpackages/apps/packages/managers.py b/gpackages/apps/package= s/managers.py index 7ebde50..5ef2ed6 100644 --- a/gpackages/apps/packages/managers.py +++ b/gpackages/apps/packages/managers.py @@ -80,9 +80,12 @@ class HerdsMixin(object):#{{{ =20 class MaintainerMixin(object):#{{{ def filter(self, *args, **kwargs): - maintainer__in =3D get_from_kwargs_and_del('maintainer__in', kwa= rgs) + maintainer__in, maintainer =3D \ + get_from_kwargs_and_del(['maintainer__in', 'maintainer'], kw= args) if maintainer__in is not None: kwargs['email__in'] =3D maintainer__in + elif maintainer is not None: + kwargs['email'] =3D maintainer.email return super(MaintainerMixin, self).filter(*args, **kwargs)#}}} =20 =20 diff --git a/gpackages/apps/packages/models.py b/gpackages/apps/packages/= models.py index 82d3ca5..dc2147c 100644 --- a/gpackages/apps/packages/models.py +++ b/gpackages/apps/packages/models.py @@ -55,7 +55,6 @@ class MaintainerModel(AbstractDateTimeModel): def init_by_maintainer(self, maintainer): self.name =3D maintainer.name self.email =3D maintainer.email - self.role =3D maintainer.role =20 def update_by_maintainer(self, maintainer): self.name =3D maintainer.name diff --git a/gpackages/apps/packages/scan.py b/gpackages/apps/packages/sc= an.py index c9645fc..db9c0e6 100644 --- a/gpackages/apps/packages/scan.py +++ b/gpackages/apps/packages/scan.py @@ -83,78 +83,315 @@ def _get_items(items_list, Model, field_name, cache_= var): return items_objects =20 =20 -def scan_maintainers(maintainers_dict): - existend_maintainers =3D models.MaintainerModel.objects.all() - main_dict =3D {} - mo_dict =3D {} - #to_del =3D [] - _update_cache_by_queryset(main_dict, maintainers_dict.keys(), 'email= ') - for maintainer_object in existend_maintainers: - if maintainer_object.email in main_dict: - maintainer_cmp =3D main_dict[maintainer_object.email] - # need update ? - if maintainer_object.check_or_need_update(maintainer_cmp): - # updating - maintainer_object.update_by_maintainer(maintainer_cmp) - maintainer_object.save(force_update =3D True) - mo_dict[maintainer_object.email] =3D maintainer_object - #else: - #to_del.append(maintainer_object.pk) - #print to_del - #print mo_dict - to_create =3D [] - for maintainer in maintainers_dict.iterkeys(): - if maintainer.email not in mo_dict: - to_create.append(maintainer) - - mobjects =3D _create_objects(models.MaintainerModel, 'maintainer', t= o_create) - _update_cache_by_queryset(mo_dict, mobjects, 'email') - - return mo_dict + +class Scanner(object): + def __init__(self, verbosity =3D 0): + # maintainers_cache: maintainer.email as key, and maintainer obj= ect as + # value + self.maintainers_cache =3D {} + self.maintainers_cache_loaded =3D False + # herds cache: herd.name as key and herd object as value + self.herds_cache =3D {} + self.herds_cache_loaded =3D False + self.licenses_cache =3D {} + self.uses_cache =3D {} + self.arches_cache =3D {} + self.homepages_cache =3D {} =20 + self.herds_cache =3D {} + self.maintainers_cache =3D {} + self.herds_object =3D herds.Herds() + + self.arches_cache =3D {} + + self.verbosity =3D verbosity + + def get_existent_maintainers(self): + return models.MaintainerModel.objects.all() + + def get_existent_herds(self): + return models.HerdsModel.objects.all() + + def get_maintainers_cache(self): + maintainers_dict =3D {} + existent_maintainers =3D self.get_existent_maintainers() + _update_cache_by_queryset(maintainers_dict, existent_maintainers= , 'email') + return maintainers_dict + + def load_maintainers_to_cache(self): + self.maintainers_cache =3D self.get_maintainers_cache() + self.maintainers_cache_loaded =3D True + =20 + def get_herds_cache(self): + herds_dict =3D {} + existent_herds =3D self.get_existent_herds() + _update_cache_by_queryset(herds_dict, existent_herds, 'name') + return herds_dict + + def load_herds_to_cache(self): + if not self.herds_cache_loaded: + self.herds_cache =3D self.get_herds_cache() + self.herds_cache_loaded =3D True + + def scan_maintainers(self, maintainers_dict): + existend_maintainers =3D self.get_existent_maintainers() + main_dict =3D {} + mo_dict =3D {} + #to_del =3D [] + _update_cache_by_queryset(main_dict, maintainers_dict.keys(), 'e= mail') + for maintainer_object in existend_maintainers: + if maintainer_object.email in main_dict: + maintainer_cmp =3D main_dict[maintainer_object.email] + # need update ? + if maintainer_object.check_or_need_update(maintainer_cmp= ): + # updating + maintainer_object.update_by_maintainer(maintainer_cm= p) + maintainer_object.save(force_update =3D True) + mo_dict[maintainer_object.email] =3D maintainer_object + + to_create =3D [] + for maintainer in maintainers_dict.iterkeys(): + if maintainer.email not in mo_dict: + to_create.append(maintainer) + + mobjects =3D _create_objects(models.MaintainerModel, 'maintainer= ', to_create) + _update_cache_by_queryset(mo_dict, mobjects, 'email') + + self.maintainers_cache =3D mo_dict + self.maitainers_cache_loaded =3D True =20 =20 -def scan_herds(): - existent_herds =3D models.HerdsModel.objects.all() - herds_object =3D herds.Herds() - herds_dict =3D herds_object.get_herds_indict() - maintainers_dict =3D herds_object.get_maintainers_with_herds() - ho_dict =3D {} - to_del =3D [] - for herd_object in existent_herds: - if herd_object.name not in herds_dict: - to_del.append(herd_object.pk) + def scan_herds(self): + existent_herds =3D self.get_existent_herds() + herds_dict =3D self.herds_object.get_herds_indict() + herds_objects_dict =3D {} + to_del =3D [] + for herd_object in existent_herds: + if herd_object.name not in herds_dict: + to_del.append(herd_object.pk) + else: + herd_cmp =3D herds_dict[herd_object.name] + # need update ? + if herd_object.check_or_need_update(herd_cmp): + # updating=20 + herd_object.update_by_herd(herd_cmp) + herd_object.save(force_update =3D True) + herds_objects_dict[herd_object.name] =3D herd_object + + models.HerdsModel.objects.filter(pk__in =3D to_del).delete() + + to_create =3D [] + for herd in herds_dict.itervalues(): + if herd.name not in herds_objects_dict: + to_create.append(herd) + + cobjects =3D _create_objects(models.HerdsModel, 'herd', to_creat= e) + _update_cache_by_queryset(herds_objects_dict, cobjects, 'name') + + # Add to cache + self.herds_cache =3D herds_objects_dict + self.herds_cache_loaded =3D True + # Add related maintainers to herds + self._add_related_to_herds() + + + def _get_maintainers_for_relation_with_herds(self, maintainers_dict)= : + res =3D defaultdict(list) + for mainteiner, herds_names in maintainers_dict.iteritems(): + for herd in herds_names: + res[herd].append(self.maintainers_cache[mainteiner.email]= ) + return res + + def _add_related_to_herds(self): + maintainers_dict =3D self.herds_object.get_maintainers_with_herd= s() + self.scan_maintainers(maintainers_dict) + #Gen data for relate with herds + res =3D self._get_maintainers_for_relation_with_herds(maintainer= s_dict) + + for herd_name, herd_object in self.herds_cache.iteritems(): + herd_object.maintainers.clear() + herd_object.maintainers.add(*res[herd_name]) + + def get_maintainers(self): + if not self.maintainers_cache_loaded: + self.load_maintainers_to_cache() + return self.maintainers_cache + + def scan_all_repos(self, scan_herds =3D True): + #cache_dict =3D anydbm.open('cache.db','c') + if scan_herds: + self.scan_herds() + for repo in portage.iter_trees(): + repo_obj, repo_created =3D models.RepositoryModel.objects.ge= t_or_create(name =3D repo.name) + self.scanpackages(repo, repo_obj) + #cache_dict.close() + + def get_licenses_objects(self, ebuild): + licenses =3D ebuild.licenses + return _get_items(licenses, models.LicensModel, 'name', self.lic= enses_cache) + + def get_uses_objects(self, ebuild): + uses =3D [ use.name for use in ebuild.iter_uses() ] + return _get_items(uses, models.UseFlagModel, 'name', self.uses_c= ache) + + def get_homepages_objects(self, ebuild): + homepages =3D ebuild.homepages + return _get_items(homepages, models.HomepageModel, 'url', self.h= omepages_cache) + + def get_maintainers_objects(self, package): + objects =3D [] + for maintainer in package.metadata.maintainers(): + if maintainer.email in self.maintainers_cache: + objects.append(self.maintainers_cache[maintainer.email]) + else: + maintainer_object, created =3D models.MaintainerModel \ + .objects.get_or_create(maintainer =3D maintainer= ) + objects.append(maintainer_object) + # Add to cache + self.maintainers_cache[maintainer_object.email] =3D main= tainer_object + return objects + + def get_herds_objects(self, package): + self.load_herds_to_cache() + herds_objects =3D [] + for herd in package.metadata.herds(): + if herd in self.herds_cache: + herds_objects.append(self.herds_cache[herd]) + + return herds_objects + + def get_arch_object(self, arch_name): + if arch_name in self.arches_cache: + arch =3D self.arches_cache[arch_name] else: - herd_cmp =3D herds_dict[herd_object.name] - # need update ? - if herd_object.check_or_need_update(herd_cmp): - # updating=20 - herd_object.update_by_herd(herd_cmp) - herd_object.save(force_update =3D True) - ho_dict[herd_object.name] =3D herd_object + arch, created =3D models.ArchesModel.objects.get_or_create(n= ame =3D arch_name) + self.arches_cache[arch_name] =3D arch + return arch + + def create_keywords_objects(self, ebuild, ebuild_object): + keywords_list =3D [] + for keyword in ebuild.get_keywords(): + keyword_object =3D models.Keyword(status =3D keyword.status, + ebuild =3D ebuild_object) + + keyword_object.arch =3D self.get_arch_object(keyword.arch) + keywords_list.append(keyword_object) + + models.Keyword.objects.bulk_create(keywords_list) + + def clean_keywords_object(self, ebuild_object): + models.Keyword.objects.filter(ebuild =3D ebuild_object).delete() =20 - models.HerdsModel.objects.filter(pk__in =3D to_del).delete() + def scanpackages(self, porttree, porttree_obj, delete =3D True, + force_update =3D False, update_cache =3D True, use_= cache =3D True): =20 - to_create =3D [] - for herd in herds_dict.itervalues(): - if herd.name not in ho_dict: - to_create.append(herd) + =20 =20 - cobjects =3D _create_objects(models.HerdsModel, 'herd', to_create) - _update_cache_by_queryset(ho_dict, cobjects, 'name') =20 - mo_dict =3D scan_maintainers(maintainers_dict) - #Gen data for relate with herds - res =3D defaultdict(list) - for mainteiner, herds_names in maintainers_dict.iteritems(): - for herd in herds_names: - res[herd].append(mo_dict[mainteiner.email]) + def add_related_to_ebuild(ebuild, ebuild_object): + # Add licenses + ebuild_object.licenses.add(*self.get_licenses_objects(ebuild= )) + ebuild_object.use_flags.add(*self.get_uses_objects(ebuild)) + ebuild_object.homepages.add(*self.get_homepages_objects(ebui= ld)) + self.create_keywords_objects(ebuild, ebuild_object) + =20 + def clear_related_to_ebuild(ebuild_object): + ebuild_object.licenses.clear() + ebuild_object.use_flags.clear() + ebuild_object.homepages.clear() + self.clean_keywords_object(ebuild_object) + + def update_related_to_ebuild(ebuild, ebuild_object): + clear_related_to_ebuild(ebuild_object) + add_related_to_ebuild(ebuild, ebuild_object) + + def create_ebuilds(package, package_object): + for ebuild in package.iter_ebuilds(): + ebuild_object =3D models.EbuildModel() + ebuild_object.init_by_ebuild(ebuild) + ebuild_object.package =3D package_object + # To Add some related objects it should have pk + ebuild_object.save(force_insert=3DTrue) + add_related_to_ebuild(ebuild, ebuild_object) + + def update_ebuilds(package, package_object, delete =3D True): + not_del =3D [] + for ebuild in package.iter_ebuilds(): + ebuild_object, ebuild_created =3D models.EbuildModel.obj= ects.get_or_create(ebuild =3D ebuild, package =3D package_object) + not_del.append(ebuild_object.pk) + if ebuild_created: + add_related_to_ebuild(ebuild, ebuild_object) + continue + if ebuild_object.check_or_need_update(ebuild): + ebuild_object.update_by_ebuild(ebuild) + update_related_to_ebuild(ebuild, ebuild_object) + ebuild_object.save(force_update =3D True) + if delete: + models.EbuildModel.objects.filter(package =3D package_ob= ject).exclude(pk__in =3D not_del).delete() + + def clear_related_to_package(package_object): + package_object.herds.clear() + package_object.maintainers.clear() + =20 + + def add_related_to_package(package, package_object): + package_object.herds.add(*self.get_herds_objects(package)) + package_object.maintainers.add(*self.get_maintainers_objects= (package)) + + def update_related_to_package(package, package_object): + clear_related_to_package(package_object) + add_related_to_package(package, package_object) =20 - for herd_name, herd_object in ho_dict.iteritems(): - herd_object.maintainers.clear() - herd_object.maintainers.add(*res[herd_name]) + def update_package(package, package_object, force_update =3D Fal= se): + if package_object.need_update_metadata(package) or force_upd= ate: + #Updating related objects to package + update_related_to_package(package, package_object) + + if package_object.need_update_ebuilds(package) or force_upda= te: + update_ebuilds(package, package_object) + + package_object.update_info(package) + package_object.save(force_update =3D True) + + # Load homepages to cache + #for homepage in models.HomepageModel.objects.all(): + #homepages_cache[homepage.url] =3D homepage + existend_categorys =3D [] + for category in porttree.iter_categories(): + existend_packages =3D [] + category_object, category_created =3D models.CategoryModel.o= bjects.get_or_create(category =3D category) + existend_categorys.append(category_object.pk) + for package in category.iter_packages(): + #if use_cache: + #key =3D str(porttree.name)+'/'+str(package) + #val =3D None + #if key in cache_dict: + #val =3D cache_dict[key] + #if val is not None and val =3D=3D package.manifest_= sha1: + #continue + print('%s [%s]' % (str(package).ljust(44), porttree)) + package_object, package_created =3D models.PackageModel.= objects.only('changelog_hash', 'manifest_hash', 'metadata_hash') \ + .get_or_create(package =3D package, category= =3D category_object, repository =3D porttree_obj) + #if update_cache: + #key =3D str(porttree.name)+'/'+str(package) + #cache_dict[key] =3D package.manifest_sha1 + =20 + existend_packages.append(package_object.pk) + if not package_created: + if package_object.check_or_need_update(package) or f= orce_update: + # need update + update_package(package, package_object) =20 - return ho_dict, mo_dict + continue + # if package_created: + add_related_to_package(package, package_object) + create_ebuilds(package, package_object) =20 + if delete: + models.PackageModel.objects.filter(category =3D category= _object, repository =3D porttree_obj).exclude(pk__in =3D existend_package= s).delete() + + + +cache_dict =3D None def update_globals_uses_descriptions(): # Need changes=20 uses_g =3D use_info.get_uses_info() @@ -203,181 +440,3 @@ def scan_uses_description(): use_desc_obj.save(force_update =3D True) models.UseFlagDescriptionModel.objects.bulk_create(to_create) =20 -licenses_cache =3D {} -uses_cache =3D {} -arches_cache =3D {} -homepages_cache =3D {} =20 -herds_cache =3D {} -maintainers_cache =3D {} - -cache_dict =3D None -def scanpackages(porttree, porttree_obj, delete =3D True, force_update =3D= False, - update_cache =3D True, use_cache =3D True): - def get_licenses_objects(ebuild): - licenses =3D ebuild.licenses - return _get_items(licenses, models.LicensModel, 'name', licenses= _cache) - - def get_uses_objects(ebuild): - uses =3D [ use.name for use in ebuild.iter_uses() ] - return _get_items(uses, models.UseFlagModel, 'name', uses_cache) - - def get_keywords_objects(ebuild, ebuild_object): - keywords_list =3D [] - for keyword in ebuild.get_keywords(): - keyword_object =3D models.Keyword(status =3D keyword.status, - ebuild =3D ebuild_object) - - if keyword.arch in arches_cache: - arch =3D arches_cache[keyword.arch] - else: - arch, created =3D models.ArchesModel.objects.get_or_crea= te(name =3D keyword.arch) - arches_cache[keyword.arch] =3D arch - =20 - keyword_object.arch =3D arch - keywords_list.append(keyword_object) - - models.Keyword.objects.bulk_create(keywords_list) - - - def get_homepages_objects(ebuild): - homepages =3D ebuild.homepages - return _get_items(homepages, models.HomepageModel, 'url', homepa= ges_cache) - - =20 - def get_maintainers_objects(package): - maintainers =3D package.metadata.maintainers() - objects =3D [] - for maintainer in maintainers: - if maintainer.email in maintainers_cache: - objects.append(maintainers_cache[maintainer.email]) - else: - maintainer_object, created =3D models.MaintainerModel \ - .objects.get_or_create(email =3D maintainer.emai= l) - if created: - maintainer_object.name =3D maintainer.name - maintainer_object.save() - objects.append(maintainer_object) - return objects - =20 - - def get_herds_objects(package): - herds =3D package.metadata.herds() - herds_objects =3D [] - for herd in herds: - if herd in herds_cache: - herds_objects.append(herds_cache[herd]) - - return herds_objects - - def add_related_to_ebuild(ebuild, ebuild_object): - # Add licenses - ebuild_object.licenses.add(*get_licenses_objects(ebuild)) - ebuild_object.use_flags.add(*get_uses_objects(ebuild)) - ebuild_object.homepages.add(*get_homepages_objects(ebuild)) - get_keywords_objects(ebuild, ebuild_object) - =20 - def clear_related_to_ebuild(ebuild_object): - ebuild_object.licenses.clear() - ebuild_object.use_flags.clear() - ebuild_object.homepages.clear() - models.Keyword.objects.filter(ebuild =3D ebuild_object).delete() - - def update_related_to_ebuild(ebuild, ebuild_object): - clear_related_to_ebuild(ebuild_object) - add_related_to_ebuild(ebuild, ebuild_object) - - def create_ebuilds(package, package_object): - for ebuild in package.iter_ebuilds(): - ebuild_object =3D models.EbuildModel() - ebuild_object.init_by_ebuild(ebuild) - ebuild_object.package =3D package_object - # To Add some related objects it should have pk - ebuild_object.save(force_insert=3DTrue) - add_related_to_ebuild(ebuild, ebuild_object) - - def update_ebuilds(package, package_object, delete =3D True): - not_del =3D [] - for ebuild in package.iter_ebuilds(): - ebuild_object, ebuild_created =3D models.EbuildModel.objects= .get_or_create(ebuild =3D ebuild, package =3D package_object) - not_del.append(ebuild_object.pk) - if ebuild_created: - add_related_to_ebuild(ebuild, ebuild_object) - continue - if ebuild_object.check_or_need_update(ebuild): - ebuild_object.update_by_ebuild(ebuild) - update_related_to_ebuild(ebuild, ebuild_object) - ebuild_object.save(force_update =3D True) - if delete: - models.EbuildModel.objects.filter(package =3D package_object= ).exclude(pk__in =3D not_del).delete() - - def clear_related_to_package(package_object): - package_object.herds.clear() - package_object.maintainers.clear() - =20 - - def add_related_to_package(package, package_object): - package_object.herds.add(*get_herds_objects(package)) - package_object.maintainers.add(*get_maintainers_objects(package)= ) - - def update_related_to_package(package, package_object): - clear_related_to_package(package_object) - add_related_to_package(package, package_object) - - def update_package(package, package_object, force_update =3D False): - if package_object.need_update_metadata(package) or force_update: - #Updating related objects to package - update_related_to_package(package, package_object) - - if package_object.need_update_ebuilds(package) or force_update: - update_ebuilds(package, package_object) - - package_object.update_info(package) - package_object.save(force_update =3D True) - - # Load homepages to cache - #for homepage in models.HomepageModel.objects.all(): - #homepages_cache[homepage.url] =3D homepage - existend_categorys =3D [] - for category in porttree.iter_categories(): - existend_packages =3D [] - category_object, category_created =3D models.CategoryModel.objec= ts.get_or_create(category =3D category) - existend_categorys.append(category_object.pk) - for package in category.iter_packages(): - #if use_cache: - #key =3D str(porttree.name)+'/'+str(package) - #val =3D None - #if key in cache_dict: - #val =3D cache_dict[key] - #if val is not None and val =3D=3D package.manifest_sha1= : - #continue - print('%s [%s]' % (str(package).ljust(44), porttree)) - package_object, package_created =3D models.PackageModel.obje= cts.only('changelog_hash', 'manifest_hash', 'metadata_hash') \ - .get_or_create(package =3D package, category =3D= category_object, repository =3D porttree_obj) - #if update_cache: - #key =3D str(porttree.name)+'/'+str(package) - #cache_dict[key] =3D package.manifest_sha1 - =20 - existend_packages.append(package_object.pk) - if not package_created: - if package_object.check_or_need_update(package) or force= _update: - # need update - update_package(package, package_object) - - continue - add_related_to_package(package, package_object) - if package_created: - create_ebuilds(package, package_object) - - if delete: - models.PackageModel.objects.filter(category =3D category_obj= ect, repository =3D porttree_obj).exclude(pk__in =3D existend_packages).d= elete() - - -def scan_all_repos(): - global herds_cache, maintainers_cache - #global cache_dict - #cache_dict =3D anydbm.open('cache.db','c') - herds_cache, maintainers_cache =3D scan_herds() - for repo in portage.iter_trees(): - repo_obj, repo_created =3D models.RepositoryModel.objects.get_or= _create(name =3D repo.name) - scanpackages(repo, repo_obj) - #cache_dict.close()