* [gentoo-commits] proj/gentoo-packages:master commit in: gpackages/apps/packages/, gpackages/apps/packages/management/commands/
@ 2012-06-04 20:09 Slava Bacherikov
0 siblings, 0 replies; 20+ messages in thread
From: Slava Bacherikov @ 2012-06-04 20:09 UTC (permalink / raw
To: gentoo-commits
commit: 0d89ed82a1d69ca7bc5b7796fb3260c50798263f
Author: Slava Bacherikov <slava <AT> bacher09 <DOT> org>
AuthorDate: Mon Jun 4 17:02:20 2012 +0000
Commit: Slava Bacherikov <slava <AT> bacherikov <DOT> org <DOT> ua>
CommitDate: Mon Jun 4 17:02:20 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/gentoo-packages.git;a=commit;h=0d89ed82
Add updating herds if some data changed.
---
.../packages/management/commands/scanpackages.py | 7 ++++++-
gpackages/apps/packages/models.py | 8 ++++++++
2 files changed, 14 insertions(+), 1 deletions(-)
diff --git a/gpackages/apps/packages/management/commands/scanpackages.py b/gpackages/apps/packages/management/commands/scanpackages.py
index 364bd5f..2fe93e2 100644
--- a/gpackages/apps/packages/management/commands/scanpackages.py
+++ b/gpackages/apps/packages/management/commands/scanpackages.py
@@ -112,7 +112,12 @@ def scan_herds():
if herd_object.name not in herds_dict:
to_del.append(herd_object.pk)
else:
- # to update ?
+ herd_cmp = herds_dict[herd_object.name]
+ # need update ?
+ if herd_object.check_or_need_update(herd_cmp):
+ # updating
+ herd_object.update_by_herd(herd_cmp)
+ herd_object.save(force_update = True)
ho_dict[herd_object.name] = herd_object
models.HerdsModel.objects.filter(pk__in = to_del).delete()
diff --git a/gpackages/apps/packages/models.py b/gpackages/apps/packages/models.py
index cf1cf1b..6b2fc1b 100644
--- a/gpackages/apps/packages/models.py
+++ b/gpackages/apps/packages/models.py
@@ -88,9 +88,17 @@ class HerdsModel(models.Model):
def init_by_herd(self, herd):
self.name = herd.name
+ self.update_by_herd(herd)
+
+ def update_by_herd(self, herd):
self.email = herd.email
self.description = herd.description
+ def check_or_need_update(self, herd):
+ return not (self.name == herd.name and \
+ self.email == herd.email and \
+ self.description == herd.description)
+
def __unicode__(self):
return self.name
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/gentoo-packages:master commit in: gpackages/apps/packages/, gpackages/apps/packages/management/commands/
@ 2012-08-26 23:00 Slava Bacherikov
0 siblings, 0 replies; 20+ messages in thread
From: Slava Bacherikov @ 2012-08-26 23:00 UTC (permalink / raw
To: gentoo-commits
commit: 06ec53cb34cf47659e6b914a1525801845cde754
Author: Slava Bacherikov <slava <AT> bacher09 <DOT> org>
AuthorDate: Sun Aug 26 22:31:00 2012 +0000
Commit: Slava Bacherikov <slava <AT> bacherikov <DOT> org <DOT> ua>
CommitDate: Sun Aug 26 22:31:00 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/gentoo-packages.git;a=commit;h=06ec53cb
Add scan only overlays option
---
.../packages/management/commands/scanpackages.py | 5 +++
gpackages/apps/packages/scan.py | 37 ++++++++++++++-----
2 files changed, 32 insertions(+), 10 deletions(-)
diff --git a/gpackages/apps/packages/management/commands/scanpackages.py b/gpackages/apps/packages/management/commands/scanpackages.py
index 32e85b0..c0c57a4 100644
--- a/gpackages/apps/packages/management/commands/scanpackages.py
+++ b/gpackages/apps/packages/management/commands/scanpackages.py
@@ -24,6 +24,11 @@ class Command(BaseCommand):
dest='scan_all',
default=False,
help='Scan all repositories'),
+ make_option('-o', '--only-overlays',
+ action='store_true',
+ dest='scan_overlays',
+ default=False,
+ help='Scan only overlays repositories'),
make_option('-r', '--update-repo',
action='store_true',
dest='update_repo',
diff --git a/gpackages/apps/packages/scan.py b/gpackages/apps/packages/scan.py
index 84d73f5..792853b 100644
--- a/gpackages/apps/packages/scan.py
+++ b/gpackages/apps/packages/scan.py
@@ -120,6 +120,7 @@ class Scanner(object):
self.verbosity = toint(kwargs.get('verbosity',1),1)
self.traceback = bool_get('traceback',False)
self.s_all = bool_get('scan_all', False)
+ self.s_overlyas = bool_get('scan_overlays', False)
self.s_packages = bool_get('packages', False)
self.s_only_repo_info = bool_get('only_repo_info', False)
self.is_show_time = bool_get('show_time', True)
@@ -148,17 +149,23 @@ class Scanner(object):
if self.is_scan_herds:
self.scan_herds()
+ scan_kwargs = {'force_update' : self.force_update,
+ 'delete' : self.delete,
+ 'update_repo' : self.update_repo
+ }
+
if self.s_all and self.s_packages:
- self.scan_all_repos(force_update = self.force_update,
- delete = self.delete,
- update_repo = self.update_repo)
+ self.scan_all_repos(**scan_kwargs)
+
+ elif self.s_overlyas and self.s_packages:
+ self.scan_all_repos(exclude = ['gentoo'], **scan_kwargs)
+
elif len(self.scan_repos_name) > 0 and self.s_packages:
- self.scan_repos_by_name(self.scan_repos_name,
- force_update = self.force_update,
- delete = self.delete,
- update_repo = self.update_repo)
+ self.scan_repos_by_name(self.scan_repos_name, **scan_kwargs)
+
elif self.s_only_repo_info and self.s_all:
self.scan_all_repo_info(delete = self.delete)
+
elif self.s_only_repo_info and len(self.scan_repos_name) > 0:
self.scan_repo_info_by_names(self.scan_repos_name)
@@ -345,11 +352,15 @@ class Scanner(object):
self.load_maintainers_to_cache()
return self.maintainers_cache
- def scan_all_repos(self, **kwargs):
+ def scan_all_repos(self, exclude = None, **kwargs):
"Scan packages in all available trees"
#cache_dict = anydbm.open('cache.db','c')
+ if exclude is not None:
+ exclude = frozenset(exclude)
for repo in portage.iter_trees():
+ if exclude is not None and repo.name in exclude:
+ continue
self.scan_repo(repo, **kwargs)
#cache_dict.close()
@@ -395,13 +406,19 @@ class Scanner(object):
return repo_obj
- def scan_all_repo_info(self, delete = False):
+ def scan_all_repo_info(self, exclude = None, delete = False):
"""Scan all info (metada) of all available repositories
Args:
delete -- if this true that repository will delete
- if portage would be not available"""
+ if portage would be not available
+ exclude -- exlude this overlays
+ """
+ if exclude is not None:
+ exclude = frozenset(exclude)
ex_pk = []
for repo in portage.iter_trees():
+ if exclude is not None and repo.name in exclude:
+ continue
repo_obj = self.get_repo_obj(repo, update_repo = True)
self.output("Scaned [%s]\n", repo.name)
ex_pk.append(repo_obj.pk)
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/gentoo-packages:master commit in: gpackages/apps/packages/, gpackages/apps/packages/management/commands/
@ 2012-08-22 17:55 Slava Bacherikov
0 siblings, 0 replies; 20+ messages in thread
From: Slava Bacherikov @ 2012-08-22 17:55 UTC (permalink / raw
To: gentoo-commits
commit: fb52b4806558ffcc31630ed0721028f933809b4c
Author: Slava Bacherikov <slava <AT> bacher09 <DOT> org>
AuthorDate: Tue Aug 21 17:19:25 2012 +0000
Commit: Slava Bacherikov <slava <AT> bacherikov <DOT> org <DOT> ua>
CommitDate: Tue Aug 21 17:19:25 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/gentoo-packages.git;a=commit;h=fb52b480
Improve updateebuildmask command
---
.../management/commands/updateebuildmask.py | 10 +++++-
gpackages/apps/packages/scan.py | 31 +++++++++++++-------
2 files changed, 28 insertions(+), 13 deletions(-)
diff --git a/gpackages/apps/packages/management/commands/updateebuildmask.py b/gpackages/apps/packages/management/commands/updateebuildmask.py
index a26554d..15a0ec5 100644
--- a/gpackages/apps/packages/management/commands/updateebuildmask.py
+++ b/gpackages/apps/packages/management/commands/updateebuildmask.py
@@ -10,8 +10,14 @@ class Command(BaseCommand):
dest='show_time',
default=True,
help='Show time of scanning'),
+ make_option('-m', '--update-missing',
+ action='store_true',
+ dest='missing',
+ default=False,
+ help='Add missing ebuilds'),
)
args = ''
- help = 'Will add missing ebuilds'
+ help = 'Will update ebuild mask status'
def handle(self, *args, **options):
- Scanner(**options).update_ebuild_mask()
+ missing = options.get('missing', False) # maybe move it to function ?
+ Scanner(**options).update_ebuild_mask(missing = missing)
diff --git a/gpackages/apps/packages/scan.py b/gpackages/apps/packages/scan.py
index 660fcc3..84d73f5 100644
--- a/gpackages/apps/packages/scan.py
+++ b/gpackages/apps/packages/scan.py
@@ -773,27 +773,35 @@ class Scanner(object):
models.PackageModel.objects.filter(pk = package_id). \
update(latest_ebuild = latest_ebuild)
+ def missing_ebuild(self, ebuild):
+ try:
+ package_obj = models.PackageModel.objects. \
+ get(package = ebuild.package)
+
+ except models.PackageModel.DoesNotExist:
+ pass
+ else:
+ # Don't run this for updated packages
+ if package_obj.manifest_hash == ebuild.package.manifest_sha1:
+ self.output('%-44s [%s]M\n', (ebuild,
+ ebuild.package.category.porttree_name))
+ self.update_ebuilds(ebuild.package, package_obj)
+
def add_mising_ebuilds(self):
for ebuild in porttree.iter_ebuilds():
try:
ebuild_obj = models.EbuildModel.objects.get(ebuild = ebuild)
except models.EbuildModel.DoesNotExist:
- self.output('%-44s [%s]\n', (ebuild,
- ebuild.package.category.porttree_name))
- try:
- package_obj = models.PackageModel.objects. \
- get(package = ebuild.package)
-
- except models.PackageModel.DoesNotExist:
- pass
- else:
- self.update_ebuilds(ebuild.package, package_obj)
+ self.missing_ebuild(ebuild)
- def update_ebuild_mask(self):
+ def update_ebuild_mask(self, missing = False):
for ebuild in porttree.iter_ebuilds():
try:
ebuild_obj = models.EbuildModel.objects.get(ebuild = ebuild)
except models.EbuildModel.DoesNotExist:
+ if missing:
+ self.missing_ebuild(ebuild)
+
continue
if ebuild_obj.is_hard_masked != ebuild.is_hard_masked:
self.output('%-44s [%s]\n', (ebuild,
@@ -801,3 +809,4 @@ class Scanner(object):
ebuild_obj.is_hard_masked = ebuild.is_hard_masked
ebuild_obj.save(force_update = True)
+
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/gentoo-packages:master commit in: gpackages/apps/packages/, gpackages/apps/packages/management/commands/
@ 2012-07-18 23:03 Slava Bacherikov
0 siblings, 0 replies; 20+ messages in thread
From: Slava Bacherikov @ 2012-07-18 23:03 UTC (permalink / raw
To: gentoo-commits
commit: a283371444516c287f9110056d2e84fc55c83345
Author: Slava Bacherikov <slava <AT> bacher09 <DOT> org>
AuthorDate: Wed Jul 18 23:03:16 2012 +0000
Commit: Slava Bacherikov <slava <AT> bacherikov <DOT> org <DOT> ua>
CommitDate: Wed Jul 18 23:03:16 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/gentoo-packages.git;a=commit;h=a2833714
Add utility for setting link to latest ebuild in package
---
.../management/commands/setlatestebuilds.py | 17 +++++++++++++++++
gpackages/apps/packages/scan.py | 15 +++++++++++++++
2 files changed, 32 insertions(+), 0 deletions(-)
diff --git a/gpackages/apps/packages/management/commands/setlatestebuilds.py b/gpackages/apps/packages/management/commands/setlatestebuilds.py
new file mode 100644
index 0000000..9b64673
--- /dev/null
+++ b/gpackages/apps/packages/management/commands/setlatestebuilds.py
@@ -0,0 +1,17 @@
+from django.core.management.base import BaseCommand, CommandError
+from packages.scan import Scanner
+from optparse import make_option
+
+
+class Command(BaseCommand):
+ option_list = BaseCommand.option_list + (
+ make_option('--not-show-time',
+ action='store_false',
+ dest='show_time',
+ default=True,
+ help='Show time of scanning'),
+ )
+ args = ''
+ help = 'Will set link to latest ebuilds in package model'
+ def handle(self, *args, **options):
+ Scanner(set_latest_ebuilds = True, **options).scan()
diff --git a/gpackages/apps/packages/scan.py b/gpackages/apps/packages/scan.py
index 1460698..a02975b 100644
--- a/gpackages/apps/packages/scan.py
+++ b/gpackages/apps/packages/scan.py
@@ -133,6 +133,7 @@ class Scanner(object):
self.is_scan_news = bool_get('scan_news', False)
self.is_scan_license_text = bool_get('license_text', False)
self.is_cache_clear = bool_get('cache_clear', False)
+ self.is_set_latest_ebuilds = bool_get('set_latest_ebuilds', False)
def show_time(self):
"Prints scan time"
@@ -160,6 +161,9 @@ class Scanner(object):
elif self.s_only_repo_info and len(self.scan_repos_name) > 0:
self.scan_repo_info_by_names(self.scan_repos_name)
+ if self.is_set_latest_ebuilds:
+ self.set_latest_ebuilds_to_package()
+
if self.scan_global_use_descr:
self.update_all_globals_uses_descriptions()
@@ -724,3 +728,14 @@ class Scanner(object):
license.text = smart_unicode(text, errors='ignore')
license.save(force_update = True)
+ @staticmethod
+ def set_latest_ebuilds_to_package():
+ for package in models.PackageModel.objects.only('pk').iterator():
+ # search latest ebuild should done by version and revison
+ # and database could not allow smart order by version becaouse
+ # version are compared as string
+ ebuilds = package.ebuildmodel_set.only('version', 'revision').all()
+ latest_ebuild = max(ebuilds, key = lambda x: x.version_cmp)
+ # Hack for not update last modified datetime
+ models.PackageModel.objects.filter(pk = package.pk). \
+ update(latest_ebuild = latest_ebuild)
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/gentoo-packages:master commit in: gpackages/apps/packages/, gpackages/apps/packages/management/commands/
@ 2012-07-17 9:42 Slava Bacherikov
0 siblings, 0 replies; 20+ messages in thread
From: Slava Bacherikov @ 2012-07-17 9:42 UTC (permalink / raw
To: gentoo-commits
commit: 5d4b702043c85469bf2d7ffc80221c8ae7f12462
Author: Slava Bacherikov <slava <AT> bacher09 <DOT> org>
AuthorDate: Mon Jul 16 15:30:12 2012 +0000
Commit: Slava Bacherikov <slava <AT> bacherikov <DOT> org <DOT> ua>
CommitDate: Mon Jul 16 15:30:12 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/gentoo-packages.git;a=commit;h=5d4b7020
Clear cache after update packages
---
.../packages/management/commands/scanpackages.py | 5 +++++
gpackages/apps/packages/scan.py | 5 +++++
2 files changed, 10 insertions(+), 0 deletions(-)
diff --git a/gpackages/apps/packages/management/commands/scanpackages.py b/gpackages/apps/packages/management/commands/scanpackages.py
index ce694fe..f3b1e2c 100644
--- a/gpackages/apps/packages/management/commands/scanpackages.py
+++ b/gpackages/apps/packages/management/commands/scanpackages.py
@@ -39,6 +39,11 @@ class Command(BaseCommand):
dest='delete',
default=True,
help='Not delete'),
+ make_option('--clear-cache',
+ action='store_true',
+ dest='cache_clear',
+ default=False,
+ help='Clear cache'),
make_option('--not-license-groups',
action='store_false',
dest='scan_license_groups',
diff --git a/gpackages/apps/packages/scan.py b/gpackages/apps/packages/scan.py
index 9135021..1460698 100644
--- a/gpackages/apps/packages/scan.py
+++ b/gpackages/apps/packages/scan.py
@@ -7,6 +7,7 @@ from django.utils.encoding import smart_unicode
from collections import defaultdict
from package_info.generic import StrThatIgnoreCase, toint
from package_info.porttree import porttree
+from django.core.cache import cache as cache_backend
import anydbm
@@ -131,6 +132,7 @@ class Scanner(object):
self.is_scan_license_groups = bool_get('scan_license_groups', False)
self.is_scan_news = bool_get('scan_news', False)
self.is_scan_license_text = bool_get('license_text', False)
+ self.is_cache_clear = bool_get('cache_clear', False)
def show_time(self):
"Prints scan time"
@@ -173,6 +175,9 @@ class Scanner(object):
if self.is_scan_license_text:
self.scan_license_text()
+ if self.is_cache_clear:
+ cache_backend.clear()
+
if self.is_show_time:
self.show_time()
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/gentoo-packages:master commit in: gpackages/apps/packages/, gpackages/apps/packages/management/commands/
@ 2012-07-06 23:09 Slava Bacherikov
0 siblings, 0 replies; 20+ messages in thread
From: Slava Bacherikov @ 2012-07-06 23:09 UTC (permalink / raw
To: gentoo-commits
commit: a5f76db6b3de800069672038421943b26a6b1bd2
Author: Slava Bacherikov <slava <AT> bacher09 <DOT> org>
AuthorDate: Fri Jul 6 16:04:00 2012 +0000
Commit: Slava Bacherikov <slava <AT> bacherikov <DOT> org <DOT> ua>
CommitDate: Fri Jul 6 16:04:00 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/gentoo-packages.git;a=commit;h=a5f76db6
Save license text to database
---
.../management/commands/scanlicensetext.py | 17 +++++++++++++++++
gpackages/apps/packages/models.py | 3 ++-
gpackages/apps/packages/scan.py | 12 ++++++++++++
3 files changed, 31 insertions(+), 1 deletions(-)
diff --git a/gpackages/apps/packages/management/commands/scanlicensetext.py b/gpackages/apps/packages/management/commands/scanlicensetext.py
new file mode 100644
index 0000000..40f9de4
--- /dev/null
+++ b/gpackages/apps/packages/management/commands/scanlicensetext.py
@@ -0,0 +1,17 @@
+from django.core.management.base import BaseCommand, CommandError
+from packages.scan import Scanner
+from optparse import make_option
+
+
+class Command(BaseCommand):
+ option_list = BaseCommand.option_list + (
+ make_option('--not-show-time',
+ action='store_false',
+ dest='show_time',
+ default=True,
+ help='Show time of scanning'),
+ )
+ args = ''
+ help = 'Will scan license text'
+ def handle(self, *args, **options):
+ Scanner(license_text = True, **options).scan()
diff --git a/gpackages/apps/packages/models.py b/gpackages/apps/packages/models.py
index 1bfc2ba..c89db88 100644
--- a/gpackages/apps/packages/models.py
+++ b/gpackages/apps/packages/models.py
@@ -494,7 +494,8 @@ class LicenseModel(StatsModel):
)
name = models.CharField(unique = True, max_length = 60, db_index = True)
- #description = TextField()
+ text = models.TextField(null = True)
+ #description = models.TextField()
# Autogenerated fields for fast stats
#ebuilds_count = models.PositiveIntegerField(default = 0)
diff --git a/gpackages/apps/packages/scan.py b/gpackages/apps/packages/scan.py
index 0fdf488..e0cc09c 100644
--- a/gpackages/apps/packages/scan.py
+++ b/gpackages/apps/packages/scan.py
@@ -129,6 +129,7 @@ class Scanner(object):
self.scan_local_use_descr = bool_get('scan_local_use', False)
self.is_scan_license_groups = bool_get('scan_license_groups', False)
self.is_scan_news = bool_get('scan_news', False)
+ self.is_scan_license_text = bool_get('license_text', False)
def show_time(self):
"Prints scan time"
@@ -168,6 +169,9 @@ class Scanner(object):
if self.is_scan_news:
self.scan_news()
+ if self.is_scan_license_text:
+ self.scan_license_text()
+
if self.is_show_time:
self.show_time()
@@ -702,3 +706,11 @@ class Scanner(object):
n_obj.save(force_update = True)
self.update_related_to_news(n, n_obj)
+ def scan_license_text(self):
+ licenses = models.LicenseModel.objects.all()
+ for license in licenses:
+ text = porttree.licenses.get_license(license.name)
+ if text:
+ license.text = smart_unicode(text, errors='ignore')
+ license.save(force_update = True)
+
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/gentoo-packages:master commit in: gpackages/apps/packages/, gpackages/apps/packages/management/commands/
@ 2012-07-05 23:27 Slava Bacherikov
0 siblings, 0 replies; 20+ messages in thread
From: Slava Bacherikov @ 2012-07-05 23:27 UTC (permalink / raw
To: gentoo-commits
commit: a9a583d239e6c5e7c07e648ad780975c4be47652
Author: Slava Bacherikov <slava <AT> bacher09 <DOT> org>
AuthorDate: Thu Jul 5 22:32:45 2012 +0000
Commit: Slava Bacherikov <slava <AT> bacherikov <DOT> org <DOT> ua>
CommitDate: Thu Jul 5 22:32:45 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/gentoo-packages.git;a=commit;h=a9a583d2
Add metaclass for autogenerate stats fields
---
.../packages/management/commands/update_stats.py | 13 +----
gpackages/apps/packages/models.py | 63 ++++++++------------
gpackages/apps/packages/stats.py | 56 ++++++++++++++++-
3 files changed, 79 insertions(+), 53 deletions(-)
diff --git a/gpackages/apps/packages/management/commands/update_stats.py b/gpackages/apps/packages/management/commands/update_stats.py
index 39b7155..37502fd 100644
--- a/gpackages/apps/packages/management/commands/update_stats.py
+++ b/gpackages/apps/packages/management/commands/update_stats.py
@@ -1,19 +1,10 @@
from django.core.management.base import BaseCommand, CommandError
from optparse import make_option
-from packages import models
-from packages.stats import StatsMixin
-import inspect
-
-
-def get_predicate(obj):
- if inspect.isclass(obj) and issubclass(obj, StatsMixin):
- return obj != StatsMixin
+from packages.stats import update_all_stats
class Command(BaseCommand):
args = ''
help = 'Will update stats in models'
def handle(self, *args, **options):
- models_list = inspect.getmembers(models, get_predicate)
- for model_name, model in models_list:
- model.calc_stats()
+ update_all_stats()
diff --git a/gpackages/apps/packages/models.py b/gpackages/apps/packages/models.py
index efb4638..73b1f42 100644
--- a/gpackages/apps/packages/models.py
+++ b/gpackages/apps/packages/models.py
@@ -6,7 +6,7 @@ from package_info.generic import get_from_kwargs_and_del
from package_info.repo_info import REPOS_TYPE
# relative
from .keywords import KeywordRepr
-from .stats import StatsMixin
+from .stats import StatsModel
#from utils import get_link
from package_info.validators import validate_url, validate_email, \
validate_name
@@ -91,7 +91,7 @@ class ArchesModel(models.Model):
def __unicode__(self):
return self.name
-class RepositoryModel(StatsMixin, AbstractDateTimeModel):
+class RepositoryModel(StatsModel, AbstractDateTimeModel):
QUALITY_CHOICES = ( (0, 'stable'),
(1, 'testing'),
(2, 'experimental'),
@@ -119,10 +119,8 @@ class RepositoryModel(StatsMixin, AbstractDateTimeModel):
official = models.BooleanField(default = False)
quality = models.PositiveSmallIntegerField(choices = QUALITY_CHOICES)
- # Maybe in future auto generate this field by metaclass ?
- # For fast stats
- packages_count = models.PositiveIntegerField(default = 0)
- ebuilds_count = models.PositiveIntegerField(default = 0)
+ # Autogenerated fields for fast stats:
+ # packages_count, ebuilds_count
objects = managers.RepositoryManager()
@@ -206,7 +204,7 @@ class RepositorySourceModel(models.Model):
def __unicode__(self):
return self.url
-class CategoryModel(StatsMixin, models.Model):
+class CategoryModel(StatsModel):
def __init__(self, *args, **kwargs):
super(CategoryModel, self).__init__(*args, **kwargs)
@@ -232,17 +230,13 @@ class CategoryModel(StatsMixin, models.Model):
description = models.TextField(blank = True, null = True)
metadata_hash = models.CharField(max_length = 128, null = True)
- # Maybe in future auto generate this field by metaclass ?
- # For fast stats
- virtual_packages_count = models.PositiveIntegerField(default = 0)
- packages_count = models.PositiveIntegerField(default = 0)
- repositories_count = models.PositiveIntegerField(default = 0)
- ebuilds_count = models.PositiveIntegerField(default = 0)
+ # Autogenerated fields for fast stats
+ #virtual_packages_count, packages_count, repositories_count, ebuilds_count
def __unicode__(self):
return unicode(self.category)
-class MaintainerModel(StatsMixin, AbstractDateTimeModel):
+class MaintainerModel(StatsModel, AbstractDateTimeModel):
def __init__(self, *args, **kwargs):
maintainer = get_from_kwargs_and_del('maintainer', kwargs)
@@ -254,7 +248,7 @@ class MaintainerModel(StatsMixin, AbstractDateTimeModel):
('packages_count', 'packagemodel'),
('herds_count', 'herdsmodel'),
('ebuilds_count', 'packagemodel__ebuildmodel'),
- ('repositories_count', 'packagemodel__ebuildmodel'),
+ ('repositories_count', 'packagemodel__repository'),
('news_author_count', 'author_news_set'),
('news_translator_count', 'translator_news_set')
)
@@ -262,15 +256,9 @@ class MaintainerModel(StatsMixin, AbstractDateTimeModel):
name = models.CharField(max_length = 255, blank = True, null = True)
email = models.EmailField(unique = True, validators = [validate_email], db_index = True)
- # For fast stats
- # Maybe use django-composition ?
- # Maybe in future auto generate this field by metaclass ?
- herds_count = models.PositiveIntegerField(default = 0)
- packages_count = models.PositiveIntegerField(default = 0)
- ebuilds_count = models.PositiveIntegerField(default = 0)
- repositories_count = models.PositiveIntegerField(default = 0)
- news_author_count = models.PositiveIntegerField(default = 0)
- news_translator_count = models.PositiveIntegerField(default = 0)
+ # Autogenerated fields for fast stats
+ # herds_count packages_count ebuilds_count repositories_count
+ # news_author_count news_translator_count
objects = managers.MaintainerManager()
@@ -291,7 +279,7 @@ class MaintainerModel(StatsMixin, AbstractDateTimeModel):
class Meta:
ordering = ('name',)
-class HerdsModel(StatsMixin, AbstractDateTimeModel):
+class HerdsModel(StatsModel, AbstractDateTimeModel):
def __init__(self, *args, **kwargs):
herd = get_from_kwargs_and_del('herd', kwargs)
@@ -311,13 +299,8 @@ class HerdsModel(StatsMixin, AbstractDateTimeModel):
description = models.TextField(blank = True, null = True)
maintainers = models.ManyToManyField(MaintainerModel, blank = True)
- # For fast stats
- # Maybe use django-composition ?
- # Maybe in future auto generate this field by metaclass ?
- maintainers_count = models.PositiveIntegerField(default = 0)
- packages_count = models.PositiveIntegerField(default = 0)
- ebuilds_count = models.PositiveIntegerField(default = 0)
- repositories_count = models.PositiveIntegerField(default = 0)
+ # Autogenerated fields for fast stats
+ # maintainers_count packages_count ebuilds_count repositories_count
objects = managers.HerdsManager()
@@ -364,7 +347,7 @@ class VirtualPackageModel(models.Model):
class Meta:
unique_together = ('name', 'category')
-class PackageModel(StatsMixin, AbstractDateTimeModel):
+class PackageModel(StatsModel, AbstractDateTimeModel):
def __init__(self, *args, **kwargs):
package_object, category = \
@@ -394,7 +377,8 @@ class PackageModel(StatsMixin, AbstractDateTimeModel):
repository = models.ForeignKey(RepositoryModel, db_index = True)
# Different versions can have different licenses, or homepages.
- ebuilds_count = models.PositiveIntegerField(default = 0)
+ # Autogenerated fields for fast stats
+ # ebuilds_count = models.PositiveIntegerField(default = 0)
objects = managers.PackageManager()
@@ -473,7 +457,7 @@ class PackageModel(StatsMixin, AbstractDateTimeModel):
unique_together = ('virtual_package', 'repository')
ordering = ('-updated_datetime',)
-class UseFlagModel(StatsMixin, models.Model):
+class UseFlagModel(StatsModel):
stats_params = (
('ebuilds_count', 'ebuildmodel'),
)
@@ -481,7 +465,8 @@ class UseFlagModel(StatsMixin, models.Model):
name = models.CharField(unique = True, max_length = 60, db_index = True)
description = models.TextField(blank = True)
- ebuilds_count = models.PositiveIntegerField(default = 0)
+ # Autogenerated fields for fast stats
+ #ebuilds_count = models.PositiveIntegerField(default = 0)
def __unicode__(self):
return self.name
@@ -503,14 +488,16 @@ class UseFlagDescriptionModel(models.Model):
class Meta:
unique_together = ('use_flag', 'package')
-class LicenseModel(StatsMixin, models.Model):
+class LicenseModel(StatsModel):
stats_params = (
('ebuilds_count', 'ebuildmodel'),
)
name = models.CharField(unique = True, max_length = 60, db_index = True)
#description = TextField()
- ebuilds_count = models.PositiveIntegerField(default = 0)
+
+ # Autogenerated fields for fast stats
+ #ebuilds_count = models.PositiveIntegerField(default = 0)
def __unicode__(self):
return self.name
diff --git a/gpackages/apps/packages/stats.py b/gpackages/apps/packages/stats.py
index 9bf3efa..b506793 100644
--- a/gpackages/apps/packages/stats.py
+++ b/gpackages/apps/packages/stats.py
@@ -1,12 +1,22 @@
-from django.db.models import Count
-from itertools import starmap
+from django.db import models
+from itertools import starmap, ifilter
+import inspect
+from django.db.models.base import ModelBase
+from django.db.models.loading import get_models
+
+def get_predicate(obj):
+ if inspect.isclass(obj) and issubclass(obj, StatsMixin):
+ return obj != StatsMixin
+
+def filter_models(models_list):
+ return ifilter(get_predicate, models_list)
gen_prefix = lambda x: x + '_prefix'
def gen_query_dict(params):
query_dict = {}
for key, value in starmap(lambda x,y: (gen_prefix(x), y), params):
- query_dict[key] = Count(value, distinct = True)
+ query_dict[key] = models.Count(value, distinct = True)
return query_dict
@@ -24,8 +34,46 @@ def update_stats(model, params):
class StatsMixin(object):
- stats_params = ((),)
+ #iterable of 2-tuples
+ stats_params = ()
@classmethod
def calc_stats(cls):
update_stats(cls, cls.stats_params)
+
+def get_stats_models(module):
+ return inspect.getmembers(module, get_predicate)
+
+def update_stats_for_models(models_list):
+ for model in models_list:
+ model.calc_stats()
+
+def update_all_stats():
+ update_stats_for_models(filter_models(get_models()))
+
+class StatsModelBase(ModelBase):
+
+ def __init__(cls, name, bases, attrs):
+ super(StatsModelBase, cls).__init__(name, bases, attrs)
+ stats_m = False
+ for base in bases:
+ if issubclass(base, StatsMixin):
+ stats_m = True
+ break
+ #if not stats_m or not hasattr(cls, stats_params):
+ if not (stats_m and hasattr(cls, 'stats_params')):
+ return None
+
+ if not cls.stats_params:
+ return None
+
+ for key, value in cls.stats_params:
+ if not hasattr(cls, key):
+ cls.add_to_class(key, models. \
+ PositiveIntegerField(default = 0, editable = False))
+
+class StatsModel(StatsMixin, models.Model):
+ __metaclass__ = StatsModelBase
+
+ class Meta:
+ abstract = True
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/gentoo-packages:master commit in: gpackages/apps/packages/, gpackages/apps/packages/management/commands/
@ 2012-06-19 0:12 Slava Bacherikov
0 siblings, 0 replies; 20+ messages in thread
From: Slava Bacherikov @ 2012-06-19 0:12 UTC (permalink / raw
To: gentoo-commits
commit: cc5fc55fac985559a64b8cfffbe183b2928a4674
Author: Slava Bacherikov <slava <AT> bacher09 <DOT> org>
AuthorDate: Tue Jun 19 00:11:54 2012 +0000
Commit: Slava Bacherikov <slava <AT> bacherikov <DOT> org <DOT> ua>
CommitDate: Tue Jun 19 00:11:54 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/gentoo-packages.git;a=commit;h=cc5fc55f
Add scaning utility for repo info
---
.../packages/management/commands/scanpackages.py | 3 +-
.../packages/management/commands/scanrepoinfo.py | 29 +++++++++++
gpackages/apps/packages/scan.py | 51 +++++++++++++++++---
3 files changed, 74 insertions(+), 9 deletions(-)
diff --git a/gpackages/apps/packages/management/commands/scanpackages.py b/gpackages/apps/packages/management/commands/scanpackages.py
index 71c308c..3809ff4 100644
--- a/gpackages/apps/packages/management/commands/scanpackages.py
+++ b/gpackages/apps/packages/management/commands/scanpackages.py
@@ -49,5 +49,4 @@ class Command(BaseCommand):
args = '<repository names ...>'
help = 'Will scan package tree and update info about it in database'
def handle(self, *args, **options):
- verbosity = int(options['verbosity'])
- Scanner(repos = args, **options).scan()
+ Scanner(repos = args, packages = True, **options).scan()
diff --git a/gpackages/apps/packages/management/commands/scanrepoinfo.py b/gpackages/apps/packages/management/commands/scanrepoinfo.py
new file mode 100644
index 0000000..19049c4
--- /dev/null
+++ b/gpackages/apps/packages/management/commands/scanrepoinfo.py
@@ -0,0 +1,29 @@
+from django.core.management.base import BaseCommand, CommandError
+from packages.scan import Scanner
+from optparse import make_option
+
+
+class Command(BaseCommand):
+ option_list = BaseCommand.option_list + (
+ make_option('--not-show-time',
+ action='store_false',
+ dest='show_time',
+ default=True,
+ help='Show time of scanning'),
+ make_option('-d' ,'--del',
+ action='store_true',
+ dest='delete',
+ default=False,
+ help='Delete'),
+ )
+
+ args = '<repository names ...>'
+ help = 'Will scan repositories info'
+ def handle(self, *args, **options):
+ if len(args) > 0:
+ scan_all = False
+ else:
+ scan_all = True
+
+ Scanner(repos = args, scan_all = scan_all,
+ only_repo_info = True, **options).scan()
diff --git a/gpackages/apps/packages/scan.py b/gpackages/apps/packages/scan.py
index 8d8fed9..3e5cf81 100644
--- a/gpackages/apps/packages/scan.py
+++ b/gpackages/apps/packages/scan.py
@@ -122,6 +122,8 @@ class Scanner(object):
self.verbosity = toint(kwargs.get('verbosity',1),1)
self.traceback = bool(kwargs.get('traceback',False))
self.s_all = bool(kwargs.get('scan_all', False))
+ self.s_packages = bool(kwargs.get('packages', False))
+ self.s_only_repo_info = bool(kwargs.get('only_repo_info', False))
self.is_show_time = bool(kwargs.get('show_time', True))
self.is_scan_herds = bool(kwargs.get('scan_herds', True))
self.force_update = bool(kwargs.get('force_update', False))
@@ -140,15 +142,19 @@ class Scanner(object):
if self.is_scan_herds:
self.scan_herds()
- if self.s_all:
+ if self.s_all and self.s_packages:
self.scan_all_repos(force_update = self.force_update,
delete = self.delete,
update_repo = self.update_repo)
- elif len(self.scan_repos_name) > 0:
+ elif len(self.scan_repos_name) > 0 and self.s_packages:
self.scan_repos_by_name(self.scan_repos_name,
force_update = self.force_update,
delete = self.delete,
update_repo = self.update_repo)
+ elif self.s_only_repo_info and self.s_all:
+ self.scan_all_repo_info(delete = self.delete)
+ elif self.s_only_repo_info and len(self.scan_repos_name) > 0:
+ self.scan_repo_info_by_names(self.scan_repos_name)
if self.scan_global_use_descr:
self.update_all_globals_uses_descriptions()
@@ -301,17 +307,25 @@ class Scanner(object):
for repo_name in repo_names:
self.scan_repo_by_name(repo_name, **kwargs)
- def scan_repo_by_name(self, repo_name, **kwargs):
+ def get_repo_by_name(self, repo_name, quiet = False, trace = False):
try:
repo = portage.get_tree_by_name(repo_name)
except ValueError:
- self.output("Bad repository name '%s'\n", repo_name, 1)
+ if not quiet:
+ self.output("Bad repository name '%s'\n", repo_name, 1)
+ if trace:
+ raise
+ return None
else:
- self.scan_repo(repo, **kwargs)
+ return repo
- def scan_repo(self, repo, update_repo = False, **kwargs):
- self.output("Scaning repository '%s'\n", repo.name, 3)
+ def scan_repo_by_name(self, repo_name, **kwargs):
+ repo = self.get_repo_by_name(repo_name)
+
+ if repo is not None:
+ self.scan_repo(repo, **kwargs)
+ def get_repo_obj(self, repo, update_repo = False):
repo_obj, repo_created = models.RepositoryModel \
.objects.get_or_create(repo = repo)
@@ -324,6 +338,29 @@ class Scanner(object):
else:
repo_obj.add_related(repo)
+ return repo_obj
+
+ def scan_all_repo_info(self, delete = False):
+ ex_pk = []
+ for repo in portage.iter_trees():
+ repo_obj = self.get_repo_obj(repo, update_repo = True)
+ self.output("Scaned [%s]\n", repo.name)
+ ex_pk.append(repo_obj.pk)
+
+ if delete:
+ models.RepositoryModel.objects.exclude(pk__in = ex_pk).delete()
+
+ def scan_repo_info_by_names(self, repo_list):
+ for repo_name in repo_list:
+ repo = self.get_repo_by_name(repo_name)
+ if repo is not None:
+ self.get_repo_obj(repo, update_repo = True)
+ self.output("Scaned [%s]\n", repo.name)
+
+ def scan_repo(self, repo, update_repo = False, **kwargs):
+ self.output("Scaning repository '%s'\n", repo.name, 3)
+
+ repo_obj = self.get_repo_obj(repo, update_repo = update_repo)
self.scanpackages(repo, repo_obj, **kwargs)
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/gentoo-packages:master commit in: gpackages/apps/packages/, gpackages/apps/packages/management/commands/
@ 2012-06-18 23:00 Slava Bacherikov
0 siblings, 0 replies; 20+ messages in thread
From: Slava Bacherikov @ 2012-06-18 23:00 UTC (permalink / raw
To: gentoo-commits
commit: b3156c48319c9773cf38b5ad3cc77c061c14bb49
Author: Slava Bacherikov <slava <AT> bacher09 <DOT> org>
AuthorDate: Mon Jun 18 23:00:41 2012 +0000
Commit: Slava Bacherikov <slava <AT> bacherikov <DOT> org <DOT> ua>
CommitDate: Mon Jun 18 23:00:41 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/gentoo-packages.git;a=commit;h=b3156c48
Add updating repository info to scan utility
---
.../packages/management/commands/scanpackages.py | 5 +++++
gpackages/apps/packages/models.py | 10 +++++++++-
gpackages/apps/packages/scan.py | 12 +++++++++---
3 files changed, 23 insertions(+), 4 deletions(-)
diff --git a/gpackages/apps/packages/management/commands/scanpackages.py b/gpackages/apps/packages/management/commands/scanpackages.py
index 97ed11e..71c308c 100644
--- a/gpackages/apps/packages/management/commands/scanpackages.py
+++ b/gpackages/apps/packages/management/commands/scanpackages.py
@@ -24,6 +24,11 @@ class Command(BaseCommand):
dest='scan_all',
default=False,
help='Force updating'),
+ make_option('-r', '--update-repo',
+ action='store_true',
+ dest='update_repo',
+ default=False,
+ help='Update repository info'),
make_option('--not-show-time',
action='store_false',
dest='show_time',
diff --git a/gpackages/apps/packages/models.py b/gpackages/apps/packages/models.py
index 5140911..1f7c6e8 100644
--- a/gpackages/apps/packages/models.py
+++ b/gpackages/apps/packages/models.py
@@ -91,7 +91,7 @@ class RepositoryModel(AbstractDateTimeModel):
RepositorySourceModel.objects.bulk_create(ret)
def clear_related_sources(self):
- self.repositorysourcemodel_set.clear()
+ RepositorySourceModel.objects.filter(repository = self).delete()
def update_related_sources(self, repo):
self.clear_related_sources()
@@ -101,6 +101,14 @@ class RepositoryModel(AbstractDateTimeModel):
self.add_related_feeds(repo)
self.add_related_sources(repo)
+ def clear_related(self):
+ self.clear_related_feeds()
+ self.clear_related_sources()
+
+ def update_related(self, repo):
+ self.clear_related()
+ self.add_related(repo)
+
def __unicode__(self):
return self.name
diff --git a/gpackages/apps/packages/scan.py b/gpackages/apps/packages/scan.py
index 76573ce..8d8fed9 100644
--- a/gpackages/apps/packages/scan.py
+++ b/gpackages/apps/packages/scan.py
@@ -125,6 +125,7 @@ class Scanner(object):
self.is_show_time = bool(kwargs.get('show_time', True))
self.is_scan_herds = bool(kwargs.get('scan_herds', True))
self.force_update = bool(kwargs.get('force_update', False))
+ self.update_repo = bool(kwargs.get('update_repo', False))
self.delete = bool(kwargs.get('delete', True))
self.scan_repos_name = tuple(kwargs.get('repos',[]))
self.scan_global_use_descr = bool(kwargs.get('scan_global_use', False))
@@ -141,11 +142,13 @@ class Scanner(object):
if self.s_all:
self.scan_all_repos(force_update = self.force_update,
- delete = self.delete)
+ delete = self.delete,
+ update_repo = self.update_repo)
elif len(self.scan_repos_name) > 0:
self.scan_repos_by_name(self.scan_repos_name,
force_update = self.force_update,
- delete = self.delete)
+ delete = self.delete,
+ update_repo = self.update_repo)
if self.scan_global_use_descr:
self.update_all_globals_uses_descriptions()
@@ -306,13 +309,16 @@ class Scanner(object):
else:
self.scan_repo(repo, **kwargs)
- def scan_repo(self, repo, **kwargs):
+ def scan_repo(self, repo, update_repo = False, **kwargs):
self.output("Scaning repository '%s'\n", repo.name, 3)
repo_obj, repo_created = models.RepositoryModel \
.objects.get_or_create(repo = repo)
if not repo_created:
+ if update_repo:
+ repo_obj.update_metadata(repo)
+ repo_obj.update_related(repo)
#Update modification time
repo_obj.save(force_update = True)
else:
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/gentoo-packages:master commit in: gpackages/apps/packages/, gpackages/apps/packages/management/commands/
@ 2012-06-13 22:15 Slava Bacherikov
0 siblings, 0 replies; 20+ messages in thread
From: Slava Bacherikov @ 2012-06-13 22:15 UTC (permalink / raw
To: gentoo-commits
commit: d34d69c598241b44661c494cd85fcf20244ce910
Author: Slava Bacherikov <slava <AT> bacher09 <DOT> org>
AuthorDate: Wed Jun 13 15:17:53 2012 +0000
Commit: Slava Bacherikov <slava <AT> bacherikov <DOT> org <DOT> ua>
CommitDate: Wed Jun 13 15:17:53 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/gentoo-packages.git;a=commit;h=d34d69c5
Update console interface fo scanherds and scanusedesc
---
.../apps/packages/management/commands/scanherds.py | 16 +++++++----
.../packages/management/commands/scanusedesc.py | 27 ++++++++++++++-----
gpackages/apps/packages/managers.py | 4 +-
gpackages/apps/packages/scan.py | 23 +++++++++++++---
4 files changed, 50 insertions(+), 20 deletions(-)
diff --git a/gpackages/apps/packages/management/commands/scanherds.py b/gpackages/apps/packages/management/commands/scanherds.py
index 43f10c3..31bc107 100644
--- a/gpackages/apps/packages/management/commands/scanherds.py
+++ b/gpackages/apps/packages/management/commands/scanherds.py
@@ -1,13 +1,17 @@
from django.core.management.base import BaseCommand, CommandError
-import datetime
-import logging
-from packages import scan
+from packages.scan import Scanner
+from optparse import make_option
class Command(BaseCommand):
+ option_list = BaseCommand.option_list + (
+ make_option('--not-show-time',
+ action='store_false',
+ dest='show_time',
+ default=True,
+ help='Show time of scanning'),
+ )
args = ''
help = 'Will scan only herds and maintainers'
def handle(self, *args, **options):
- st = datetime.datetime.now()
- scan.scan_herds()
- print (datetime.datetime.now() - st).total_seconds()
+ Scanner(scan_herds = True, **options).scan()
diff --git a/gpackages/apps/packages/management/commands/scanusedesc.py b/gpackages/apps/packages/management/commands/scanusedesc.py
index e0ca2b8..705314f 100644
--- a/gpackages/apps/packages/management/commands/scanusedesc.py
+++ b/gpackages/apps/packages/management/commands/scanusedesc.py
@@ -1,14 +1,27 @@
from django.core.management.base import BaseCommand, CommandError
-import datetime
-import logging
-from packages import scan
+from packages.scan import Scanner
+from optparse import make_option
class Command(BaseCommand):
+ option_list = BaseCommand.option_list + (
+ make_option('-g','--no-globals',
+ action='store_false',
+ dest='scan_global_use',
+ default=True,
+ help='Don\'t scan globals use descriptions'),
+ make_option('-l', '--no-locals',
+ action='store_false',
+ dest='scan_local_use',
+ default=True,
+ help='Don\'t scan globals use descriptions'),
+ make_option('--not-show-time',
+ action='store_false',
+ dest='show_time',
+ default=True,
+ help='Show time of scanning'),
+ )
args = ''
help = 'Will scan use descriptions'
def handle(self, *args, **options):
- st = datetime.datetime.now()
- scan.update_globals_uses_descriptions()
- scan.scan_uses_description()
- print (datetime.datetime.now() - st).total_seconds()
+ Scanner(**options).scan()
diff --git a/gpackages/apps/packages/managers.py b/gpackages/apps/packages/managers.py
index 7d02c05..ca1561c 100644
--- a/gpackages/apps/packages/managers.py
+++ b/gpackages/apps/packages/managers.py
@@ -104,13 +104,13 @@ def get_name_and_category_by_cp(package):
class VirtualPackageMixin(object):
- def filter(self, **kwargs):
+ def filter(self,*args, **kwargs):
package = get_from_kwargs_and_del('package', kwargs)
if package is not None:
category, name = get_name_and_category_by_cp(package)
kwargs.update({'name': name, 'category__category': category})
- return super(VirtualPackageMixin, self).filter(**kwargs)
+ return super(VirtualPackageMixin, self).filter(*args, **kwargs)
_gen_all_query_and_manager('Mixin', 'QuerySet', 'Manager',
diff --git a/gpackages/apps/packages/scan.py b/gpackages/apps/packages/scan.py
index 3417c12..cdd9538 100644
--- a/gpackages/apps/packages/scan.py
+++ b/gpackages/apps/packages/scan.py
@@ -127,6 +127,8 @@ class Scanner(object):
self.force_update = bool(kwargs.get('force_update', False))
self.delete = bool(kwargs.get('delete', True))
self.scan_repos_name = tuple(kwargs.get('repos',[]))
+ self.scan_global_use_descr = bool(kwargs.get('scan_global_use', False))
+ self.scan_local_use_descr = bool(kwargs.get('scan_local_use', False))
def show_time(self):
end = datetime.now()
@@ -140,11 +142,17 @@ class Scanner(object):
if self.s_all:
self.scan_all_repos(force_update = self.force_update,
delete = self.delete)
- else:
+ elif len(self.scan_repos_name) > 0:
self.scan_repos_by_name(self.scan_repos_name,
force_update = self.force_update,
delete = self.delete)
+ if self.scan_global_use_descr:
+ self.update_all_globals_uses_descriptions()
+
+ if self.scan_local_use_descr:
+ self.scan_all_uses_description()
+
if self.is_show_time:
self.show_time()
@@ -497,7 +505,7 @@ class Scanner(object):
.filter(name__in = use_dict.keys())
for use_object in existend_use_objects:
- use_object.description = use_dict[use_object.name]
+ use_object.description = use_dict[use_object.name.lower()]
use_object.save(force_update = True)
def update_all_globals_uses_descriptions(self):
@@ -507,7 +515,9 @@ class Scanner(object):
self.scan_uses_description(portage.get_all_use_local_desc())
def scan_uses_description(self, use_local_desc):
- existent_use_objects = models.UseFlagModel.objects.filter(name__in = use_local_desc.keys())
+ existent_use_objects = models.UseFlagModel.objects \
+ .filter(name__in = use_local_desc.keys())
+
existent_use_local_descr = models.UseFlagDescriptionModel.objects.all()
cache_uses = {}
_update_cache_by_queryset(cache_uses, existent_use_objects, 'name')
@@ -522,7 +532,8 @@ class Scanner(object):
# If this use flag not in database than create it
if use_flag not in cache_uses:
# Maybe get_or_create ?
- use_flag_object = models.UseFlagModel.objects.create(name = use_flag)
+ use_flag_object = models.UseFlagModel.objects \
+ .create(name = use_flag)
# Add to cache
cache_uses[use_flag.lower()] = use_flag_object
else:
@@ -534,7 +545,9 @@ class Scanner(object):
package_object = package_cache[package]
else:
try:
- package_object = models.VirtualPackageModel.objects.get(package = package)
+ package_object = models.VirtualPackageModel.objects \
+ .get(package = package)
+
except models.VirtualPackageModel.DoesNotExist:
continue
else:
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/gentoo-packages:master commit in: gpackages/apps/packages/, gpackages/apps/packages/management/commands/
@ 2012-06-10 22:23 Slava Bacherikov
0 siblings, 0 replies; 20+ messages in thread
From: Slava Bacherikov @ 2012-06-10 22:23 UTC (permalink / raw
To: gentoo-commits
commit: 80008434dd3688ad1b150080dfd29f61725e83d2
Author: Slava Bacherikov <slava <AT> bacher09 <DOT> org>
AuthorDate: Sun Jun 10 21:21:18 2012 +0000
Commit: Slava Bacherikov <slava <AT> bacherikov <DOT> org <DOT> ua>
CommitDate: Sun Jun 10 21:21:18 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/gentoo-packages.git;a=commit;h=80008434
Add support of delete and force_update in scanpackages
---
.../packages/management/commands/scanpackages.py | 8 +++--
gpackages/apps/packages/scan.py | 38 ++++++++++++--------
2 files changed, 28 insertions(+), 18 deletions(-)
diff --git a/gpackages/apps/packages/management/commands/scanpackages.py b/gpackages/apps/packages/management/commands/scanpackages.py
index a1d5cd6..97ed11e 100644
--- a/gpackages/apps/packages/management/commands/scanpackages.py
+++ b/gpackages/apps/packages/management/commands/scanpackages.py
@@ -29,6 +29,11 @@ class Command(BaseCommand):
dest='show_time',
default=True,
help='Show time of scanning'),
+ make_option('--not-del',
+ action='store_false',
+ dest='delete',
+ default=True,
+ help='Not delete'),
#make_option('-r', '--repo',
#action='store',
#type="string",
@@ -41,6 +46,3 @@ class Command(BaseCommand):
def handle(self, *args, **options):
verbosity = int(options['verbosity'])
Scanner(repos = args, **options).scan()
- #Scanner(verbosity = verbosity).scan_all_repos()
- #self.stdout.write(unicode((datetime.datetime.now() - st).total_seconds()))
- #self.stdout.write("\n")
diff --git a/gpackages/apps/packages/scan.py b/gpackages/apps/packages/scan.py
index 411500f..ef28293 100644
--- a/gpackages/apps/packages/scan.py
+++ b/gpackages/apps/packages/scan.py
@@ -125,7 +125,8 @@ class Scanner(object):
self.s_all = bool(kwargs.get('scan_all', False))
self.is_show_time = bool(kwargs.get('show_time', True))
self.is_scan_herds = bool(kwargs.get('scan_herds', True))
- #self.force_update = bool(kwargs.get('force_update', False))
+ self.force_update = bool(kwargs.get('force_update', False))
+ self.delete = bool(kwargs.get('delete', True))
self.scan_repos_name = tuple(kwargs.get('repos',[]))
def show_time(self):
@@ -138,9 +139,12 @@ class Scanner(object):
self.scan_herds()
if self.s_all:
- self.scan_all_repos()
+ self.scan_all_repos(force_update = self.force_update,
+ delete = self.delete)
else:
- self.scan_repos_by_name(self.scan_repos_name)
+ self.scan_repos_by_name(self.scan_repos_name,
+ force_update = self.force_update,
+ delete = self.delete)
if self.is_show_time:
self.show_time()
@@ -276,32 +280,32 @@ class Scanner(object):
self.load_maintainers_to_cache()
return self.maintainers_cache
- def scan_all_repos(self):
+ def scan_all_repos(self, **kwargs):
#cache_dict = anydbm.open('cache.db','c')
for repo in portage.iter_trees():
- self.scan_repo(repo)
+ self.scan_repo(repo, **kwargs)
#cache_dict.close()
- def scan_repos_by_name(self, repo_names):
+ def scan_repos_by_name(self, repo_names, **kwargs):
for repo_name in repo_names:
- self.scan_repo_by_name(repo_name)
+ self.scan_repo_by_name(repo_name, **kwargs)
- def scan_repo_by_name(self, repo_name):
+ def scan_repo_by_name(self, repo_name, **kwargs):
try:
repo = portage.get_tree_by_name(repo_name)
except ValueError:
self.output("Bad repository name '%s'", repo.name, 1)
else:
- self.scan_repo(repo)
+ self.scan_repo(repo, **kwargs)
- def scan_repo(self, repo):
+ def scan_repo(self, repo, **kwargs):
self.output("Scaning repository '%s'\n", repo.name, 3)
repo_obj, repo_created = models.RepositoryModel \
.objects.get_or_create(name = repo.name)
- self.scanpackages(repo, repo_obj)
+ self.scanpackages(repo, repo_obj, **kwargs)
def get_licenses_objects(self, ebuild):
@@ -402,7 +406,9 @@ class Scanner(object):
self.clear_related_to_package(package_object)
self.add_related_to_package(package, package_object)
- def update_ebuilds(self, package, package_object, delete = True):
+ def update_ebuilds(self, package, package_object, delete = True,
+ force_update = False):
+
not_del = []
for ebuild in package.iter_ebuilds():
ebuild_object, ebuild_created = models.EbuildModel.objects \
@@ -413,7 +419,7 @@ class Scanner(object):
self.add_related_to_ebuild(ebuild, ebuild_object)
self.output("ebuild created '%s'\n", ebuild_object, 3)
- if ebuild_object.check_or_need_update(ebuild):
+ if ebuild_object.check_or_need_update(ebuild) or force_update:
ebuild_object.update_by_ebuild(ebuild)
self.update_related_to_ebuild(ebuild, ebuild_object)
ebuild_object.save(force_update = True)
@@ -429,7 +435,8 @@ class Scanner(object):
self.update_related_to_package(package, package_object)
if package_object.need_update_ebuilds(package) or force_update:
- self.update_ebuilds(package, package_object)
+ self.update_ebuilds(package, package_object,
+ force_update = force_update)
package_object.update_info(package)
package_object.save(force_update = True)
@@ -466,7 +473,8 @@ class Scanner(object):
if not package_created:
if package_object.check_or_need_update(package) or force_update:
# need update
- self.update_package(package, package_object)
+ self.update_package(package, package_object,
+ force_update = force_update)
continue
# if package_created:
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/gentoo-packages:master commit in: gpackages/apps/packages/, gpackages/apps/packages/management/commands/
@ 2012-06-10 17:51 Slava Bacherikov
0 siblings, 0 replies; 20+ messages in thread
From: Slava Bacherikov @ 2012-06-10 17:51 UTC (permalink / raw
To: gentoo-commits
commit: 1885cf27372ec2f34964d3323c2ffdb0322c6123
Author: Slava Bacherikov <slava <AT> bacher09 <DOT> org>
AuthorDate: Sun Jun 10 17:10:26 2012 +0000
Commit: Slava Bacherikov <slava <AT> bacherikov <DOT> org <DOT> ua>
CommitDate: Sun Jun 10 17:10:26 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/gentoo-packages.git;a=commit;h=1885cf27
Add support for command modificators
---
.../packages/management/commands/scanpackages.py | 41 +++++++++--
gpackages/apps/packages/scan.py | 73 +++++++++++++++++---
2 files changed, 95 insertions(+), 19 deletions(-)
diff --git a/gpackages/apps/packages/management/commands/scanpackages.py b/gpackages/apps/packages/management/commands/scanpackages.py
index 8f47848..a1d5cd6 100644
--- a/gpackages/apps/packages/management/commands/scanpackages.py
+++ b/gpackages/apps/packages/management/commands/scanpackages.py
@@ -1,5 +1,4 @@
from django.core.management.base import BaseCommand, CommandError
-import datetime
import logging
from packages.scan import Scanner
from optparse import make_option
@@ -9,13 +8,39 @@ from optparse import make_option
class Command(BaseCommand):
- args = ''
+ option_list = BaseCommand.option_list + (
+ make_option('--force-update',
+ action='store_true',
+ dest='force_update',
+ default=False,
+ help='Force updating'),
+ make_option('--not-scan-herds',
+ action='store_false',
+ dest='scan_herds',
+ default=True,
+ help='Force updating'),
+ make_option('-a', '--all',
+ action='store_true',
+ dest='scan_all',
+ default=False,
+ help='Force updating'),
+ make_option('--not-show-time',
+ action='store_false',
+ dest='show_time',
+ default=True,
+ help='Show time of scanning'),
+ #make_option('-r', '--repo',
+ #action='store',
+ #type="string",
+ #dest='reponame',
+ #help='Scan only this repository'),
+ )
+
+ args = '<repository names ...>'
help = 'Will scan package tree and update info about it in database'
def handle(self, *args, **options):
verbosity = int(options['verbosity'])
- st = datetime.datetime.now()
- #scan.scanpackages()
- #scan.scan_all_repos()
- Scanner(verbosity = verbosity).scan_all_repos()
- self.stdout.write(unicode((datetime.datetime.now() - st).total_seconds()))
- self.stdout.write("\n")
+ Scanner(repos = args, **options).scan()
+ #Scanner(verbosity = verbosity).scan_all_repos()
+ #self.stdout.write(unicode((datetime.datetime.now() - st).total_seconds()))
+ #self.stdout.write("\n")
diff --git a/gpackages/apps/packages/scan.py b/gpackages/apps/packages/scan.py
index 37ba159..411500f 100644
--- a/gpackages/apps/packages/scan.py
+++ b/gpackages/apps/packages/scan.py
@@ -1,3 +1,4 @@
+from datetime import datetime
from packages import models
import sys
from django.db import IntegrityError
@@ -86,9 +87,15 @@ def _get_items(items_list, Model, field_name, cache_var):
return items_objects
+def toint(val, defval):
+ try:
+ return int(val)
+ except ValueError:
+ return defval
+
class Scanner(object):
- def __init__(self, verbosity = 1):
+ def __init__(self, **kwargs):
# maintainers_cache: maintainer.email as key, and maintainer object as
# value
self.maintainers_cache = {}
@@ -106,7 +113,37 @@ class Scanner(object):
self.arches_cache = {}
- self.verbosity = int(verbosity)
+ self.update_options(**kwargs)
+ self.reset_timer()
+
+ def reset_timer(self):
+ self.start_time = datetime.now()
+
+ def update_options(self, **kwargs):
+ self.verbosity = toint(kwargs.get('verbosity',1),1)
+ self.traceback = bool(kwargs.get('traceback',False))
+ self.s_all = bool(kwargs.get('scan_all', False))
+ self.is_show_time = bool(kwargs.get('show_time', True))
+ self.is_scan_herds = bool(kwargs.get('scan_herds', True))
+ #self.force_update = bool(kwargs.get('force_update', False))
+ self.scan_repos_name = tuple(kwargs.get('repos',[]))
+
+ def show_time(self):
+ end = datetime.now()
+ t_time = end - self.start_time
+ self.output("Scanning time is: %s secconds.\n", t_time.total_seconds())
+
+ def scan(self):
+ if self.is_scan_herds:
+ self.scan_herds()
+
+ if self.s_all:
+ self.scan_all_repos()
+ else:
+ self.scan_repos_by_name(self.scan_repos_name)
+
+ if self.is_show_time:
+ self.show_time()
def write(self, what, verbosity = 1):
if verbosity <= self.verbosity:
@@ -177,6 +214,7 @@ class Scanner(object):
def scan_herds(self):
+ self.write('Scaning herds\n', 3)
existent_herds = self.get_existent_herds()
herds_dict = self.herds_object.get_herds_indict()
herds_objects_dict = {}
@@ -238,20 +276,33 @@ class Scanner(object):
self.load_maintainers_to_cache()
return self.maintainers_cache
- def scan_all_repos(self, scan_herds = True):
+ def scan_all_repos(self):
#cache_dict = anydbm.open('cache.db','c')
- if scan_herds:
- self.write('Scaning herds\n', 3)
- self.scan_herds()
for repo in portage.iter_trees():
- self.output("Scaning repository '%s'\n", repo.name, 3)
+ self.scan_repo(repo)
+ #cache_dict.close()
- repo_obj, repo_created = models.RepositoryModel.objects \
- .get_or_create(name = repo.name)
+ def scan_repos_by_name(self, repo_names):
+ for repo_name in repo_names:
+ self.scan_repo_by_name(repo_name)
- self.scanpackages(repo, repo_obj)
- #cache_dict.close()
+ def scan_repo_by_name(self, repo_name):
+ try:
+ repo = portage.get_tree_by_name(repo_name)
+ except ValueError:
+ self.output("Bad repository name '%s'", repo.name, 1)
+ else:
+ self.scan_repo(repo)
+
+ def scan_repo(self, repo):
+ self.output("Scaning repository '%s'\n", repo.name, 3)
+
+ repo_obj, repo_created = models.RepositoryModel \
+ .objects.get_or_create(name = repo.name)
+
+ self.scanpackages(repo, repo_obj)
+
def get_licenses_objects(self, ebuild):
licenses = ebuild.licenses
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/gentoo-packages:master commit in: gpackages/apps/packages/, gpackages/apps/packages/management/commands/
@ 2012-06-09 18:19 Slava Bacherikov
0 siblings, 0 replies; 20+ messages in thread
From: Slava Bacherikov @ 2012-06-09 18:19 UTC (permalink / raw
To: gentoo-commits
commit: d482308972446f7649ff7fbf446512160e57cfb2
Author: Slava Bacherikov <slava <AT> bacher09 <DOT> org>
AuthorDate: Sat Jun 9 15:45:24 2012 +0000
Commit: Slava Bacherikov <slava <AT> bacherikov <DOT> org <DOT> ua>
CommitDate: Sat Jun 9 15:45:24 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/gentoo-packages.git;a=commit;h=d4823089
Refactoring scan.py
---
.../packages/management/commands/scanpackages.py | 9 +-
gpackages/apps/packages/managers.py | 5 +-
gpackages/apps/packages/models.py | 1 -
gpackages/apps/packages/scan.py | 539 +++++++++++---------
4 files changed, 309 insertions(+), 245 deletions(-)
diff --git a/gpackages/apps/packages/management/commands/scanpackages.py b/gpackages/apps/packages/management/commands/scanpackages.py
index 35a117b..3f5010c 100644
--- a/gpackages/apps/packages/management/commands/scanpackages.py
+++ b/gpackages/apps/packages/management/commands/scanpackages.py
@@ -1,7 +1,8 @@
from django.core.management.base import BaseCommand, CommandError
import datetime
import logging
-from packages import scan
+from packages.scan import Scanner
+from optparse import make_option
#l = logging.getLogger('django.db.backends')
#l.setLevel(logging.DEBUG)
#l.addHandler(logging.FileHandler('database.log'))
@@ -13,5 +14,7 @@ class Command(BaseCommand):
def handle(self, *args, **options):
st = datetime.datetime.now()
#scan.scanpackages()
- scan.scan_all_repos()
- print (datetime.datetime.now() - st).total_seconds()
+ #scan.scan_all_repos()
+ Scanner().scan_all_repos()
+ self.stdout.write(unicode((datetime.datetime.now() - st).total_seconds()))
+ self.stdout.write("\n")
diff --git a/gpackages/apps/packages/managers.py b/gpackages/apps/packages/managers.py
index 7ebde50..5ef2ed6 100644
--- a/gpackages/apps/packages/managers.py
+++ b/gpackages/apps/packages/managers.py
@@ -80,9 +80,12 @@ class HerdsMixin(object):#{{{
class MaintainerMixin(object):#{{{
def filter(self, *args, **kwargs):
- maintainer__in = get_from_kwargs_and_del('maintainer__in', kwargs)
+ maintainer__in, maintainer = \
+ get_from_kwargs_and_del(['maintainer__in', 'maintainer'], kwargs)
if maintainer__in is not None:
kwargs['email__in'] = maintainer__in
+ elif maintainer is not None:
+ kwargs['email'] = maintainer.email
return super(MaintainerMixin, self).filter(*args, **kwargs)#}}}
diff --git a/gpackages/apps/packages/models.py b/gpackages/apps/packages/models.py
index 82d3ca5..dc2147c 100644
--- a/gpackages/apps/packages/models.py
+++ b/gpackages/apps/packages/models.py
@@ -55,7 +55,6 @@ class MaintainerModel(AbstractDateTimeModel):
def init_by_maintainer(self, maintainer):
self.name = maintainer.name
self.email = maintainer.email
- self.role = maintainer.role
def update_by_maintainer(self, maintainer):
self.name = maintainer.name
diff --git a/gpackages/apps/packages/scan.py b/gpackages/apps/packages/scan.py
index c9645fc..db9c0e6 100644
--- a/gpackages/apps/packages/scan.py
+++ b/gpackages/apps/packages/scan.py
@@ -83,78 +83,315 @@ def _get_items(items_list, Model, field_name, cache_var):
return items_objects
-def scan_maintainers(maintainers_dict):
- existend_maintainers = models.MaintainerModel.objects.all()
- main_dict = {}
- mo_dict = {}
- #to_del = []
- _update_cache_by_queryset(main_dict, maintainers_dict.keys(), 'email')
- for maintainer_object in existend_maintainers:
- if maintainer_object.email in main_dict:
- maintainer_cmp = main_dict[maintainer_object.email]
- # need update ?
- if maintainer_object.check_or_need_update(maintainer_cmp):
- # updating
- maintainer_object.update_by_maintainer(maintainer_cmp)
- maintainer_object.save(force_update = True)
- mo_dict[maintainer_object.email] = maintainer_object
- #else:
- #to_del.append(maintainer_object.pk)
- #print to_del
- #print mo_dict
- to_create = []
- for maintainer in maintainers_dict.iterkeys():
- if maintainer.email not in mo_dict:
- to_create.append(maintainer)
-
- mobjects = _create_objects(models.MaintainerModel, 'maintainer', to_create)
- _update_cache_by_queryset(mo_dict, mobjects, 'email')
-
- return mo_dict
+
+class Scanner(object):
+ def __init__(self, verbosity = 0):
+ # maintainers_cache: maintainer.email as key, and maintainer object as
+ # value
+ self.maintainers_cache = {}
+ self.maintainers_cache_loaded = False
+ # herds cache: herd.name as key and herd object as value
+ self.herds_cache = {}
+ self.herds_cache_loaded = False
+ self.licenses_cache = {}
+ self.uses_cache = {}
+ self.arches_cache = {}
+ self.homepages_cache = {}
+ self.herds_cache = {}
+ self.maintainers_cache = {}
+ self.herds_object = herds.Herds()
+
+ self.arches_cache = {}
+
+ self.verbosity = verbosity
+
+ def get_existent_maintainers(self):
+ return models.MaintainerModel.objects.all()
+
+ def get_existent_herds(self):
+ return models.HerdsModel.objects.all()
+
+ def get_maintainers_cache(self):
+ maintainers_dict = {}
+ existent_maintainers = self.get_existent_maintainers()
+ _update_cache_by_queryset(maintainers_dict, existent_maintainers, 'email')
+ return maintainers_dict
+
+ def load_maintainers_to_cache(self):
+ self.maintainers_cache = self.get_maintainers_cache()
+ self.maintainers_cache_loaded = True
+
+ def get_herds_cache(self):
+ herds_dict = {}
+ existent_herds = self.get_existent_herds()
+ _update_cache_by_queryset(herds_dict, existent_herds, 'name')
+ return herds_dict
+
+ def load_herds_to_cache(self):
+ if not self.herds_cache_loaded:
+ self.herds_cache = self.get_herds_cache()
+ self.herds_cache_loaded = True
+
+ def scan_maintainers(self, maintainers_dict):
+ existend_maintainers = self.get_existent_maintainers()
+ main_dict = {}
+ mo_dict = {}
+ #to_del = []
+ _update_cache_by_queryset(main_dict, maintainers_dict.keys(), 'email')
+ for maintainer_object in existend_maintainers:
+ if maintainer_object.email in main_dict:
+ maintainer_cmp = main_dict[maintainer_object.email]
+ # need update ?
+ if maintainer_object.check_or_need_update(maintainer_cmp):
+ # updating
+ maintainer_object.update_by_maintainer(maintainer_cmp)
+ maintainer_object.save(force_update = True)
+ mo_dict[maintainer_object.email] = maintainer_object
+
+ to_create = []
+ for maintainer in maintainers_dict.iterkeys():
+ if maintainer.email not in mo_dict:
+ to_create.append(maintainer)
+
+ mobjects = _create_objects(models.MaintainerModel, 'maintainer', to_create)
+ _update_cache_by_queryset(mo_dict, mobjects, 'email')
+
+ self.maintainers_cache = mo_dict
+ self.maitainers_cache_loaded = True
-def scan_herds():
- existent_herds = models.HerdsModel.objects.all()
- herds_object = herds.Herds()
- herds_dict = herds_object.get_herds_indict()
- maintainers_dict = herds_object.get_maintainers_with_herds()
- ho_dict = {}
- to_del = []
- for herd_object in existent_herds:
- if herd_object.name not in herds_dict:
- to_del.append(herd_object.pk)
+ def scan_herds(self):
+ existent_herds = self.get_existent_herds()
+ herds_dict = self.herds_object.get_herds_indict()
+ herds_objects_dict = {}
+ to_del = []
+ for herd_object in existent_herds:
+ if herd_object.name not in herds_dict:
+ to_del.append(herd_object.pk)
+ else:
+ herd_cmp = herds_dict[herd_object.name]
+ # need update ?
+ if herd_object.check_or_need_update(herd_cmp):
+ # updating
+ herd_object.update_by_herd(herd_cmp)
+ herd_object.save(force_update = True)
+ herds_objects_dict[herd_object.name] = herd_object
+
+ models.HerdsModel.objects.filter(pk__in = to_del).delete()
+
+ to_create = []
+ for herd in herds_dict.itervalues():
+ if herd.name not in herds_objects_dict:
+ to_create.append(herd)
+
+ cobjects = _create_objects(models.HerdsModel, 'herd', to_create)
+ _update_cache_by_queryset(herds_objects_dict, cobjects, 'name')
+
+ # Add to cache
+ self.herds_cache = herds_objects_dict
+ self.herds_cache_loaded = True
+ # Add related maintainers to herds
+ self._add_related_to_herds()
+
+
+ def _get_maintainers_for_relation_with_herds(self, maintainers_dict):
+ res = defaultdict(list)
+ for mainteiner, herds_names in maintainers_dict.iteritems():
+ for herd in herds_names:
+ res[herd].append(self.maintainers_cache[mainteiner.email])
+ return res
+
+ def _add_related_to_herds(self):
+ maintainers_dict = self.herds_object.get_maintainers_with_herds()
+ self.scan_maintainers(maintainers_dict)
+ #Gen data for relate with herds
+ res = self._get_maintainers_for_relation_with_herds(maintainers_dict)
+
+ for herd_name, herd_object in self.herds_cache.iteritems():
+ herd_object.maintainers.clear()
+ herd_object.maintainers.add(*res[herd_name])
+
+ def get_maintainers(self):
+ if not self.maintainers_cache_loaded:
+ self.load_maintainers_to_cache()
+ return self.maintainers_cache
+
+ def scan_all_repos(self, scan_herds = True):
+ #cache_dict = anydbm.open('cache.db','c')
+ if scan_herds:
+ self.scan_herds()
+ for repo in portage.iter_trees():
+ repo_obj, repo_created = models.RepositoryModel.objects.get_or_create(name = repo.name)
+ self.scanpackages(repo, repo_obj)
+ #cache_dict.close()
+
+ def get_licenses_objects(self, ebuild):
+ licenses = ebuild.licenses
+ return _get_items(licenses, models.LicensModel, 'name', self.licenses_cache)
+
+ def get_uses_objects(self, ebuild):
+ uses = [ use.name for use in ebuild.iter_uses() ]
+ return _get_items(uses, models.UseFlagModel, 'name', self.uses_cache)
+
+ def get_homepages_objects(self, ebuild):
+ homepages = ebuild.homepages
+ return _get_items(homepages, models.HomepageModel, 'url', self.homepages_cache)
+
+ def get_maintainers_objects(self, package):
+ objects = []
+ for maintainer in package.metadata.maintainers():
+ if maintainer.email in self.maintainers_cache:
+ objects.append(self.maintainers_cache[maintainer.email])
+ else:
+ maintainer_object, created = models.MaintainerModel \
+ .objects.get_or_create(maintainer = maintainer)
+ objects.append(maintainer_object)
+ # Add to cache
+ self.maintainers_cache[maintainer_object.email] = maintainer_object
+ return objects
+
+ def get_herds_objects(self, package):
+ self.load_herds_to_cache()
+ herds_objects = []
+ for herd in package.metadata.herds():
+ if herd in self.herds_cache:
+ herds_objects.append(self.herds_cache[herd])
+
+ return herds_objects
+
+ def get_arch_object(self, arch_name):
+ if arch_name in self.arches_cache:
+ arch = self.arches_cache[arch_name]
else:
- herd_cmp = herds_dict[herd_object.name]
- # need update ?
- if herd_object.check_or_need_update(herd_cmp):
- # updating
- herd_object.update_by_herd(herd_cmp)
- herd_object.save(force_update = True)
- ho_dict[herd_object.name] = herd_object
+ arch, created = models.ArchesModel.objects.get_or_create(name = arch_name)
+ self.arches_cache[arch_name] = arch
+ return arch
+
+ def create_keywords_objects(self, ebuild, ebuild_object):
+ keywords_list = []
+ for keyword in ebuild.get_keywords():
+ keyword_object = models.Keyword(status = keyword.status,
+ ebuild = ebuild_object)
+
+ keyword_object.arch = self.get_arch_object(keyword.arch)
+ keywords_list.append(keyword_object)
+
+ models.Keyword.objects.bulk_create(keywords_list)
+
+ def clean_keywords_object(self, ebuild_object):
+ models.Keyword.objects.filter(ebuild = ebuild_object).delete()
- models.HerdsModel.objects.filter(pk__in = to_del).delete()
+ def scanpackages(self, porttree, porttree_obj, delete = True,
+ force_update = False, update_cache = True, use_cache = True):
- to_create = []
- for herd in herds_dict.itervalues():
- if herd.name not in ho_dict:
- to_create.append(herd)
+
- cobjects = _create_objects(models.HerdsModel, 'herd', to_create)
- _update_cache_by_queryset(ho_dict, cobjects, 'name')
- mo_dict = scan_maintainers(maintainers_dict)
- #Gen data for relate with herds
- res = defaultdict(list)
- for mainteiner, herds_names in maintainers_dict.iteritems():
- for herd in herds_names:
- res[herd].append(mo_dict[mainteiner.email])
+ def add_related_to_ebuild(ebuild, ebuild_object):
+ # Add licenses
+ ebuild_object.licenses.add(*self.get_licenses_objects(ebuild))
+ ebuild_object.use_flags.add(*self.get_uses_objects(ebuild))
+ ebuild_object.homepages.add(*self.get_homepages_objects(ebuild))
+ self.create_keywords_objects(ebuild, ebuild_object)
+
+ def clear_related_to_ebuild(ebuild_object):
+ ebuild_object.licenses.clear()
+ ebuild_object.use_flags.clear()
+ ebuild_object.homepages.clear()
+ self.clean_keywords_object(ebuild_object)
+
+ def update_related_to_ebuild(ebuild, ebuild_object):
+ clear_related_to_ebuild(ebuild_object)
+ add_related_to_ebuild(ebuild, ebuild_object)
+
+ def create_ebuilds(package, package_object):
+ for ebuild in package.iter_ebuilds():
+ ebuild_object = models.EbuildModel()
+ ebuild_object.init_by_ebuild(ebuild)
+ ebuild_object.package = package_object
+ # To Add some related objects it should have pk
+ ebuild_object.save(force_insert=True)
+ add_related_to_ebuild(ebuild, ebuild_object)
+
+ def update_ebuilds(package, package_object, delete = True):
+ not_del = []
+ for ebuild in package.iter_ebuilds():
+ ebuild_object, ebuild_created = models.EbuildModel.objects.get_or_create(ebuild = ebuild, package = package_object)
+ not_del.append(ebuild_object.pk)
+ if ebuild_created:
+ add_related_to_ebuild(ebuild, ebuild_object)
+ continue
+ if ebuild_object.check_or_need_update(ebuild):
+ ebuild_object.update_by_ebuild(ebuild)
+ update_related_to_ebuild(ebuild, ebuild_object)
+ ebuild_object.save(force_update = True)
+ if delete:
+ models.EbuildModel.objects.filter(package = package_object).exclude(pk__in = not_del).delete()
+
+ def clear_related_to_package(package_object):
+ package_object.herds.clear()
+ package_object.maintainers.clear()
+
+
+ def add_related_to_package(package, package_object):
+ package_object.herds.add(*self.get_herds_objects(package))
+ package_object.maintainers.add(*self.get_maintainers_objects(package))
+
+ def update_related_to_package(package, package_object):
+ clear_related_to_package(package_object)
+ add_related_to_package(package, package_object)
- for herd_name, herd_object in ho_dict.iteritems():
- herd_object.maintainers.clear()
- herd_object.maintainers.add(*res[herd_name])
+ def update_package(package, package_object, force_update = False):
+ if package_object.need_update_metadata(package) or force_update:
+ #Updating related objects to package
+ update_related_to_package(package, package_object)
+
+ if package_object.need_update_ebuilds(package) or force_update:
+ update_ebuilds(package, package_object)
+
+ package_object.update_info(package)
+ package_object.save(force_update = True)
+
+ # Load homepages to cache
+ #for homepage in models.HomepageModel.objects.all():
+ #homepages_cache[homepage.url] = homepage
+ existend_categorys = []
+ for category in porttree.iter_categories():
+ existend_packages = []
+ category_object, category_created = models.CategoryModel.objects.get_or_create(category = category)
+ existend_categorys.append(category_object.pk)
+ for package in category.iter_packages():
+ #if use_cache:
+ #key = str(porttree.name)+'/'+str(package)
+ #val = None
+ #if key in cache_dict:
+ #val = cache_dict[key]
+ #if val is not None and val == package.manifest_sha1:
+ #continue
+ print('%s [%s]' % (str(package).ljust(44), porttree))
+ package_object, package_created = models.PackageModel.objects.only('changelog_hash', 'manifest_hash', 'metadata_hash') \
+ .get_or_create(package = package, category = category_object, repository = porttree_obj)
+ #if update_cache:
+ #key = str(porttree.name)+'/'+str(package)
+ #cache_dict[key] = package.manifest_sha1
+
+ existend_packages.append(package_object.pk)
+ if not package_created:
+ if package_object.check_or_need_update(package) or force_update:
+ # need update
+ update_package(package, package_object)
- return ho_dict, mo_dict
+ continue
+ # if package_created:
+ add_related_to_package(package, package_object)
+ create_ebuilds(package, package_object)
+ if delete:
+ models.PackageModel.objects.filter(category = category_object, repository = porttree_obj).exclude(pk__in = existend_packages).delete()
+
+
+
+cache_dict = None
def update_globals_uses_descriptions():
# Need changes
uses_g = use_info.get_uses_info()
@@ -203,181 +440,3 @@ def scan_uses_description():
use_desc_obj.save(force_update = True)
models.UseFlagDescriptionModel.objects.bulk_create(to_create)
-licenses_cache = {}
-uses_cache = {}
-arches_cache = {}
-homepages_cache = {}
-herds_cache = {}
-maintainers_cache = {}
-
-cache_dict = None
-def scanpackages(porttree, porttree_obj, delete = True, force_update = False,
- update_cache = True, use_cache = True):
- def get_licenses_objects(ebuild):
- licenses = ebuild.licenses
- return _get_items(licenses, models.LicensModel, 'name', licenses_cache)
-
- def get_uses_objects(ebuild):
- uses = [ use.name for use in ebuild.iter_uses() ]
- return _get_items(uses, models.UseFlagModel, 'name', uses_cache)
-
- def get_keywords_objects(ebuild, ebuild_object):
- keywords_list = []
- for keyword in ebuild.get_keywords():
- keyword_object = models.Keyword(status = keyword.status,
- ebuild = ebuild_object)
-
- if keyword.arch in arches_cache:
- arch = arches_cache[keyword.arch]
- else:
- arch, created = models.ArchesModel.objects.get_or_create(name = keyword.arch)
- arches_cache[keyword.arch] = arch
-
- keyword_object.arch = arch
- keywords_list.append(keyword_object)
-
- models.Keyword.objects.bulk_create(keywords_list)
-
-
- def get_homepages_objects(ebuild):
- homepages = ebuild.homepages
- return _get_items(homepages, models.HomepageModel, 'url', homepages_cache)
-
-
- def get_maintainers_objects(package):
- maintainers = package.metadata.maintainers()
- objects = []
- for maintainer in maintainers:
- if maintainer.email in maintainers_cache:
- objects.append(maintainers_cache[maintainer.email])
- else:
- maintainer_object, created = models.MaintainerModel \
- .objects.get_or_create(email = maintainer.email)
- if created:
- maintainer_object.name = maintainer.name
- maintainer_object.save()
- objects.append(maintainer_object)
- return objects
-
-
- def get_herds_objects(package):
- herds = package.metadata.herds()
- herds_objects = []
- for herd in herds:
- if herd in herds_cache:
- herds_objects.append(herds_cache[herd])
-
- return herds_objects
-
- def add_related_to_ebuild(ebuild, ebuild_object):
- # Add licenses
- ebuild_object.licenses.add(*get_licenses_objects(ebuild))
- ebuild_object.use_flags.add(*get_uses_objects(ebuild))
- ebuild_object.homepages.add(*get_homepages_objects(ebuild))
- get_keywords_objects(ebuild, ebuild_object)
-
- def clear_related_to_ebuild(ebuild_object):
- ebuild_object.licenses.clear()
- ebuild_object.use_flags.clear()
- ebuild_object.homepages.clear()
- models.Keyword.objects.filter(ebuild = ebuild_object).delete()
-
- def update_related_to_ebuild(ebuild, ebuild_object):
- clear_related_to_ebuild(ebuild_object)
- add_related_to_ebuild(ebuild, ebuild_object)
-
- def create_ebuilds(package, package_object):
- for ebuild in package.iter_ebuilds():
- ebuild_object = models.EbuildModel()
- ebuild_object.init_by_ebuild(ebuild)
- ebuild_object.package = package_object
- # To Add some related objects it should have pk
- ebuild_object.save(force_insert=True)
- add_related_to_ebuild(ebuild, ebuild_object)
-
- def update_ebuilds(package, package_object, delete = True):
- not_del = []
- for ebuild in package.iter_ebuilds():
- ebuild_object, ebuild_created = models.EbuildModel.objects.get_or_create(ebuild = ebuild, package = package_object)
- not_del.append(ebuild_object.pk)
- if ebuild_created:
- add_related_to_ebuild(ebuild, ebuild_object)
- continue
- if ebuild_object.check_or_need_update(ebuild):
- ebuild_object.update_by_ebuild(ebuild)
- update_related_to_ebuild(ebuild, ebuild_object)
- ebuild_object.save(force_update = True)
- if delete:
- models.EbuildModel.objects.filter(package = package_object).exclude(pk__in = not_del).delete()
-
- def clear_related_to_package(package_object):
- package_object.herds.clear()
- package_object.maintainers.clear()
-
-
- def add_related_to_package(package, package_object):
- package_object.herds.add(*get_herds_objects(package))
- package_object.maintainers.add(*get_maintainers_objects(package))
-
- def update_related_to_package(package, package_object):
- clear_related_to_package(package_object)
- add_related_to_package(package, package_object)
-
- def update_package(package, package_object, force_update = False):
- if package_object.need_update_metadata(package) or force_update:
- #Updating related objects to package
- update_related_to_package(package, package_object)
-
- if package_object.need_update_ebuilds(package) or force_update:
- update_ebuilds(package, package_object)
-
- package_object.update_info(package)
- package_object.save(force_update = True)
-
- # Load homepages to cache
- #for homepage in models.HomepageModel.objects.all():
- #homepages_cache[homepage.url] = homepage
- existend_categorys = []
- for category in porttree.iter_categories():
- existend_packages = []
- category_object, category_created = models.CategoryModel.objects.get_or_create(category = category)
- existend_categorys.append(category_object.pk)
- for package in category.iter_packages():
- #if use_cache:
- #key = str(porttree.name)+'/'+str(package)
- #val = None
- #if key in cache_dict:
- #val = cache_dict[key]
- #if val is not None and val == package.manifest_sha1:
- #continue
- print('%s [%s]' % (str(package).ljust(44), porttree))
- package_object, package_created = models.PackageModel.objects.only('changelog_hash', 'manifest_hash', 'metadata_hash') \
- .get_or_create(package = package, category = category_object, repository = porttree_obj)
- #if update_cache:
- #key = str(porttree.name)+'/'+str(package)
- #cache_dict[key] = package.manifest_sha1
-
- existend_packages.append(package_object.pk)
- if not package_created:
- if package_object.check_or_need_update(package) or force_update:
- # need update
- update_package(package, package_object)
-
- continue
- add_related_to_package(package, package_object)
- if package_created:
- create_ebuilds(package, package_object)
-
- if delete:
- models.PackageModel.objects.filter(category = category_object, repository = porttree_obj).exclude(pk__in = existend_packages).delete()
-
-
-def scan_all_repos():
- global herds_cache, maintainers_cache
- #global cache_dict
- #cache_dict = anydbm.open('cache.db','c')
- herds_cache, maintainers_cache = scan_herds()
- for repo in portage.iter_trees():
- repo_obj, repo_created = models.RepositoryModel.objects.get_or_create(name = repo.name)
- scanpackages(repo, repo_obj)
- #cache_dict.close()
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/gentoo-packages:master commit in: gpackages/apps/packages/, gpackages/apps/packages/management/commands/
@ 2012-06-09 18:19 Slava Bacherikov
0 siblings, 0 replies; 20+ messages in thread
From: Slava Bacherikov @ 2012-06-09 18:19 UTC (permalink / raw
To: gentoo-commits
commit: e0b3afa49d0b5b1e99c92e5d766a1a443dc5aa77
Author: Slava Bacherikov <slava <AT> bacher09 <DOT> org>
AuthorDate: Sat Jun 9 18:17:40 2012 +0000
Commit: Slava Bacherikov <slava <AT> bacherikov <DOT> org <DOT> ua>
CommitDate: Sat Jun 9 18:17:40 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/gentoo-packages.git;a=commit;h=e0b3afa4
Add support verbosity levels in scan
---
.../packages/management/commands/scanpackages.py | 3 +-
gpackages/apps/packages/models.py | 6 +-
gpackages/apps/packages/scan.py | 47 ++++++++++++++++----
3 files changed, 43 insertions(+), 13 deletions(-)
diff --git a/gpackages/apps/packages/management/commands/scanpackages.py b/gpackages/apps/packages/management/commands/scanpackages.py
index 3f5010c..8f47848 100644
--- a/gpackages/apps/packages/management/commands/scanpackages.py
+++ b/gpackages/apps/packages/management/commands/scanpackages.py
@@ -12,9 +12,10 @@ class Command(BaseCommand):
args = ''
help = 'Will scan package tree and update info about it in database'
def handle(self, *args, **options):
+ verbosity = int(options['verbosity'])
st = datetime.datetime.now()
#scan.scanpackages()
#scan.scan_all_repos()
- Scanner().scan_all_repos()
+ Scanner(verbosity = verbosity).scan_all_repos()
self.stdout.write(unicode((datetime.datetime.now() - st).total_seconds()))
self.stdout.write("\n")
diff --git a/gpackages/apps/packages/models.py b/gpackages/apps/packages/models.py
index 713cb70..3486399 100644
--- a/gpackages/apps/packages/models.py
+++ b/gpackages/apps/packages/models.py
@@ -36,7 +36,7 @@ class CategoryModel(models.Model):
category = models.CharField(unique = True, max_length = 70)
def __unicode__(self):
- return self.category
+ return unicode(self.category)
class MaintainerModel(AbstractDateTimeModel):
@@ -126,11 +126,11 @@ class PackageModel(AbstractDateTimeModel):
objects = managers.PackageManager()
def __unicode__(self):
- return self.cp
+ return unicode(self.cp)
@property
def cp(self):
- return '%s/%s' % (self.category, self.name)
+ return "%s/%s" % (unicode(self.category), self.name)
def init_by_package(self, package, category = None):
self.name = package.name
diff --git a/gpackages/apps/packages/scan.py b/gpackages/apps/packages/scan.py
index d0f476e..a7a77b9 100644
--- a/gpackages/apps/packages/scan.py
+++ b/gpackages/apps/packages/scan.py
@@ -1,4 +1,5 @@
from packages import models
+import sys
from django.db import IntegrityError
from collections import defaultdict
from generic import StrThatIgnoreCase
@@ -87,7 +88,7 @@ def _get_items(items_list, Model, field_name, cache_var):
class Scanner(object):
- def __init__(self, verbosity = 0):
+ def __init__(self, verbosity = 1):
# maintainers_cache: maintainer.email as key, and maintainer object as
# value
self.maintainers_cache = {}
@@ -105,7 +106,16 @@ class Scanner(object):
self.arches_cache = {}
- self.verbosity = verbosity
+ self.verbosity = int(verbosity)
+
+ def write(self, what, verbosity = 1):
+ if verbosity <= self.verbosity:
+ sys.stdout.write(what)
+
+ def output(self, format_str, whats, verbosity = 1):
+ # Maybe implement lazy format string ?
+ if verbosity <= self.verbosity:
+ sys.stdout.write(format_str % whats)
def get_existent_maintainers(self):
return models.MaintainerModel.objects.all()
@@ -148,11 +158,15 @@ class Scanner(object):
# updating
maintainer_object.update_by_maintainer(maintainer_cmp)
maintainer_object.save(force_update = True)
+
+ # Show info if need
+ self.output("update maintainer '%s'\n", maintainer_object, 2)
mo_dict[maintainer_object.email] = maintainer_object
to_create = []
for maintainer in maintainers_dict.iterkeys():
if maintainer.email not in mo_dict:
+ self.output("create maintainer '%s'\n", maintainer, 2)
to_create.append(maintainer)
mobjects = _create_objects(models.MaintainerModel, 'maintainer', to_create)
@@ -170,6 +184,7 @@ class Scanner(object):
for herd_object in existent_herds:
if herd_object.name not in herds_dict:
to_del.append(herd_object.pk)
+ self.output("herd to del '%s'\n", herd_object, 2)
else:
herd_cmp = herds_dict[herd_object.name]
# need update ?
@@ -177,6 +192,7 @@ class Scanner(object):
# updating
herd_object.update_by_herd(herd_cmp)
herd_object.save(force_update = True)
+ self.output("update herd '%s'\n", herd_object, 2)
herds_objects_dict[herd_object.name] = herd_object
models.HerdsModel.objects.filter(pk__in = to_del).delete()
@@ -185,6 +201,7 @@ class Scanner(object):
for herd in herds_dict.itervalues():
if herd.name not in herds_objects_dict:
to_create.append(herd)
+ self.output("create herd '%s'\n", herd, 2)
cobjects = _create_objects(models.HerdsModel, 'herd', to_create)
_update_cache_by_queryset(herds_objects_dict, cobjects, 'name')
@@ -213,6 +230,9 @@ class Scanner(object):
herd_object.maintainers.clear()
herd_object.maintainers.add(*res[herd_name])
+ self.output("add maintainers '%s' to mantainer '%s'\n",
+ (res[herd_name], herd_object), 3)
+
def get_maintainers(self):
if not self.maintainers_cache_loaded:
self.load_maintainers_to_cache()
@@ -221,8 +241,12 @@ class Scanner(object):
def scan_all_repos(self, scan_herds = True):
#cache_dict = anydbm.open('cache.db','c')
if scan_herds:
+ self.write('Scaning herds\n', 3)
self.scan_herds()
+
for repo in portage.iter_trees():
+ self.output("Scaning repository '%s'\n", repo.name, 3)
+
repo_obj, repo_created = models.RepositoryModel.objects.get_or_create(name = repo.name)
self.scanpackages(repo, repo_obj)
#cache_dict.close()
@@ -265,7 +289,8 @@ class Scanner(object):
if arch_name in self.arches_cache:
arch = self.arches_cache[arch_name]
else:
- arch, created = models.ArchesModel.objects.get_or_create(name = arch_name)
+ arch, created = models.ArchesModel.objects \
+ .get_or_create(name = arch_name)
self.arches_cache[arch_name] = arch
return arch
@@ -309,6 +334,8 @@ class Scanner(object):
ebuild_object.save(force_insert=True)
self.add_related_to_ebuild(ebuild, ebuild_object)
+ self.output("ebuild created '%s'\n", ebuild_object, 3)
+
def clear_related_to_package(self, package_object):
package_object.herds.clear()
package_object.maintainers.clear()
@@ -325,15 +352,20 @@ class Scanner(object):
def update_ebuilds(self, package, package_object, delete = True):
not_del = []
for ebuild in package.iter_ebuilds():
- ebuild_object, ebuild_created = models.EbuildModel.objects.get_or_create(ebuild = ebuild, package = package_object)
+ ebuild_object, ebuild_created = models.EbuildModel.objects \
+ .get_or_create(ebuild = ebuild, package = package_object)
+
not_del.append(ebuild_object.pk)
if ebuild_created:
self.add_related_to_ebuild(ebuild, ebuild_object)
- continue
+ self.output("ebuild created '%s'\n", ebuild_object, 3)
+
if ebuild_object.check_or_need_update(ebuild):
ebuild_object.update_by_ebuild(ebuild)
self.update_related_to_ebuild(ebuild, ebuild_object)
ebuild_object.save(force_update = True)
+
+ self.output("ebuild updated '%s'\n", ebuild_object, 3)
if delete:
models.EbuildModel.objects.filter(package = package_object).exclude(pk__in = not_del).delete()
@@ -351,9 +383,6 @@ class Scanner(object):
def scanpackages(self, porttree, porttree_obj, delete = True,
force_update = False, update_cache = True, use_cache = True):
- # Load homepages to cache
- #for homepage in models.HomepageModel.objects.all():
- #homepages_cache[homepage.url] = homepage
existend_categorys = []
for category in porttree.iter_categories():
existend_packages = []
@@ -367,7 +396,7 @@ class Scanner(object):
#val = cache_dict[key]
#if val is not None and val == package.manifest_sha1:
#continue
- print('%-44s [%s]' % (package, porttree))
+ self.output('%-44s [%s]\n', (package, porttree))
package_object, package_created = models.PackageModel.objects.only('changelog_hash', 'manifest_hash', 'metadata_hash') \
.get_or_create(package = package, category = category_object, repository = porttree_obj)
#if update_cache:
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/gentoo-packages:master commit in: gpackages/apps/packages/, gpackages/apps/packages/management/commands/
@ 2012-06-04 20:09 Slava Bacherikov
0 siblings, 0 replies; 20+ messages in thread
From: Slava Bacherikov @ 2012-06-04 20:09 UTC (permalink / raw
To: gentoo-commits
commit: c9515a663684817d00640bd658863d44bc0bcc5b
Author: Slava Bacherikov <slava <AT> bacher09 <DOT> org>
AuthorDate: Mon Jun 4 20:08:58 2012 +0000
Commit: Slava Bacherikov <slava <AT> bacherikov <DOT> org <DOT> ua>
CommitDate: Mon Jun 4 20:08:58 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/gentoo-packages.git;a=commit;h=c9515a66
Create scan module, move code from scanpackages.py
---
.../packages/management/commands/scanpackages.py | 290 +-------------------
.../commands/scanpackages.py => scan.py} | 202 +++++++--------
2 files changed, 96 insertions(+), 396 deletions(-)
diff --git a/gpackages/apps/packages/management/commands/scanpackages.py b/gpackages/apps/packages/management/commands/scanpackages.py
index 0c875c1..27cc782 100644
--- a/gpackages/apps/packages/management/commands/scanpackages.py
+++ b/gpackages/apps/packages/management/commands/scanpackages.py
@@ -1,302 +1,16 @@
from django.core.management.base import BaseCommand, CommandError
-from packages import models
-from collections import defaultdict
-import porttree
-import herds
-import use_info
-
import datetime
import logging
+from packages import scan
#l = logging.getLogger('django.db.backends')
#l.setLevel(logging.DEBUG)
#l.addHandler(logging.FileHandler('database.log'))
-def _get_from_cache(cache, what):
- save_to = []
- geted_items = set()
- for item in what:
- if item in cache:
- geted_items.add(item)
- save_to.append(cache[item])
- return save_to, geted_items
-
-def _get_from_database(Model, field_name, request_items):
- request_items = list(request_items)
- if not request_items:
- return None
- return Model.objects.filter(**{field_name + '__in': request_items})
-
-def _update_cache_by_queryset(cache, queryset, field_name):
- geted_items = set()
- for item in queryset:
- cache[getattr(item, field_name)] = item
- geted_items.add(getattr(item, field_name))
- return geted_items
-
-def _get_from_database_and_update_cache(Model, field_name, request_items, cache):
- queryset = _get_from_database(Model, field_name, request_items)
- if queryset is None:
- return None, set()
- geted = _update_cache_by_queryset(cache, queryset, field_name)
- return queryset, geted
-
-def _create_objects(Model, field_name, need_create):
- if not need_create:
- return None
- creating_list = []
- for item in need_create:
- creating_list.append(Model(**{field_name: item}))
-
- Model.objects.bulk_create(creating_list)
- created = Model.objects.filter(**{field_name + '__in': need_create})
- return created
-
-def _create_objects_and_update_cache(Model, field_name, need_create, cache):
- created = _create_objects(Model, field_name, need_create)
- if created is None:
- return None, None
- geted = _update_cache_by_queryset(cache, created, field_name)
- return created, geted
-
-def _get_items(items_list, Model, field_name, cache_var):
- items_set = frozenset(items_list)
- # Getting from cache
- items_objects, geted_items = _get_from_cache(cache_var, items_set)
- # Getting from database
- queryset, geted = _get_from_database_and_update_cache(Model, field_name, items_set - geted_items, cache_var)
- if queryset is None:
- return items_objects
- geted_items = geted_items | geted
-
- # Add to list with items
- items_objects.extend(queryset)
- # Create not existend items fast using bulk_create, works only in django 1.4 or gt
- need_create = list(items_set - geted_items)
- created, geted = _create_objects_and_update_cache(Model, field_name, need_create, cache_var)
- if created is None:
- return items_objects
- items_objects.extend(created)
- geted_items = geted_items | geted
- return items_objects
-
-
-def scan_maintainers(maintainers_dict):
- existend_maintainers = models.MaintainerModel.objects.all()
- mo_dict = {}
- to_del = []
- for maintainer_object in existend_maintainers:
- if maintainer_object in maintainers_dict:
- maintainer_cmp = maintainers_dict[maintainer_object]
- # need update ?
- if maintainer_object.check_or_need_update(maintainer_cmp):
- # updating
- maintainer_object.update_by_maintainer(maintainer_cmp)
- maintainer_object.save(force_update = True)
- mo_dict[maintainer_object.email] = maintainer_object
- else:
- to_del.append(maintainer_object.pk)
-
- to_create = []
- for maintainer in maintainers_dict.iterkeys():
- if maintainer.email not in mo_dict:
- to_create.append(maintainer)
-
- mobjects = _create_objects(models.MaintainerModel, 'maintainer', to_create)
- _update_cache_by_queryset(mo_dict, mobjects, 'email')
-
- return mo_dict
-
-
-def scan_herds():
- existent_herds = models.HerdsModel.objects.all()
- herds_object = herds.Herds()
- herds_dict = herds_object.get_herds_indict()
- maintainers_dict = herds_object.get_maintainers_with_herds()
- ho_dict = {}
- to_del = []
- for herd_object in existent_herds:
- if herd_object.name not in herds_dict:
- to_del.append(herd_object.pk)
- else:
- herd_cmp = herds_dict[herd_object.name]
- # need update ?
- if herd_object.check_or_need_update(herd_cmp):
- # updating
- herd_object.update_by_herd(herd_cmp)
- herd_object.save(force_update = True)
- ho_dict[herd_object.name] = herd_object
-
- models.HerdsModel.objects.filter(pk__in = to_del).delete()
-
- to_create = []
- for herd in herds_dict.itervalues():
- if herd.name not in ho_dict:
- to_create.append(herd)
-
- cobjects = _create_objects(models.HerdsModel, 'herd', to_create)
- _update_cache_by_queryset(ho_dict, cobjects, 'name')
-
- mo_dict = scan_maintainers(maintainers_dict)
- #Gen data for relate with herds
- res = defaultdict(list)
- for mainteiner, herds_names in maintainers_dict.iteritems():
- for herd in herds_names:
- res[herd].append(mo_dict[mainteiner.email])
-
- for herd_name, herd_object in ho_dict.iteritems():
- herd_object.maintainers.clear()
- herd_object.maintainers.add(*res[herd_name])
-
- return ho_dict, mo_dict
-
-def update_globals_uses_descriptions():
- # Need changes
- uses_g = use_info.get_uses_info()
- existend_use_objects = models.UseFlagModel.objects.filter(name__in = uses_g.keys())
- for use_object in existend_use_objects:
- use_object.description = uses_g[use_object.name]
- use_object.save(force_update = True)
-
-
-def scan_uses_description():
- uses_local = use_info.get_local_uses_info()
- existend_use_objects = models.UseFlagModel.objects.filter(name__in = uses_local.keys())
- existend_use_local_descr = models.UseFlagDescriptionModel.objects.all()
- cache_uses = {}
- _update_cache_by_queryset(cache_uses, existend_use_objects, 'name')
- use_local_cache = defaultdict(dict)
- for use_obj in existend_use_local_descr:
- use_local_cache[use_obj.use_flag.name][use_obj.package.cp] = use_obj
-
- package_cache = dict()
- for use_flag, packages_dict in uses_local.iteritems():
- if use_flag not in cache_uses:
- continue
- use_flag_object = cache_uses[use_flag]
- to_create = []
- for package, description in packages_dict.iteritems():
- if package in package_cache:
- package_object = package_cache[package]
- else:
- try:
- package_object = models.PackageModel.objects.get(package = package)
- except models.PackageModel.DoesNotExist:
- continue
- else:
- package_cache[package] = package_object
- if package not in use_local_cache[use_flag]:
- to_create.append(
- models.UseFlagDescriptionModel(package = package_object,
- use_flag = use_flag_object,
- description = description))
- else:
- use_desc_obj = use_local_cache[use_flag][package]
- if use_desc_obj.check_or_need_update(description):
- use_desc_obj.description = description
- use_desc_obj.save(force_update = True)
- models.UseFlagDescriptionModel.objects.bulk_create(to_create)
-
-
-
class Command(BaseCommand):
args = ''
help = 'Will scan package tree and update info about it in database'
- porttree = porttree.PortTree()
def handle(self, *args, **options):
- licenses_cache = {}
- def get_licenses_objects(ebuild):
- licenses = ebuild.licenses
- return _get_items(licenses, models.LicensModel, 'name', licenses_cache)
-
- uses_cache = {}
- def get_uses_objects(ebuild):
- uses = [ use.name for use in ebuild.iter_uses() ]
- return _get_items(uses, models.UseFlagModel, 'name', uses_cache)
-
- arches_cache = {}
- def get_keywords_objects(ebuild, ebuild_object):
- keywords_list = []
- for keyword in ebuild.get_keywords():
- keyword_object = models.Keyword(status = keyword.status,
- ebuild = ebuild_object)
-
- if keyword.arch in arches_cache:
- arch = arches_cache[keyword.arch]
- else:
- arch, created = models.ArchesModel.objects.get_or_create(name = keyword.arch)
- arches_cache[keyword.arch] = arch
-
- keyword_object.arch = arch
- keywords_list.append(keyword_object)
-
- models.Keyword.objects.bulk_create(keywords_list)
-
-
- homepages_cache = {}
- def get_homepages_objects(ebuild):
- homepages = ebuild.homepages
- return _get_items(homepages, models.HomepageModel, 'url', homepages_cache)
-
-
st = datetime.datetime.now()
- herds_cache, maintainers_cache = scan_herds()
- def get_maintainers_objects(package):
- maintainers = package.metadata.maintainers()
- objects = []
- for maintainer in maintainers:
- if maintainer.email in maintainers_cache:
- objects.append(maintainers_cache[maintainer.email])
- else:
- maintainer_object, created = models.MaintainerModel \
- .objects.get_or_create(email = maintainer.email)
- if created:
- maintainer_object.name = maintainer.name
- maintainer_object.save()
- objects.append(maintainer_object)
- return objects
-
-
- def get_herds_objects(package):
- herds = package.metadata.herds()
- herds_objects = []
- for herd in herds:
- if herd in herds_cache:
- herds_objects.append(herds_cache[herd])
-
- return herds_objects
- # Load homepages to cache
- #for homepage in models.HomepageModel.objects.all():
- #homepages_cache[homepage.url] = homepage
-
- for category in self.porttree.iter_categories():
- category_object, category_created = models.CategoryModel.objects.get_or_create(category = category)
- for package in category.iter_packages():
- print package
- package_object, package_created = models.PackageModel.objects.get_or_create(package = package, category = category_object)
- if not package_created:
- if package_object.check_or_need_update(package):
- # need update
- pass
- package_object.herds.add(*get_herds_objects(package))
- package_object.maintainers.add(*get_maintainers_objects(package))
- for ebuild in package.iter_ebuilds():
- ebuild_object = models.EbuildModel()
- ebuild_object.init_by_ebuild(ebuild)
- ebuild_object.package = package_object
- # To Add some related objects it should have pk
- ebuild_object.save(force_insert=True)
- # Add licenses
- ebuild_object.licenses.add(*get_licenses_objects(ebuild))
- ebuild_object.use_flags.add(*get_uses_objects(ebuild))
- ebuild_object.homepages.add(*get_homepages_objects(ebuild))
- get_keywords_objects(ebuild, ebuild_object)
- #homepages_list = []
- #for homepage in ebuild.homepages:
- #homepage_object = models.HomepageModel(url = homepage,
- #ebuild = ebuild_object)
- #homepages_list.append(homepage_object)
- #models.HomepageModel.objects.bulk_create(homepages_list)
-
-
+ scan.scanpackages()
print (datetime.datetime.now() - st).total_seconds()
diff --git a/gpackages/apps/packages/management/commands/scanpackages.py b/gpackages/apps/packages/scan.py
similarity index 59%
copy from gpackages/apps/packages/management/commands/scanpackages.py
copy to gpackages/apps/packages/scan.py
index 0c875c1..d4b4601 100644
--- a/gpackages/apps/packages/management/commands/scanpackages.py
+++ b/gpackages/apps/packages/scan.py
@@ -1,15 +1,10 @@
-from django.core.management.base import BaseCommand, CommandError
from packages import models
from collections import defaultdict
import porttree
import herds
import use_info
-import datetime
-import logging
-#l = logging.getLogger('django.db.backends')
-#l.setLevel(logging.DEBUG)
-#l.addHandler(logging.FileHandler('database.log'))
+porttree = porttree.PortTree()
def _get_from_cache(cache, what):
save_to = []
@@ -196,107 +191,98 @@ def scan_uses_description():
use_desc_obj.save(force_update = True)
models.UseFlagDescriptionModel.objects.bulk_create(to_create)
+def scanpackages():
+ licenses_cache = {}
+ def get_licenses_objects(ebuild):
+ licenses = ebuild.licenses
+ return _get_items(licenses, models.LicensModel, 'name', licenses_cache)
+
+ uses_cache = {}
+ def get_uses_objects(ebuild):
+ uses = [ use.name for use in ebuild.iter_uses() ]
+ return _get_items(uses, models.UseFlagModel, 'name', uses_cache)
+
+ arches_cache = {}
+ def get_keywords_objects(ebuild, ebuild_object):
+ keywords_list = []
+ for keyword in ebuild.get_keywords():
+ keyword_object = models.Keyword(status = keyword.status,
+ ebuild = ebuild_object)
+
+ if keyword.arch in arches_cache:
+ arch = arches_cache[keyword.arch]
+ else:
+ arch, created = models.ArchesModel.objects.get_or_create(name = keyword.arch)
+ arches_cache[keyword.arch] = arch
+
+ keyword_object.arch = arch
+ keywords_list.append(keyword_object)
-
-
-class Command(BaseCommand):
- args = ''
- help = 'Will scan package tree and update info about it in database'
- porttree = porttree.PortTree()
- def handle(self, *args, **options):
- licenses_cache = {}
- def get_licenses_objects(ebuild):
- licenses = ebuild.licenses
- return _get_items(licenses, models.LicensModel, 'name', licenses_cache)
-
- uses_cache = {}
- def get_uses_objects(ebuild):
- uses = [ use.name for use in ebuild.iter_uses() ]
- return _get_items(uses, models.UseFlagModel, 'name', uses_cache)
-
- arches_cache = {}
- def get_keywords_objects(ebuild, ebuild_object):
- keywords_list = []
- for keyword in ebuild.get_keywords():
- keyword_object = models.Keyword(status = keyword.status,
- ebuild = ebuild_object)
-
- if keyword.arch in arches_cache:
- arch = arches_cache[keyword.arch]
- else:
- arch, created = models.ArchesModel.objects.get_or_create(name = keyword.arch)
- arches_cache[keyword.arch] = arch
+ models.Keyword.objects.bulk_create(keywords_list)
+
+
+ homepages_cache = {}
+ def get_homepages_objects(ebuild):
+ homepages = ebuild.homepages
+ return _get_items(homepages, models.HomepageModel, 'url', homepages_cache)
+
+
+ herds_cache, maintainers_cache = scan_herds()
+ def get_maintainers_objects(package):
+ maintainers = package.metadata.maintainers()
+ objects = []
+ for maintainer in maintainers:
+ if maintainer.email in maintainers_cache:
+ objects.append(maintainers_cache[maintainer.email])
+ else:
+ maintainer_object, created = models.MaintainerModel \
+ .objects.get_or_create(email = maintainer.email)
+ if created:
+ maintainer_object.name = maintainer.name
+ maintainer_object.save()
+ objects.append(maintainer_object)
+ return objects
- keyword_object.arch = arch
- keywords_list.append(keyword_object)
-
- models.Keyword.objects.bulk_create(keywords_list)
-
-
- homepages_cache = {}
- def get_homepages_objects(ebuild):
- homepages = ebuild.homepages
- return _get_items(homepages, models.HomepageModel, 'url', homepages_cache)
-
-
- st = datetime.datetime.now()
- herds_cache, maintainers_cache = scan_herds()
- def get_maintainers_objects(package):
- maintainers = package.metadata.maintainers()
- objects = []
- for maintainer in maintainers:
- if maintainer.email in maintainers_cache:
- objects.append(maintainers_cache[maintainer.email])
- else:
- maintainer_object, created = models.MaintainerModel \
- .objects.get_or_create(email = maintainer.email)
- if created:
- maintainer_object.name = maintainer.name
- maintainer_object.save()
- objects.append(maintainer_object)
- return objects
-
-
- def get_herds_objects(package):
- herds = package.metadata.herds()
- herds_objects = []
- for herd in herds:
- if herd in herds_cache:
- herds_objects.append(herds_cache[herd])
-
- return herds_objects
- # Load homepages to cache
- #for homepage in models.HomepageModel.objects.all():
- #homepages_cache[homepage.url] = homepage
-
- for category in self.porttree.iter_categories():
- category_object, category_created = models.CategoryModel.objects.get_or_create(category = category)
- for package in category.iter_packages():
- print package
- package_object, package_created = models.PackageModel.objects.get_or_create(package = package, category = category_object)
- if not package_created:
- if package_object.check_or_need_update(package):
- # need update
- pass
- package_object.herds.add(*get_herds_objects(package))
- package_object.maintainers.add(*get_maintainers_objects(package))
- for ebuild in package.iter_ebuilds():
- ebuild_object = models.EbuildModel()
- ebuild_object.init_by_ebuild(ebuild)
- ebuild_object.package = package_object
- # To Add some related objects it should have pk
- ebuild_object.save(force_insert=True)
- # Add licenses
- ebuild_object.licenses.add(*get_licenses_objects(ebuild))
- ebuild_object.use_flags.add(*get_uses_objects(ebuild))
- ebuild_object.homepages.add(*get_homepages_objects(ebuild))
- get_keywords_objects(ebuild, ebuild_object)
- #homepages_list = []
- #for homepage in ebuild.homepages:
- #homepage_object = models.HomepageModel(url = homepage,
- #ebuild = ebuild_object)
- #homepages_list.append(homepage_object)
- #models.HomepageModel.objects.bulk_create(homepages_list)
-
-
- print (datetime.datetime.now() - st).total_seconds()
+
+ def get_herds_objects(package):
+ herds = package.metadata.herds()
+ herds_objects = []
+ for herd in herds:
+ if herd in herds_cache:
+ herds_objects.append(herds_cache[herd])
+
+ return herds_objects
+ # Load homepages to cache
+ #for homepage in models.HomepageModel.objects.all():
+ #homepages_cache[homepage.url] = homepage
+
+ for category in porttree.iter_categories():
+ category_object, category_created = models.CategoryModel.objects.get_or_create(category = category)
+ for package in category.iter_packages():
+ print package
+ package_object, package_created = models.PackageModel.objects.get_or_create(package = package, category = category_object)
+ if not package_created:
+ if package_object.check_or_need_update(package):
+ # need update
+ pass
+ package_object.herds.add(*get_herds_objects(package))
+ package_object.maintainers.add(*get_maintainers_objects(package))
+ for ebuild in package.iter_ebuilds():
+ ebuild_object = models.EbuildModel()
+ ebuild_object.init_by_ebuild(ebuild)
+ ebuild_object.package = package_object
+ # To Add some related objects it should have pk
+ ebuild_object.save(force_insert=True)
+ # Add licenses
+ ebuild_object.licenses.add(*get_licenses_objects(ebuild))
+ ebuild_object.use_flags.add(*get_uses_objects(ebuild))
+ ebuild_object.homepages.add(*get_homepages_objects(ebuild))
+ get_keywords_objects(ebuild, ebuild_object)
+ #homepages_list = []
+ #for homepage in ebuild.homepages:
+ #homepage_object = models.HomepageModel(url = homepage,
+ #ebuild = ebuild_object)
+ #homepages_list.append(homepage_object)
+ #models.HomepageModel.objects.bulk_create(homepages_list)
+
+
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/gentoo-packages:master commit in: gpackages/apps/packages/, gpackages/apps/packages/management/commands/
@ 2012-06-04 20:09 Slava Bacherikov
0 siblings, 0 replies; 20+ messages in thread
From: Slava Bacherikov @ 2012-06-04 20:09 UTC (permalink / raw
To: gentoo-commits
commit: 1cb21de9787358ecb667a324bca1b0f441af6628
Author: Slava Bacherikov <slava <AT> bacher09 <DOT> org>
AuthorDate: Mon Jun 4 19:44:02 2012 +0000
Commit: Slava Bacherikov <slava <AT> bacherikov <DOT> org <DOT> ua>
CommitDate: Mon Jun 4 19:44:02 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/gentoo-packages.git;a=commit;h=1cb21de9
Add scaning uses info.
---
.../packages/management/commands/scanpackages.py | 55 +++++++++++++++++++-
gpackages/apps/packages/managers.py | 3 +
2 files changed, 57 insertions(+), 1 deletions(-)
diff --git a/gpackages/apps/packages/management/commands/scanpackages.py b/gpackages/apps/packages/management/commands/scanpackages.py
index 58e12ba..0c875c1 100644
--- a/gpackages/apps/packages/management/commands/scanpackages.py
+++ b/gpackages/apps/packages/management/commands/scanpackages.py
@@ -3,6 +3,7 @@ from packages import models
from collections import defaultdict
import porttree
import herds
+import use_info
import datetime
import logging
@@ -148,6 +149,54 @@ def scan_herds():
return ho_dict, mo_dict
+def update_globals_uses_descriptions():
+ # Need changes
+ uses_g = use_info.get_uses_info()
+ existend_use_objects = models.UseFlagModel.objects.filter(name__in = uses_g.keys())
+ for use_object in existend_use_objects:
+ use_object.description = uses_g[use_object.name]
+ use_object.save(force_update = True)
+
+
+def scan_uses_description():
+ uses_local = use_info.get_local_uses_info()
+ existend_use_objects = models.UseFlagModel.objects.filter(name__in = uses_local.keys())
+ existend_use_local_descr = models.UseFlagDescriptionModel.objects.all()
+ cache_uses = {}
+ _update_cache_by_queryset(cache_uses, existend_use_objects, 'name')
+ use_local_cache = defaultdict(dict)
+ for use_obj in existend_use_local_descr:
+ use_local_cache[use_obj.use_flag.name][use_obj.package.cp] = use_obj
+
+ package_cache = dict()
+ for use_flag, packages_dict in uses_local.iteritems():
+ if use_flag not in cache_uses:
+ continue
+ use_flag_object = cache_uses[use_flag]
+ to_create = []
+ for package, description in packages_dict.iteritems():
+ if package in package_cache:
+ package_object = package_cache[package]
+ else:
+ try:
+ package_object = models.PackageModel.objects.get(package = package)
+ except models.PackageModel.DoesNotExist:
+ continue
+ else:
+ package_cache[package] = package_object
+ if package not in use_local_cache[use_flag]:
+ to_create.append(
+ models.UseFlagDescriptionModel(package = package_object,
+ use_flag = use_flag_object,
+ description = description))
+ else:
+ use_desc_obj = use_local_cache[use_flag][package]
+ if use_desc_obj.check_or_need_update(description):
+ use_desc_obj.description = description
+ use_desc_obj.save(force_update = True)
+ models.UseFlagDescriptionModel.objects.bulk_create(to_create)
+
+
class Command(BaseCommand):
@@ -221,10 +270,14 @@ class Command(BaseCommand):
#homepages_cache[homepage.url] = homepage
for category in self.porttree.iter_categories():
- category_object, categor_created = models.CategoryModel.objects.get_or_create(category = category)
+ category_object, category_created = models.CategoryModel.objects.get_or_create(category = category)
for package in category.iter_packages():
print package
package_object, package_created = models.PackageModel.objects.get_or_create(package = package, category = category_object)
+ if not package_created:
+ if package_object.check_or_need_update(package):
+ # need update
+ pass
package_object.herds.add(*get_herds_objects(package))
package_object.maintainers.add(*get_maintainers_objects(package))
for ebuild in package.iter_ebuilds():
diff --git a/gpackages/apps/packages/managers.py b/gpackages/apps/packages/managers.py
index d7caaf1..27136f8 100644
--- a/gpackages/apps/packages/managers.py
+++ b/gpackages/apps/packages/managers.py
@@ -34,6 +34,9 @@ class PackageMixin(object):#{{{
args[1] = category
else:
kwargs.update({'name': name})
+ elif package is not None:
+ category, name = package.split('/')
+ kwargs.update({'name': name, 'category__category': category})
return super(PackageMixin, self).get(*args, **kwargs)#}}}
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/gentoo-packages:master commit in: gpackages/apps/packages/, gpackages/apps/packages/management/commands/
@ 2012-06-04 20:09 Slava Bacherikov
0 siblings, 0 replies; 20+ messages in thread
From: Slava Bacherikov @ 2012-06-04 20:09 UTC (permalink / raw
To: gentoo-commits
commit: 7e5f0b70b12ba1dcb0eae128728b418d09bce040
Author: Slava Bacherikov <slava <AT> bacher09 <DOT> org>
AuthorDate: Mon Jun 4 17:10:09 2012 +0000
Commit: Slava Bacherikov <slava <AT> bacherikov <DOT> org <DOT> ua>
CommitDate: Mon Jun 4 17:10:09 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/gentoo-packages.git;a=commit;h=7e5f0b70
Updating maintainers info if it is changed
---
.../packages/management/commands/scanpackages.py | 7 ++++++-
gpackages/apps/packages/models.py | 7 +++++++
2 files changed, 13 insertions(+), 1 deletions(-)
diff --git a/gpackages/apps/packages/management/commands/scanpackages.py b/gpackages/apps/packages/management/commands/scanpackages.py
index 2fe93e2..58e12ba 100644
--- a/gpackages/apps/packages/management/commands/scanpackages.py
+++ b/gpackages/apps/packages/management/commands/scanpackages.py
@@ -85,7 +85,12 @@ def scan_maintainers(maintainers_dict):
to_del = []
for maintainer_object in existend_maintainers:
if maintainer_object in maintainers_dict:
- # to update ?
+ maintainer_cmp = maintainers_dict[maintainer_object]
+ # need update ?
+ if maintainer_object.check_or_need_update(maintainer_cmp):
+ # updating
+ maintainer_object.update_by_maintainer(maintainer_cmp)
+ maintainer_object.save(force_update = True)
mo_dict[maintainer_object.email] = maintainer_object
else:
to_del.append(maintainer_object.pk)
diff --git a/gpackages/apps/packages/models.py b/gpackages/apps/packages/models.py
index 6b2fc1b..81ac823 100644
--- a/gpackages/apps/packages/models.py
+++ b/gpackages/apps/packages/models.py
@@ -64,6 +64,13 @@ class MaintainerModel(models.Model):
self.name = maintainer.name
self.email = maintainer.email
self.role = maintainer.role
+
+ def update_by_maintainer(self, maintainer):
+ self.name = maintainer.name
+
+ def check_or_need_update(self, maintainer):
+ return not (self.name == maintainer.name and \
+ self.email == maintainer.email)
def __unicode__(self):
return ':'.join((unicode(self.name), self.email))
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/gentoo-packages:master commit in: gpackages/apps/packages/, gpackages/apps/packages/management/commands/
@ 2012-06-03 16:19 Slava Bacherikov
0 siblings, 0 replies; 20+ messages in thread
From: Slava Bacherikov @ 2012-06-03 16:19 UTC (permalink / raw
To: gentoo-commits
commit: f7fc2cca4d2f549b2a4bbd19bc9ec8be49c09ba9
Author: Slava Bacherikov <slava <AT> bacher09 <DOT> org>
AuthorDate: Sun Jun 3 16:19:05 2012 +0000
Commit: Slava Bacherikov <slava <AT> bacherikov <DOT> org <DOT> ua>
CommitDate: Sun Jun 3 16:19:05 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/gentoo-packages.git;a=commit;h=f7fc2cca
Add more info about ebuild metadata.
---
.../packages/management/commands/scanpackages.py | 18 +++++++++++++++++-
gpackages/apps/packages/models.py | 6 ++++++
2 files changed, 23 insertions(+), 1 deletions(-)
diff --git a/gpackages/apps/packages/management/commands/scanpackages.py b/gpackages/apps/packages/management/commands/scanpackages.py
index b3adb97..f63f0d0 100644
--- a/gpackages/apps/packages/management/commands/scanpackages.py
+++ b/gpackages/apps/packages/management/commands/scanpackages.py
@@ -174,7 +174,6 @@ class Command(BaseCommand):
models.Keyword.objects.bulk_create(keywords_list)
-
homepages_cache = {}
def get_homepages_objects(ebuild):
homepages = ebuild.homepages
@@ -183,6 +182,22 @@ class Command(BaseCommand):
st = datetime.datetime.now()
herds_cache, maintainers_cache = scan_herds()
+ def get_maintainers_objects(package):
+ maintainers = package.metadata.maintainers()
+ objects = []
+ for maintainer in maintainers:
+ if maintainer.email in maintainers_cache:
+ objects.append(maintainers_cache[maintainer.email])
+ else:
+ maintainer_object, created = models.MaintainerModel \
+ .objects.get_or_create(email = maintainer.email)
+ if created:
+ maintainer_object.name = maintainer.name
+ maintainer_object.save()
+ objects.append(maintainer_object)
+ return objects
+
+
def get_herds_objects(package):
herds = package.metadata.herds()
herds_objects = []
@@ -201,6 +216,7 @@ class Command(BaseCommand):
print package
package_object, package_created = models.PackageModel.objects.get_or_create(package = package, category = category_object)
package_object.herds.add(*get_herds_objects(package))
+ package_object.maintainers.add(*get_maintainers_objects(package))
for ebuild in package.iter_ebuilds():
ebuild_object = models.EbuildModel()
ebuild_object.init_by_ebuild(ebuild)
diff --git a/gpackages/apps/packages/models.py b/gpackages/apps/packages/models.py
index d3ff5f6..0fc1dc0 100644
--- a/gpackages/apps/packages/models.py
+++ b/gpackages/apps/packages/models.py
@@ -122,6 +122,7 @@ class PackageModel(models.Model):
mtime = models.DateTimeField(blank = True, null = True)
herds = models.ManyToManyField(HerdsModel, blank = True)
+ maintainers = models.ManyToManyField(MaintainerModel, blank = True)
# Different versions can have different licenses, or homepages.
objects = managers.PackageManager()
@@ -180,6 +181,11 @@ class EbuildModel(models.Model):
homepages = models.ManyToManyField(HomepageModel, blank = True)
description = models.TextField(blank = True, null = True)
+ eapi = models.PositiveSmallIntegerField(default = 0)
+ slot = models.PositiveSmallIntegerField(default = 0)
+
+
+
objects = managers.EbuildManager()
def __init__(self, *args, **kwargs ):
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/gentoo-packages:master commit in: gpackages/apps/packages/, gpackages/apps/packages/management/commands/
@ 2012-06-03 13:19 Slava Bacherikov
0 siblings, 0 replies; 20+ messages in thread
From: Slava Bacherikov @ 2012-06-03 13:19 UTC (permalink / raw
To: gentoo-commits
commit: 565c26d136ad4d149b0f069172b7d07011d594b3
Author: Slava Bacherikov <slava <AT> bacher09 <DOT> org>
AuthorDate: Sun Jun 3 13:16:35 2012 +0000
Commit: Slava Bacherikov <slava <AT> bacherikov <DOT> org <DOT> ua>
CommitDate: Sun Jun 3 13:16:35 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/gentoo-packages.git;a=commit;h=565c26d1
Add herds and maintainers to scanning.
---
.../packages/management/commands/scanpackages.py | 78 ++++++++++++++-
gpackages/apps/packages/models.py | 104 ++++++++++----------
2 files changed, 129 insertions(+), 53 deletions(-)
diff --git a/gpackages/apps/packages/management/commands/scanpackages.py b/gpackages/apps/packages/management/commands/scanpackages.py
index 2968d65..b3adb97 100644
--- a/gpackages/apps/packages/management/commands/scanpackages.py
+++ b/gpackages/apps/packages/management/commands/scanpackages.py
@@ -1,6 +1,8 @@
from django.core.management.base import BaseCommand, CommandError
from packages import models
+from collections import defaultdict
import porttree
+import herds
import datetime
import logging
@@ -77,6 +79,67 @@ def _get_items(items_list, Model, field_name, cache_var):
return items_objects
+def scan_maintainers(maintainers_dict):
+ existend_maintainers = models.MaintainerModel.objects.all()
+ mo_dict = {}
+ to_del = []
+ for maintainer_object in existend_maintainers:
+ if maintainer_object in maintainers_dict:
+ # to update ?
+ mo_dict[maintainer_object.email] = maintainer_object
+ else:
+ to_del.append(maintainer_object.pk)
+
+ to_create = []
+ for maintainer in maintainers_dict.iterkeys():
+ if maintainer.email not in mo_dict:
+ to_create.append(maintainer)
+
+ mobjects = _create_objects(models.MaintainerModel, 'maintainer', to_create)
+ _update_cache_by_queryset(mo_dict, mobjects, 'email')
+
+ return mo_dict
+
+
+def scan_herds():
+ existent_herds = models.HerdsModel.objects.all()
+ herds_object = herds.Herds()
+ herds_dict = herds_object.get_herds_indict()
+ maintainers_dict = herds_object.get_maintainers_with_hers()
+ ho_dict = {}
+ to_del = []
+ for herd_object in existent_herds:
+ if herd_object.name not in herds_dict:
+ to_del.append(herd_object.pk)
+ else:
+ # to update ?
+ ho_dict[herd_object.name] = herd_object
+
+ models.HerdsModel.objects.filter(pk__in = to_del).delete()
+
+ to_create = []
+ for herd in herds_dict.itervalues():
+ if herd.name not in ho_dict:
+ to_create.append(herd)
+
+ cobjects = _create_objects(models.HerdsModel, 'herd', to_create)
+ _update_cache_by_queryset(ho_dict, cobjects, 'name')
+
+ mo_dict = scan_maintainers(maintainers_dict)
+ #Gen data for relate with herds
+ res = defaultdict(list)
+ for mainteiner, herds_names in maintainers_dict.iteritems():
+ for herd in herds_names:
+ res[herd].append(mo_dict[mainteiner.email])
+
+ for herd_name, herd_object in ho_dict.iteritems():
+ herd_object.maintainers.clear()
+ herd_object.maintainers.add(*res[herd_name])
+
+ return ho_dict, mo_dict
+
+
+
class Command(BaseCommand):
args = ''
help = 'Will scan package tree and update info about it in database'
@@ -116,17 +179,28 @@ class Command(BaseCommand):
def get_homepages_objects(ebuild):
homepages = ebuild.homepages
return _get_items(homepages, models.HomepageModel, 'url', homepages_cache)
+
st = datetime.datetime.now()
+ herds_cache, maintainers_cache = scan_herds()
+ def get_herds_objects(package):
+ herds = package.metadata.herds()
+ herds_objects = []
+ for herd in herds:
+ if herd in herds_cache:
+ herds_objects.append(herds_cache[herd])
+
+ return herds_objects
# Load homepages to cache
- for homepage in models.HomepageModel.objects.all():
- homepages_cache[homepage.url] = homepage
+ #for homepage in models.HomepageModel.objects.all():
+ #homepages_cache[homepage.url] = homepage
for category in self.porttree.iter_categories():
category_object, categor_created = models.CategoryModel.objects.get_or_create(category = category)
for package in category.iter_packages():
print package
package_object, package_created = models.PackageModel.objects.get_or_create(package = package, category = category_object)
+ package_object.herds.add(*get_herds_objects(package))
for ebuild in package.iter_ebuilds():
ebuild_object = models.EbuildModel()
ebuild_object.init_by_ebuild(ebuild)
diff --git a/gpackages/apps/packages/models.py b/gpackages/apps/packages/models.py
index 2ad8faa..d3ff5f6 100644
--- a/gpackages/apps/packages/models.py
+++ b/gpackages/apps/packages/models.py
@@ -42,6 +42,57 @@ class CategoryModel(models.Model):
def __unicode__(self):
return self.category
+class MaintainerModel(models.Model):
+
+ def __init__(self, *args, **kwargs):
+ #TODO: Bad code, maybe use some libraries for overload methods
+ maintainer = None
+ if 'maintainer' in kwargs:
+ maintainer = kwargs['maintainer']
+ del kwargs['maintainer']
+ super(MaintainerModel, self).__init__(*args, **kwargs)
+ if maintainer is not None:
+ self.init_by_maintainer(maintainer)
+
+ name = models.CharField(max_length = 255, blank = True, null = True)
+ email = models.EmailField(unique = True)
+ role = models.TextField(blank = True, null = True)
+
+ objects = managers.MaintainerManager()
+
+ def init_by_maintainer(self, maintainer):
+ self.name = maintainer.name
+ self.email = maintainer.email
+ self.role = maintainer.role
+
+ def __unicode__(self):
+ return ':'.join((unicode(self.name), self.email))
+
+class HerdsModel(models.Model):
+
+ def __init__(self, *args, **kwargs):
+ herd = None
+ if 'herd' in kwargs:
+ herd = kwargs['herd']
+ del kwargs['herd']
+ super(HerdsModel, self).__init__(*args, **kwargs)
+ if herd is not None:
+ self.init_by_herd(herd)
+
+ name = models.CharField(unique = True, max_length = 150)
+ email = models.EmailField()
+ description = models.TextField(blank = True, null = True)
+ maintainers = models.ManyToManyField(MaintainerModel, blank = True)
+
+ objects = managers.HerdsManager()
+
+ def init_by_herd(self, herd):
+ self.name = herd.name
+ self.email = herd.email
+ self.description = herd.description
+
+ def __unicode__(self):
+ return self.name
class PackageModel(models.Model):
def __init__(self, *args, **kwargs):
@@ -69,6 +120,8 @@ class PackageModel(models.Model):
changelog_mtime = models.DateTimeField(blank = True, null = True)
manifest_mtime = models.DateTimeField(blank = True, null = True)
mtime = models.DateTimeField(blank = True, null = True)
+
+ herds = models.ManyToManyField(HerdsModel, blank = True)
# Different versions can have different licenses, or homepages.
objects = managers.PackageManager()
@@ -220,54 +273,3 @@ class Keyword(models.Model):
unique_together = ('ebuild', 'arch')
-class MaintainerModel(models.Model):
-
- def __init__(self, *args, **kwargs):
- #TODO: Bad code, maybe use some libraries for overload methods
- maintainer = None
- if 'maintainer' in kwargs:
- maintainer = kwargs['maintainer']
- del kwargs['maintainer']
- super(MaintainerModel, self).__init__(*args, **kwargs)
- if maintainer is not None:
- self.init_by_maintainer(maintainer)
-
- name = models.CharField(max_length = 255, blank = True, null = True)
- email = models.EmailField(unique = True)
- role = models.TextField(blank = True, null = True)
-
- objects = managers.MaintainerManager()
-
- def init_by_maintainer(self, maintainer):
- self.name = maintainer.name
- self.email = maintainer.email
- self.role = maintainer.role
-
- def __unicode__(self):
- return ':'.join((unicode(self.name), self.email))
-
-class HerdsModel(models.Model):
-
- def __init__(self, *args, **kwargs):
- herd = None
- if 'herd' in kwargs:
- herd = kwargs['herd']
- del kwargs['herd']
- super(HerdsModel, self).__init__(*args, **kwargs)
- if herd is not None:
- self.init_by_herd(herd)
-
- name = models.CharField(unique = True, max_length = 150)
- email = models.EmailField()
- description = models.TextField(blank = True, null = True)
- maintainers = models.ManyToManyField(MaintainerModel, blank = True)
-
- objects = managers.HerdsManager()
-
- def init_by_herd(self, herd):
- self.name = herd.name
- self.email = herd.email
- self.description = herd.description
-
- def __unicode__(self):
- return self.name
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/gentoo-packages:master commit in: gpackages/apps/packages/, gpackages/apps/packages/management/commands/
@ 2012-06-01 21:28 Slava Bacherikov
0 siblings, 0 replies; 20+ messages in thread
From: Slava Bacherikov @ 2012-06-01 21:28 UTC (permalink / raw
To: gentoo-commits
commit: ed0c2e1d641fad2209cec912be2ac3359b0625d2
Author: Slava Bacherikov <slava <AT> bacher09 <DOT> org>
AuthorDate: Fri Jun 1 20:26:39 2012 +0000
Commit: Slava Bacherikov <slava <AT> bacherikov <DOT> org <DOT> ua>
CommitDate: Fri Jun 1 20:26:39 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/gentoo-packages.git;a=commit;h=ed0c2e1d
Normalize homepages
---
.../packages/management/commands/scanpackages.py | 18 ++++++++++------
gpackages/apps/packages/models.py | 21 +++++++++++++++----
2 files changed, 27 insertions(+), 12 deletions(-)
diff --git a/gpackages/apps/packages/management/commands/scanpackages.py b/gpackages/apps/packages/management/commands/scanpackages.py
index 133f886..cbf8f66 100644
--- a/gpackages/apps/packages/management/commands/scanpackages.py
+++ b/gpackages/apps/packages/management/commands/scanpackages.py
@@ -113,7 +113,10 @@ class Command(BaseCommand):
homepages_cache = {}
-
+ def get_homepages_objects(ebuild):
+ homepages = ebuild.homepages
+ return _get_items(homepages, models.HomepageModel, 'url', homepages_cache)
+
st = datetime.datetime.now()
# Load homepages to cache
for homepage in models.HomepageModel.objects.all():
@@ -133,13 +136,14 @@ class Command(BaseCommand):
# Add licenses
ebuild_object.licenses.add(*get_licenses_objects(ebuild))
ebuild_object.use_flags.add(*get_uses_objects(ebuild))
+ ebuild_object.homepages.add(*get_homepages_objects(ebuild))
get_keywords_objects(ebuild, ebuild_object)
- homepages_list = []
- for homepage in ebuild.homepages:
- homepage_object = models.HomepageModel(url = homepage,
- ebuild = ebuild_object)
- homepages_list.append(homepage_object)
- models.HomepageModel.objects.bulk_create(homepages_list)
+ #homepages_list = []
+ #for homepage in ebuild.homepages:
+ #homepage_object = models.HomepageModel(url = homepage,
+ #ebuild = ebuild_object)
+ #homepages_list.append(homepage_object)
+ #models.HomepageModel.objects.bulk_create(homepages_list)
print (datetime.datetime.now() - st).total_seconds()
diff --git a/gpackages/apps/packages/models.py b/gpackages/apps/packages/models.py
index 9da855f..d90bbb0 100644
--- a/gpackages/apps/packages/models.py
+++ b/gpackages/apps/packages/models.py
@@ -3,6 +3,12 @@ from django.db import models
from porttree import Category, Package, Ebuild
import managers
+class HomepageModel(models.Model):
+ url = models.URLField(max_length=255, unique = True)
+
+ def __unicode__(self):
+ return self.url
+
class ArchesModel(models.Model):
name = models.CharField(unique = True, max_length = 22)
@@ -118,6 +124,7 @@ class EbuildModel(models.Model):
is_masked = models.BooleanField(default = False)
#homepage = models.URLField(blank = True, null = True, max_length=255)
+ homepages = models.ManyToManyField(HomepageModel, blank = True)
description = models.TextField(blank = True, null = True)
objects = managers.EbuildManager()
@@ -213,10 +220,14 @@ class Keyword(models.Model):
unique_together = ('ebuild', 'arch')
-class HomepageModel(models.Model):
- url = models.URLField(max_length=255)
- ebuild = models.ForeignKey(EbuildModel)
+class MaintainerModel(models.Model):
+ name = models.CharField( unique = True, max_length = 255)
+ email = models.EmailField()
+ role = models.TextField(blank = True, null = True)
- def __unicode__(self):
- return self.url
+class HerdsModel(models.Model):
+ name = models.CharField(unique = True, max_length = 150)
+ email = models.EmailField()
+ description = models.TextField(blank = True, null = True)
+ maintainers = models.ManyToManyField(MaintainerModel, blank = True)
^ permalink raw reply related [flat|nested] 20+ messages in thread
end of thread, other threads:[~2012-08-26 23:01 UTC | newest]
Thread overview: 20+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2012-06-04 20:09 [gentoo-commits] proj/gentoo-packages:master commit in: gpackages/apps/packages/, gpackages/apps/packages/management/commands/ Slava Bacherikov
-- strict thread matches above, loose matches on Subject: below --
2012-08-26 23:00 Slava Bacherikov
2012-08-22 17:55 Slava Bacherikov
2012-07-18 23:03 Slava Bacherikov
2012-07-17 9:42 Slava Bacherikov
2012-07-06 23:09 Slava Bacherikov
2012-07-05 23:27 Slava Bacherikov
2012-06-19 0:12 Slava Bacherikov
2012-06-18 23:00 Slava Bacherikov
2012-06-13 22:15 Slava Bacherikov
2012-06-10 22:23 Slava Bacherikov
2012-06-10 17:51 Slava Bacherikov
2012-06-09 18:19 Slava Bacherikov
2012-06-09 18:19 Slava Bacherikov
2012-06-04 20:09 Slava Bacherikov
2012-06-04 20:09 Slava Bacherikov
2012-06-04 20:09 Slava Bacherikov
2012-06-03 16:19 Slava Bacherikov
2012-06-03 13:19 Slava Bacherikov
2012-06-01 21:28 Slava Bacherikov
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox