* [gentoo-commits] proj/g-sorcery:master commit in: gs_pypi/
@ 2013-08-12 0:36 Jauhien Piatlicki
0 siblings, 0 replies; 20+ messages in thread
From: Jauhien Piatlicki @ 2013-08-12 0:36 UTC (permalink / raw
To: gentoo-commits
commit: c0c360efe7f98b87d9bde80a962fd5fd04502ef0
Author: Jauhien Piatlicki (jauhien) <piatlicki <AT> gmail <DOT> com>
AuthorDate: Sun Aug 11 23:40:31 2013 +0000
Commit: Jauhien Piatlicki <piatlicki <AT> gmail <DOT> com>
CommitDate: Sun Aug 11 23:40:31 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/g-sorcery.git;a=commit;h=c0c360ef
gs_pypi/pypi_db: handling wget errors
---
gs_pypi/pypi_db.py | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/gs_pypi/pypi_db.py b/gs_pypi/pypi_db.py
index 44e77a4..ef0b807 100644
--- a/gs_pypi/pypi_db.py
+++ b/gs_pypi/pypi_db.py
@@ -13,6 +13,7 @@
import bs4
+from g_sorcery.exceptions import DownloadingError
from g_sorcery.g_collections import Package
from g_sorcery.package_db import DBGenerator
@@ -44,7 +45,13 @@ class PypiDBGenerator(DBGenerator):
"output": package + "-" + version})
pkg_uries = self.decode_download_uries(pkg_uries)
for uri in pkg_uries:
- self.process_uri(uri, data)
+ while True:
+ try:
+ self.process_uri(uri, data)
+ except DownloadingError as error:
+ print(str(error))
+ continue
+ break
return data
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/g-sorcery:master commit in: gs_pypi/
@ 2013-11-21 11:30 Jauhien Piatlicki
0 siblings, 0 replies; 20+ messages in thread
From: Jauhien Piatlicki @ 2013-11-21 11:30 UTC (permalink / raw
To: gentoo-commits
commit: 7ac872366513428d234c91d11a24649cc02417ec
Author: Jauhien Piatlicki (jauhien) <piatlicki <AT> gmail <DOT> com>
AuthorDate: Wed Nov 20 22:51:59 2013 +0000
Commit: Jauhien Piatlicki <piatlicki <AT> gmail <DOT> com>
CommitDate: Wed Nov 20 22:51:59 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/g-sorcery.git;a=commit;h=7ac87236
gs-pypi: fix issue#4
---
gs_pypi/pypi_db.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/gs_pypi/pypi_db.py b/gs_pypi/pypi_db.py
index 847cbf3..2b201c6 100644
--- a/gs_pypi/pypi_db.py
+++ b/gs_pypi/pypi_db.py
@@ -137,7 +137,10 @@ class PypiDBGenerator(DBGenerator):
data["info"][entry_name] = {}
for cat_entry in entry("a"):
cat_data = cat_entry.string.split(" :: ")
- data["info"][entry_name][cat_data[0]] = cat_data[1:]
+ if not cat_data[0] in data["info"][entry_name]:
+ data["info"][entry_name][cat_data[0]] = cat_data[1:]
+ else:
+ data["info"][entry_name][cat_data[0]].extend(cat_data[1:])
continue
if entry("span"):
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/g-sorcery:master commit in: gs_pypi/
@ 2013-11-12 23:10 Jauhien Piatlicki
0 siblings, 0 replies; 20+ messages in thread
From: Jauhien Piatlicki @ 2013-11-12 23:10 UTC (permalink / raw
To: gentoo-commits
commit: 20c4bbc953b0cc5442efeed4b31e2f545b41ec01
Author: Jauhien Piatlicki (jauhien) <piatlicki <AT> gmail <DOT> com>
AuthorDate: Tue Nov 12 22:58:56 2013 +0000
Commit: Jauhien Piatlicki <piatlicki <AT> gmail <DOT> com>
CommitDate: Tue Nov 12 22:58:56 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/g-sorcery.git;a=commit;h=20c4bbc9
gs-pypi: fix issue#3
---
gs_pypi/ebuild.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/gs_pypi/ebuild.py b/gs_pypi/ebuild.py
index 7fdf145..82024c3 100644
--- a/gs_pypi/ebuild.py
+++ b/gs_pypi/ebuild.py
@@ -50,7 +50,8 @@ class PypiEbuildWithDigestGenerator(DefaultEbuildGenerator):
def __init__(self, package_db):
vars_before_inherit = \
- [{"name" : "digest_sources", "value" : "yes"}, {"name" : "python_compat", "raw" : True}]
+ ["realname", "realversion",
+ {"name" : "digest_sources", "value" : "yes"}, {"name" : "python_compat", "raw" : True}]
inherit = ["gs-pypi"]
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/g-sorcery:master commit in: gs_pypi/
@ 2013-09-08 9:06 Jauhien Piatlicki
0 siblings, 0 replies; 20+ messages in thread
From: Jauhien Piatlicki @ 2013-09-08 9:06 UTC (permalink / raw
To: gentoo-commits
commit: 00147c1d07778dfe1ebb4e6f826da9fca4244028
Author: Jauhien Piatlicki (jauhien) <piatlicki <AT> gmail <DOT> com>
AuthorDate: Sun Sep 8 09:08:18 2013 +0000
Commit: Jauhien Piatlicki <piatlicki <AT> gmail <DOT> com>
CommitDate: Sun Sep 8 09:08:18 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/g-sorcery.git;a=commit;h=00147c1d
gs_pypi/pypi_db: fix filtering
---
gs_pypi/pypi_db.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/gs_pypi/pypi_db.py b/gs_pypi/pypi_db.py
index f982fcd..120a153 100644
--- a/gs_pypi/pypi_db.py
+++ b/gs_pypi/pypi_db.py
@@ -154,12 +154,12 @@ class PypiDBGenerator(DBGenerator):
pkg_db.add_category(category)
#todo: write filter functions
- allowed_ords_pkg = set(range(ord('a'), ord('z'))) | set(range(ord('A'), ord('Z'))) | \
- set(range(ord('0'), ord('9'))) | set(list(map(ord,
+ allowed_ords_pkg = set(range(ord('a'), ord('z') + 1)) | set(range(ord('A'), ord('Z') + 1)) | \
+ set(range(ord('0'), ord('9') + 1)) | set(list(map(ord,
['+', '_', '-'])))
- allowed_ords_desc = set(range(ord('a'), ord('z'))) | set(range(ord('A'), ord('Z'))) | \
- set(range(ord('0'), ord('9'))) | set(list(map(ord,
+ allowed_ords_desc = set(range(ord('a'), ord('z') + 1)) | set(range(ord('A'), ord('Z') + 1)) | \
+ set(range(ord('0'), ord('9') + 1)) | set(list(map(ord,
['+', '_', '-', ' ', '.', '(', ')', '[', ']', '{', '}', ','])))
now = datetime.datetime.now()
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/g-sorcery:master commit in: gs_pypi/
@ 2013-09-06 15:42 Jauhien Piatlicki
0 siblings, 0 replies; 20+ messages in thread
From: Jauhien Piatlicki @ 2013-09-06 15:42 UTC (permalink / raw
To: gentoo-commits
commit: 4f81685858f43cc42c91acb3ec74868604b1f233
Author: Jauhien Piatlicki (jauhien) <piatlicki <AT> gmail <DOT> com>
AuthorDate: Fri Sep 6 15:42:02 2013 +0000
Commit: Jauhien Piatlicki <piatlicki <AT> gmail <DOT> com>
CommitDate: Fri Sep 6 15:42:02 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/g-sorcery.git;a=commit;h=4f816858
gs-pypi: fix errors
---
gs_pypi/gs_pypi_generate_db.py | 2 +-
gs_pypi/pypi_db.py | 10 +++++-----
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/gs_pypi/gs_pypi_generate_db.py b/gs_pypi/gs_pypi_generate_db.py
index 1d95a8a..a59b5e3 100644
--- a/gs_pypi/gs_pypi_generate_db.py
+++ b/gs_pypi/gs_pypi_generate_db.py
@@ -48,7 +48,7 @@ def main():
generator = PypiDBGenerator()
db_name = sys.argv[1]
temp_dir = TemporaryDirectory()
- pkg_db = generator(temp_dir.name, "pypi", config=config["repositories"]["pypi"])
+ pkg_db = generator(temp_dir.name, "pypi", config=config["repositories"]["pypi"], common_config=config["common_config"])
if os.path.exists(db_name):
os.system('rm -rf ' + db_name + '/*')
else:
diff --git a/gs_pypi/pypi_db.py b/gs_pypi/pypi_db.py
index a700e4b..f982fcd 100644
--- a/gs_pypi/pypi_db.py
+++ b/gs_pypi/pypi_db.py
@@ -200,7 +200,7 @@ class PypiDBGenerator(DBGenerator):
continue
homepage = ""
- license = ""
+ pkg_license = ""
py_versions = []
if info:
if "Home Page:" in info:
@@ -225,8 +225,8 @@ class PypiDBGenerator(DBGenerator):
py_versions.extend(['3_3'])
if "License" in categories:
- license = categories["License"][-1]
- license = self.convert([common_config, config], "licenses", license)
+ pkg_license = categories["License"][-1]
+ pkg_license = self.convert([common_config, config], "licenses", pkg_license)
if not py_versions:
py_versions = ['2_6', '2_7', '3_2', '3_3']
@@ -236,7 +236,7 @@ class PypiDBGenerator(DBGenerator):
python_compat = '( python{' + py_versions[0]
for ver in py_versions[1:]:
python_compat += ',' + ver
- python_compat += '} )'
+ python_compat += '} )'
filtered_package = "".join([x for x in package if ord(x) in allowed_ords_pkg])
description = "".join([x for x in description if ord(x) in allowed_ords_desc])
@@ -262,7 +262,7 @@ class PypiDBGenerator(DBGenerator):
ebuild_data["maintainer"] = maintainer
ebuild_data["homepage"] = homepage
- ebuild_data["license"] = license
+ ebuild_data["license"] = pkg_license
ebuild_data["source_uri"] = source_uri
ebuild_data["md5"] = md5
ebuild_data["python_compat"] = python_compat
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/g-sorcery:master commit in: gs_pypi/
@ 2013-09-06 7:06 Jauhien Piatlicki
0 siblings, 0 replies; 20+ messages in thread
From: Jauhien Piatlicki @ 2013-09-06 7:06 UTC (permalink / raw
To: gentoo-commits
commit: 778c454c0cd11fdfa155f2f2bbde71d057dcacbe
Author: Jauhien Piatlicki (jauhien) <piatlicki <AT> gmail <DOT> com>
AuthorDate: Fri Sep 6 07:07:32 2013 +0000
Commit: Jauhien Piatlicki <piatlicki <AT> gmail <DOT> com>
CommitDate: Fri Sep 6 07:07:32 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/g-sorcery.git;a=commit;h=778c454c
gs_pypi/pypi_db: fix typo
---
gs_pypi/pypi_db.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/gs_pypi/pypi_db.py b/gs_pypi/pypi_db.py
index a4a2e78..a700e4b 100644
--- a/gs_pypi/pypi_db.py
+++ b/gs_pypi/pypi_db.py
@@ -210,7 +210,7 @@ class PypiDBGenerator(DBGenerator):
categories = info["Categories"]
if 'Programming Language' in categories:
- for entry in ebuild_data['info']['Categories']['Programming Language']:
+ for entry in categories['Programming Language']:
if entry == '2':
py_versions.extend(['2_6', '2_7'])
elif entry == '3':
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/g-sorcery:master commit in: gs_pypi/
@ 2013-09-05 16:33 Jauhien Piatlicki
0 siblings, 0 replies; 20+ messages in thread
From: Jauhien Piatlicki @ 2013-09-05 16:33 UTC (permalink / raw
To: gentoo-commits
commit: 5c6b3a04afb268653b3595026e100bbff5084a50
Author: Jauhien Piatlicki (jauhien) <piatlicki <AT> gmail <DOT> com>
AuthorDate: Thu Sep 5 16:32:38 2013 +0000
Commit: Jauhien Piatlicki <piatlicki <AT> gmail <DOT> com>
CommitDate: Thu Sep 5 16:32:38 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/g-sorcery.git;a=commit;h=5c6b3a04
pypi: use realname and realversion
---
gs_pypi/ebuild.py | 7 ++++---
gs_pypi/pypi_db.py | 3 +++
2 files changed, 7 insertions(+), 3 deletions(-)
diff --git a/gs_pypi/ebuild.py b/gs_pypi/ebuild.py
index d3f5342..666b55f 100644
--- a/gs_pypi/ebuild.py
+++ b/gs_pypi/ebuild.py
@@ -24,8 +24,9 @@ class PypiEbuildWithoutDigestGenerator(DefaultEbuildGenerator):
def __init__(self, package_db):
vars_before_inherit = \
- [{"name" : "repo_uri", "value" : 'http://pypi.python.org/packages/source/${PN:0:1}/${PN}/'},
- {"name" : "sourcefile", "value" : '${P}.tar.gz'}, {"name" : "python_compat", "raw" : True}]
+ ["realname", "realversion",
+ {"name" : "repo_uri", "value" : 'http://pypi.python.org/packages/source/${REALNAME:0:1}/${REALNAME}/'},
+ {"name" : "sourcefile", "value" : '${REALNAME}-${REALVERSION}.tar.gz'}, {"name" : "python_compat", "raw" : True}]
inherit = ["gs-pypi"]
@@ -49,7 +50,7 @@ class PypiEbuildWithDigestGenerator(DefaultEbuildGenerator):
vars_after_description = \
["homepage",
- {"name" : "src_uri", "value" : 'http://pypi.python.org/packages/source/${PN:0:1}/${PN}/${P}.tar.gz'}]
+ {"name" : "src_uri", "value" : 'http://pypi.python.org/packages/source/${REALNAME:0:1}/${REALNAME}/${REALNAME}-${REALVERSION}.tar.gz'}]
vars_after_keywords = \
[]
diff --git a/gs_pypi/pypi_db.py b/gs_pypi/pypi_db.py
index 188be01..a9addea 100644
--- a/gs_pypi/pypi_db.py
+++ b/gs_pypi/pypi_db.py
@@ -251,6 +251,9 @@ class PypiDBGenerator(DBGenerator):
'name' : 'Jauhien Piatlicki'}]
ebuild_data = {}
+ ebuild_data["realname"] = package
+ ebuild_data["realversion"] = version
+
ebuild_data["description"] = description
ebuild_data["longdescription"] = description
ebuild_data["dependencies"] = dependencies
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/g-sorcery:master commit in: gs_pypi/
@ 2013-08-30 16:08 Jauhien Piatlicki
0 siblings, 0 replies; 20+ messages in thread
From: Jauhien Piatlicki @ 2013-08-30 16:08 UTC (permalink / raw
To: gentoo-commits
commit: 4ac11d5a3fd8a2111104e6c71c8758f1e9571036
Author: Jauhien Piatlicki (jauhien) <piatlicki <AT> gmail <DOT> com>
AuthorDate: Fri Aug 30 16:07:53 2013 +0000
Commit: Jauhien Piatlicki <piatlicki <AT> gmail <DOT> com>
CommitDate: Fri Aug 30 16:07:53 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/g-sorcery.git;a=commit;h=4ac11d5a
gs_pypi/ebuild: homepage and src_uri
---
gs_pypi/ebuild.py | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/gs_pypi/ebuild.py b/gs_pypi/ebuild.py
index 1acccae..08ca188 100644
--- a/gs_pypi/ebuild.py
+++ b/gs_pypi/ebuild.py
@@ -30,7 +30,7 @@ class PypiEbuildWithoutDigestGenerator(DefaultEbuildGenerator):
inherit = ["gs-pypi"]
vars_after_description = \
- []
+ ["homepage"]
vars_after_keywords = \
[]
@@ -43,14 +43,13 @@ class PypiEbuildWithDigestGenerator(DefaultEbuildGenerator):
def __init__(self, package_db):
vars_before_inherit = \
- [("digest_sources", "yes"),
- ("repo_uri", '"http://pypi.python.org/packages/source/${PN:0:1}/${PN}/"'),
- ("sourcefile", '"${P}.tar.gz"')]
+ [("digest_sources", "yes")]
inherit = ["gs-pypi"]
vars_after_description = \
- []
+ ["homepage",
+ ("src_uri", '"http://pypi.python.org/packages/source/${PN:0:1}/${PN}/${P}.tar.gz"')]
vars_after_keywords = \
[]
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/g-sorcery:master commit in: gs_pypi/
@ 2013-08-29 23:29 Jauhien Piatlicki
0 siblings, 0 replies; 20+ messages in thread
From: Jauhien Piatlicki @ 2013-08-29 23:29 UTC (permalink / raw
To: gentoo-commits
commit: 30e078d0ce12f907e88bbaadeb55d312e3a6f63d
Author: Jauhien Piatlicki (jauhien) <piatlicki <AT> gmail <DOT> com>
AuthorDate: Thu Aug 29 23:28:43 2013 +0000
Commit: Jauhien Piatlicki <piatlicki <AT> gmail <DOT> com>
CommitDate: Thu Aug 29 23:28:43 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/g-sorcery.git;a=commit;h=30e078d0
gs_pypi/pypi_db: undo accidentially commited debug restriction
---
gs_pypi/pypi_db.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/gs_pypi/pypi_db.py b/gs_pypi/pypi_db.py
index b3e5854..f614c4a 100644
--- a/gs_pypi/pypi_db.py
+++ b/gs_pypi/pypi_db.py
@@ -53,7 +53,7 @@ class PypiDBGenerator(DBGenerator):
soup.decompose()
pkg_uries = self.decode_download_uries(pkg_uries)
- for uri in pkg_uries[:10]:
+ for uri in pkg_uries:
attempts = 0
while True:
try:
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/g-sorcery:master commit in: gs_pypi/
@ 2013-08-29 19:23 Jauhien Piatlicki
0 siblings, 0 replies; 20+ messages in thread
From: Jauhien Piatlicki @ 2013-08-29 19:23 UTC (permalink / raw
To: gentoo-commits
commit: 401ab7a946fffff4cf31769160838b7ad176ec23
Author: Jauhien Piatlicki (jauhien) <piatlicki <AT> gmail <DOT> com>
AuthorDate: Thu Aug 29 16:25:14 2013 +0000
Commit: Jauhien Piatlicki <piatlicki <AT> gmail <DOT> com>
CommitDate: Thu Aug 29 16:25:14 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/g-sorcery.git;a=commit;h=401ab7a9
gs-pypi: correct directory structure in generated DB
---
gs_pypi/gs_pypi_generate_db.py | 8 ++++++--
gs_pypi/pypi_db.py | 2 +-
2 files changed, 7 insertions(+), 3 deletions(-)
diff --git a/gs_pypi/gs_pypi_generate_db.py b/gs_pypi/gs_pypi_generate_db.py
index 3f2fcd1..a4f8395 100644
--- a/gs_pypi/gs_pypi_generate_db.py
+++ b/gs_pypi/gs_pypi_generate_db.py
@@ -14,7 +14,8 @@
import os
import sys
-from g_sorcery.fileutils import FileJSON
+from g_sorcery.compatibility import TemporaryDirectory
+from g_sorcery.fileutils import copy_all, FileJSON
from g_sorcery.logger import Logger
from .pypi_db import PypiDBGenerator
@@ -45,7 +46,10 @@ def main():
generator = PypiDBGenerator()
db_name = sys.argv[1]
- pkg_db = generator(db_name, "pypi", config=config["repositories"]["pypi"])
+ temp_dir = TemporaryDirectory()
+ pkg_db = generator(temp_dir.name, "pypi", config=config["repositories"]["pypi"])
+ os.mkdir(db_name)
+ copy_all(os.path.join(temp_dir.name, "pypi/db"), db_name)
os.system('tar cvzf ' + db_name + '.tar.gz ' + db_name)
if __name__ == "__main__":
diff --git a/gs_pypi/pypi_db.py b/gs_pypi/pypi_db.py
index f614c4a..b3e5854 100644
--- a/gs_pypi/pypi_db.py
+++ b/gs_pypi/pypi_db.py
@@ -53,7 +53,7 @@ class PypiDBGenerator(DBGenerator):
soup.decompose()
pkg_uries = self.decode_download_uries(pkg_uries)
- for uri in pkg_uries:
+ for uri in pkg_uries[:10]:
attempts = 0
while True:
try:
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/g-sorcery:master commit in: gs_pypi/
@ 2013-08-14 19:21 Jauhien Piatlicki
0 siblings, 0 replies; 20+ messages in thread
From: Jauhien Piatlicki @ 2013-08-14 19:21 UTC (permalink / raw
To: gentoo-commits
commit: b952cdebc16a3602a17336eef26e441f5c4c9dfe
Author: Jauhien Piatlicki (jauhien) <piatlicki <AT> gmail <DOT> com>
AuthorDate: Wed Aug 14 19:22:09 2013 +0000
Commit: Jauhien Piatlicki <piatlicki <AT> gmail <DOT> com>
CommitDate: Wed Aug 14 19:22:09 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/g-sorcery.git;a=commit;h=b952cdeb
gs_pypi/pypi_db.py: fix data initializing
---
gs_pypi/pypi_db.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/gs_pypi/pypi_db.py b/gs_pypi/pypi_db.py
index 07fc3bc..f614c4a 100644
--- a/gs_pypi/pypi_db.py
+++ b/gs_pypi/pypi_db.py
@@ -143,6 +143,8 @@ class PypiDBGenerator(DBGenerator):
print("There was an error during parsing: " + str(error))
print("Ignoring this package.")
data = {}
+ data["files"] = []
+ data["info"] = {}
soup.decompose()
return data
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/g-sorcery:master commit in: gs_pypi/
@ 2013-08-14 13:34 Jauhien Piatlicki
0 siblings, 0 replies; 20+ messages in thread
From: Jauhien Piatlicki @ 2013-08-14 13:34 UTC (permalink / raw
To: gentoo-commits
commit: 0c6eb9864124cee25a5ba407ad01d0bb930b4701
Author: Jauhien Piatlicki (jauhien) <piatlicki <AT> gmail <DOT> com>
AuthorDate: Wed Aug 14 13:33:26 2013 +0000
Commit: Jauhien Piatlicki <piatlicki <AT> gmail <DOT> com>
CommitDate: Wed Aug 14 13:33:26 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/g-sorcery.git;a=commit;h=0c6eb986
gs_pypi/pypi_db: restrict number of attempts to download package page
---
gs_pypi/pypi_db.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/gs_pypi/pypi_db.py b/gs_pypi/pypi_db.py
index 69ad476..07fc3bc 100644
--- a/gs_pypi/pypi_db.py
+++ b/gs_pypi/pypi_db.py
@@ -54,13 +54,16 @@ class PypiDBGenerator(DBGenerator):
pkg_uries = self.decode_download_uries(pkg_uries)
for uri in pkg_uries:
+ attempts = 0
while True:
try:
+ attempts += 1
self.process_uri(uri, data)
except DownloadingError as error:
print(str(error))
- time.sleep(2)
- continue
+ time.sleep(5)
+ if attempts < 100:
+ continue
break
return data
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/g-sorcery:master commit in: gs_pypi/
@ 2013-08-13 23:47 Jauhien Piatlicki
0 siblings, 0 replies; 20+ messages in thread
From: Jauhien Piatlicki @ 2013-08-13 23:47 UTC (permalink / raw
To: gentoo-commits
commit: 9361f780802cde1563aeb5dbe5da301d6fc00985
Author: Jauhien Piatlicki (jauhien) <piatlicki <AT> gmail <DOT> com>
AuthorDate: Tue Aug 13 23:48:29 2013 +0000
Commit: Jauhien Piatlicki <piatlicki <AT> gmail <DOT> com>
CommitDate: Tue Aug 13 23:48:29 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/g-sorcery.git;a=commit;h=9361f780
gs_pypi/pypi_db: memory leak fixed (I hope so)
---
gs_pypi/pypi_db.py | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/gs_pypi/pypi_db.py b/gs_pypi/pypi_db.py
index ee5c2d5..fca3432 100644
--- a/gs_pypi/pypi_db.py
+++ b/gs_pypi/pypi_db.py
@@ -46,6 +46,11 @@ class PypiDBGenerator(DBGenerator):
pkg_uries.append({"uri": self.repo_uri + "pypi/" + package + "/" + version,
"parser": self.parse_package_page,
"output": package + "-" + version})
+ entry.decompose()
+
+ packages.decompose()
+ soup.decompose()
+
pkg_uries = self.decode_download_uries(pkg_uries)
for uri in pkg_uries:
while True:
@@ -94,6 +99,8 @@ class PypiDBGenerator(DBGenerator):
"pyversion": file_pyversion,
"uploaded": file_uploaded,
"size": file_size})
+ entry.decompose()
+ table.decompose()
uls = soup("ul", class_ = "nodot")
if uls:
@@ -123,7 +130,10 @@ class PypiDBGenerator(DBGenerator):
if entry("a"):
data["info"][entry_name] = entry("a")[0]["href"]
continue
+ entry.decompose()
+ ul.decompose()
+ soup.decompose()
return data
def process_data(self, pkg_db, data, common_config, config):
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/g-sorcery:pypi commit in: gs_pypi/
@ 2013-08-12 18:38 Jauhien Piatlicki
2013-08-12 15:45 ` [gentoo-commits] proj/g-sorcery:master " Jauhien Piatlicki
0 siblings, 1 reply; 20+ messages in thread
From: Jauhien Piatlicki @ 2013-08-12 18:38 UTC (permalink / raw
To: gentoo-commits
commit: 78ec3e8545829801ec9e0bc97e0fdf84c23e328b
Author: Jauhien Piatlicki (jauhien) <piatlicki <AT> gmail <DOT> com>
AuthorDate: Mon Aug 12 15:45:13 2013 +0000
Commit: Jauhien Piatlicki <piatlicki <AT> gmail <DOT> com>
CommitDate: Mon Aug 12 15:45:13 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/g-sorcery.git;a=commit;h=78ec3e85
gs_pypi/pypi_db: remove accidentally commited debug code
---
gs_pypi/pypi_db.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/gs_pypi/pypi_db.py b/gs_pypi/pypi_db.py
index 1a42f92..52e83e3 100644
--- a/gs_pypi/pypi_db.py
+++ b/gs_pypi/pypi_db.py
@@ -47,7 +47,7 @@ class PypiDBGenerator(DBGenerator):
"parser": self.parse_package_page,
"output": package + "-" + version})
pkg_uries = self.decode_download_uries(pkg_uries)
- for uri in pkg_uries[:10]:
+ for uri in pkg_uries:
while True:
try:
self.process_uri(uri, data)
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/g-sorcery:master commit in: gs_pypi/
@ 2013-08-12 18:38 Jauhien Piatlicki
0 siblings, 0 replies; 20+ messages in thread
From: Jauhien Piatlicki @ 2013-08-12 18:38 UTC (permalink / raw
To: gentoo-commits
commit: 5db4a26f73b50f8ef398709dc605b20191dacb5e
Author: Jauhien Piatlicki (jauhien) <piatlicki <AT> gmail <DOT> com>
AuthorDate: Mon Aug 12 18:37:57 2013 +0000
Commit: Jauhien Piatlicki <piatlicki <AT> gmail <DOT> com>
CommitDate: Mon Aug 12 18:37:57 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/g-sorcery.git;a=commit;h=5db4a26f
gs_pypi/pypi_db: fix parsing and store info in database
---
gs_pypi/pypi_db.py | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/gs_pypi/pypi_db.py b/gs_pypi/pypi_db.py
index 52e83e3..ee5c2d5 100644
--- a/gs_pypi/pypi_db.py
+++ b/gs_pypi/pypi_db.py
@@ -63,7 +63,10 @@ class PypiDBGenerator(DBGenerator):
data = {}
data["files"] = []
data["info"] = {}
- for table in soup("table")[-1:]:
+ for table in soup("table", class_ = "list")[-1:]:
+ if not "File" in table("th")[0].string:
+ continue
+
for entry in table("tr")[1:-1]:
fields = entry("td")
@@ -151,10 +154,12 @@ class PypiDBGenerator(DBGenerator):
continue
files_src_uri = ""
+ md5 = ""
if pkg_data["files"]:
for file_entry in pkg_data["files"]:
if file_entry["type"] == "\n Source\n ":
files_src_uri = file_entry["url"]
+ md5 = file_entry["md5"]
break
download_url = ""
@@ -205,5 +210,8 @@ class PypiDBGenerator(DBGenerator):
ebuild_data["homepage"] = homepage
ebuild_data["license"] = license
ebuild_data["source_uri"] = source_uri
+ ebuild_data["md5"] = md5
+
+ ebuild_data["info"] = info
pkg_db.add_package(Package(category, filtered_package, filtered_version), ebuild_data)
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/g-sorcery:master commit in: gs_pypi/
@ 2013-08-12 15:41 Jauhien Piatlicki
0 siblings, 0 replies; 20+ messages in thread
From: Jauhien Piatlicki @ 2013-08-12 15:41 UTC (permalink / raw
To: gentoo-commits
commit: 82a4549a97cd62c2896e8ba7d15409fc526b0a82
Author: Jauhien Piatlicki (jauhien) <piatlicki <AT> gmail <DOT> com>
AuthorDate: Mon Aug 12 15:40:39 2013 +0000
Commit: Jauhien Piatlicki <piatlicki <AT> gmail <DOT> com>
CommitDate: Mon Aug 12 15:40:39 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/g-sorcery.git;a=commit;h=82a4549a
gs_pypi/pypi_db: fix URI and parsing
---
gs_pypi/pypi_db.py | 14 ++++++++++----
1 file changed, 10 insertions(+), 4 deletions(-)
diff --git a/gs_pypi/pypi_db.py b/gs_pypi/pypi_db.py
index 1feb59e..1a42f92 100644
--- a/gs_pypi/pypi_db.py
+++ b/gs_pypi/pypi_db.py
@@ -24,7 +24,7 @@ class PypiDBGenerator(DBGenerator):
def get_download_uries(self, common_config, config):
self.repo_uri = config["repo_uri"]
- return [{"uri": self.repo_uri + "?%3Aaction=index", "output": "packages"}]
+ return [{"uri": self.repo_uri + "pypi?%3Aaction=index", "output": "packages"}]
def parse_data(self, data_f):
soup = bs4.BeautifulSoup(data_f.read())
@@ -47,7 +47,7 @@ class PypiDBGenerator(DBGenerator):
"parser": self.parse_package_page,
"output": package + "-" + version})
pkg_uries = self.decode_download_uries(pkg_uries)
- for uri in pkg_uries:
+ for uri in pkg_uries[:10]:
while True:
try:
self.process_uri(uri, data)
@@ -91,8 +91,14 @@ class PypiDBGenerator(DBGenerator):
"pyversion": file_pyversion,
"uploaded": file_uploaded,
"size": file_size})
-
- for ul in soup("ul", class_ = "nodot")[:1]:
+
+ uls = soup("ul", class_ = "nodot")
+ if uls:
+ if "Downloads (All Versions):" in uls[0]("strong")[0].string:
+ ul = uls[1]
+ else:
+ ul = uls[0]
+
for entry in ul.contents:
if not hasattr(entry, "name") or entry.name != "li":
continue
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/g-sorcery:master commit in: gs_pypi/
@ 2013-08-12 0:36 Jauhien Piatlicki
0 siblings, 0 replies; 20+ messages in thread
From: Jauhien Piatlicki @ 2013-08-12 0:36 UTC (permalink / raw
To: gentoo-commits
commit: 2fbcbf9d09708ae66ab0022d4dda10b55cd80a3b
Author: Jauhien Piatlicki (jauhien) <piatlicki <AT> gmail <DOT> com>
AuthorDate: Mon Aug 12 00:35:52 2013 +0000
Commit: Jauhien Piatlicki <piatlicki <AT> gmail <DOT> com>
CommitDate: Mon Aug 12 00:35:52 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/g-sorcery.git;a=commit;h=2fbcbf9d
gs_pypi/pypi_db: working database generator
---
gs_pypi/pypi_db.py | 105 ++++++++++++++++++++++++++++++++++++++---------------
1 file changed, 76 insertions(+), 29 deletions(-)
diff --git a/gs_pypi/pypi_db.py b/gs_pypi/pypi_db.py
index ef0b807..1feb59e 100644
--- a/gs_pypi/pypi_db.py
+++ b/gs_pypi/pypi_db.py
@@ -11,10 +11,13 @@
:license: GPL-2, see LICENSE for more details.
"""
+import datetime
+import re
+
import bs4
from g_sorcery.exceptions import DownloadingError
-from g_sorcery.g_collections import Package
+from g_sorcery.g_collections import Package, serializable_elist
from g_sorcery.package_db import DBGenerator
class PypiDBGenerator(DBGenerator):
@@ -98,9 +101,10 @@ class PypiDBGenerator(DBGenerator):
continue
if entry_name == "Categories":
- data["info"][entry_name] = []
+ data["info"][entry_name] = {}
for cat_entry in entry("a"):
- data["info"][entry_name].append(cat_entry.string.split(" :: "))
+ cat_data = cat_entry.string.split(" :: ")
+ data["info"][entry_name][cat_data[0]] = cat_data[1:]
continue
if entry("span"):
@@ -128,29 +132,72 @@ class PypiDBGenerator(DBGenerator):
now = datetime.datetime.now()
pseudoversion = "%04d%02d%02d" % (now.year, now.month, now.day)
-
- for package, versions in data.items():
- package = "".join([x for x in package if ord(x) in allowed_ords_pkg])
- for version, ebuild_data in versions.items():
- description = ebuild_data["summary"]
- description = "".join([x for x in description if ord(x) in allowed_ords_desc])
- longdescription = ebuild_data["description"]
- longdescription = "".join([x for x in longdescription if ord(x) in allowed_ords_desc])
-
- pkgver = version
- match_object = re.match("(^[0-9]+[a-z]?$)|(^[0-9][0-9\.]+[0-9][a-z]?$)", pkgver)
- if not match_object:
- pkgver = pseudoversion
-
- dependencies = serializable_elist(separator="\n\t")
- eclasses = ['gs-pypi']
- maintainer = [{'email' : 'piatlicki@gmail.com',
- 'name' : 'Jauhien Piatlicki'}]
-
- ebuild_data["description"] = description
- ebuild_data["longdescription"] = longdescription
- ebuild_data["dependencies"] = dependencies
- ebuild_data["eclasses"] = eclasses
- ebuild_data["maintainer"] = maintainer
-
- pkg_db.add_package(Package(category, package, pkgver), ebuild_data)
+
+ for (package, version), description in data["packages"]["index"].items():
+
+ pkg = package + "-" + version
+ if not pkg in data["packages"]:
+ continue
+
+ pkg_data = data["packages"][pkg]
+
+ if not pkg_data["files"] and not pkg_data["info"]:
+ continue
+
+ files_src_uri = ""
+ if pkg_data["files"]:
+ for file_entry in pkg_data["files"]:
+ if file_entry["type"] == "\n Source\n ":
+ files_src_uri = file_entry["url"]
+ break
+
+ download_url = ""
+ info = pkg_data["info"]
+ if info:
+ if "Download URL:" in info:
+ download_url = pkg_data["info"]["Download URL:"]
+
+ if download_url:
+ source_uri = download_url #todo: find how to define src_uri
+ else:
+ source_uri = files_src_uri
+
+ if not source_uri:
+ continue
+
+ homepage = ""
+ license = ""
+ if info:
+ if "Home Page:" in info:
+ homepage = info["Home Page:"]
+ categories = {}
+ if "Categories" in info:
+ categories = info["Categories"]
+ if "License" in categories:
+ license = categories["License"][-1]
+
+ filtered_package = "".join([x for x in package if ord(x) in allowed_ords_pkg])
+ description = "".join([x for x in description if ord(x) in allowed_ords_desc])
+ filtered_version = version
+ match_object = re.match("(^[0-9]+[a-z]?$)|(^[0-9][0-9\.]+[0-9][a-z]?$)",
+ filtered_version)
+ if not match_object:
+ filtered_version = pseudoversion
+
+ dependencies = serializable_elist(separator="\n\t")
+ eclasses = ['gs-pypi']
+ maintainer = [{'email' : 'piatlicki@gmail.com',
+ 'name' : 'Jauhien Piatlicki'}]
+
+ ebuild_data = {}
+ ebuild_data["description"] = description
+ ebuild_data["longdescription"] = description
+ ebuild_data["dependencies"] = dependencies
+ ebuild_data["eclasses"] = eclasses
+ ebuild_data["maintainer"] = maintainer
+
+ ebuild_data["homepage"] = homepage
+ ebuild_data["license"] = license
+ ebuild_data["source_uri"] = source_uri
+
+ pkg_db.add_package(Package(category, filtered_package, filtered_version), ebuild_data)
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/g-sorcery:master commit in: gs_pypi/
@ 2013-08-12 0:36 Jauhien Piatlicki
0 siblings, 0 replies; 20+ messages in thread
From: Jauhien Piatlicki @ 2013-08-12 0:36 UTC (permalink / raw
To: gentoo-commits
commit: b8e92db3d4cde97bb2ebcd181cba3841cd789432
Author: Jauhien Piatlicki (jauhien) <piatlicki <AT> gmail <DOT> com>
AuthorDate: Sun Aug 11 23:33:59 2013 +0000
Commit: Jauhien Piatlicki <piatlicki <AT> gmail <DOT> com>
CommitDate: Sun Aug 11 23:33:59 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/g-sorcery.git;a=commit;h=b8e92db3
gs_pypi/pypi_db: parse data
---
gs_pypi/pypi_db.py | 148 +++++++++++++++++++++++++++++++----------------------
1 file changed, 88 insertions(+), 60 deletions(-)
diff --git a/gs_pypi/pypi_db.py b/gs_pypi/pypi_db.py
index af3e65a..44e77a4 100644
--- a/gs_pypi/pypi_db.py
+++ b/gs_pypi/pypi_db.py
@@ -11,72 +11,100 @@
:license: GPL-2, see LICENSE for more details.
"""
-from g_sorcery.compatibility import py2k
+import bs4
-if py2k:
- import xmlrpclib
-else:
- import xmlrpc.client as xmlrpclib
-
-import datetime
-import re
-import sys
-
-from g_sorcery.g_collections import Package, serializable_elist
-from g_sorcery.logger import Logger
+from g_sorcery.g_collections import Package
from g_sorcery.package_db import DBGenerator
class PypiDBGenerator(DBGenerator):
def get_download_uries(self, common_config, config):
- return [config["repo_uri"] + "/pypi"]
-
- def process_uri(self, uri, data):
- url = uri["uri"]
- client = xmlrpclib.ServerProxy(url)
- logger = Logger()
- logger.info("downloading packages data")
- pkg_list = client.list_packages()
-
- number_of_packages = len(pkg_list)
- downloaded_number = 0
-
- for pkg in pkg_list:
- data[pkg] = {}
-
- chars = ['-','\\','|','/']
- show = chars[downloaded_number % 4]
- percent = (downloaded_number * 100)//number_of_packages
- length = 70
- progress = (percent * length)//100
- blank = length - progress
-
- sys.stdout.write("\r %s [%s%s] %s%%" % (show, "#" * progress, " " * blank, percent))
- sys.stdout.flush()
- downloaded_number += 1
-
- versions = []
- while not versions:
- try:
- versions = client.package_releases(pkg)
- except Exception as error:
- logger.warn("Something went wrong: " + str(error))
- logger.info("Trying again...")
- versions = []
-
- for version in versions:
- data[pkg][version] = {}
- while not data[pkg][version]:
- try:
- data[pkg][version] = client.release_data(pkg, version)
- except Exception as error:
- logger.warn("Something went wrong: " + str(error))
- logger.info("Trying again...")
- data[pkg][version] = {}
-
- sys.stdout.write("\r %s [%s] %s%%" % ("-", "#" * length, 100))
- sys.stdout.flush()
- print("")
+ self.repo_uri = config["repo_uri"]
+ return [{"uri": self.repo_uri + "?%3Aaction=index", "output": "packages"}]
+
+ def parse_data(self, data_f):
+ soup = bs4.BeautifulSoup(data_f.read())
+ packages = soup.table
+ data = {}
+ data["index"] = {}
+
+ pkg_uries = []
+
+ for entry in packages.find_all("tr")[1:-1]:
+ package, description = entry.find_all("td")
+
+ if description.contents:
+ description = description.contents[0]
+ else:
+ description = ""
+ package, version = package.a["href"].split("/")[2:]
+ data["index"][(package, version)] = description
+ pkg_uries.append({"uri": self.repo_uri + "pypi/" + package + "/" + version,
+ "parser": self.parse_package_page,
+ "output": package + "-" + version})
+ pkg_uries = self.decode_download_uries(pkg_uries)
+ for uri in pkg_uries:
+ self.process_uri(uri, data)
+
+ return data
+
+ def parse_package_page(self, data_f):
+ soup = bs4.BeautifulSoup(data_f.read())
+ data = {}
+ data["files"] = []
+ data["info"] = {}
+ for table in soup("table")[-1:]:
+ for entry in table("tr")[1:-1]:
+ fields = entry("td")
+
+ FILE = 0
+ URL = 0
+ MD5 = 1
+
+ TYPE = 1
+ PYVERSION = 2
+ UPLOADED = 3
+ SIZE = 4
+
+ file_inf = fields[FILE]("a")[0]["href"].split("#")
+ file_url = file_inf[URL]
+ file_md5 = file_inf[MD5][4:]
+
+ file_type = fields[TYPE].string
+ file_pyversion = fields[PYVERSION].string
+ file_uploaded = fields[UPLOADED].string
+ file_size = fields[SIZE].string
+
+ data["files"].append({"url": file_url,
+ "md5": file_md5,
+ "type": file_type,
+ "pyversion": file_pyversion,
+ "uploaded": file_uploaded,
+ "size": file_size})
+
+ for ul in soup("ul", class_ = "nodot")[:1]:
+ for entry in ul.contents:
+ if not hasattr(entry, "name") or entry.name != "li":
+ continue
+ entry_name = entry("strong")[0].string
+ if not entry_name:
+ continue
+
+ if entry_name == "Categories":
+ data["info"][entry_name] = []
+ for cat_entry in entry("a"):
+ data["info"][entry_name].append(cat_entry.string.split(" :: "))
+ continue
+
+ if entry("span"):
+ data["info"][entry_name] = entry("span")[0].string
+ continue
+
+ if entry("a"):
+ data["info"][entry_name] = entry("a")[0]["href"]
+ continue
+
+ return data
def process_data(self, pkg_db, data, common_config, config):
category = "dev-python"
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/g-sorcery:master commit in: gs_pypi/
@ 2013-07-26 14:46 Jauhien Piatlicki
0 siblings, 0 replies; 20+ messages in thread
From: Jauhien Piatlicki @ 2013-07-26 14:46 UTC (permalink / raw
To: gentoo-commits
commit: 05665d33bc3c6880aaf26aae91ccdecaabbed498
Author: Jauhien Piatlicki (jauhien) <piatlicki <AT> gmail <DOT> com>
AuthorDate: Fri Jul 26 14:46:25 2013 +0000
Commit: Jauhien Piatlicki <piatlicki <AT> gmail <DOT> com>
CommitDate: Fri Jul 26 14:46:25 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/g-sorcery.git;a=commit;h=05665d33
gs_pypi/pypi_db; try download again in case of failure
---
gs_pypi/pypi_db.py | 19 +++++++++++++++++--
1 file changed, 17 insertions(+), 2 deletions(-)
diff --git a/gs_pypi/pypi_db.py b/gs_pypi/pypi_db.py
index d350e39..af3e65a 100644
--- a/gs_pypi/pypi_db.py
+++ b/gs_pypi/pypi_db.py
@@ -55,9 +55,24 @@ class PypiDBGenerator(DBGenerator):
sys.stdout.flush()
downloaded_number += 1
- versions = client.package_releases(pkg)
+ versions = []
+ while not versions:
+ try:
+ versions = client.package_releases(pkg)
+ except Exception as error:
+ logger.warn("Something went wrong: " + str(error))
+ logger.info("Trying again...")
+ versions = []
+
for version in versions:
- data[pkg][version] = client.release_data(pkg, version)
+ data[pkg][version] = {}
+ while not data[pkg][version]:
+ try:
+ data[pkg][version] = client.release_data(pkg, version)
+ except Exception as error:
+ logger.warn("Something went wrong: " + str(error))
+ logger.info("Trying again...")
+ data[pkg][version] = {}
sys.stdout.write("\r %s [%s] %s%%" % ("-", "#" * length, 100))
sys.stdout.flush()
^ permalink raw reply related [flat|nested] 20+ messages in thread* [gentoo-commits] proj/g-sorcery:master commit in: gs_pypi/
@ 2013-07-26 9:40 Jauhien Piatlicki
0 siblings, 0 replies; 20+ messages in thread
From: Jauhien Piatlicki @ 2013-07-26 9:40 UTC (permalink / raw
To: gentoo-commits
commit: 7fc51241db06c67860d1d0d12175ec8ca1b36eff
Author: Jauhien Piatlicki (jauhien) <piatlicki <AT> gmail <DOT> com>
AuthorDate: Fri Jul 26 09:40:03 2013 +0000
Commit: Jauhien Piatlicki <piatlicki <AT> gmail <DOT> com>
CommitDate: Fri Jul 26 09:40:03 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/g-sorcery.git;a=commit;h=7fc51241
gs_pypi/pypi_db: usable ebuild_data
---
gs_pypi/pypi_db.py | 34 ++++++++++++++++++++++++++++++++--
1 file changed, 32 insertions(+), 2 deletions(-)
diff --git a/gs_pypi/pypi_db.py b/gs_pypi/pypi_db.py
index d0b57d4..d350e39 100644
--- a/gs_pypi/pypi_db.py
+++ b/gs_pypi/pypi_db.py
@@ -18,9 +18,11 @@ if py2k:
else:
import xmlrpc.client as xmlrpclib
+import datetime
+import re
import sys
-from g_sorcery.g_collections import Package
+from g_sorcery.g_collections import Package, serializable_elist
from g_sorcery.logger import Logger
from g_sorcery.package_db import DBGenerator
@@ -69,8 +71,36 @@ class PypiDBGenerator(DBGenerator):
allowed_ords_pkg = set(range(ord('a'), ord('z'))) | set(range(ord('A'), ord('Z'))) | \
set(range(ord('0'), ord('9'))) | set(list(map(ord,
['+', '_', '-'])))
+
+ allowed_ords_desc = set(range(ord('a'), ord('z'))) | set(range(ord('A'), ord('Z'))) | \
+ set(range(ord('0'), ord('9'))) | set(list(map(ord,
+ ['+', '_', '-', ' ', '.', '(', ')', '[', ']', '{', '}', ','])))
+
+ now = datetime.datetime.now()
+ pseudoversion = "%04d%02d%02d" % (now.year, now.month, now.day)
for package, versions in data.items():
package = "".join([x for x in package if ord(x) in allowed_ords_pkg])
for version, ebuild_data in versions.items():
- pkg_db.add_package(Package(category, package, version), ebuild_data)
+ description = ebuild_data["summary"]
+ description = "".join([x for x in description if ord(x) in allowed_ords_desc])
+ longdescription = ebuild_data["description"]
+ longdescription = "".join([x for x in longdescription if ord(x) in allowed_ords_desc])
+
+ pkgver = version
+ match_object = re.match("(^[0-9]+[a-z]?$)|(^[0-9][0-9\.]+[0-9][a-z]?$)", pkgver)
+ if not match_object:
+ pkgver = pseudoversion
+
+ dependencies = serializable_elist(separator="\n\t")
+ eclasses = ['gs-pypi']
+ maintainer = [{'email' : 'piatlicki@gmail.com',
+ 'name' : 'Jauhien Piatlicki'}]
+
+ ebuild_data["description"] = description
+ ebuild_data["longdescription"] = longdescription
+ ebuild_data["dependencies"] = dependencies
+ ebuild_data["eclasses"] = eclasses
+ ebuild_data["maintainer"] = maintainer
+
+ pkg_db.add_package(Package(category, package, pkgver), ebuild_data)
^ permalink raw reply related [flat|nested] 20+ messages in thread
end of thread, other threads:[~2013-11-21 11:30 UTC | newest]
Thread overview: 20+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2013-08-12 0:36 [gentoo-commits] proj/g-sorcery:master commit in: gs_pypi/ Jauhien Piatlicki
-- strict thread matches above, loose matches on Subject: below --
2013-11-21 11:30 Jauhien Piatlicki
2013-11-12 23:10 Jauhien Piatlicki
2013-09-08 9:06 Jauhien Piatlicki
2013-09-06 15:42 Jauhien Piatlicki
2013-09-06 7:06 Jauhien Piatlicki
2013-09-05 16:33 Jauhien Piatlicki
2013-08-30 16:08 Jauhien Piatlicki
2013-08-29 23:29 Jauhien Piatlicki
2013-08-29 19:23 Jauhien Piatlicki
2013-08-14 19:21 Jauhien Piatlicki
2013-08-14 13:34 Jauhien Piatlicki
2013-08-13 23:47 Jauhien Piatlicki
2013-08-12 18:38 [gentoo-commits] proj/g-sorcery:pypi " Jauhien Piatlicki
2013-08-12 15:45 ` [gentoo-commits] proj/g-sorcery:master " Jauhien Piatlicki
2013-08-12 18:38 Jauhien Piatlicki
2013-08-12 15:41 Jauhien Piatlicki
2013-08-12 0:36 Jauhien Piatlicki
2013-08-12 0:36 Jauhien Piatlicki
2013-07-26 14:46 Jauhien Piatlicki
2013-07-26 9:40 Jauhien Piatlicki
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox