* [gentoo-commits] proj/g-sorcery:master commit in: /, gs_pypi/
@ 2013-07-23 23:13 Jauhien Piatlicki
0 siblings, 0 replies; 6+ messages in thread
From: Jauhien Piatlicki @ 2013-07-23 23:13 UTC (permalink / raw
To: gentoo-commits
commit: 3e4df382f7391e344a9658fbbef84569c25eb4a8
Author: Jauhien Piatlicki (jauhien) <piatlicki <AT> gmail <DOT> com>
AuthorDate: Tue Jul 23 22:03:18 2013 +0000
Commit: Jauhien Piatlicki <piatlicki <AT> gmail <DOT> com>
CommitDate: Tue Jul 23 22:03:18 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/g-sorcery.git;a=commit;h=3e4df382
PyPI backend: initial commit
---
gs-pypi.json | 8 ++++++++
gs_pypi/__init__.py | 2 ++
gs_pypi/backend.py | 32 ++++++++++++++++++++++++++++++
gs_pypi/ebuild.py | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++
gs_pypi/pypi_db.py | 17 ++++++++++++++++
5 files changed, 116 insertions(+)
diff --git a/gs-pypi.json b/gs-pypi.json
new file mode 100644
index 0000000..eb26221
--- /dev/null
+++ b/gs-pypi.json
@@ -0,0 +1,8 @@
+{
+ "package": "gs_pypi",
+ "repositories": {
+ "pypi": {
+ "repo_uri": "https://pypi.python.org/pypi"
+ }
+ }
+}
diff --git a/gs_pypi/__init__.py b/gs_pypi/__init__.py
new file mode 100644
index 0000000..cf529d7
--- /dev/null
+++ b/gs_pypi/__init__.py
@@ -0,0 +1,2 @@
+#!/usr/bin/env python
+
diff --git a/gs_pypi/backend.py b/gs_pypi/backend.py
new file mode 100644
index 0000000..9fd6547
--- /dev/null
+++ b/gs_pypi/backend.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+ backend.py
+ ~~~~~~~~~~
+
+ PyPI backend
+
+ :copyright: (c) 2013 by Jauhien Piatlicki
+ :license: GPL-2, see LICENSE for more details.
+"""
+
+import os
+
+from g_sorcery.backend import Backend
+from g_sorcery.metadata import MetadataGenerator
+from g_sorcery.eclass import EclassGenerator
+from g_sorcery.fileutils import get_pkgpath
+
+from .pypi_db import PypiDBGenerator
+from .ebuild import PypiEbuildWithoutDigestGenerator, PypiEbuildWithDigestGenerator
+
+
+class PypiEclassGenerator(EclassGenerator):
+ def __init__(self):
+ super(PypiEclassGenerator, self).__init__(os.path.join(get_pkgpath(__file__), 'data'))
+
+
+instance = Backend(PypiDBGenerator,
+ PypiEbuildWithDigestGenerator, PypiEbuildWithoutDigestGenerator,
+ PypiEclassGenerator, MetadataGenerator)
diff --git a/gs_pypi/ebuild.py b/gs_pypi/ebuild.py
new file mode 100644
index 0000000..bbc0357
--- /dev/null
+++ b/gs_pypi/ebuild.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+ ebuild.py
+ ~~~~~~~~~
+
+ ebuild generation
+
+ :copyright: (c) 2013 by Jauhien Piatlicki
+ :license: GPL-2, see LICENSE for more details.
+"""
+
+import collections
+import os
+
+from g_sorcery.ebuild import DefaultEbuildGenerator
+
+Layout = collections.namedtuple("Layout",
+ ["vars_before_inherit", "inherit", "vars_after_description", "vars_after_keywords"])
+
+
+class PypiEbuildWithoutDigestGenerator(DefaultEbuildGenerator):
+ def __init__(self, package_db):
+
+ vars_before_inherit = \
+ []
+
+ inherit = ["gs-pypi"]
+
+ vars_after_description = \
+ []
+
+ vars_after_keywords = \
+ []
+
+ layout = Layout(vars_before_inherit, inherit, vars_after_description, vars_after_keywords)
+
+ super(PypiEbuildWithoutDigestGenerator, self).__init__(package_db, layout)
+
+class PypiEbuildWithDigestGenerator(DefaultEbuildGenerator):
+ def __init__(self, package_db):
+
+ vars_before_inherit = \
+ []
+
+ inherit = ["gs-pypi"]
+
+ vars_after_description = \
+ []
+
+ vars_after_keywords = \
+ []
+
+ layout = Layout(vars_before_inherit, inherit, vars_after_description, vars_after_keywords)
+
+ super(PypiEbuildWithDigestGenerator, self).__init__(package_db, layout)
diff --git a/gs_pypi/pypi_db.py b/gs_pypi/pypi_db.py
new file mode 100644
index 0000000..938f79c
--- /dev/null
+++ b/gs_pypi/pypi_db.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+ pypi_db.py
+ ~~~~~~~~~~
+
+ PyPI package database
+
+ :copyright: (c) 2013 by Jauhien Piatlicki
+ :license: GPL-2, see LICENSE for more details.
+"""
+
+from g_sorcery.package_db import DBGenerator
+
+class PypiDBGenerator(DBGenerator):
+ pass
^ permalink raw reply related [flat|nested] 6+ messages in thread
* [gentoo-commits] proj/g-sorcery:master commit in: /, gs_pypi/
@ 2013-07-23 23:13 Jauhien Piatlicki
0 siblings, 0 replies; 6+ messages in thread
From: Jauhien Piatlicki @ 2013-07-23 23:13 UTC (permalink / raw
To: gentoo-commits
commit: 65e4510b7409a122a8744f1ced20286ca9ca0a3c
Author: Jauhien Piatlicki (jauhien) <piatlicki <AT> gmail <DOT> com>
AuthorDate: Tue Jul 23 23:14:02 2013 +0000
Commit: Jauhien Piatlicki <piatlicki <AT> gmail <DOT> com>
CommitDate: Tue Jul 23 23:14:02 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/g-sorcery.git;a=commit;h=65e4510b
PyPI backend: experiments
---
gs-pypi.json | 2 +-
gs_pypi/pypi_db.py | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++-
2 files changed, 55 insertions(+), 2 deletions(-)
diff --git a/gs-pypi.json b/gs-pypi.json
index eb26221..06a23a4 100644
--- a/gs-pypi.json
+++ b/gs-pypi.json
@@ -2,7 +2,7 @@
"package": "gs_pypi",
"repositories": {
"pypi": {
- "repo_uri": "https://pypi.python.org/pypi"
+ "repo_uri": "http://pypi.python.org/pypi"
}
}
}
diff --git a/gs_pypi/pypi_db.py b/gs_pypi/pypi_db.py
index 938f79c..71e5397 100644
--- a/gs_pypi/pypi_db.py
+++ b/gs_pypi/pypi_db.py
@@ -11,7 +11,60 @@
:license: GPL-2, see LICENSE for more details.
"""
+from g_sorcery.compatibility import py2k
+
+if py2k:
+ import xmlrpclib
+ import httplib
+ urlparse
+else:
+ import xmlrpc.client as xmlrpclib
+ import http.client as httplib
+ import urllib.parse as urlparse
+
+import sys
+
+from g_sorcery.logger import Logger
from g_sorcery.package_db import DBGenerator
class PypiDBGenerator(DBGenerator):
- pass
+
+ def process_uri(self, uri, data):
+ url = uri["uri"]
+ client = xmlrpclib.ServerProxy(url)
+ logger = Logger()
+ logger.info("downloading packages data")
+ pkg_list = client.list_packages()
+
+ number_of_packages = len(pkg_list)
+ downloaded_number = 0
+
+ connection = httplib.HTTPConnection(urlparse.urlparse(url).netloc)
+
+ connection.request("GET", "/pypi/zmqpy/json")
+ response = connection.getresponse()
+ print(response.getheaders())
+
+ for pkg in pkg_list:
+ data[pkg] = {}
+
+ chars = ['-','\\','|','/']
+ show = chars[downloaded_number % 4]
+ percent = (downloaded_number * 100)//number_of_packages
+ length = 70
+ progress = (percent * length)//100
+ blank = length - progress
+
+ sys.stdout.write("\r %s [%s%s] %s%%" % (show, "#" * progress, " " * blank, percent))
+ sys.stdout.flush()
+ downloaded_number += 1
+
+ connection.close()
+
+ sys.stdout.write("\r %s [%s] %s%%" % ("-", "#" * length, 100))
+ sys.stdout.flush()
+ print("")
+
+ def process_data(self, pkg_db, data, common_config, config):
+ #print(data)
+ pass
^ permalink raw reply related [flat|nested] 6+ messages in thread
* [gentoo-commits] proj/g-sorcery:master commit in: /, gs_pypi/
@ 2013-07-24 23:32 Jauhien Piatlicki
0 siblings, 0 replies; 6+ messages in thread
From: Jauhien Piatlicki @ 2013-07-24 23:32 UTC (permalink / raw
To: gentoo-commits
commit: 189f6b962da3a77695a3ee7d13e2104b51637580
Author: Jauhien Piatlicki (jauhien) <piatlicki <AT> gmail <DOT> com>
AuthorDate: Wed Jul 24 23:32:43 2013 +0000
Commit: Jauhien Piatlicki <piatlicki <AT> gmail <DOT> com>
CommitDate: Wed Jul 24 23:32:43 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/g-sorcery.git;a=commit;h=189f6b96
PyPI backend: obtain metadata from the site
---
gs-pypi.json | 2 +-
gs_pypi/pypi_db.py | 69 +++++++++++++++---------------------------------------
2 files changed, 20 insertions(+), 51 deletions(-)
diff --git a/gs-pypi.json b/gs-pypi.json
index 06a23a4..14656ce 100644
--- a/gs-pypi.json
+++ b/gs-pypi.json
@@ -2,7 +2,7 @@
"package": "gs_pypi",
"repositories": {
"pypi": {
- "repo_uri": "http://pypi.python.org/pypi"
+ "repo_uri": "http://testpypi.python.org/pypi"
}
}
}
diff --git a/gs_pypi/pypi_db.py b/gs_pypi/pypi_db.py
index 71e5397..3c3e491 100644
--- a/gs_pypi/pypi_db.py
+++ b/gs_pypi/pypi_db.py
@@ -11,60 +11,29 @@
:license: GPL-2, see LICENSE for more details.
"""
-from g_sorcery.compatibility import py2k
+import bs4
-if py2k:
- import xmlrpclib
- import httplib
- urlparse
-else:
- import xmlrpc.client as xmlrpclib
- import http.client as httplib
- import urllib.parse as urlparse
-
-import sys
-
-from g_sorcery.logger import Logger
from g_sorcery.package_db import DBGenerator
class PypiDBGenerator(DBGenerator):
- def process_uri(self, uri, data):
- url = uri["uri"]
- client = xmlrpclib.ServerProxy(url)
- logger = Logger()
- logger.info("downloading packages data")
- pkg_list = client.list_packages()
-
- number_of_packages = len(pkg_list)
- downloaded_number = 0
-
- connection = httplib.HTTPConnection(urlparse.urlparse(url).netloc)
-
- connection.request("GET", "/pypi/zmqpy/json")
- response = connection.getresponse()
- print(response.getheaders())
-
- for pkg in pkg_list:
- data[pkg] = {}
-
- chars = ['-','\\','|','/']
- show = chars[downloaded_number % 4]
- percent = (downloaded_number * 100)//number_of_packages
- length = 70
- progress = (percent * length)//100
- blank = length - progress
-
- sys.stdout.write("\r %s [%s%s] %s%%" % (show, "#" * progress, " " * blank, percent))
- sys.stdout.flush()
- downloaded_number += 1
-
- connection.close()
-
- sys.stdout.write("\r %s [%s] %s%%" % ("-", "#" * length, 100))
- sys.stdout.flush()
- print("")
+ def get_download_uries(self, common_config, config):
+ return [config["repo_uri"] + "?%3Aaction=index"]
+ def parse_data(self, data_f):
+ soup = bs4.BeautifulSoup(data_f.read())
+ return soup.table
+
+
def process_data(self, pkg_db, data, common_config, config):
- #print(data)
- pass
+ data = data["pypi?:action=index"]
+ for entry in data.find_all("tr")[1:-1]:
+ package, description = entry.find_all("td")
+
+ if description.contents:
+ description = description.contents[0]
+ else:
+ description = ""
+ package, version = package.a["href"].split("/")[2:]
+
+ print(package, version, description)
^ permalink raw reply related [flat|nested] 6+ messages in thread
* [gentoo-commits] proj/g-sorcery:master commit in: /, gs_pypi/
@ 2013-08-30 16:58 Jauhien Piatlicki
0 siblings, 0 replies; 6+ messages in thread
From: Jauhien Piatlicki @ 2013-08-30 16:58 UTC (permalink / raw
To: gentoo-commits
commit: d848d85e2f47131c17193d3f0b8c191446ee3c91
Author: Jauhien Piatlicki (jauhien) <piatlicki <AT> gmail <DOT> com>
AuthorDate: Fri Aug 30 16:58:17 2013 +0000
Commit: Jauhien Piatlicki <piatlicki <AT> gmail <DOT> com>
CommitDate: Fri Aug 30 16:58:17 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/g-sorcery.git;a=commit;h=d848d85e
fixes
---
gs_pypi/ebuild.py | 6 +++---
setup.py | 4 +++-
2 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/gs_pypi/ebuild.py b/gs_pypi/ebuild.py
index 08ca188..7f8fb5f 100644
--- a/gs_pypi/ebuild.py
+++ b/gs_pypi/ebuild.py
@@ -24,8 +24,8 @@ class PypiEbuildWithoutDigestGenerator(DefaultEbuildGenerator):
def __init__(self, package_db):
vars_before_inherit = \
- [("repo_uri", '"http://pypi.python.org/packages/source/${PN:0:1}/${PN}/"'),
- ("sourcefile", '"${P}.tar.gz"')]
+ [("repo_uri", 'http://pypi.python.org/packages/source/${PN:0:1}/${PN}/'),
+ ("sourcefile", '${P}.tar.gz')]
inherit = ["gs-pypi"]
@@ -49,7 +49,7 @@ class PypiEbuildWithDigestGenerator(DefaultEbuildGenerator):
vars_after_description = \
["homepage",
- ("src_uri", '"http://pypi.python.org/packages/source/${PN:0:1}/${PN}/${P}.tar.gz"')]
+ ("src_uri", 'http://pypi.python.org/packages/source/${PN:0:1}/${PN}/${P}.tar.gz')]
vars_after_keywords = \
[]
diff --git a/setup.py b/setup.py
index e8a43e4..a447f82 100644
--- a/setup.py
+++ b/setup.py
@@ -8,7 +8,9 @@ setup(name = 'g-sorcery',
author = 'Jauhien Piatlicki',
author_email = 'piatlicki@gmail.com',
packages = ['g_sorcery', 'gs_db_tool', 'gs_elpa', 'gs_ctan', 'gs_pypi'],
- package_data = {'gs_elpa': ['data/*'],
+ package_data = {'g_sorcery': ['data/*'],
+ 'gs_pypy': ['data/*'],
+ 'gs_elpa': ['data/*'],
'gs_ctan': ['data/*']},
scripts = ['bin/g-sorcery', 'bin/gs-db-tool', 'bin/gs-elpa',
'bin/gs-ctan', 'bin/gs-pypi-generate-db', 'bin/gs-pypi'],
^ permalink raw reply related [flat|nested] 6+ messages in thread
* [gentoo-commits] proj/g-sorcery:master commit in: /, gs_pypi/
@ 2013-09-05 17:09 Jauhien Piatlicki
0 siblings, 0 replies; 6+ messages in thread
From: Jauhien Piatlicki @ 2013-09-05 17:09 UTC (permalink / raw
To: gentoo-commits
commit: b72296309417bc121a493a792212006dfb02aac9
Author: Jauhien Piatlicki (jauhien) <piatlicki <AT> gmail <DOT> com>
AuthorDate: Thu Sep 5 17:08:59 2013 +0000
Commit: Jauhien Piatlicki <piatlicki <AT> gmail <DOT> com>
CommitDate: Thu Sep 5 17:08:59 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/g-sorcery.git;a=commit;h=b7229630
pypi: add license entry to ebuild
---
gs-pypi.json | 37 ++++++++++++++++++++++++++++++++++++-
gs_pypi/ebuild.py | 4 ++--
gs_pypi/pypi_db.py | 1 +
3 files changed, 39 insertions(+), 3 deletions(-)
diff --git a/gs-pypi.json b/gs-pypi.json
index b43af2e..58586ce 100644
--- a/gs-pypi.json
+++ b/gs-pypi.json
@@ -5,5 +5,40 @@
"repo_uri": "https://pypi.python.org/",
"db_uri": "https://github.com/jauhien/gs-pypi-db/archive/master.tar.gz"
}
- }
+ },
+ "common_config": {
+ "licenses": {
+ "Academic Free License (AFL)": "AFL-3.0",
+ "Aladdin Free Public License (AFPL)": "Aladdin",
+ "Apache Software License": "Apache-2.0",
+ "Artistic License": "Artistic",
+ "BSD License": "BSD",
+ "CC0 1.0 Universal (CC0 1.0) Public Domain Dedication": "CC0-1.0",
+ "Common Public License": "CPL-1.0",
+ "GNU Affero General Public License v3": "AGPL-3",
+ "GNU Affero General Public License v3 or later (AGPLv3+)": "AGPL-3",
+ "GNU Free Documentation License (FDL)": "FDL-1.1+",
+ "GNU General Public License (GPL)": "GPL-1+",
+ "GNU General Public License v2 (GPLv2)": "GPL-2",
+ "GNU General Public License v2 or later (GPLv2+)": "GPL-2+",
+ "GNU General Public License v3 (GPLv3)": "GPL-3",
+ "GNU General Public License v3 or later (GPLv3+)": "GPL-3+",
+ "GNU Lesser General Public License v2 (LGPLv2)": "LGPL-2",
+ "GNU Lesser General Public License v2 or later (LGPLv2+)": "LGPL-2+",
+ "GNU Lesser General Public License v3 (LGPLv3)": "LGPL-3",
+ "GNU Lesser General Public License v3 or later (LGPLv3+)": "LGPL-3+",
+ "GNU Library or Lesser General Public License (LGPL)": "LGPL-2+",
+ "ISC License (ISCL)": "ISC",
+ "MIT License": "MIT",
+ "Mozilla Public License 1.0 (MPL)": "MPL-1.0",
+ "Mozilla Public License 1.1 (MPL 1.1)": "MPL-1.1",
+ "Mozilla Public License 2.0 (MPL 2.0)": "MPL-2.0",
+ "Public Domain": "public-domain",
+ "Python License (CNRI Python License)": "CNRI",
+ "Python Software Foundation License": "PYTHON",
+ "Repoze Public License": "repoze",
+ "W3C License": "W3C",
+ "Zope Public License": "ZPL",
+ "zlib/libpng License": "ZLIB"
+ }
}
diff --git a/gs_pypi/ebuild.py b/gs_pypi/ebuild.py
index 666b55f..0be2999 100644
--- a/gs_pypi/ebuild.py
+++ b/gs_pypi/ebuild.py
@@ -31,7 +31,7 @@ class PypiEbuildWithoutDigestGenerator(DefaultEbuildGenerator):
inherit = ["gs-pypi"]
vars_after_description = \
- ["homepage"]
+ ["homepage", "license"]
vars_after_keywords = \
[]
@@ -49,7 +49,7 @@ class PypiEbuildWithDigestGenerator(DefaultEbuildGenerator):
inherit = ["gs-pypi"]
vars_after_description = \
- ["homepage",
+ ["homepage", , "license",
{"name" : "src_uri", "value" : 'http://pypi.python.org/packages/source/${REALNAME:0:1}/${REALNAME}/${REALNAME}-${REALVERSION}.tar.gz'}]
vars_after_keywords = \
diff --git a/gs_pypi/pypi_db.py b/gs_pypi/pypi_db.py
index a9addea..a4a2e78 100644
--- a/gs_pypi/pypi_db.py
+++ b/gs_pypi/pypi_db.py
@@ -226,6 +226,7 @@ class PypiDBGenerator(DBGenerator):
if "License" in categories:
license = categories["License"][-1]
+ license = self.convert([common_config, config], "licenses", license)
if not py_versions:
py_versions = ['2_6', '2_7', '3_2', '3_3']
^ permalink raw reply related [flat|nested] 6+ messages in thread
* [gentoo-commits] proj/g-sorcery:master commit in: /, gs_pypi/
@ 2013-09-05 17:21 Jauhien Piatlicki
0 siblings, 0 replies; 6+ messages in thread
From: Jauhien Piatlicki @ 2013-09-05 17:21 UTC (permalink / raw
To: gentoo-commits
commit: fb684f013326283da1c9c574b2c7bf6699a27ccf
Author: Jauhien Piatlicki (jauhien) <piatlicki <AT> gmail <DOT> com>
AuthorDate: Thu Sep 5 17:20:44 2013 +0000
Commit: Jauhien Piatlicki <piatlicki <AT> gmail <DOT> com>
CommitDate: Thu Sep 5 17:20:44 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/g-sorcery.git;a=commit;h=fb684f01
gs-pypi: fix errors
---
gs-pypi.json | 1 +
gs_pypi/ebuild.py | 2 +-
gs_pypi/gs_pypi_generate_db.py | 1 +
3 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/gs-pypi.json b/gs-pypi.json
index 58586ce..052b490 100644
--- a/gs-pypi.json
+++ b/gs-pypi.json
@@ -41,4 +41,5 @@
"Zope Public License": "ZPL",
"zlib/libpng License": "ZLIB"
}
+ }
}
diff --git a/gs_pypi/ebuild.py b/gs_pypi/ebuild.py
index 0be2999..b25c1a4 100644
--- a/gs_pypi/ebuild.py
+++ b/gs_pypi/ebuild.py
@@ -49,7 +49,7 @@ class PypiEbuildWithDigestGenerator(DefaultEbuildGenerator):
inherit = ["gs-pypi"]
vars_after_description = \
- ["homepage", , "license",
+ ["homepage", "license",
{"name" : "src_uri", "value" : 'http://pypi.python.org/packages/source/${REALNAME:0:1}/${REALNAME}/${REALNAME}-${REALVERSION}.tar.gz'}]
vars_after_keywords = \
diff --git a/gs_pypi/gs_pypi_generate_db.py b/gs_pypi/gs_pypi_generate_db.py
index 52585f3..1d95a8a 100644
--- a/gs_pypi/gs_pypi_generate_db.py
+++ b/gs_pypi/gs_pypi_generate_db.py
@@ -15,6 +15,7 @@ import os
import sys
from g_sorcery.compatibility import TemporaryDirectory
+from g_sorcery.exceptions import FileJSONError
from g_sorcery.fileutils import copy_all, FileJSON
from g_sorcery.logger import Logger
^ permalink raw reply related [flat|nested] 6+ messages in thread
end of thread, other threads:[~2013-09-05 17:21 UTC | newest]
Thread overview: 6+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2013-09-05 17:21 [gentoo-commits] proj/g-sorcery:master commit in: /, gs_pypi/ Jauhien Piatlicki
-- strict thread matches above, loose matches on Subject: below --
2013-09-05 17:09 Jauhien Piatlicki
2013-08-30 16:58 Jauhien Piatlicki
2013-07-24 23:32 Jauhien Piatlicki
2013-07-23 23:13 Jauhien Piatlicki
2013-07-23 23:13 Jauhien Piatlicki
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox