[gentoo-commits] proj/g-sorcery:master commit in: /, gs_pypi/

["Jauhien Piatlicki" <piatlicki@gmail.com>]

commit:     189f6b962da3a77695a3ee7d13e2104b51637580
Author:     Jauhien Piatlicki (jauhien) <piatlicki <AT> gmail <DOT> com>
AuthorDate: Wed Jul 24 23:32:43 2013 +0000
Commit:     Jauhien Piatlicki <piatlicki <AT> gmail <DOT> com>
CommitDate: Wed Jul 24 23:32:43 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/g-sorcery.git;a=commit;h=189f6b96

PyPI backend: obtain metadata from the site

---
 gs-pypi.json       |  2 +-
 gs_pypi/pypi_db.py | 69 +++++++++++++++---------------------------------------
 2 files changed, 20 insertions(+), 51 deletions(-)

diff --git a/gs-pypi.json b/gs-pypi.json
index 06a23a4..14656ce 100644
--- a/gs-pypi.json
+++ b/gs-pypi.json
@@ -2,7 +2,7 @@
   "package": "gs_pypi", 
   "repositories": {
     "pypi": {
-      "repo_uri": "http://pypi.python.org/pypi"
+      "repo_uri": "http://testpypi.python.org/pypi"
     }
   }
 }

diff --git a/gs_pypi/pypi_db.py b/gs_pypi/pypi_db.py
index 71e5397..3c3e491 100644
--- a/gs_pypi/pypi_db.py
+++ b/gs_pypi/pypi_db.py
@@ -11,60 +11,29 @@
     :license: GPL-2, see LICENSE for more details.
 """
 
-from g_sorcery.compatibility import py2k
+import bs4
 
-if py2k:
-    import xmlrpclib
-    import httplib
-    urlparse
-else:
-    import xmlrpc.client as xmlrpclib
-    import http.client as httplib
-    import urllib.parse as urlparse
-
-import sys
-
-from g_sorcery.logger import Logger
 from g_sorcery.package_db import DBGenerator
 
 class PypiDBGenerator(DBGenerator):
 
-    def process_uri(self, uri, data):
-        url = uri["uri"]
-        client = xmlrpclib.ServerProxy(url)
-        logger = Logger()
-        logger.info("downloading packages data")
-        pkg_list = client.list_packages()
-
-        number_of_packages = len(pkg_list)
-        downloaded_number = 0
-
-        connection = httplib.HTTPConnection(urlparse.urlparse(url).netloc)
-
-        connection.request("GET", "/pypi/zmqpy/json")
-        response = connection.getresponse()
-        print(response.getheaders())
-        
-        for pkg in pkg_list:
-            data[pkg] = {}
-
-            chars = ['-','\\','|','/']
-            show = chars[downloaded_number % 4]
-            percent = (downloaded_number * 100)//number_of_packages
-            length = 70
-            progress = (percent * length)//100
-            blank = length - progress
-
-            sys.stdout.write("\r %s [%s%s] %s%%" % (show, "#" * progress, " " * blank, percent))
-            sys.stdout.flush()
-            downloaded_number += 1
-
-        connection.close()
-
-        sys.stdout.write("\r %s [%s] %s%%" % ("-", "#" * length, 100))
-        sys.stdout.flush()
-        print("")
+    def get_download_uries(self, common_config, config):
+        return [config["repo_uri"] + "?%3Aaction=index"]
 
+    def parse_data(self, data_f):
+        soup = bs4.BeautifulSoup(data_f.read())
+        return soup.table
+            
+                
     def process_data(self, pkg_db, data, common_config, config):
-        #print(data)
-        pass
+        data = data["pypi?:action=index"]
+        for entry in data.find_all("tr")[1:-1]:
+            package, description = entry.find_all("td")
+            
+            if description.contents:
+                description = description.contents[0]
+            else:
+                description = ""
+            package, version = package.a["href"].split("/")[2:]
+
+            print(package, version, description)