From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id 3B8001381F3 for ; Wed, 24 Jul 2013 23:32:45 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id D1D52E0982; Wed, 24 Jul 2013 23:32:41 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 4E17FE0982 for ; Wed, 24 Jul 2013 23:32:41 +0000 (UTC) Received: from hornbill.gentoo.org (hornbill.gentoo.org [94.100.119.163]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id B88B233E60F for ; Wed, 24 Jul 2013 23:32:39 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by hornbill.gentoo.org (Postfix) with ESMTP id 1B341E5458 for ; Wed, 24 Jul 2013 23:32:37 +0000 (UTC) From: "Jauhien Piatlicki" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Jauhien Piatlicki" Message-ID: <1374708763.189f6b962da3a77695a3ee7d13e2104b51637580.jauhien@gentoo> Subject: [gentoo-commits] proj/g-sorcery:master commit in: /, gs_pypi/ X-VCS-Repository: proj/g-sorcery X-VCS-Files: gs-pypi.json gs_pypi/pypi_db.py X-VCS-Directories: / gs_pypi/ X-VCS-Committer: jauhien X-VCS-Committer-Name: Jauhien Piatlicki X-VCS-Revision: 189f6b962da3a77695a3ee7d13e2104b51637580 X-VCS-Branch: master Date: Wed, 24 Jul 2013 23:32:37 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Archives-Salt: f9c63205-71c6-43b8-b356-992d471d337b X-Archives-Hash: 3898917f0bc9ea363679aa4ee4548300 commit: 189f6b962da3a77695a3ee7d13e2104b51637580 Author: Jauhien Piatlicki (jauhien) gmail com> AuthorDate: Wed Jul 24 23:32:43 2013 +0000 Commit: Jauhien Piatlicki gmail com> CommitDate: Wed Jul 24 23:32:43 2013 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=proj/g-sorcery.git;a=commit;h=189f6b96 PyPI backend: obtain metadata from the site --- gs-pypi.json | 2 +- gs_pypi/pypi_db.py | 69 +++++++++++++++--------------------------------------- 2 files changed, 20 insertions(+), 51 deletions(-) diff --git a/gs-pypi.json b/gs-pypi.json index 06a23a4..14656ce 100644 --- a/gs-pypi.json +++ b/gs-pypi.json @@ -2,7 +2,7 @@ "package": "gs_pypi", "repositories": { "pypi": { - "repo_uri": "http://pypi.python.org/pypi" + "repo_uri": "http://testpypi.python.org/pypi" } } } diff --git a/gs_pypi/pypi_db.py b/gs_pypi/pypi_db.py index 71e5397..3c3e491 100644 --- a/gs_pypi/pypi_db.py +++ b/gs_pypi/pypi_db.py @@ -11,60 +11,29 @@ :license: GPL-2, see LICENSE for more details. """ -from g_sorcery.compatibility import py2k +import bs4 -if py2k: - import xmlrpclib - import httplib - urlparse -else: - import xmlrpc.client as xmlrpclib - import http.client as httplib - import urllib.parse as urlparse - -import sys - -from g_sorcery.logger import Logger from g_sorcery.package_db import DBGenerator class PypiDBGenerator(DBGenerator): - def process_uri(self, uri, data): - url = uri["uri"] - client = xmlrpclib.ServerProxy(url) - logger = Logger() - logger.info("downloading packages data") - pkg_list = client.list_packages() - - number_of_packages = len(pkg_list) - downloaded_number = 0 - - connection = httplib.HTTPConnection(urlparse.urlparse(url).netloc) - - connection.request("GET", "/pypi/zmqpy/json") - response = connection.getresponse() - print(response.getheaders()) - - for pkg in pkg_list: - data[pkg] = {} - - chars = ['-','\\','|','/'] - show = chars[downloaded_number % 4] - percent = (downloaded_number * 100)//number_of_packages - length = 70 - progress = (percent * length)//100 - blank = length - progress - - sys.stdout.write("\r %s [%s%s] %s%%" % (show, "#" * progress, " " * blank, percent)) - sys.stdout.flush() - downloaded_number += 1 - - connection.close() - - sys.stdout.write("\r %s [%s] %s%%" % ("-", "#" * length, 100)) - sys.stdout.flush() - print("") + def get_download_uries(self, common_config, config): + return [config["repo_uri"] + "?%3Aaction=index"] + def parse_data(self, data_f): + soup = bs4.BeautifulSoup(data_f.read()) + return soup.table + + def process_data(self, pkg_db, data, common_config, config): - #print(data) - pass + data = data["pypi?:action=index"] + for entry in data.find_all("tr")[1:-1]: + package, description = entry.find_all("td") + + if description.contents: + description = description.contents[0] + else: + description = "" + package, version = package.a["href"].split("/")[2:] + + print(package, version, description)