From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from pigeon.gentoo.org ([208.92.234.80] helo=lists.gentoo.org) by finch.gentoo.org with esmtp (Exim 4.60) (envelope-from ) id 1Qv9ED-0004uS-G9 for garchives@archives.gentoo.org; Sun, 21 Aug 2011 14:42:23 +0000 Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 2A72721C22D; Sun, 21 Aug 2011 14:42:14 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) by pigeon.gentoo.org (Postfix) with ESMTP id BD50B21C22D for ; Sun, 21 Aug 2011 14:42:13 +0000 (UTC) Received: from pelican.gentoo.org (unknown [66.219.59.40]) (using TLSv1 with cipher ADH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id E7DAD1B4050 for ; Sun, 21 Aug 2011 14:42:12 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by pelican.gentoo.org (Postfix) with ESMTP id C323880051 for ; Sun, 21 Aug 2011 14:42:11 +0000 (UTC) From: "Vikraman Choudhury" To: gentoo-commits@lists.gentoo.org Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Vikraman Choudhury" Message-ID: <858ed9350a3115195509d0fbc1cf11a5b9a80750.vikraman@gentoo> Subject: [gentoo-commits] proj/gentoostats:master commit in: server/, server/tests/ X-VCS-Repository: proj/gentoostats X-VCS-Files: server/__init__.py server/about.py server/arch.py server/config.py server/dbconfig.py server/feature.py server/helpers.py server/host.py server/index.py server/kwd.py server/lang.py server/mirror.py server/package.py server/repo.py server/search.py server/tests/test_host.py server/tests/test_index.py X-VCS-Directories: server/ server/tests/ X-VCS-Committer: vikraman X-VCS-Committer-Name: Vikraman Choudhury X-VCS-Revision: 858ed9350a3115195509d0fbc1cf11a5b9a80750 Date: Sun, 21 Aug 2011 14:42:11 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: quoted-printable X-Archives-Salt: X-Archives-Hash: 69830e04d332ff7740b774e9e684b78a commit: 858ed9350a3115195509d0fbc1cf11a5b9a80750 Author: Vikraman Choudhury gmail com> AuthorDate: Sun Aug 21 14:41:37 2011 +0000 Commit: Vikraman Choudhury gmail com> CommitDate: Sun Aug 21 14:41:37 2011 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=3Dproj/gentoostats.g= it;a=3Dcommit;h=3D858ed935 commented server code --- server/__init__.py | 1 + server/about.py | 3 ++ server/arch.py | 1 + server/config.py | 6 +++++ server/dbconfig.py | 6 +++++ server/feature.py | 3 +- server/helpers.py | 49 +++++++++++++++++++++++++++++++++++---= ------ server/host.py | 25 +++++++++++++++++----- server/index.py | 1 + server/kwd.py | 7 +++++- server/lang.py | 3 +- server/mirror.py | 3 +- server/package.py | 12 ++++++++++ server/repo.py | 4 ++- server/search.py | 9 ++++++++ server/tests/test_host.py | 18 ++++++++++++++++ server/tests/test_index.py | 12 ++++++++++ 17 files changed, 142 insertions(+), 21 deletions(-) diff --git a/server/__init__.py b/server/__init__.py index e69de29..3c6cfa2 100644 --- a/server/__init__.py +++ b/server/__init__.py @@ -0,0 +1 @@ +# Make this a python package diff --git a/server/about.py b/server/about.py index c808341..62eb30c 100644 --- a/server/about.py +++ b/server/about.py @@ -4,4 +4,7 @@ import web class About(object): =20 def GET(self): + """ + Redirect to static about page + """ raise web.seeother('/static/about.html') diff --git a/server/arch.py b/server/arch.py index 814c7c4..b6ddc9f 100644 --- a/server/arch.py +++ b/server/arch.py @@ -11,6 +11,7 @@ class Arch(object): if helpers.is_json_request(): return helpers.serialize(arch_data) else: + # generate plot x_ticklabels =3D arch_data.keys() y_values =3D [ arch_data[a]['HOSTS'] for a in x_ticklabels ] arch_plot =3D helpers.barchart(title =3D 'Hosts per arch', x= _label =3D 'Arch', diff --git a/server/config.py b/server/config.py index 98326ae..cca5c0e 100644 --- a/server/config.py +++ b/server/config.py @@ -17,7 +17,13 @@ db =3D web.database( render =3D web.template.render(rootdir + 'templates/', base=3D'layout') =20 def notfound(): + """ + Rendered for HTTP 404 errors + """ return web.notfound(render.error_404()) =20 def internalerror(): + """ + Rendered for HTTP 500 errors + """ return web.internalerror(render.error_500()) diff --git a/server/dbconfig.py b/server/dbconfig.py index e5eb42c..f0a8418 100644 --- a/server/dbconfig.py +++ b/server/dbconfig.py @@ -5,12 +5,18 @@ import ConfigParser class DBConfig(object): =20 def __init__(self, configfile): + """ + Initialie db config from configfile + """ self.config =3D ConfigParser.ConfigParser() if len(self.config.read(configfile)) =3D=3D 0: sys.stderr.write('Cannot read ' + configfile) sys.exit(1) =20 def get_config(self): + """ + Return db config as dict + """ ret =3D dict() try: ret['DB'] =3D self.config.get('MYSQL', 'DB') diff --git a/server/feature.py b/server/feature.py index 1ea1c36..68e8657 100644 --- a/server/feature.py +++ b/server/feature.py @@ -4,7 +4,8 @@ from config import render, db =20 class Feature(object): def GET(self): - feature_count =3D db.query('SELECT FEATURE,COUNT(UUID) AS HOSTS = FROM HOST_FEATURES NATURAL JOIN FEATURES GROUP BY FEATURE') + feature_count =3D db.query('SELECT FEATURE,COUNT(UUID) AS HOSTS\ + FROM HOST_FEATURES NATURAL JOIN FEATURES GROUP BY FEATUR= E') feature_data =3D dict() for t in feature_count: feature_data[t['FEATURE']] =3D {'HOSTS':t['HOSTS']} diff --git a/server/helpers.py b/server/helpers.py index 5b81cb3..6e12ae1 100644 --- a/server/helpers.py +++ b/server/helpers.py @@ -13,22 +13,24 @@ os.environ['HOME'] =3D '/tmp' from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCan= vas from matplotlib.figure import Figure =20 -# check valid uuid - def is_uuid(uuid): + """ + Check uuid validity + """ regex =3D re.compile(r'^(\w{8})-(\w{4})-(\w{4})-(\w{4})-(\w{12})$') return regex.search(uuid) =20 -# convert uuid string to raw bytes - def uuidbin(string): - #TODO: string is not a valid uuid + """ + Convert uuid string to raw bytes + """ u =3D uuid.UUID(string) return u.bytes =20 -# custom pkgsplit - def pkgsplit(pkgname): + """ + Custom pkgsplit + """ cpv=3D{'cat':'','pkg':'','ver':''} cpvr =3D catpkgsplit(pkgname) if cpvr is None: @@ -43,10 +45,10 @@ def pkgsplit(pkgname): cpv['ver'] =3D cpv['ver'] + '-' + cpvr[3] return cpv =20 -# get functions for index keys -# lookup key and insert if not found - def get_kwkey(db, keyword): + """ + Lookup keyword and return index key. Insert keyword if not found. + """ if keyword is None: return None db_keyword =3D db.select('KEYWORDS', vars=3D{'keyword':keyword}, whe= re=3D'KEYWORD=3D$keyword') @@ -57,6 +59,9 @@ def get_kwkey(db, keyword): return kwkey =20 def get_lkey(db, lang): + """ + Lookup lang and return index key. Insert lang if not found. + """ if lang is None: return None db_lang =3D db.select('LANG', vars=3D{'lang':lang}, where=3D'LANG=3D= $lang') @@ -67,6 +72,9 @@ def get_lkey(db, lang): return lkey =20 def get_fkey(db, feature): + """ + Lookup feature and return index key. Insert feature if not found. + """ if feature is None: return None db_feature =3D db.select('FEATURES', vars=3D{'feature':feature}, whe= re=3D'FEATURE=3D$feature') @@ -77,6 +85,9 @@ def get_fkey(db, feature): return fkey =20 def get_mkey(db, mirror): + """ + Lookup mirror and return index key. Insert mirror if not found. + """ if mirror is None: return None db_mirror =3D db.select('GENTOO_MIRRORS', vars=3D{'mirror':mirror}, = where=3D'MIRROR=3D$mirror') @@ -87,6 +98,9 @@ def get_mkey(db, mirror): return mkey =20 def get_ukey(db, useflag): + """ + Lookup useflag and return index key. Insert useflag if not found. + """ if useflag is None: return None db_useflag =3D db.select('USEFLAGS', vars=3D{'useflag':useflag}, whe= re=3D'USEFLAG=3D$useflag') @@ -97,6 +111,9 @@ def get_ukey(db, useflag): return ukey =20 def get_pkey(db, package): + """ + Lookup package and return index key. Insert package if not found. + """ if package is None: return None cpv =3D pkgsplit(package) @@ -108,6 +125,9 @@ def get_pkey(db, package): return pkey =20 def get_rkey(db, repo): + """ + Lookup repo and return index key. Insert repo if not found. + """ if repo is None: return None db_repo =3D db.select('REPOSITORIES', vars=3D{'repo':repo}, where=3D= 'REPO=3D$repo') @@ -118,9 +138,15 @@ def get_rkey(db, repo): return rkey =20 def is_json_request(): + """ + Check for json headers + """ return web.ctx.environ['HTTP_ACCEPT'].find('json') !=3D -1 =20 def serialize(object, human=3DTrue): + """ + Encode object in json + """ if human: indent =3D 2 else: @@ -128,6 +154,9 @@ def serialize(object, human=3DTrue): return json.JSONEncoder(indent=3Dindent).encode(object) =20 def barchart(title, x_label, x_ticklabels, y_label, y_values): + """ + Generate a barchart and return base64 encoded image data + """ fig =3D Figure() canvas =3D FigureCanvas(fig) ind =3D range(len(y_values)) diff --git a/server/host.py b/server/host.py index 7fd8132..c354939 100644 --- a/server/host.py +++ b/server/host.py @@ -6,6 +6,7 @@ import config from web import form from config import render, db =20 +# host search form host_form =3D form.Form( form.Textbox('uuid', description =3D 'UUID'), form.Button('submit', description =3D 'Submit') @@ -36,32 +37,44 @@ class Host(object): host_data[var] =3D e[var] =20 host_data['FEATURES'] =3D list() - features =3D db.query('SELECT FEATURE FROM HOST_FEATURES NATURAL= JOIN FEATURES WHERE UUID=3D$uuid', vars=3D{'uuid':uuid}) + features =3D db.query('SELECT FEATURE\ + FROM HOST_FEATURES NATURAL JOIN FEATURES\ + WHERE UUID=3D$uuid', vars=3D{'uuid':uuid}) for f in features: host_data['FEATURES'].append(f['FEATURE']) =20 host_data['ACCEPT_KEYWORDS'] =3D list() - keywords =3D db.query('SELECT KEYWORD FROM GLOBAL_KEYWORDS NATUR= AL JOIN KEYWORDS WHERE UUID=3D$uuid', vars=3D{'uuid':uuid}) + keywords =3D db.query('SELECT KEYWORD\ + FROM GLOBAL_KEYWORDS NATURAL JOIN KEYWORDS\ + WHERE UUID=3D$uuid', vars=3D{'uuid':uuid}) for k in keywords: host_data['ACCEPT_KEYWORDS'].append(k['KEYWORD']) =20 host_data['USE'] =3D list() - useflags =3D db.query('SELECT USEFLAG FROM GLOBAL_USEFLAGS NATUR= AL JOIN USEFLAGS WHERE UUID=3D$uuid', vars=3D{'uuid':uuid}) + useflags =3D db.query('SELECT USEFLAG\ + FROM GLOBAL_USEFLAGS NATURAL JOIN USEFLAGS\ + WHERE UUID=3D$uuid', vars=3D{'uuid':uuid}) for u in useflags: host_data['USE'].append(u['USEFLAG']) =20 host_data['LANG'] =3D list() - lang =3D db.query('SELECT LANG FROM HOST_LANG NATURAL JOIN LANG = WHERE UUID=3D$uuid', vars=3D{'uuid':uuid}) + lang =3D db.query('SELECT LANG\ + FROM HOST_LANG NATURAL JOIN LANG\ + WHERE UUID=3D$uuid', vars=3D{'uuid':uuid}) for l in lang: host_data['LANG'].append(l['LANG']) =20 host_data['GENTOO_MIRRORS'] =3D list() - mirrors =3D db.query('SELECT MIRROR FROM HOST_MIRRORS NATURAL JO= IN GENTOO_MIRRORS WHERE UUID=3D$uuid', vars=3D{'uuid':uuid}) + mirrors =3D db.query('SELECT MIRROR\ + FROM HOST_MIRRORS NATURAL JOIN GENTOO_MIRRORS\ + WHERE UUID=3D$uuid', vars=3D{'uuid':uuid}) for m in mirrors: host_data['GENTOO_MIRRORS'].append(m['MIRROR']) =20 host_data['PACKAGES'] =3D dict() - packages =3D db.query('SELECT CAT, PKG, VER FROM INSTALLED_PACKA= GES NATURAL JOIN PACKAGES WHERE UUID=3D$uuid ORDER BY CAT, PKG, VER', var= s=3D{'uuid':uuid}) + packages =3D db.query('SELECT CAT, PKG, VER\ + FROM INSTALLED_PACKAGES NATURAL JOIN PACKAGES\ + WHERE UUID=3D$uuid ORDER BY CAT, PKG, VER', vars=3D{'uui= d':uuid}) for p in packages: cpv =3D p['CAT'] + '/' + p['PKG'] + '-' + p['VER'] host_data['PACKAGES'][cpv] =3D dict() diff --git a/server/index.py b/server/index.py index 2204bd3..808759c 100644 --- a/server/index.py +++ b/server/index.py @@ -2,6 +2,7 @@ from web import form from config import render, db =20 +# package search form search_form =3D form.Form( form.Textbox('cat', value =3D 'any', description =3D 'Category')= , form.Textbox('pkg', value =3D 'any', description =3D 'Package'), diff --git a/server/kwd.py b/server/kwd.py index 779ac61..d9b30e2 100644 --- a/server/kwd.py +++ b/server/kwd.py @@ -4,13 +4,18 @@ from config import render, db =20 class Keyword(object): def GET(self): - keyword_count =3D db.query('SELECT KEYWORD, COUNT(DISTINCT IPKEY= ) AS PACKAGES, COUNT(DISTINCT UUID) AS HOSTS FROM GLOBAL_KEYWORDS NATURAL= JOIN KEYWORDS NATURAL JOIN INSTALLED_PACKAGES GROUP BY KEYWORD') + keyword_count =3D db.query('SELECT KEYWORD,\ + COUNT(DISTINCT IPKEY) AS PACKAGES,\ + COUNT(DISTINCT UUID) AS HOSTS\ + FROM GLOBAL_KEYWORDS NATURAL JOIN KEYWORDS\ + NATURAL JOIN INSTALLED_PACKAGES GROUP BY KEYWORD') keyword_data =3D dict() for t in keyword_count: keyword_data[t['KEYWORD']] =3D {'HOSTS':t['HOSTS'], 'PACKAGE= S':t['PACKAGES']} if helpers.is_json_request(): return helpers.serialize(keyword_data) else: + # generate plot x_ticklabels =3D keyword_data.keys() y_values =3D [ keyword_data[k]['PACKAGES'] for k in x_tickla= bels ] keyword_plot =3D helpers.barchart(title =3D 'Installed packa= ges per keyword', diff --git a/server/lang.py b/server/lang.py index de6c625..5052378 100644 --- a/server/lang.py +++ b/server/lang.py @@ -4,7 +4,8 @@ from config import render, db =20 class Lang(object): def GET(self): - lang_count =3D db.query('SELECT LANG,COUNT(UUID) AS HOSTS FROM H= OST_LANG NATURAL JOIN LANG GROUP BY LANG') + lang_count =3D db.query('SELECT LANG,COUNT(UUID) AS HOSTS\ + FROM HOST_LANG NATURAL JOIN LANG GROUP BY LANG') lang_data =3D dict() for t in lang_count: lang_data[t['LANG']] =3D {'HOSTS':t['HOSTS']} diff --git a/server/mirror.py b/server/mirror.py index e701404..4055c09 100644 --- a/server/mirror.py +++ b/server/mirror.py @@ -4,7 +4,8 @@ from config import render, db =20 class Mirror(object): def GET(self): - mirror_count =3D db.query('SELECT MIRROR,COUNT(UUID) AS HOSTS FR= OM HOST_MIRRORS NATURAL JOIN GENTOO_MIRRORS GROUP BY MIRROR') + mirror_count =3D db.query('SELECT MIRROR,COUNT(UUID) AS HOSTS\ + FROM HOST_MIRRORS NATURAL JOIN GENTOO_MIRRORS GROUP BY M= IRROR') mirror_data =3D dict() for t in mirror_count: mirror_data[t['MIRROR']] =3D {'HOSTS':t['HOSTS']} diff --git a/server/package.py b/server/package.py index aa037e5..e73a139 100644 --- a/server/package.py +++ b/server/package.py @@ -24,6 +24,9 @@ class Package(object): return config.internalerror() =20 def __GET(self, top): + """ + Get category + """ p_query =3D db.query('SELECT COUNT(DISTINCT UUID) AS HOST_COUNT,= \ COUNT(DISTINCT CAT) AS C_COUNT, \ COUNT(DISTINCT CAT, PKG) AS CP_COUNT, \ @@ -44,6 +47,9 @@ class Package(object): return render.package(p_data) =20 def __GET_C(self, top, cat): + """ + Get category/package + """ p_query =3D db.query('SELECT COUNT(DISTINCT UUID) AS HOST_COUNT,= \ COUNT(DISTINCT CAT, PKG) AS CP_COUNT, \ COUNT(DISTINCT CAT, PKG, VER) AS CPV_COUNT\ @@ -63,6 +69,9 @@ class Package(object): return render.package_c(cat, p_data) =20 def __GET_CP(self, top, cat, pkg): + """ + Get category/package-version + """ p_query =3D db.query('SELECT COUNT(DISTINCT UUID) AS HOST_COUNT,= \ COUNT(DISTINCT CAT, PKG, VER) AS CPV_COUNT\ FROM INSTALLED_PACKAGES RIGHT OUTER JOIN PACKAGES\ @@ -94,6 +103,9 @@ class Package(object): return render.package_cpv(cat, pkg, ver, p_data) =20 def __top(self, count, *args): + """ + Find top entries + """ t_list =3D list() if len(args) =3D=3D 0: tc_query =3D db.query('SELECT CAT, COUNT(DISTINCT UUID) AS H= OST_COUNT\ diff --git a/server/repo.py b/server/repo.py index 7be99ac..f129fa0 100644 --- a/server/repo.py +++ b/server/repo.py @@ -4,7 +4,9 @@ from config import render, db =20 class Repo(object): def GET(self): - repo_count =3D db.query('select REPO,COUNT(DISTINCT IPKEY) AS PA= CKAGES,COUNT(DISTINCT UUID) AS HOSTS from INSTALLED_PACKAGES natural join= REPOSITORIES group by REPO') + repo_count =3D db.query('SELECT REPO,COUNT(DISTINCT IPKEY) AS PA= CKAGES,\ + COUNT(DISTINCT UUID) AS HOSTS\ + FROM INSTALLED_PACKAGES NATURAL JOIN REPOSITORIES GROUP = BY REPO') repo_data =3D dict() for t in repo_count: repo_data[t['REPO']] =3D {'HOSTS':t['HOSTS'], 'PACKAGES':t['= PACKAGES']} diff --git a/server/search.py b/server/search.py index d3d35d5..5543047 100644 --- a/server/search.py +++ b/server/search.py @@ -49,6 +49,9 @@ class Search(object): return render.search(search_tuples) =20 def _build_query(self, where, having): + """ + Build SELECT clause + """ sep =3D ' ' query =3D '' query +=3D 'SELECT' + sep + ','.join(what) + sep @@ -66,6 +69,9 @@ class Search(object): return query.strip() =20 def _build_where(self): + """ + Build WHERE clause + """ where =3D [] cat =3D string.lower(self.args.cat) if cat !=3D 'any': @@ -85,6 +91,9 @@ class Search(object): return where =20 def _build_having(self): + """ + Build HAVING clause + """ having =3D [] if self.min_hosts !=3D -1: having.append('HOSTS>=3D$min_hosts') diff --git a/server/tests/test_host.py b/server/tests/test_host.py index b781a09..1b68050 100644 --- a/server/tests/test_host.py +++ b/server/tests/test_host.py @@ -7,14 +7,23 @@ from app import app class TestHost(unittest.TestCase): =20 def setUp(self): + """ + Initialize browser + """ self.b =3D app.browser() =20 def test_basic(self): + """ + Test /host + """ self.b.open('/host') self.assertEqual(self.b.path, '/host') self.assertEqual(self.b.status, 404) =20 def test_get(self): + """ + Test GET + """ uri =3D '/host/' + str(uuid.uuid4()) self.b.open(uri) self.assertEqual(self.b.path, uri) @@ -25,6 +34,9 @@ class TestHost(unittest.TestCase): self.assertEqual(self.b.status, 404) =20 def test_post_empty(self): + """ + Test empty POST=20 + """ str_uuid =3D str(uuid.uuid4()) uri =3D '/host/' + str_uuid # post with empty string @@ -47,6 +59,9 @@ class TestHost(unittest.TestCase): self.assertEqual(self.b.status, 500) =20 def test_post_bad(self): + """ + Test bad POST + """ str_uuid =3D str(uuid.uuid4()) uri =3D '/host/' + str_uuid # different uuid in payload @@ -61,6 +76,9 @@ class TestHost(unittest.TestCase): self.assertTrue('Invalid uuid' in self.b.data) =20 def test_post_get(self): + """ + Test GET using POST + """ str_uuid =3D str(uuid.uuid4()) uri =3D '/host/' + str_uuid payload =3D { diff --git a/server/tests/test_index.py b/server/tests/test_index.py index f77a60d..0fdc740 100644 --- a/server/tests/test_index.py +++ b/server/tests/test_index.py @@ -5,17 +5,29 @@ from app import app class TestIndex(unittest.TestCase): =20 def setUp(self): + """ + Initialize browser + """ self.b =3D app.browser() self.b.open('/') =20 def test_basic(self): + """ + Test / + """ self.assertEqual(self.b.path, '/') self.assertEqual(self.b.status, 200) =20 def test_content(self): + """ + Test html + """ self.assertTrue('Welcome to the gentoostats webapp' in self.b.da= ta) =20 def test_hosts(self): + """ + Test host count from html + """ self.assertTrue('Number of hosts' in self.b.data) lines =3D self.b.data.split('\n') for line in lines: