From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from pigeon.gentoo.org ([208.92.234.80] helo=lists.gentoo.org) by finch.gentoo.org with esmtp (Exim 4.60) (envelope-from ) id 1Qccpv-0002ZE-MZ for garchives@archives.gentoo.org; Fri, 01 Jul 2011 12:28:44 +0000 Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 057A11C09C; Fri, 1 Jul 2011 12:28:35 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) by pigeon.gentoo.org (Postfix) with ESMTP id B16BA1C09C for ; Fri, 1 Jul 2011 12:28:35 +0000 (UTC) Received: from pelican.gentoo.org (unknown [66.219.59.40]) (using TLSv1 with cipher ADH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id E4A0C1B402B for ; Fri, 1 Jul 2011 12:28:34 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by pelican.gentoo.org (Postfix) with ESMTP id 008ED8003D for ; Fri, 1 Jul 2011 12:28:34 +0000 (UTC) From: "Andrea Arteaga" To: gentoo-commits@lists.gentoo.org Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Andrea Arteaga" Message-ID: <1599222a06e321570be756d36e03132bd8359d3b.spiros@gentoo> Subject: [gentoo-commits] proj/auto-numerical-bench:master commit in: app-benchmarks/autobench/files/python/ X-VCS-Repository: proj/auto-numerical-bench X-VCS-Files: app-benchmarks/autobench/files/python/PortageUtils.py app-benchmarks/autobench/files/python/btlbase.py app-benchmarks/autobench/files/python/btlutils.py app-benchmarks/autobench/files/python/main.py X-VCS-Directories: app-benchmarks/autobench/files/python/ X-VCS-Committer: spiros X-VCS-Committer-Name: Andrea Arteaga X-VCS-Revision: 1599222a06e321570be756d36e03132bd8359d3b Date: Fri, 1 Jul 2011 12:28:34 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: quoted-printable X-Archives-Salt: X-Archives-Hash: 780eae94bbc7147688eea1fcb32df7ec commit: 1599222a06e321570be756d36e03132bd8359d3b Author: spiros gmail com> AuthorDate: Fri Jul 1 12:27:51 2011 +0000 Commit: Andrea Arteaga gmail com> CommitDate: Fri Jul 1 12:27:51 2011 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=3Dproj/auto-numerica= l-bench.git;a=3Dcommit;h=3D1599222a Emerge, compilation and run are logged. More readable output. --- .../autobench/files/python/PortageUtils.py | 23 ++++++++++++++= +--- app-benchmarks/autobench/files/python/btlbase.py | 22 ++++++++++++--= --- app-benchmarks/autobench/files/python/btlutils.py | 14 +++++++++- app-benchmarks/autobench/files/python/main.py | 25 ++++++++++++++= ++--- 4 files changed, 67 insertions(+), 17 deletions(-) diff --git a/app-benchmarks/autobench/files/python/PortageUtils.py b/app-= benchmarks/autobench/files/python/PortageUtils.py index 4877cbd..c162e1d 100644 --- a/app-benchmarks/autobench/files/python/PortageUtils.py +++ b/app-benchmarks/autobench/files/python/PortageUtils.py @@ -1,4 +1,5 @@ import commands as cmd +import subprocess as sp import portage import os =20 @@ -17,7 +18,8 @@ def available_packages(pattern): for l in cmd.getoutput('equery -q list -po ' + pattern).split()] =20 =20 -def install_package(package, env=3D{}, root=3D'/', pkgdir=3D'usr/portage= /packages'): +def install_package(package, env=3D{}, root=3D'/', pkgdir=3D'usr/portage= /packages', + logfile=3DNone): """Emerge a package in the given root. =20 package is the package to be emerged. It has to be a tuple @@ -50,9 +52,22 @@ def install_package(package, env=3D{}, root=3D'/', pkg= dir=3D'usr/portage/packages'): envl +=3D i + '=3D"' + env[i] + '" ' cl =3D envl + 'emerge --ignore-default-opts -OB "=3D' + pkg + '"' =20 - # Execute emerge command - so =3D cmd.getstatusoutput(cl) - if so[0] !=3D 0: + # Execute emerge command and log the results + if logfile is not None: + fout =3D file(logfile, 'w') + fout.write(cl+'\n'+80*'-'+'\n') + fout.flush() + else: + fout =3D sp.PIPE + p =3D sp.Popen( \ + ['emerge', '--ignore-default-opts', '-OB', '=3D' + pkg], \ + env =3D env, \ + stdout =3D fout, stderr =3D fout \ + ) + p.wait() + if logfile is not None: + fout.close() + if p.returncode !=3D 0: # In case of error, print the whole emerge command raise InstallException(cl) =20 diff --git a/app-benchmarks/autobench/files/python/btlbase.py b/app-bench= marks/autobench/files/python/btlbase.py index a8d8c1e..a8dc3f4 100644 --- a/app-benchmarks/autobench/files/python/btlbase.py +++ b/app-benchmarks/autobench/files/python/btlbase.py @@ -39,7 +39,7 @@ class BTLBase: self._parse_args(passargs) =20 =20 - def run_test(self, root, impl, testdir, env): + def run_test(self, root, impl, testdir, env, logdir): # Convenient renames and definition of report files=20 Print =3D self.Print libdir =3D self.libdir @@ -94,6 +94,7 @@ class BTLBase: # Compile # TODO: use CXX instead of g++ btldir =3D 'btl/' + logfile =3D os.path.join(logdir, name+"_comp.log") returncode, compilecl =3D btl.btlcompile( exe =3D testdir + "/test", source =3D btldir + self._btl_source(), @@ -102,19 +103,24 @@ class BTLBase: defines =3D self._btl_defines(), libs =3D [], libdirs =3D [root+libdir], - other =3D self._get_flags(root, impl, libdir) + other =3D self._get_flags(root, impl, libdir), + logfile =3D logfile ) if returncode !=3D 0: - raise Exception("Compilation failed: " + compilecl) - Print("Compilation successful: " + compilecl) + Print("Compilation failed") + Print("See log: " + logfile) + return + Print("Compilation successful") =20 # Run test - args =3D [testdir + "/test"] + self.tests + logfile =3D file(os.path.join(logdir, name+"_run.log"), 'w') + args =3D [os.path.join(testdir,"test")] + self.tests proc =3D sp.Popen(args, bufsize=3D1, stdout=3Dsp.PIPE, stderr=3D= sp.PIPE,=20 cwd =3D testdir) results =3D {} while True: errline =3D proc.stderr.readline() + logfile.write(errline) if not errline: break resfile =3D errline.split()[-1] @@ -123,11 +129,13 @@ class BTLBase: Print(resfile) Print.down() for i in xrange(100): - outline =3D proc.stdout.readline().rstrip() - Print(outline) + outline =3D proc.stdout.readline() + logfile.write(outline) + Print(outline.rstrip()) Print.up() Print.up() proc.wait() + logfile.close() if proc.returncode !=3D 0: Print('Test failed') else: diff --git a/app-benchmarks/autobench/files/python/btlutils.py b/app-benc= hmarks/autobench/files/python/btlutils.py index 752096e..d2207cd 100644 --- a/app-benchmarks/autobench/files/python/btlutils.py +++ b/app-benchmarks/autobench/files/python/btlutils.py @@ -3,7 +3,8 @@ import shlex =20 run_cmd =3D lambda c : sp.Popen(c, stdout=3Dsp.PIPE).communicate()[0] =20 -def btlcompile(exe, source, btldir, includes, defines, libs, libdirs, ot= her): +def btlcompile(exe, source, btldir, includes, defines, libs, libdirs, ot= her, \ + logfile=3DNone): incs =3D ( "%s/actions" % btldir, "%s/generic_bench" % btldir, @@ -25,7 +26,16 @@ def btlcompile(exe, source, btldir, includes, defines,= libs, libdirs, other): # TODO: use CXX instead of g++ cl =3D "g++ -o %s %s %s %s %s %s %s %s" \ % (exe, source, incs, defs, libs, libdirs, cxxflags, otherflags) + =20 + if logfile is None: + fout =3D sp.PIPE + else: + fout =3D file(logfile, 'w') + fout.write(cl + "\n" + 80*'-' + "\n") + fout.flush() cl =3D shlex.split(cl) - cp =3D sp.Popen(cl, stdout=3Dsp.PIPE, stderr=3Dsp.PIPE) + cp =3D sp.Popen(cl, stdout=3Dfout, stderr=3Dsp.STDOUT) cp.communicate() + if logfile is not None: + fout.close() return (cp.returncode, ' '.join(cl)) diff --git a/app-benchmarks/autobench/files/python/main.py b/app-benchmar= ks/autobench/files/python/main.py index 474f9bd..21e3c34 100644 --- a/app-benchmarks/autobench/files/python/main.py +++ b/app-benchmarks/autobench/files/python/main.py @@ -20,6 +20,17 @@ testsdir =3D "/var/tmp/benchmarks/tests/" libdir =3D sp.Popen \ ('ABI=3D$(portageq envvar ABI); echo /usr/`portageq envvar LIBDIR_$ABI= `/', \ stdout=3Dsp.PIPE, shell=3DTrue).communicate()[0].strip() +logdir =3D "/var/log/benchmarks/" + time.strftime('%Y-%m-%d') +if os.path.exists(logdir): + n =3D 1 + while True: + logdir =3D "/var/log/benchmarks/" + time.strftime('%Y-%m-%d') + = "_%i"%n + if not os.path.exists(logdir): + os.makedirs(logdir) + break + n +=3D 1 +else: + os.makedirs(logdir) =20 def print_usage(): print "Usage: benchmarks [blas|cblas|lapack] file args" =20 @@ -150,6 +161,8 @@ for tn,(name,test) in enumerate(tests.items(),1): =20 pkgdir =3D "%s/%s/" % (pkgsdir, name) root =3D "%s/%s/" % (rootsdir, name) + tlogdir =3D os.path.join(logdir, name) + os.path.exists(tlogdir) or os.makedirs(tlogdir) =20 # Emerge package Print.down() @@ -160,8 +173,11 @@ for tn,(name,test) in enumerate(tests.items(),1): Print("Package already emerged - skipping") else: try: + logfile =3D os.path.join(tlogdir, 'emerge.log') install_package( \ - test['package'], env=3Dtest['env'], root=3Droot, pkgdir=3D= pkgdir) + test['package'], env=3Dtest['env'], root=3Droot, pkgdir=3D= pkgdir, \ + logfile=3Dlogfile + ) # Unpack the archive onto the given root directory archive =3D pkgdir + package + '.tbz2' os.path.exists(root) or os.makedirs(root) @@ -172,7 +188,8 @@ for tn,(name,test) in enumerate(tests.items(),1): raise InstallException(tarcmd) =20 except InstallException as e: - Print("Package %s failed to emerge: %s" % (package, e.comman= d)) + Print("Package %s failed to emerge" % package) + Print("See emerge log: " + logfile) Print.up() print continue @@ -181,7 +198,7 @@ for tn,(name,test) in enumerate(tests.items(),1): # Find implementations impls =3D mod.get_impls(root) test['implementations'] =3D impls - =20 + =20 # Test every implementation test['results'] =3D {} for impl in impls: @@ -191,7 +208,7 @@ for tn,(name,test) in enumerate(tests.items(),1): # Run the test suite testdir =3D "%s/%s/%s" % (testsdir, name, impl) test['results'][impl] =3D \ - mod.run_test(root=3Droot, impl=3Dimpl, testdir=3Dtestdir, env=3D= test['env']) + mod.run_test(root, impl, testdir, env=3Dtest['env'], logdir=3D= tlogdir) Print.up() =20 Print.up()