[gentoo-commits] proj/auto-numerical-bench:master commit in: /, btl/libs/BLAS/

["Andrea Arteaga" <andyspiros@gmail.com>]

commit:     6fbea9e9eb755b2ca0336b54daf52ffc365f72b6
Author:     spiros <andyspiros <AT> gmail <DOT> com>
AuthorDate: Thu Jun  9 00:37:29 2011 +0000
Commit:     Andrea Arteaga <andyspiros <AT> gmail <DOT> com>
CommitDate: Thu Jun  9 00:37:29 2011 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/auto-numerical-bench.git;a=commit;h=6fbea9e9

Input from file added. Finished CBLAS suite. Added Base module for BLAS and CBLAS. Support for selecting single tests. Modified BTL main.cpp.
Many bugs solved. Porting to subprocess almost finished.

---
 PortageUtils.py                      |    9 +---
 blas.py                              |  108 +++------------------------------
 btl/libs/BLAS/blas_interface.hh      |   31 ++++------
 btl/libs/BLAS/blas_interface_impl.hh |   72 ----------------------
 btl/libs/BLAS/main.cpp               |   81 ++++++++++++++++++--------
 main.py                              |   69 +++++++++++++++-------
 6 files changed, 127 insertions(+), 243 deletions(-)

diff --git a/PortageUtils.py b/PortageUtils.py
index c27dc79..4877cbd 100644
--- a/PortageUtils.py
+++ b/PortageUtils.py
@@ -56,13 +56,6 @@ def install_package(package, env={}, root='/', pkgdir='usr/portage/packages'):
         # In case of error, print the whole emerge command
         raise InstallException(cl)
     
-    # Unpack the archive onto the given root directory
-    archive = pkgdir + pkg + '.tbz2'
-    os.path.exists(root) or os.makedirs(root)
-    so = cmd.getstatusoutput("tar xjf " + archive + " -C " + root);
-    if so[0] != 0:
-        raise InstallException(so[1])
-    
 if __name__ == '__main__':
     # Just a test
-    print available_packages('blas-reference')
\ No newline at end of file
+    print available_packages('blas-reference')

diff --git a/blas.py b/blas.py
index 2cf1ec8..295e833 100644
--- a/blas.py
+++ b/blas.py
@@ -1,108 +1,20 @@
-import os
-import commands as cmd
+import os, blasbase
 import subprocess as sp
 
-run_cmd = lambda c : sp.Popen(c, stdout=sp.PIPE).communicate()[0]
-
-class Module:
-    def __init__(self, Print, libdir):
-        self.Print = Print
-        self.libdir = libdir
-    
+class Module(blasbase.ModuleBase): 
     @staticmethod
     def get_impls(root):
         return [i for i in os.listdir(root + "/etc/env.d/alternatives/blas") \
           if i[0] != '_']
-          
-    def run_test(self, root, impl, testdir):
-        Print = self.Print
-        libdir = self.libdir
-        name = 'blas'
-        files = ['%s/bench_%s_%s.dat' %(testdir, op, name) for op in (
-              'axpy', 'axpby', 'matrix_vector', 'atv', 'symv',
-              'syr2', 'ger', 'rot', 'matrix_matrix', 'aat',
-              'trisolve_vector', 'trisolve_matrix', 'trmm')]
-        
-        # Create dir. If all results already exist use them, otherwise
-        # remove old results
-        runtests = False
-        if os.path.exists(testdir):
-             runtests = all([os.path.exists(i) for i in files])
-        else:
-            os.makedirs(testdir)
-            runtests = True
-        
-        if not runtests:
-            Print("Not testing: result exist")
-            return files
-        
-        for i in files:
-            if os.path.exists(i): os.remove(i)
-        
-        # Setup environment for testing
-        oldenv = {}
-        for v in ('LIBRARY_PATH', 'INCLUDE_PATH', 'LD_LIBRARY_PATH'):
-            # Backup old environment variables
-            oldenv[v] = \
-              (os.environ.has_key(v) and (os.environ[v],) or (None,))[0]
-        os.environ['LIBRARY_PATH'] = root + libdir
-        os.environ['INCLUDE_PATH'] = root + '/usr/include'
-        os.environ['LD_LIBRARY_PATH']= root+libdir+":"+oldenv['LD_LIBRARY_PATH']
-        
+           
+    def _get_flags(self, root, impl, libdir):
         # Retrieve pkgconfig settings and map the directories to the new root
-        os.environ['PKG_CONFIG_PATH'] = \
+        path = \
           "%s/etc/env.d/alternatives/blas/%s/%s/pkgconfig" % (root,impl,libdir)
-        pkgconf = cmd.getoutput('pkg-config --libs --cflags blas')
-        del os.environ['PKG_CONFIG_PATH']
+        pkgconf = sp.Popen('pkg-config --libs --cflags blas', shell=True, \
+          stdout=sp.PIPE, env={'PKG_CONFIG_PATH':path}).communicate()[0]
         pkgconf = pkgconf.replace('-L/', '-L'+root+'/')
         pkgconf = pkgconf.replace('-I/', '-I'+root+'/')
-        
-        # Compile
-        exe = testdir + "/test"
-        inc = """
-        -Ibtl/actions
-        -Ibtl/generic_bench
-        -Ibtl/generic_bench/utils
-        -Ibtl/libs/BLAS
-        -Ibtl/libs/STL
-        """.replace('\n', ' ')
-        libs = "-lrt -L" + root+libdir
-        cxxflags = run_cmd(['portageq', 'envvar', 'CXXFLAGS']).strip()
-        defines = "-DCBLASNAME=" + name
-        cl = "g++ %s %s %s %s %s btl/libs/BLAS/main.cpp -o %s" \
-            % (pkgconf, inc, libs, cxxflags, defines, exe)
-        so = cmd.getstatusoutput(cl)
-        if so[0] != 0:
-            raise Exception("Compilation failed: " + cl)
-        Print("Compilation successful: %s" % cl)
-        
-        # Run test        
-        proc = sp.Popen(exe, bufsize=1, stdout=sp.PIPE, stderr=sp.PIPE, 
-          cwd = testdir)
-        results = []
-        while True:
-            errline = proc.stderr.readline()
-            if not errline:
-                break
-            resfile = errline.split()[-1]
-            results.append(resfile)
-            Print(resfile)
-            Print.down()
-            for i in xrange(100):
-                outline = proc.stdout.readline().rstrip()
-                Print(outline)
-            Print.up()
-        Print.up()
-        proc.wait()
-        if proc.returncode != 0:
-            Print('Test failed')
-        else:
-            Print('Test successful')
-        
-        # Restore old environment variables
-        for v in ('LIBRARY_PATH', 'INCLUDE_PATH', 'LD_LIBRARY_PATH'):
-            if oldenv[v] != None:
-                os.environ[v] = oldenv[v]
-            elif os.environ.has_key(v):
-                del os.environ[v]
-        return files
\ No newline at end of file
+        return pkgconf + " -DBLAS_INTERFACE"
+
+del blasbase

diff --git a/btl/libs/BLAS/blas_interface.hh b/btl/libs/BLAS/blas_interface.hh
index 6510546..86881a2 100644
--- a/btl/libs/BLAS/blas_interface.hh
+++ b/btl/libs/BLAS/blas_interface.hh
@@ -25,22 +25,8 @@
 extern "C"
 {
 #include "blas.h"
-
-  // Cholesky Factorization
-//   void spotrf_(const char* uplo, const int* n, float *a, const int* ld, int* info);
-//   void dpotrf_(const char* uplo, const int* n, double *a, const int* ld, int* info);
-  void ssytrd_(char *uplo, const int *n, float *a, const int *lda, float *d, float *e, float *tau, float *work, int *lwork, int *info );
-  void dsytrd_(char *uplo, const int *n, double *a, const int *lda, double *d, double *e, double *tau, double *work, int *lwork, int *info );
-  void sgehrd_( const int *n, int *ilo, int *ihi, float *a, const int *lda, float *tau, float *work, int *lwork, int *info );
-  void dgehrd_( const int *n, int *ilo, int *ihi, double *a, const int *lda, double *tau, double *work, int *lwork, int *info );
-
-  // LU row pivoting
-//   void dgetrf_( int *m, int *n, double *a, int *lda, int *ipiv, int *info );
-//   void sgetrf_(const int* m, const int* n, float *a, const int* ld, int* ipivot, int* info);
-  // LU full pivoting
-  void sgetc2_(const int* n, float *a, const int *lda, int *ipiv, int *jpiv, int*info );
-  void dgetc2_(const int* n, double *a, const int *lda, int *ipiv, int *jpiv, int*info );
-#ifdef HAS_LAPACK
+#ifdef CBLAS_INTERFACE
+#  include "cblas.h"
 #endif
 }
 
@@ -63,17 +49,24 @@ static char left = 'L';
 static int intone = 1;
 
 
-
 #define SCALAR        float
 #define SCALAR_PREFIX s
-#include "blas_interface_impl.hh"
+#ifdef CBLAS_INTERFACE
+#  include "cblas_interface_impl.hh"
+#else
+#  include "blas_interface_impl.hh"
+#endif
 #undef SCALAR
 #undef SCALAR_PREFIX
 
 
 #define SCALAR        double
 #define SCALAR_PREFIX d
-#include "blas_interface_impl.hh"
+#ifdef CBLAS_INTERFACE
+#  include "cblas_interface_impl.hh"
+#else
+#  include "blas_interface_impl.hh"
+#endif
 #undef SCALAR
 #undef SCALAR_PREFIX
 

diff --git a/btl/libs/BLAS/blas_interface_impl.hh b/btl/libs/BLAS/blas_interface_impl.hh
index 0e84df0..97fb631 100644
--- a/btl/libs/BLAS/blas_interface_impl.hh
+++ b/btl/libs/BLAS/blas_interface_impl.hh
@@ -46,10 +46,6 @@ public :
     BLAS_FUNC(gemm)(&notrans,&notrans,&N,&N,&N,&fone,A,&N,B,&N,&fzero,X,&N);
   }
 
-//   static inline void ata_product(gene_matrix & A, gene_matrix & X, int N){
-//     ssyrk_(&lower,&trans,&N,&N,&fone,A,&N,&fzero,X,&N);
-//   }
-
   static inline void aat_product(gene_matrix & A, gene_matrix & X, int N){
     BLAS_FUNC(syrk)(&lower,&notrans,&N,&N,&fone,A,&N,&fzero,X,&N);
   }
@@ -63,25 +59,6 @@ public :
     BLAS_FUNC(axpy)(&N,&a,X,&intone,Y,&intone);
   }
 
-  static inline void cholesky(const gene_matrix & X, gene_matrix & C, int N){
-    int N2 = N*N;
-    BLAS_FUNC(copy)(&N2, X, &intone, C, &intone);
-    char uplo = 'L';
-    int info = 0;
-    BLAS_FUNC(potrf)(&uplo, &N, C, &N, &info);
-    if(info!=0) std::cerr << "potrf_ error " << info << "\n";
-  }
-
-  static inline void partial_lu_decomp(const gene_matrix & X, gene_matrix & C, int N){
-    int N2 = N*N;
-    BLAS_FUNC(copy)(&N2, X, &intone, C, &intone);
-    char uplo = 'L';
-    int info = 0;
-    int * ipiv = (int*)alloca(sizeof(int)*N);
-    BLAS_FUNC(getrf)(&N, &N, C, &N, ipiv, &info);
-    if(info!=0) std::cerr << "getrf_ error " << info << "\n";
-  }
-  
   static inline void trisolve_lower(const gene_matrix & L, const gene_vector& B, gene_vector & X, int N){
     BLAS_FUNC(copy)(&N, B, &intone, X, &intone);
     BLAS_FUNC(trsv)(&lower, &notrans, &nonunit, &N, L, &N, X, &intone);
@@ -96,55 +73,6 @@ public :
     BLAS_FUNC(trmm)(&left, &lower, &notrans,&nonunit, &N,&N,&fone,A,&N,B,&N);
   }
 
-  #ifdef HAS_LAPACK
-
-  static inline void lu_decomp(const gene_matrix & X, gene_matrix & C, int N){
-    int N2 = N*N;
-    BLAS_FUNC(copy)(&N2, X, &intone, C, &intone);
-    char uplo = 'L';
-    int info = 0;
-    int * ipiv = (int*)alloca(sizeof(int)*N);
-    int * jpiv = (int*)alloca(sizeof(int)*N);
-    BLAS_FUNC(getc2)(&N, C, &N, ipiv, jpiv, &info);
-  }
-
-
-
-  static inline void hessenberg(const gene_matrix & X, gene_matrix & C, int N){
-    {
-      int N2 = N*N;
-      int inc = 1;
-      BLAS_FUNC(copy)(&N2, X, &inc, C, &inc);
-    }
-    int info = 0;
-    int ilo = 1;
-    int ihi = N;
-    int bsize = 64;
-    int worksize = N*bsize;
-    SCALAR* d = new SCALAR[N+worksize];
-    BLAS_FUNC(gehrd)(&N, &ilo, &ihi, C, &N, d, d+N, &worksize, &info);
-    delete[] d;
-  }
-
-  static inline void tridiagonalization(const gene_matrix & X, gene_matrix & C, int N){
-    {
-      int N2 = N*N;
-      int inc = 1;
-      BLAS_FUNC(copy)(&N2, X, &inc, C, &inc);
-    }
-    char uplo = 'U';
-    int info = 0;
-    int ilo = 1;
-    int ihi = N;
-    int bsize = 64;
-    int worksize = N*bsize;
-    SCALAR* d = new SCALAR[3*N+worksize];
-    BLAS_FUNC(sytrd)(&uplo, &N, C, &N, d, d+N, d+2*N, d+3*N, &worksize, &info);
-    delete[] d;
-  }
-  
-  #endif // HAS_LAPACK
-
 };
 
 SCALAR blas_interface<SCALAR>::fone = SCALAR(1);

diff --git a/btl/libs/BLAS/main.cpp b/btl/libs/BLAS/main.cpp
index ebe1966..7cea4b6 100644
--- a/btl/libs/BLAS/main.cpp
+++ b/btl/libs/BLAS/main.cpp
@@ -17,56 +17,89 @@
 // along with this program; if not, write to the Free Software
 // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 //
+
+#ifndef BLAS_INTERFACE
+#  ifndef CBLAS_INTERFACE
+#    define BLAS_INTERFACE
+#  endif
+#endif
+
 #include "utilities.h"
 #include "blas_interface.hh"
 #include "bench.hh"
 #include "basic_actions.hh"
-
-#include "action_cholesky.hh"
-#include "action_lu_decomp.hh"
-#include "action_partial_lu.hh"
 #include "action_trisolve_matrix.hh"
 
-#ifdef HAS_LAPACK
-#include "action_hessenberg.hh"
-#endif
+#include <string>
 
 BTL_MAIN;
 
-int main()
+int main(int argv, char **argc)
 {
+  bool
+  axpy=false, axpby=false, rot=false,
+  matrix_vector=false, atv=false, symv=false, syr2=false, ger=false, trisolve_vector=false,
+  matrix_matrix=false, aat=false, trisolve_matrix=false, trmm=false  
+  ;
+  
+  
+  for (int i = 1; i < argv; ++i) {
+    std::string arg = argc[i];
+    if (arg == "axpy") axpy = true;
+    else if (arg == "axpby") axpby = true;
+    else if (arg == "rot") rot = true;
+    else if (arg == "matrix_vector") matrix_vector = true;
+    else if (arg == "atv") atv = true;
+    else if (arg == "symv") symv = true;
+    else if (arg == "syr2") syr2 = true;
+    else if (arg == "ger") ger = true;
+    else if (arg == "trisolve_vector") trisolve_vector = true;
+    else if (arg == "matrix_matrix") matrix_matrix = true;
+    else if (arg == "aat") aat = true;
+    else if (arg == "trisolve_matrix") trisolve_matrix = true;
+    else if (arg == "trmm") trmm = true;
+    
+    else if (arg[0] == '1' && arg[1] == '\0') {
+      axpy = true; axpby = true; rot = true;
+    }
+    else if (arg[0] == '2' && arg[1] == '\0') {
+      matrix_vector=true; atv=true; symv=true; syr2=true; ger=true; trisolve_vector=true;
+    }
+    else if (arg[0] == '3' && arg[1] == '\0') {
+      matrix_matrix=true; aat=true; trisolve_matrix=true; trmm=true;
+    }
+  }
 
+  if (axpy)
   bench<Action_axpy<blas_interface<REAL_TYPE> > >(MIN_AXPY,MAX_AXPY,NB_POINT);
+  if (axpby)
   bench<Action_axpby<blas_interface<REAL_TYPE> > >(MIN_AXPY,MAX_AXPY,NB_POINT);
+  if (rot)
+  bench<Action_rot<blas_interface<REAL_TYPE> > >(MIN_AXPY,MAX_AXPY,NB_POINT);
 
+  if (matrix_vector)
   bench<Action_matrix_vector_product<blas_interface<REAL_TYPE> > >(MIN_MV,MAX_MV,NB_POINT);
+  if (atv)
   bench<Action_atv_product<blas_interface<REAL_TYPE> > >(MIN_MV,MAX_MV,NB_POINT);
+  if (symv)
   bench<Action_symv<blas_interface<REAL_TYPE> > >(MIN_MV,MAX_MV,NB_POINT);
+  if (syr2)
   bench<Action_syr2<blas_interface<REAL_TYPE> > >(MIN_MV,MAX_MV,NB_POINT);
-
+  if (ger)
   bench<Action_ger<blas_interface<REAL_TYPE> > >(MIN_MV,MAX_MV,NB_POINT);
-  bench<Action_rot<blas_interface<REAL_TYPE> > >(MIN_AXPY,MAX_AXPY,NB_POINT);
+  if (trisolve_vector)
+  bench<Action_trisolve<blas_interface<REAL_TYPE> > >(MIN_MM,MAX_MM,NB_POINT);
 
+  if (matrix_matrix)
   bench<Action_matrix_matrix_product<blas_interface<REAL_TYPE> > >(MIN_MM,MAX_MM,NB_POINT);
-//   bench<Action_ata_product<blas_interface<REAL_TYPE> > >(MIN_MM,MAX_MM,NB_POINT);
+  if (aat)
   bench<Action_aat_product<blas_interface<REAL_TYPE> > >(MIN_MM,MAX_MM,NB_POINT);
-
-  bench<Action_trisolve<blas_interface<REAL_TYPE> > >(MIN_MM,MAX_MM,NB_POINT);
+  if (trisolve_matrix)
   bench<Action_trisolve_matrix<blas_interface<REAL_TYPE> > >(MIN_MM,MAX_MM,NB_POINT);
-
+  if (trmm)
   bench<Action_trmm<blas_interface<REAL_TYPE> > >(MIN_MM,MAX_MM,NB_POINT);
-/*
-  bench<Action_cholesky<blas_interface<REAL_TYPE> > >(MIN_MM,MAX_MM,NB_POINT);
-  bench<Action_partial_lu<blas_interface<REAL_TYPE> > >(MIN_MM,MAX_MM,NB_POINT);
 
-  #ifdef HAS_LAPACK
-  bench<Action_lu_decomp<blas_interface<REAL_TYPE> > >(MIN_MM,MAX_MM,NB_POINT);
-  bench<Action_hessenberg<blas_interface<REAL_TYPE> > >(MIN_MM,MAX_MM,NB_POINT);
-  bench<Action_tridiagonalization<blas_interface<REAL_TYPE> > >(MIN_MM,MAX_MM,NB_POINT);
-  #endif
 
-  //bench<Action_lu_solve<blas_LU_solve_interface<REAL_TYPE> > >(MIN_LU,MAX_LU,NB_POINT);
-*/
   return 0;
 }
 

diff --git a/main.py b/main.py
index 5f2f0d3..7b3c3a9 100644
--- a/main.py
+++ b/main.py
@@ -1,12 +1,15 @@
 import os, sys, shlex
-import commands as cmd
 from PortageUtils import *
+import subprocess as sp
   
 class _Print:
-    def __init__(self):
+    def __init__(self, maxlevel=10):
         self._level = 0
+        self._maxlevel = maxlevel
     
     def __call__(self, arg):
+        if self._level > self._maxlevel:
+            return
         if self._level <= 0:
             print str(arg)
             return
@@ -18,7 +21,7 @@ class _Print:
     def down(self, n=1):
         self._level = max(self._level+n, 0)
      
-Print = _Print()
+Print = _Print(2)
     
     
 # Retrieve relevant directories
@@ -28,37 +31,48 @@ else:
     pkgsdir = os.environ['HOME'] + "/.benchmarks/packages/"
 rootsdir = "/var/tmp/benchmarks/roots/"
 testsdir = "/var/tmp/benchmarks/tests/"
-libdir = cmd.getoutput \
-  ('ABI=$(portageq envvar ABI); echo /usr/`portageq envvar LIBDIR_$ABI`/')
+libdir = sp.Popen \
+  ('ABI=$(portageq envvar ABI); echo /usr/`portageq envvar LIBDIR_$ABI`/', \
+  stdout=sp.PIPE, shell=True).communicate()[0].strip()
 
 
 
 def print_usage():
-    print "Usage: benchmarks [blas|cblas|lapack]"
+    print "Usage: benchmarks [blas|cblas|lapack] file args"
     
     
 def tests_from_input(input):
     tests = {}
     for line in input.split('\n'):
         line = line.strip()
-        spl = shlex.split(line)
+        spl = [i.strip() for i in shlex.split(line)]
         if len(spl) < 2:
             continue
-        package = available_packages(spl[1])
+        if line[0] == '#':
+            continue
+        avail = available_packages(spl[1])
         env = {}
+        # TODO: add @file for env set based on external file
+        # TODO: add -impl for skipping implementation
         for var in spl[2:]:
             s = var.split('=')
             env[s[0]] = s[1]
-        tests[spl[0]] = {'package' : package , 'env' : env}
+	avail = available_packages(spl[1])
+	if len(avail) > 1:
+            for n,p in enumerate(avail):
+	            tests[spl[0]+"_%02i"%n] = {'package':p , 'env':env}
+	else:
+	    tests[spl[0]] = {'package':avail[0] , 'env':env}
     return tests
     
     
 # Import the desired module or print help and exit
 try:
     tmp = __import__(sys.argv[1], fromlist = ['Module'])
-    mod = tmp.Module(Print, libdir)
+    mod = tmp.Module(Print, libdir, sys.argv[3:])
     del tmp
-except:
+    testsfname = sys.argv[2]
+except ImportError, IndexError:
     print_usage()
     exit(1)
 
@@ -101,7 +115,7 @@ which can contain any type of data and will be used for the final report.
 
 
 """
-The test variable is generated from a string which can be read from input.
+The test variable is generated from a string which can be read from the file.
 Here is an example of the parsed input.
 Every line contains a configuration and will be an entry in the tests
 dictionary; the line has to contain:
@@ -109,16 +123,11 @@ dictionary; the line has to contain:
 - a package description, which can, but does not must to, contain a version
 - a list of environment variables separated by means of spaces 
 """
-input = '''
-abcde blas-reference-3.3.1-r1 FC=gfortran
-fghij dev-cpp/eigen-3.0.0-r1 CXX=gcc CXXFLAGS='-O2'
-klmno dev-cpp/eigen-3.0.0-r1 CXX=icc CXXFLAGS='-O3'
-pqrst sci-libs/blas-reference-3.3.1-r1 FC=ifort
-'''
+input = file(testsfname).read()
 tests = tests_from_input(input)
 
 for tn,(name,test) in enumerate(tests.items(),1):
-    Print("BEGIN TEST %i" % tn)
+    Print("BEGIN TEST %i - %s" % (tn, name))
     
     pkgdir = "%s/%s/" % (pkgsdir, name)
     root = "%s/%s/" % (rootsdir, name)
@@ -126,13 +135,23 @@ for tn,(name,test) in enumerate(tests.items(),1):
     # Emerge package
     Print.down()
     package = "%s/%s-%s-%s" % test['package']
+    archive = pkgdir+package+".tbz2"
     Print("Emerging package %s" % package)
-    if os.exists(pkgdir+package+".tbz2"):
+    if os.path.exists(archive):
         Print("Package already emerged - skipping")
     else:
         try:
             install_package( \
               test['package'], env=test['env'], root=root, pkgdir=pkgdir)
+            # Unpack the archive onto the given root directory
+            archive = pkgdir + package + '.tbz2'
+            os.path.exists(root) or os.makedirs(root)
+            tarcmd = "tar xjf " + archive + " -C " + root
+            tarp = sp.Popen(tarcmd, stdout=sp.PIPE, stderr=sp.PIPE, shell=True)
+            tarp.communicate()
+            if tarp.returncode != 0:
+                raise InstallException(tarcmd)
+                
         except InstallException as e:
             Print("Package %s failed to emerge: %s" % (package, e.command))
             Print.up()
@@ -142,6 +161,7 @@ for tn,(name,test) in enumerate(tests.items(),1):
     
     # Find implementations
     impls = mod.get_impls(root)
+    test['implementations'] = impls
       
     # Test every implementation
     test['results'] = {}
@@ -158,5 +178,10 @@ for tn,(name,test) in enumerate(tests.items(),1):
     print
     
 
-print tests
-exit(0)
+results = {}
+for (name,test) in tests.items():
+    for impl in test['implementations']:
+        results[(name, impl)] = test['results'][impl]
+
+for r,rr in results.items():
+    print r, rr