[gentoo-commits] proj/R_overlay:gsoc13/next commit in: roverlay/util/

[gentoo-commits] proj/R_overlay:gsoc13/next commit in: roverlay/util/

[André Erdmann]

commit:     b703c26878c27f803d22d6a6b80bbaf2464c3dfe
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Jun 20 23:26:56 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Jun 20 23:26:56 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=b703c268

roverlay/util: fix synop(False)

---
 roverlay/util/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roverlay/util/common.py b/roverlay/util/common.py
index d8ad4b7..6deee14 100644
--- a/roverlay/util/common.py
+++ b/roverlay/util/common.py
@@ -147,7 +147,7 @@ def sysnop ( nop_returns_success=True, format_str=None, old_formatting=False ):
    if nop_returns_success:
       candidates = ( '/bin/true', '/bin/echo' )
    else:
-      candidates = ( '/bin/false' )
+      candidates = ( '/bin/false', )
 
    for c in candidates:
       if os.path.isfile ( c ):

[gentoo-commits] proj/R_overlay:master commit in: roverlay/util/

[André Erdmann]

commit:     b703c26878c27f803d22d6a6b80bbaf2464c3dfe
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Jun 20 23:26:56 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Jun 20 23:26:56 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=b703c268

roverlay/util: fix synop(False)

---
 roverlay/util/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roverlay/util/common.py b/roverlay/util/common.py
index d8ad4b7..6deee14 100644
--- a/roverlay/util/common.py
+++ b/roverlay/util/common.py
@@ -147,7 +147,7 @@ def sysnop ( nop_returns_success=True, format_str=None, old_formatting=False ):
    if nop_returns_success:
       candidates = ( '/bin/true', '/bin/echo' )
    else:
-      candidates = ( '/bin/false' )
+      candidates = ( '/bin/false', )
 
    for c in candidates:
       if os.path.isfile ( c ):

[gentoo-commits] proj/R_overlay:gsoc13/next commit in: roverlay/util/

[André Erdmann]

commit:     932c509c1e622b28c7f0d4a3640bb9b424c59309
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Mon Jul  8 22:33:04 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Mon Jul  8 22:33:04 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=932c509c

roverlay/util/common: is_vcs_dir(), getsize()

---
 roverlay/util/common.py | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/roverlay/util/common.py b/roverlay/util/common.py
index 6deee14..9ecdcc9 100644
--- a/roverlay/util/common.py
+++ b/roverlay/util/common.py
@@ -7,7 +7,8 @@
 """provides utility functions commonly used"""
 
 __all__= [
-   'dodir', 'for_all_files', 'get_dict_hash', 'keepenv', 'priosort', 'sysnop'
+   'dodir', 'for_all_files', 'get_dict_hash', 'keepenv', 'priosort', 'sysnop',
+   'getsize', 'is_vcs_dir',
 ]
 
 import os
@@ -16,6 +17,10 @@ import logging
 
 LOGGER = logging.getLogger ( 'util' )
 
+# COULDFIX: add .svn et al.
+#
+VCS_DIRNAMES = frozenset ({ '.git', })
+
 def for_all_files (
    files_or_dirs, func,
    args=(), kwargs={}, file_filter=None, ignore_missing=False
@@ -186,3 +191,22 @@ def dodir ( directory, mkdir_p=False, **makedirs_kw ):
       return os.path.isdir ( directory )
 
 # --- end of dodir (...) ---
+
+def getsize ( filepath ):
+   """Returns the size of the given file.
+
+   arguments:
+   * filepath --
+   """
+   return os.stat ( filepath ).st_size
+# --- end of getsize (...) ---
+
+def is_vcs_dir ( dirpath ):
+   """Returns True if dirpath could be a directory maintained by a version
+   control system, e.g. git.
+
+   arguments:
+   * dirpath --
+   """
+   return os.path.basename ( dirpath.rstrip ( os.sep ) ) in VCS_DIRNAMES
+# --- end of is_vcs_dir (...) ---

[gentoo-commits] proj/R_overlay:gsoc13/next commit in: roverlay/util/

[André Erdmann]

commit:     8492fe2902389562be1a6d86e801f22df7dffdf5
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jul 10 08:10:32 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jul 10 08:10:32 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=8492fe29

roverlay/util/common: dodir_for_file()

---
 roverlay/util/common.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/roverlay/util/common.py b/roverlay/util/common.py
index 9ecdcc9..aff6d94 100644
--- a/roverlay/util/common.py
+++ b/roverlay/util/common.py
@@ -192,6 +192,10 @@ def dodir ( directory, mkdir_p=False, **makedirs_kw ):
 
 # --- end of dodir (...) ---
 
+def dodir_for_file ( filepath, mkdir_p=True, **kw ):
+   return dodir ( os.path.basename ( filepath ), mkdir_p=mkdir_p, **kw )
+# --- end of dodir_for_file (...) ---
+
 def getsize ( filepath ):
    """Returns the size of the given file.
 

[gentoo-commits] proj/R_overlay:gsoc13/next commit in: roverlay/util/

[André Erdmann]

commit:     e890ffc98194c6afd73c05022ca3fc19252ef6f5
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jul 10 08:12:22 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jul 10 08:12:22 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=e890ffc9

roverlay/util/: mapreader module

mapreader provides some classes (MapFileParser, DictFileParser,
ReverseDictFileParser) for reading  "key -> multiple values" mappings from
from files. The syntax is the same as for dependency rules.

---
 roverlay/util/mapreader.py | 207 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 207 insertions(+)

diff --git a/roverlay/util/mapreader.py b/roverlay/util/mapreader.py
new file mode 100644
index 0000000..2958616
--- /dev/null
+++ b/roverlay/util/mapreader.py
@@ -0,0 +1,207 @@
+# R overlay -- util, map file reader
+# -*- coding: utf-8 -*-
+# Copyright (C) 2012 André Erdmann <dywi@mailerd.de>
+# Distributed under the terms of the GNU General Public License;
+# either version 2 of the License, or (at your option) any later version.
+
+import logging
+import re
+
+import roverlay.util.fileio
+
+class MapFileParser ( object ):
+   """Parser for reading <key,value^n> maps from files or text input."""
+
+   # shared attr
+   logger                = logging.getLogger ( 'MapFileParser' )
+   log_unparseable       = logger.warning
+   multiline_start_char  = '{'
+   multiline_stop_char   = '}'
+   comment_char          = '#'
+   single_line_separator = re.compile ( '\s+[:][:]\s+' )
+
+   def __init__ ( self, value_separator=None ):
+      super ( MapFileParser, self ).__init__()
+
+      if value_separator is not None:
+         self.single_line_separator = re.compile ( value_separator )
+         # else use shared
+
+      self._next  = None
+      # items = list ( tuple ( <key>, list ( <value(s)> ) ) )
+      self._items = list()
+   # --- end of __init__ (...) ---
+
+   def zap ( self ):
+      if self._next is not None:
+         self.logger.warning (
+            "Multi line entry does not end at EOF - ignored"
+         )
+
+      self.stop_reading = False
+      self._next        = None
+      self._items       = list()
+   # --- end of zap (...) ---
+
+   def make_result ( self ):
+      return self._items
+   # --- end of make_result (...) ---
+
+   def done ( self, *args, **kwargs ):
+      ret = self.make_result ( *args, **kwargs )
+      self.zap()
+      return ret
+   # --- end of done (...) ---
+
+   def preparse_line ( self, line ):
+      return line.strip()
+   # --- end of preparse_line (...) ---
+
+   def handle_multiline_begin ( self, line ):
+      self._next = ( line, list() )
+      return True
+   # --- end of handle_multiline_begin (...) ---
+
+   def handle_multiline_end ( self, line ):
+      self._items.append ( self._next )
+      self._next = None
+      return True
+   # --- end of handle_multiline_end (...) ---
+
+   def handle_multiline_append ( self, line ):
+      self._next[1].append ( line )
+      return True
+   # --- end of handle_multiline_append (...) ---
+
+   def handle_comment_line ( self, line ):
+      return True
+   # --- end of handle_comment_line (...) ---
+
+   def handle_option_line ( self, line ):
+      return False
+   # --- end of handle_option_line (...) ---
+
+   def handle_entry_line ( self, key=None, value=None ):
+      if key and value:
+         self._items.append ( ( key, [ value ] ) )
+         return True
+      else:
+         return False
+   # --- end of handle_entry_line (...) ---
+
+   def add ( self, line ):
+      l = self.preparse_line ( line )
+
+      if not l:
+         return True
+      elif self._next is not None:
+         if l[0] == self.multiline_stop_char:
+            # end of a multiline entry
+            #  add it to self._items and set next to None
+            return self.handle_multiline_end ( l[1:].lstrip() )
+         else:
+            return self.handle_multiline_append ( l )
+
+      elif l[0] == self.comment_char:
+         #  it's intentional that multi line rules cannot contain comments
+         return self.handle_comment_line ( l[1:].lstrip() )
+
+      elif self.handle_option_line ( l ):
+         return True
+
+      elif len ( l ) > 1 and l[-1] == self.multiline_start_char:
+         return self.handle_multiline_begin ( l[:-1].rstrip() )
+
+      else:
+         return self.handle_entry_line (
+            *self.single_line_separator.split ( l, 1 )
+         )
+   # --- end of add (...) ---
+
+   def read_lines ( self, lines ):
+      self.stop_reading = False
+      ret = True
+      for lino, line in enumerate ( lines ):
+         if not self.add ( line ):
+            ret = False
+            self.log_unparseable (
+               "{f}: cannot parse line {n:d}".format ( f=filepath, n=lino+1 )
+            )
+
+         if self.stop_reading:
+            break
+   # --- end of read_lines (...) ---
+
+   def read_file ( self, filepath, handle_compressed=True ):
+      try:
+         if handle_compressed:
+            ret = self.read_lines (
+               roverlay.util.fileio.read_text_file ( filepath )
+            )
+         else:
+            with open ( filepath, 'rt' ) as FH:
+               ret = self.read_lines ( FH.readlines() )
+      except IOError:
+         self.logger.error (
+            "Could not read file {!r}.".format ( filepath )
+         )
+         raise
+
+      return ret
+   # --- end of read_file (...) ---
+
+# --- end of MapFileParser ---
+
+
+class DictFileParser ( MapFileParser ):
+   """MapFileParser that creates a dict as result."""
+
+   def make_result ( self, unpack_values=True, strict_unpack=False ):
+      def unpacked ( value ):
+         if value:
+            return value[0] if len ( value ) == 1 else value
+         else:
+            return None
+      # --- end of unpacked (...) ---
+
+      can_unpack = True
+
+      if strict_unpack and can_unpack:
+         for k, v in self._items:
+            if len ( v ) > 1:
+               assert not isinstance ( v, str )
+               can_unpack = False
+               break
+
+
+      if can_unpack:
+         return { k: unpacked ( v ) for k, v in self._items }
+      else:
+         return { k: v for k, v in self._items }
+   # --- end of make_result (...) ---
+
+# --- end of DictFileParser ---
+
+
+class ReverseDictFileParser ( MapFileParser ):
+   """Like DictFileParser,
+   but creates a "<value,key> for value in value^n" dict.
+   """
+
+   # this is the common case for map files used in roverlay,
+   #  multiple values should be mapped to a single key
+   #  (e.g. dependency rules, but the depres parser doesn't use this class)
+   #
+
+   def iter_result_pairs ( self ):
+      for value, keys in self._items:
+         for key in keys:
+            if key:
+               yield ( key, value )
+   # --- end of iter_pairs (...) ---
+
+   def make_result ( self ):
+      return { k: v for k, v in self.iter_result_pairs() }
+   # --- end of make_result (...) ---
+
+# --- end of ReverseDictFileParser ---

[gentoo-commits] proj/R_overlay:gsoc13/next commit in: roverlay/util/

[André Erdmann]

commit:     6297df8496584410eab08ef7b99145c8e0353dec
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jul 10 08:11:05 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jul 10 08:11:05 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=6297df84

roverlay/util/: fileio

Common functions for reading/writing text files. Supports compression
(bzip2,gzip).

---
 roverlay/util/fileio.py | 87 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 87 insertions(+)

diff --git a/roverlay/util/fileio.py b/roverlay/util/fileio.py
new file mode 100644
index 0000000..546a2ef
--- /dev/null
+++ b/roverlay/util/fileio.py
@@ -0,0 +1,87 @@
+# R overlay -- util, file read operations
+# -*- coding: utf-8 -*-
+# Copyright (C) 2012 André Erdmann <dywi@mailerd.de>
+# Distributed under the terms of the GNU General Public License;
+# either version 2 of the License, or (at your option) any later version.
+
+import gzip
+import bz2
+import mimetypes
+import sys
+
+import roverlay.util.common
+
+if sys.hexversion >= 0x3000000:
+   def iter_decode ( lv ):
+      for l in lv:
+         yield l.decode()
+else:
+   def iter_decode ( lv ):
+      return lv
+
+
+_MIME = mimetypes.MimeTypes()
+
+guess_filetype = _MIME.guess_type
+
+COMP_GZIP  = 1
+COMP_BZIP2 = 2
+
+SUPPORTED_COMPRESSION = {
+   'gzip'     : gzip.GzipFile,
+   'gz'       : gzip.GzipFile,
+   COMP_GZIP  : gzip.GzipFile,
+   'bzip2'    : bz2.BZ2File,
+   'bz2'      : bz2.BZ2File,
+   COMP_BZIP2 : bz2.BZ2File,
+}
+
+
+def read_text_file ( filepath, preparse=None ):
+   ftype         = guess_filetype ( filepath )
+   compress_open = SUPPORTED_COMPRESSION.get ( ftype[1], None )
+
+   if compress_open is not None:
+      with compress_open ( filepath, mode='r' ) as CH:
+         for line in iter_decode ( CH.readlines() ):
+            yield line if preparse is None else preparse ( line )
+   else:
+      with open ( filepath, 'rt' ) as FH:
+         for line in FH.readlines():
+            yield line if preparse is None else preparse ( line )
+
+
+def write_text_file (
+   filepath, lines, compression=None, mode='wt',
+   append_newlines=True, append_newline_eof=False, create_dir=True,
+   newline='\n'
+):
+   compress_open = (
+      SUPPORTED_COMPRESSION [compression] if compression else None
+   )
+
+   if create_dir:
+      roverlay.util.common.dodir_for_file ( filepath )
+
+   if compress_open:
+      NL = '\n'.encode()
+      with compress_open ( filepath, mode.rstrip ( 'tu' ) ) as CH:
+         for line in lines:
+            CH.write ( str ( line ).encode() )
+            if append_newlines:
+               CH.write ( NL )
+         else:
+            if append_newline_eof:
+               FH.write ( NL )
+   else:
+      with open ( filepath, mode ) as FH:
+         for line in lines:
+            FH.write ( str ( line ) )
+            if append_newlines:
+               FH.write ( newline )
+         else:
+            if append_newline_eof:
+               FH.write ( newline )
+
+   return True
+# --- end of write_text_file (...) ---

[gentoo-commits] proj/R_overlay:gsoc13/next commit in: roverlay/util/

[André Erdmann]

commit:     261ca9169d73c7ae3799b7c3b0506457af6fe3f2
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Fri Jul 12 13:43:08 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Fri Jul 12 13:43:08 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=261ca916

roverlay/util/mapreader: fix log_unparseable

---
 roverlay/util/mapreader.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/roverlay/util/mapreader.py b/roverlay/util/mapreader.py
index c2cce1f..19d4129 100644
--- a/roverlay/util/mapreader.py
+++ b/roverlay/util/mapreader.py
@@ -126,7 +126,7 @@ class MapFileParser ( object ):
       pass
    # --- end of read_lines_begin (...) ---
 
-   def read_lines ( self, lines ):
+   def read_lines ( self, lines, src=None ):
       self.read_lines_begin()
       self.stop_reading = False
       ret = True
@@ -134,7 +134,9 @@ class MapFileParser ( object ):
          if not self.add ( line ):
             ret = False
             self.log_unparseable (
-               "{f}: cannot parse line {n:d}".format ( f=filepath, n=lino+1 )
+               "{f}: cannot parse line {n:d}: {txt!r}".format (
+                  f=( src or "<input>" ), n=lino+1, txt=line
+               )
             )
 
          if self.stop_reading:
@@ -148,11 +150,11 @@ class MapFileParser ( object ):
       try:
          if handle_compressed:
             ret = self.read_lines (
-               roverlay.util.fileio.read_text_file ( filepath )
+               roverlay.util.fileio.read_text_file ( filepath ), filepath
             )
          else:
             with open ( filepath, 'rt' ) as FH:
-               ret = self.read_lines ( FH.readlines() )
+               ret = self.read_lines ( FH.readlines(), filepath )
       except IOError:
          self.logger.error (
             "Could not read file {!r}.".format ( filepath )

[gentoo-commits] proj/R_overlay:gsoc13/next commit in: roverlay/util/

[André Erdmann]

commit:     316ff643c54198004c14dd579863dedc80c99112
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Jul 18 19:21:27 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Jul 18 19:21:27 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=316ff643

roverlay/util/mapread: has_context()

---
 roverlay/util/mapreader.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/roverlay/util/mapreader.py b/roverlay/util/mapreader.py
index 19d4129..5333bd9 100644
--- a/roverlay/util/mapreader.py
+++ b/roverlay/util/mapreader.py
@@ -43,6 +43,13 @@ class MapFileParser ( object ):
       self._items       = list()
    # --- end of zap (...) ---
 
+   def has_context ( self ):
+      return (
+         self._deptype_once != deptype.NONE
+         or self._next is not None
+      )
+   # --- end of has_context (...) ---
+
    def make_result ( self ):
       return self._items
    # --- end of make_result (...) ---

[gentoo-commits] proj/R_overlay:gsoc13/next commit in: roverlay/util/

[André Erdmann]

commit:     eb054c34b1af6b908d4df808742113d42e218d12
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jul 23 09:31:34 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jul 23 09:31:34 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=eb054c34

roverlay/util/common: try_unlink()

Remove a file if it exists and ignore errors caused by non-existence.

---
 roverlay/util/common.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/roverlay/util/common.py b/roverlay/util/common.py
index a2c155a..f27593a 100644
--- a/roverlay/util/common.py
+++ b/roverlay/util/common.py
@@ -8,9 +8,10 @@
 
 __all__= [
    'dodir', 'for_all_files', 'get_dict_hash', 'keepenv', 'priosort', 'sysnop',
-   'getsize', 'is_vcs_dir', 'headtail'
+   'getsize', 'is_vcs_dir', 'headtail', 'try_unlink',
 ]
 
+import errno
 import os
 import sys
 import logging
@@ -26,6 +27,21 @@ def headtail ( iterable ):
    return ( iterable[0], iterable[1:] )
 # --- end of headtail #py2 (...) ---
 
+def try_unlink ( fspath ):
+   """Tries to remove a file. Does not fail if the file did not exist.
+
+   arguments:
+   * fspath --
+   """
+   try:
+      os.unlink ( fspath )
+   except OSError as oserr:
+      if oserr.errno == errno.ENOENT:
+         pass
+      else:
+         raise
+# --- end of try_unlink (...) ---
+
 def for_all_files (
    files_or_dirs, func,
    args=(), kwargs={}, file_filter=None, ignore_missing=False

[gentoo-commits] proj/R_overlay:gsoc13/next commit in: roverlay/util/

[André Erdmann]

commit:     db75bfc7ca865feff6421995ed6baf7638a437d9
Author:     André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jul 23 13:31:02 2013 +0000
Commit:     André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jul 23 13:31:02 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=db75bfc7

roverlay/util, try_unlink(): return true/false

---
 roverlay/util/common.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/roverlay/util/common.py b/roverlay/util/common.py
index f27593a..f3126ad 100644
--- a/roverlay/util/common.py
+++ b/roverlay/util/common.py
@@ -30,6 +30,8 @@ def headtail ( iterable ):
 def try_unlink ( fspath ):
    """Tries to remove a file. Does not fail if the file did not exist.
 
+   Returns: True if a file has been removed, else False.
+
    arguments:
    * fspath --
    """
@@ -37,9 +39,11 @@ def try_unlink ( fspath ):
       os.unlink ( fspath )
    except OSError as oserr:
       if oserr.errno == errno.ENOENT:
-         pass
+         return False
       else:
          raise
+   else:
+      return True
 # --- end of try_unlink (...) ---
 
 def for_all_files (