From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from pigeon.gentoo.org ([208.92.234.80] helo=lists.gentoo.org) by finch.gentoo.org with esmtp (Exim 4.60) (envelope-from ) id 1SkH4k-00019S-B5 for garchives@archives.gentoo.org; Thu, 28 Jun 2012 15:56:10 +0000 Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 70A81E05F9; Thu, 28 Jun 2012 15:55:58 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) by pigeon.gentoo.org (Postfix) with ESMTP id 2F74CE05F9 for ; Thu, 28 Jun 2012 15:55:58 +0000 (UTC) Received: from hornbill.gentoo.org (hornbill.gentoo.org [94.100.119.163]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 60B551B405D for ; Thu, 28 Jun 2012 15:55:57 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by hornbill.gentoo.org (Postfix) with ESMTP id 04A91E543C for ; Thu, 28 Jun 2012 15:55:55 +0000 (UTC) From: "André Erdmann" To: gentoo-commits@lists.gentoo.org Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "André Erdmann" Message-ID: <1340898713.4d7bcbfae3c037d56ac315c393bd9b30cf57a37b.dywi@gentoo> Subject: [gentoo-commits] proj/R_overlay:master commit in: roverlay/rpackage/ X-VCS-Repository: proj/R_overlay X-VCS-Files: roverlay/rpackage/descriptionreader.py X-VCS-Directories: roverlay/rpackage/ X-VCS-Committer: dywi X-VCS-Committer-Name: André Erdmann X-VCS-Revision: 4d7bcbfae3c037d56ac315c393bd9b30cf57a37b X-VCS-Branch: master Date: Thu, 28 Jun 2012 15:55:55 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: quoted-printable X-Archives-Salt: fb46152c-b799-4168-98b6-16b44213f50e X-Archives-Hash: 39266a24391274fbb1807d136a941f13 commit: 4d7bcbfae3c037d56ac315c393bd9b30cf57a37b Author: Andr=C3=A9 Erdmann mailerd de> AuthorDate: Thu Jun 28 15:51:53 2012 +0000 Commit: Andr=C3=A9 Erdmann mailerd de> CommitDate: Thu Jun 28 15:51:53 2012 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=3Dproj/R_overlay.git= ;a=3Dcommit;h=3D4d7bcbfa fix for description reader * read field lines as one string and parse it after reading this fixes reading entries like << Depends: a (>1.0) b (>0.9) c >> that should be read as {'Depends':('a (>1.0)','b (>0.9)','c')} and _not_ as {'Depends':('a (>1.0)','b','(>0.9)','c')} modified: roverlay/rpackage/descriptionreader.py --- roverlay/rpackage/descriptionreader.py | 368 +++++++++++++++++---------= ------ 1 files changed, 197 insertions(+), 171 deletions(-) diff --git a/roverlay/rpackage/descriptionreader.py b/roverlay/rpackage/d= escriptionreader.py index 8aa35b3..1f6bd3e 100644 --- a/roverlay/rpackage/descriptionreader.py +++ b/roverlay/rpackage/descriptionreader.py @@ -10,12 +10,31 @@ import time from roverlay import config, util from roverlay.rpackage import descriptionfields =20 +def make_desc_packageinfo ( filepath ): + """Creates a minimal dict that can be used as package info in the + DescriptionReader (for testing/debugging). + + arguments: + * filepath -- + """ + name, sep, ver =3D filepath.partition ( '_' ) + return dict ( + package_file =3D filepath, + package_name =3D name, + ebuild_verstr =3D ver, + name =3D name, + ) + + class DescriptionReader ( object ): """Description Reader""" =20 WRITE_DESCFILES_DIR =3D config.get ( 'DESCRIPTION.descfiles_dir', None = ) =20 - def __init__ ( self, package_info, logger, read_now=3DFalse ): + def __init__ ( self, + package_info, logger, + read_now=3DFalse, write_desc=3DTrue + ): """Initializes a DESCRIPTION file reader.""" =20 if not config.access().get_field_definition(): @@ -26,9 +45,8 @@ class DescriptionReader ( object ): self.field_definition =3D config.access().get_field_definition() self.fileinfo =3D package_info self.logger =3D logger.getChild ( 'desc_reader' ) - self.desc_data =3D None =20 - if DescriptionReader.WRITE_DESCFILES_DIR is not None: + if write_desc and DescriptionReader.WRITE_DESCFILES_DIR is not None: self.write_desc_file =3D os.path.join ( DescriptionReader.WRITE_DESCFILES_DIR, '%s_%s.desc' % ( @@ -36,72 +54,114 @@ class DescriptionReader ( object ): ) ) =20 - if read_now: self.run() =20 # --- end of __init__ (...) --- =20 def get_desc ( self, run_if_unset=3DTrue ): - if self.desc_data is None: - self.run () + if not hasattr ( self, 'desc_data' ): + if run_if_unset: + self.run() + else: + raise Exception ( "no desc data" ) =20 return self.desc_data # --- end of get_desc (...) --- =20 - def _parse_read_data ( self, read_data ): - """Verifies and parses/fixes read data. + def _make_read_data ( self, raw ): + """Create read data (value or list of values per field) for the given + raw data (list of text lines per field). =20 arguments: - * read_data -- data from file, will be modified + * raw -- + + returns: read data """ + # catch None + if raw is None: return None + + # this dict will be returned as result later + read =3D dict() + + flags =3D self.field_definition.get_fields_with_flag =20 # insert default values default_values =3D self.field_definition.get_fields_with_default_value= () =20 for field_name in default_values.keys(): - if not field_name in read_data: - read_data [field_name] =3D default_values [field_name] + if not field_name in raw: + read [field_name] =3D default_values [field_name] + + + # transfer fields from raw as string or list + fields_join =3D flags ( 'joinValues' ) + fields_isList =3D flags ( 'isList' ) + fields_wsList =3D flags ( 'isWhitespaceList' ) + + list_split =3D re.compile ( + config.get_or_fail ( 'DESCRIPTION.list_split_regex' ) + ).split + slist_split =3D re.compile ( '\s+' ).split + + make_list =3D lambda l : tuple ( filter ( None, list_split ( l, 0 ) = ) ) + make_slist =3D lambda l : tuple ( filter ( None, slist_split ( l, 0 ) = ) ) + + for field in raw.keys(): + value_line =3D ' '.join ( filter ( None, raw [field] ) ) + + # join > isList > wsList [... >=3D join (implicit)] + + if field in fields_join: + read [field] =3D value_line + + elif field in fields_isList: + read [field] =3D make_list ( value_line ) + + elif field in fields_wsList: + read [field] =3D make_slist ( value_line ) + + else: + read [field] =3D value_line =20 =20 - # join values to a single string - for field_name in \ - self.field_definition.get_fields_with_flag ( 'joinValues' ) \ - : - if field_name in read_data: - read_data [field_name] =3D ' ' . join ( read_data [field_name] ) + return read + # --- end of _make_read_data (...) --- + + def _verify_read_data ( self, read ): + """Verifies read data. + Checks that all mandatory fields are set and all fields have suitable + values. + + Returns True (^=3D valid data) or False (^=3D cannot use package) + """ + fref =3D self.field_definition =20 # ensure that all mandatory fields are set missing_fields =3D set () =20 - for field_name in \ - self.field_definition.get_fields_with_flag ( 'mandatory' ): + for field in fref.get_fields_with_flag ( 'mandatory' ): =20 - if field_name in read_data: - if read_data [field_name] is None or \ - len ( read_data [field_name] ) < 1 \ - : - missing_fields.add ( field_name ) + if field in read: + if read [field] is None or len ( read [field] ) < 1: + missing_fields.add ( field ) #else: ok else: - missing_fields.add ( field_name ) + missing_fields.add ( field ) =20 =20 # check for fields that allow only certain values unsuitable_fields =3D set() =20 - restricted_fields =3D \ - self.field_definition.get_fields_with_allowed_values() + restricted_fields =3D fref.get_fields_with_allowed_values() =20 - for field_name in restricted_fields: - if field_name in read_data: - if not self.field_definition.get ( field_name ) . value_allowed ( - read_data [field_name] - ): - unsuitable_fields.add ( field_name ) + for field in restricted_fields: + if field in read: + if not fref.get ( field ).value_allowed ( read [field] ): + unsuitable_fields.add ( field ) =20 # summarize results - valid =3D not bool ( len ( missing_fields ) or len ( unsuitable_fields= ) ) + valid =3D not len ( missing_fields ) and not len ( unsuitable_fields ) if not valid: self.logger.info ( "Cannot use R package" ) # name? if len ( missing_fields ): @@ -117,149 +177,87 @@ class DescriptionReader ( object ): =20 return valid =20 - # --- end of _parse_read_data (...) --- + # --- end of _verify_read_data (...) --- =20 - def run ( self ): - """Reads a DESCRIPTION file and returns the read data if successful, - else None. + def _get_desc_from_file ( self, filepath, pkg_name=3D'.' ): + """Reads a file returns the description data. =20 arguments: - * file -- path to the tarball file (containing the description file) - that should be read + * filepath -- file to read (str; path to tarball or file) + * pkg_name -- name of the package, in tarballs the description file + is located in / and thus this argument + is required. Defaults to '.', set to None to disable. + + All exceptions are passed to the caller (TarError, IOErr, ). + can either be a tarball in which case the real DESCRIPTION + file is read (/DESCRIPTION) or a normal file. + """ =20 - It does some pre-parsing, inter alia - -> assigning field identifiers from the file to real field names - -> split field values - -> filter out unwanted/useless fields + self.logger.debug ( "Starting to read file '%s' ...\n" % filepath ) =20 - The return value is a description_data dict or None if the read data - are "useless" (not suited to create an ebuild for it, - e.g. if OS_TYPE is not unix). - """ + if not ( isinstance ( filepath, str ) and filepath ): + raise Exception ( "bad usage" ) =20 - def make_values ( value_str, field_context=3DNone ): - """Extracts relevant data from value_str and returns them as list. - - arguments: - * value_str -- string that represents the (just read) values - * field_context -- field name the value belongs to; - optional, defaults to None - - It's useful to set field_context 'cause several fields ('Depends') - have multiple values arranged in a list (dep0, dep1 [, depK]*). - """ - - svalue_str =3D value_str.strip() - - if not svalue_str: - # empty value(s) - return [] - - elif field_context is None: - # default return if no context given - return [ svalue_str ] - - elif field_context in \ - self.field_definition.get_fields_with_flag ( 'isList' ) \ - : - # split up this list (separated by commata and/or semicolons) - # *beware*/fixme: py3, filter returns filter object - return filter ( None, re.split ( - config.get ( 'DESCRIPTION.list_split_regex' ), - svalue_str, - 0 - ) ) - - elif field_context in \ - self.field_definition.get_fields_with_flag ( 'isWhitespaceList' ) \ - : - # split up this list (separated by whitespace) - return filter ( None, re.split ( '\s+', svalue_str, 0 ) ) - - # default return - return [ svalue_str ] - - # --- end of make_values (...) --- - - def get_desc_from_file ( filepath, pkg_name=3D'.' ): - """Reads a file returns the description data. - - arguments: - * filepath -- file to read (str; path to tarball or file) - * pkg_name -- name of the package, in tarballs the description file - is located in / and thus this argument - is required. Defaults to '.', set to None to disable. - - All exceptions are passed to the caller (TarError, IOErr, ). - can either be a tarball in which case the real DESCRIPTION - file is read (/DESCRIPTION) or a normal file. - """ - - self.logger.debug ( "Starting to read file '%s' ...\n" % filepath ) - - if not ( isinstance ( filepath, str ) and filepath ): - raise Exception ( "bad usage" ) - - # read describes how to import the lines from a file (e.g. rstrip()) - # fh, th are file/tar handles - read =3D th =3D fh =3D None - - if tarfile.is_tarfile ( filepath ): - # filepath is a tarball, open tar handle + file handle - th =3D tarfile.open ( filepath, 'r' ) - if pkg_name: - fh =3D th.extractfile ( os.path.join ( - pkg_name, - config.get ( 'DESCRIPTION.file_name' ) - ) ) - else: - fh =3D th.extractfile ( config.get ( 'DESCRIPTION.file_name' ) ) + # read describes how to import the lines from a file (e.g. rstrip()) + # fh, th are file/tar handles + read =3D th =3D fh =3D None =20 - # have to decode the lines - read =3D lambda lines : [ line.decode().rstrip() for line in lines ] + if tarfile.is_tarfile ( filepath ): + # filepath is a tarball, open tar handle + file handle + th =3D tarfile.open ( filepath, 'r' ) + if pkg_name: + fh =3D th.extractfile ( os.path.join ( + pkg_name, + config.get ( 'DESCRIPTION.file_name' ) + ) ) else: - # open file handle only - fh =3D open ( filepath, 'r' ) - read =3D lambda lines : [ line.rstrip() for line in lines ] - - x =3D None - read_lines =3D read ( fh.readlines() ) - del x, read - - fh.close() - if not th is None: th.close() - del fh, th - - if hasattr ( self, 'write_desc_file' ): - try: - util.dodir ( DescriptionReader.WRITE_DESCFILES_DIR ) - fh =3D open ( self.write_desc_file, 'w' ) - fh.write ( - '=3D=3D=3D This is debug output (%s) =3D=3D=3D\n' - % time.strftime ( '%F %H:%M:%S' ) - ) - fh.write ( '\n'.join ( read_lines ) ) - fh.write ( '\n' ) - finally: - if 'fh' in locals() and fh: fh.close() - + fh =3D th.extractfile ( config.get ( 'DESCRIPTION.file_name' ) ) + + # have to decode the lines + read =3D lambda lines : [ line.decode().rstrip() for line in lines ] + else: + # open file handle only + fh =3D open ( filepath, 'r' ) + read =3D lambda lines : [ line.rstrip() for line in lines ] + + x =3D None + read_lines =3D read ( fh.readlines() ) + del x, read + + fh.close() + if not th is None: th.close() + del fh, th + + if hasattr ( self, 'write_desc_file' ): + try: + util.dodir ( DescriptionReader.WRITE_DESCFILES_DIR ) + fh =3D open ( self.write_desc_file, 'w' ) + fh.write ( + '=3D=3D=3D This is debug output (%s) =3D=3D=3D\n' + % time.strftime ( '%F %H:%M:%S' ) + ) + fh.write ( '\n'.join ( read_lines ) ) + fh.write ( '\n' ) + finally: + if 'fh' in locals() and fh: fh.close() =20 - return read_lines =20 - # --- end of get_desc_from_file (...) --- + return read_lines =20 - self.desc_data =3D None - read_data =3D dict () + # --- end of _get_desc_from_file (...) --- =20 + def _get_raw_data ( self ): try: - desc_lines =3D get_desc_from_file ( + desc_lines =3D self._get_desc_from_file ( self.fileinfo ['package_file'], self.fileinfo ['package_name'] ) =20 - except IOError as err: + except Exception as err: self.logger.exception ( err ) - return self.desc_data + return None + + raw =3D dict() =20 field_context =3D None =20 @@ -281,8 +279,8 @@ class DescriptionReader ( object ): if field_context: # context is set =3D> append values =20 - for val in make_values ( sline, field_context ): - read_data [field_context] . append ( val ) + raw [field_context].append ( sline ) + else: # no valid context =3D> ignore line pass @@ -292,7 +290,7 @@ class DescriptionReader ( object ): field_context =3D None =20 line_components =3D sline.partition ( - config.get ( 'DESCRIPTION.field_separator' ) + config.get ( 'DESCRIPTION.field_separator', ':' ) ) =20 if line_components [1]: @@ -319,15 +317,13 @@ class DescriptionReader ( object ): 'already been catched in DescriptionField...' ) =20 - # create a new empty list for this field_context - read_data [field_context] =3D [] + if field_context in raw: + raise Exception ( "field %s exists!" % field_context ) =20 # add values to read_data, no need to check # line_components [2] 'cause [1] was a true str - for val in \ - make_values ( line_components [2], field_context ) \ - : - read_data [field_context] . append ( val ) + # create a new empty list for this field_context + raw[field_context] =3D [ line_components [2].lstrip() ] =20 else: # reaching this branch means that @@ -341,14 +337,44 @@ class DescriptionReader ( object ): =20 # -- end for -- =20 - if self._parse_read_data ( read_data ): + return raw + # --- end of _get_raw_data (...) --- + + def run ( self ): + """Reads a DESCRIPTION file and returns the read data if successful, + else None. + + arguments: + * file -- path to the tarball file (containing the description file) + that should be read + + It does some pre-parsing, inter alia + -> assigning field identifiers from the file to real field names + -> split field values + -> filter out unwanted/useless fields + + The return value is a description_data dict or None if the read data + are "useless" (not suited to create an ebuild for it, + e.g. if OS_TYPE is not unix). + """ + + raw_data =3D self._get_raw_data() + read_data =3D self._make_read_data ( raw_data ) + + self.desc_data =3D None + + if read_data is None: + self.logger.warning ( + "Failed to read file '%s'." % self.fileinfo ['package_file'] + ) + + elif self._verify_read_data ( read_data ): self.logger.debug ( "Successfully read file '%s' with data =3D %s." % ( self.fileinfo ['package_file'], read_data ) ) self.desc_data =3D read_data =20 - # get_desc() is preferred, but this method returns the desc data, too - return self.desc_data + # else have log entries from _verify() =20 # --- end of run (...) ---