From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from pigeon.gentoo.org ([208.92.234.80] helo=lists.gentoo.org) by finch.gentoo.org with esmtp (Exim 4.60) (envelope-from ) id 1SYGpl-0001dT-7G for garchives@archives.gentoo.org; Sat, 26 May 2012 13:15:05 +0000 Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 46B72E0652; Sat, 26 May 2012 13:14:53 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) by pigeon.gentoo.org (Postfix) with ESMTP id 042FEE0652 for ; Sat, 26 May 2012 13:14:52 +0000 (UTC) Received: from hornbill.gentoo.org (hornbill.gentoo.org [94.100.119.163]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 3487D1B4007 for ; Sat, 26 May 2012 13:14:52 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by hornbill.gentoo.org (Postfix) with ESMTP id A83A7E5434 for ; Sat, 26 May 2012 13:14:49 +0000 (UTC) From: "André Erdmann" To: gentoo-commits@lists.gentoo.org Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "André Erdmann" Message-ID: <1338038044.9298e87377b76ecd093535458d4566d890c7b2e8.dywi@gentoo> Subject: [gentoo-commits] proj/R_overlay:master commit in: roverlay/ X-VCS-Repository: proj/R_overlay X-VCS-Files: roverlay/fileio.py X-VCS-Directories: roverlay/ X-VCS-Committer: dywi X-VCS-Committer-Name: André Erdmann X-VCS-Revision: 9298e87377b76ecd093535458d4566d890c7b2e8 X-VCS-Branch: master Date: Sat, 26 May 2012 13:14:49 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: quoted-printable X-Archives-Salt: d1c2e623-b775-4ea8-bb84-f57255c7c332 X-Archives-Hash: 7b1fe4ca63f860855aabc90931dd8552 commit: 9298e87377b76ecd093535458d4566d890c7b2e8 Author: Andre Erdmann mailerd de> AuthorDate: Sat May 26 13:14:04 2012 +0000 Commit: Andr=C3=A9 Erdmann mailerd de> CommitDate: Sat May 26 13:14:04 2012 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=3Dproj/R_overlay.git= ;a=3Dcommit;h=3D9298e873 roverlay,fileio, DescriptionReader: field identifier aliases now in field ['alias'] [] add ability to set default values for fields field flag checking improved modified: roverlay/fileio.py --- roverlay/fileio.py | 336 +++++++++++++++++++++++++++++-----------------= ------ 1 files changed, 189 insertions(+), 147 deletions(-) diff --git a/roverlay/fileio.py b/roverlay/fileio.py index 88cca0f..6e905b9 100644 --- a/roverlay/fileio.py +++ b/roverlay/fileio.py @@ -9,39 +9,34 @@ import tarfile # temporary import until logging is implemented from sys import stderr as logging =20 - - # temporary import until config and real constants are implemented from roverlay import tmpconst as const =20 class DescriptionReader: + """Description Reader""" =20 @classmethod def __init__ ( self ): """Initializes a DESCRIPTION file reader.""" - self._mandatory_fields =3D [] + pass =20 @classmethod - def _get_mandatory_fields ( self, force_update=3DFalse ): - """Returns a list of the fields a DESCRIPTION file must define. + def _get_fields_with_flag ( self, flag, foce_update=3DFalse ): =20 - arguments: - * force_update -- enforce creation of a new list - """ + matching_fields =3D [] =20 - # use previously calculated results (if item count > 0) - if force_update or ( not len (self._mandatory_fields) ): - field =3D None - mandatory =3D [] - for field in const.DESCRIPTION_FIELD_MAP.keys(): - if 'flags' in const.DESCRIPTION_FIELD_MAP [field]: - if 'mandatory' in const.DESCRIPTION_FIELD_MAP [field] ['flags']: - mandatory.append ( field ) + field =3D None + for field in const.DESCRIPTION_FIELD_MAP.keys(): + if flag is None: + matching_fields.append ( field ) + + elif 'flags' in const.DESCRIPTION_FIELD_MAP [field]: + if flag in const.DESCRIPTION_FIELD_MAP [field] ['flags']: + matching_fields.append ( field ) =20 - self._mandatory_fields =3D mandatory - del field, mandatory + del field + return matching_fields =20 - return self._mandatory_fields =20 @classmethod def _find_field ( self , field_identifier ): @@ -62,7 +57,7 @@ class DescriptionReader: =20 # save some time by prevent searching if field_id is empty if not field_identifier: - return '' + return None =20 # search for real field names first for field in const.DESCRIPTION_FIELD_MAP.keys(): @@ -73,29 +68,23 @@ class DescriptionReader: =20 for field in const.DESCRIPTION_FIELD_MAP.keys(): =20 - # ?TODO : DESCRIPTION_FIELD_MAP ['alias'] [] instead of = the current structure? - # does extra information (-> alias(es)) for this field exist? - if isinstance ( const.DESCRIPTION_FIELD_MAP [field], dict ): - - for alias_type in const.DESCRIPTION_FIELD_MAP [field] . keys(): - - if alias_type =3D=3D 'withcase': - - for alias in const.DESCRIPTION_FIELD_MAP [field] [alias_type]: - if field_identifier =3D=3D alias: - return field + if 'alias' in const.DESCRIPTION_FIELD_MAP [field]: =20 - elif alias_type =3D=3D 'nocase': + if 'withcase' in const.DESCRIPTION_FIELD_MAP [field] ['alias']: + for alias in const.DESCRIPTION_FIELD_MAP [field] ['alias'] ['withca= se']: + if field_identifier =3D=3D alias: + return field =20 - for alias in const.DESCRIPTION_FIELD_MAP [field] [alias_type]: - if field_id_lower =3D=3D alias.lower(): - return field + if 'nocase' in const.DESCRIPTION_FIELD_MAP [field] ['alias']: + for alias in const.DESCRIPTION_FIELD_MAP [field] ['alias'] ['nocase= ']: + if field_id_lower =3D=3D alias.lower(): + return field =20 - #elif other_alias_type: + #if 'other_alias_type' in const.DESCRIPTION_FIELD_MAP [field] ['alia= s']: =20 - # returning empty string if no valid field identifier matches - return '' + # returning None if no valid field identifier matches + return None =20 @classmethod def _make_values ( self, value_str, field_context=3DNone ): @@ -111,12 +100,16 @@ class DescriptionReader: =20 svalue_str =3D value_str.strip() =20 - if field_context is None: + if not svalue_str: + # empty value(s) + return [] + + elif field_context is None: # default return if no context given return [ svalue_str ] =20 - if self._check_fieldflag ( field_context ): - # have flags for field_context, check these + elif self._check_fieldflag ( field_context ): + # value str is not empty and have flags for field_context, check thes= e =20 if self._check_fieldflag ( field_context, 'isList' ): # split up this list (that is separated by commata and/or semicolon= s) @@ -142,12 +135,9 @@ class DescriptionReader: """ =20 if field in const.DESCRIPTION_FIELD_MAP: - if 'flags' in const.DESCRIPTION_FIELD_MAP [field]: - if flag_to_check in const.DESCRIPTION_FIELD_MAP [field] ['flags']: return True - elif flag_to_check is None: # 'flags' exist, return true return True @@ -155,52 +145,146 @@ class DescriptionReader: return False =20 @staticmethod - def _get_desc_from_tarball ( tarball, pkg_name=3D'.' ): - """ - Extracts the contents of the description file in the given tarball - and returns them as list of str. + def _get_desc_from_file ( filepath, pkg_name=3D'.' ): + """Reads a file returns the description data. =20 arguments: - * tarball -- tarball to read - * pkg_name -- name of the package, usually the description file is - /DESCRIPTION and this that arguments is - required. Defaults to '.', set to None to disable. + * filepath -- file to read (str; path to tarball or file) + * pkg_name -- name of the package, in tarballs the description file + is located in / and thus this argument is requ= ired. + Defaults to '.', set to None to disable. =20 All exceptions are passed to the caller (TarError, IOErr, ). + can either be a tarball in which case the real DESCRIPTION + file is read (/DESCRIPTION) or a normal file. """ =20 - logging.write ( "Starting to read tarball file '" + tarball + "' ...\n= " ) + logging.write ( "Starting to read file '" + str ( filepath ) + "' ...\= n" ) =20 - if not tarfile.is_tarfile ( tarball ): - # not a tarball, real exception - raise Exception ("tarball expected") + if not ( isinstance ( filepath, str ) and filepath ): + raise Exception ( "bad usage" ) =20 - # open a file handle for the DESCRIPTION file using a tar handle = - th =3D fh =3D None + # read describes how to import the lines from a file (e.g. rstrip()) + # fh, th are file/tar handles + read =3D th =3D fh =3D None =20 - th =3D tarfile.open ( tarball, 'r' ) - if pkg_name: - fh =3D th.extractfile ( - os.path.join ( pkg_name, const.DESCRIPTION_FILE_NAME ) - ) + if tarfile.is_tarfile ( filepath ): + # filepath is a tarball, open tar handle + file handle + th =3D tarfile.open ( filepath, 'r' ) + if pkg_name: + fh =3D th.extractfile ( os.path.join ( pkg_name, const.DESCRIPTION_F= ILE_NAME ) ) + else: + fh =3D th.extractfile ( const.DESCRIPTION_FILE_NAME ) + + # have to decode the lines + read =3D lambda lines : [ line.decode().rstrip() for line in lines ] else: - fh =3D th.extractfile ( const.DESCRIPTION_FILE_NAME ) + # open file handle only + fh =3D open ( filepath, 'r' ) + read =3D lambda lines : [ line.rstrip() for line in lines ] =20 x =3D None - # get lines from , decode and remove end of line whitespace - read_lines =3D [ x.decode().rstrip() for x in fh.readlines() ] - del x + read_lines =3D read ( fh.readlines() ) + del x, read =20 fh.close() - th.close() - + if not th is None: th.close() del fh, th + return read_lines =20 + @staticmethod + def _get_fileinfo ( filepath ): + """Returns some info about the given filepath as dict whose contents a= re + the file path, the file name ([as package_file with suffix and] + as filename with tarball suffix removed), the package name + and the package_version. + + arguments: + * filepath -- + """ + + package_file =3D os.path.basename ( filepath ) + + filename =3D re.sub ( const.RPACKAGE_SUFFIX_REGEX + '$', '', package_f= ile ) + + # todo move that separator to const + package_name, sepa, package_version =3D filename.partition ( '_' ) + + if not sepa: + # file name unexpected, tarball extraction will (probably) fail + #raise Exception ("file name unexpected") + logging.write ( "unexpected file name '" + filename + "'.\n" ) + + return dict ( + filepath =3D filepath, + filename =3D filename, + #package_file =3D package_file, + package_name =3D package_name, + package_version =3D package_version, + ) + =20 =20 @classmethod - def readfile ( self, file ): + def _verify_read_data ( self, read_data ): + """Verifies and fixes (e.g. add default values) read data""" + + def stats ( data ): + """Temporary function that prints some info about the given data.""" + field =3D None + logging.write ( "=3D=3D=3D this is the list of read data =3D=3D=3D\n"= ) + for field in read_data.keys(): + logging.write ( field + " =3D " + str ( read_data [field] ) + "\n" ) + logging.write ( "=3D=3D=3D end of list =3D=3D=3D\n" ) + del field + + stats ( read_data ) + + # "finalize" data + logging.write ( "Fixing data...\n" ) + field =3D None + + # join values to a single str + for field in self._get_fields_with_flag ( 'joinValues' ): + if field in read_data.keys(): + read_data [field] =3D ' ' . join ( read_data [field] ) + + # verify that all necessary fields have been added and are set + missing_fields =3D dict() + for field in self._get_fields_with_flag ( 'mandatory' ): + if field in read_data: + if not len (read_data [field]): + missing_fields [field] =3D 'unset' + else: + missing_fields [field] =3D 'missing' + + del field + + if len (missing_fields): + logging.write ( + "Verification of mandatory fields failed, the result leading to this= was: " + + str (missing_fields) + "\n" + ) + + # + raise Exception ("^^^look above") + + del missing_fields + + # add/insert default values + for field in const.DESCRIPTION_FIELD_MAP.keys(): + if not field in read_data and 'default_value' in const.DESCRIPTION_FI= ELD_MAP [field]: + read_data [field] =3D const.DESCRIPTION_FIELD_MAP [field] ['default_= value'] + + + stats ( read_data ) + + return True + + + @classmethod + def readfile ( self, filepath ): """Reads a DESCRIPTION file and returns the read data if successful, e= lse None. =20 arguments: @@ -216,56 +300,50 @@ class DescriptionReader: with as str and as list. """ =20 - # todo move that regex to const - filename =3D re.sub ('[.](tgz|tbz2|(tar[.](gz|bz2)))', - '', - os.path.basename ( file ) + read_data =3D dict ( + _ =3D DescriptionReader._get_fileinfo ( filepath ) ) - # todo move that separator to const - package_name, sepa, package_version =3D filename.partition ( '_' ) - if not sepa: - # file name unexpected - raise Exception ("file name unexpected") =20 =20 try: - desc_lines =3D DescriptionReader._get_desc_from_tarball ( file, packa= ge_name ) + desc_lines =3D DescriptionReader._get_desc_from_file ( + filepath, read_data ['_'] ['package_name'] + ) + =20 except IOError as err: # raise =20 - read_data =3D dict() =20 field_context =3D val =3D line =3D sline =3D None for line in desc_lines: + # using s(tripped)line whenever whitespace doesn't matter sline =3D line.lstrip() =20 - if not sline: - # empty line + if (not sline) or (line [0] =3D=3D const.DESCRIPTION_COMMENT_CHAR): + # empty line or comment pass =20 - elif line [0] =3D=3D const.DESCRIPTION_COMMENT_CHAR: - pass - - elif field_context and line [0] !=3D sline [0]: - # line starts with whitespace and context is set =3D> append values - for val in self._make_values ( sline, field_context ): - read_data [field_context] . append ( val ) + elif line [0] !=3D sline [0]: + # line starts with whitespace =20 + if field_context: + # context is set =3D> append values =20 - elif line [0] !=3D sline [0]: - # line starts with whitespace and context is not set =3D> ignore - pass + for val in self._make_values ( sline, field_context ): + read_data [field_context] . append ( val ) + else: + # no valid context =3D> ignore line + pass =20 else: - # new context, forget last one + # line introduces a new field context, forget last one field_context =3D None =20 line_components =3D sline.partition ( const.DESCRIPTION_FIELD_SEPARA= TOR ) =20 - if line_components [1]: # line contains a field separator, set field context field_context =3D self._find_field ( line_components [0] ) @@ -274,10 +352,10 @@ class DescriptionReader: # create a new empty list for field_context read_data [field_context] =3D [] =20 - if len ( line_components ) =3D=3D 3: - # add values to read_data - for val in self._make_values ( line_components [2], field_context= ): - read_data [field_context] . append ( val ) + # add values to read_data + # no need to check line_components [2] 'cause [1] was a true str + for val in self._make_values ( line_components [2], field_context = ): + read_data [field_context] . append ( val ) =20 else: # useless line, skip @@ -286,57 +364,21 @@ class DescriptionReader: + line_components [0] + "'\n" ) =20 - del line_components - else: - # how to reach this block? -- remove later - raise Exception ( "should-be unreachable code" ) - - del val, line, sline, field_context - - def stats ( data ): - """Temporary function that prints some info about the given data.""" - field =3D None - logging.write ( "=3D=3D=3D this is the list of read data =3D=3D=3D\n"= ) - for field in read_data.keys(): - logging.write ( field + " =3D " + str ( read_data [field] ) + "\n" ) - logging.write ( "=3D=3D=3D end of list =3D=3D=3D\n" ) - del field + # reaching this branch means that + # (a) line has no leading whitespace + # (b) line has no separator (:) + # this should not occur in description files (bad syntax?) + logging.write ( "***" + line_components [0] + "***\n") + raise Exception ( "bad file" ) =20 - stats ( read_data ) + del line_components =20 - # "finalize" data - field =3D None - for field in read_data.keys(): - if self._check_fieldflag ( field ): - # has flags - if self._check_fieldflag ( field, 'joinValues' ): - read_data [field] =3D ' ' . join ( read_data [field] ) + del sline, line, val, field_context =20 - # verify that all necessary fields have been added and are set - missing_fields =3D dict() - for field in self._get_mandatory_fields(): - if field in read_data: - if not len (read_data [field]): - missing_fields [field] =3D 'unset' =20 - else: - missing_fields [field] =3D 'missing' - - if len (missing_fields): - logging.write ("Verification of mandatory fields failed, the result l= eading to this is: " + - str (missing_fields) + "\n" - ) - - # - raise Exception ("^^^look above") - - del field, missing_fields - - # add default values - - logging.write ( "Fixing data...\n" ) - stats ( read_data ) - - return read_data + if self._verify_read_data ( read_data ): + return read_data + else: + return None =20