From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id 15EF31381F3 for ; Wed, 28 Aug 2013 15:54:39 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 92F80E0C38; Wed, 28 Aug 2013 15:54:33 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 050E9E0C8F for ; Wed, 28 Aug 2013 15:54:32 +0000 (UTC) Received: from hornbill.gentoo.org (hornbill.gentoo.org [94.100.119.163]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id DE50033E0B3 for ; Wed, 28 Aug 2013 15:54:31 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by hornbill.gentoo.org (Postfix) with ESMTP id 91569E545F for ; Wed, 28 Aug 2013 15:54:30 +0000 (UTC) From: "André Erdmann" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "André Erdmann" Message-ID: <1377705107.c55acecf0d7d58ec898a5c9bbd36086fc5ee52f6.dywi@gentoo> Subject: [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/pkgdir/ X-VCS-Repository: proj/R_overlay X-VCS-Files: roverlay/overlay/pkgdir/ebuildparser.py X-VCS-Directories: roverlay/overlay/pkgdir/ X-VCS-Committer: dywi X-VCS-Committer-Name: André Erdmann X-VCS-Revision: c55acecf0d7d58ec898a5c9bbd36086fc5ee52f6 X-VCS-Branch: master Date: Wed, 28 Aug 2013 15:54:30 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Archives-Salt: 39c878e4-9235-4907-a846-a19b029b4fd4 X-Archives-Hash: 350521e06f882c9845b4aab7a1a1370c commit: c55acecf0d7d58ec898a5c9bbd36086fc5ee52f6 Author: André Erdmann mailerd de> AuthorDate: Wed Aug 28 15:51:47 2013 +0000 Commit: André Erdmann mailerd de> CommitDate: Wed Aug 28 15:51:47 2013 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=c55acecf overlay: (partially) parse ebuild files This commit adds a parser that parses an ebuild file partially. More concrete, it reads $SRC_URI, which is required distmap-related operations, e.g. for reliably handling package file collisions due to multiple categories. --- roverlay/overlay/pkgdir/ebuildparser.py | 283 ++++++++++++++++++++++++++++++++ 1 file changed, 283 insertions(+) diff --git a/roverlay/overlay/pkgdir/ebuildparser.py b/roverlay/overlay/pkgdir/ebuildparser.py new file mode 100644 index 0000000..79e6f15 --- /dev/null +++ b/roverlay/overlay/pkgdir/ebuildparser.py @@ -0,0 +1,283 @@ +# R overlay -- overlay package, package directory, "minimal" ebuild parser +# -*- coding: utf-8 -*- +# Copyright (C) 2012, 2013 André Erdmann +# Distributed under the terms of the GNU General Public License; +# either version 2 of the License, or (at your option) any later version. + +from __future__ import print_function + +import os +import shlex +import string + +import roverlay.util.objects +import roverlay.strutil + + +STR_FORMATTER = string.Formatter() +VFORMAT = STR_FORMATTER.vformat + +class ParserException ( Exception ): + pass +# --- end of ParserException --- + + +class SrcUriEntry ( object ): + + def __init__ ( self, src_uri, output_file=None ): + super ( SrcUriEntry, self ).__init__() + self.uri = src_uri + if output_file: + self.have_local = True + self.local_file = output_file + else: + self.have_local = False + self.local_file = src_uri.rpartition ( '/' ) [-1] + # --- end of __init__ (...) --- + + def __str__ ( self ): + if self.have_local: + return "{URI} -> {f}".format ( URI=self.uri, f=self.local_file ) + else: + return self.uri + # --- end of __str__ (...) --- + +# --- end of SrcUriEntry --- + + + +class EbuildParser ( object ): + + @classmethod + def from_file ( cls, filepath, vartable=None ): + instance = cls ( filepath, vartable=vartable ) + instance.read() + return instance + # --- end of from_file (...) --- + + def __init__ ( self, filepath, vartable=None ): + super ( EbuildParser, self ).__init__() + self.filepath = filepath + self.vartable = vartable + # --- end of __init__ (...) --- + + def _read_tokens ( self, breakparse=None ): + with open ( self.filepath, 'rt' ) as FH: + reader = shlex.shlex ( FH ) + reader.whitespace_split = False + reader.wordchars += ' ,./$()[]:+-@*~<>' + + token = reader.get_token() + if breakparse is None: + while token: + yield token + token = reader.get_token() + else: + while token and not breakparse ( token ): + yield token + token = reader.get_token() + # --- end of _read_tokens (...) --- + + def _read_variables ( self, do_unquote=True ): + # assumption: no (important) variables after the first function + + + # read all tokens and store them in a list + # this allows to look back/ahead + tokens = list ( self._read_tokens ( + breakparse=( lambda s: ( len(s) > 2 and s[-2:] == '()' ) ) + ) ) + + + varname = None + data = dict() + + last_index = len ( tokens ) - 1 + ignore_next = False + is_bash_array = False + # 0 -> no value read, 1-> have value(s), 2 -> reject token + value_mode = 0 + + for index, token in enumerate ( tokens ): + if ignore_next: + ignore_next = False + pass + + elif index < last_index and tokens [index+1] == '=': + # lookahead result: token is a varname + ignore_next = True + is_bash_array = False + value_mode = 0 + varname = token + data [token] = None + + elif value_mode == 0: + if value_mode == 0 and token == '()': + is_bash_array = True + value_mode = 2 + data [varname] = [] + + elif value_mode == 0 and token == '(': + is_bash_array = True + value_mode = 1 + data [varname] = [] + + else: + data [varname] = token + value_mode = 1 + + elif value_mode > 1: + pass + + elif is_bash_array: + # impiles value_mode != 0 + + if token == ')': + value_mode = 2 + else: + data [varname].append ( token ) + +# else: +# pass + + + if do_unquote: + return { + varname: ( + roverlay.strutil.unquote ( value ) if isinstance ( value, str ) + else list ( + roverlay.strutil.unquote ( item ) for item in value + ) + ) + for varname, value in data.items() + } + else: + return data + # --- end of _read_variables (...) --- + + def _get_src_uri_entries ( self, value ): + assert isinstance ( value, str ) + + src_uri = None + want_local_file = False + + for s in value.split(): + if not s or s[-1] == '?' or s in { '(', ')' }: + pass + + elif want_local_file: + yield SrcUriEntry ( src_uri, s ) + want_local_file = False + src_uri = None + + elif s == '->': + if src_uri is None: + raise Exception ( + "SRC_URI: arrow operator -> without preceding URI" + ) + else: + want_local_file = True + + else: + if src_uri is not None: + yield SrcUriEntry ( src_uri ) + src_uri = s + + # -- end for + + if want_local_file: + raise Exception ( "SRC_URI: arrow operator -> without local file" ) + + elif src_uri is not None: + yield SrcUriEntry ( src_uri ) + # --- end of _get_src_uri_entries (...) --- + + @roverlay.util.objects.abstractmethod + def read ( self ): + pass + # --- end of read (...) --- + +# --- end of EbuildParser --- + + +class SrcUriParser ( EbuildParser ): + + def __init__ ( self, filepath, vartable=None ): + super ( SrcUriParser, self ).__init__ ( filepath, vartable=vartable ) + self.src_uri = None + # --- end of __init__ (...) --- + + def iter_entries ( self ): + if self.src_uri: + for entry in self.src_uri: + yield entry + # --- end of _iterate (...) --- + + def iter_local_files ( self, ignore_unparseable=False ): + def convert_chars_with_vars ( text ): + mode = 0 + for index, char in enumerate ( text ): + + if mode == 0: + if char == '$': + mode = 1 + else: + yield char + + elif mode == 1: + if char == '{': + yield char + mode = 2 + else: + raise ParserException ( + 'cannot convert variable starting at ' + 'position {:d} in {}'.format ( index, text ) + ) + + elif mode == 2 and char in { '/', }: + raise ParserException ( + 'unsupported char {} inside variable at ' + 'position {:d} in {}'.format ( char, index, text ) + ) + + else: + yield char + # --- end of convert_chars_with_vars (...) --- + + varstr = lambda s: VFORMAT ( + ''.join ( convert_chars_with_vars ( s ) ), (), self.vartable + ) + + if self.src_uri: + for entry in self.src_uri: + local_file = entry.local_file + if '$' in local_file: + # TODO: need vartable (dict(P->,PN->)) + # + if ignore_unparseable: + try: + yield varstr ( local_file ) + except ParserException: + if not ignore_unparseable: + raise + else: + yield varstr ( local_file ) + + else: + yield local_file + # --- end of iter_local_files (...) --- + + def __iter__ ( self ): + return self.iter_entries() + # --- end of __iter__ (...) --- + + def read ( self ): + data = self._read_variables() + + if 'SRC_URI' in data: + self.src_uri = list ( + self._get_src_uri_entries ( data ['SRC_URI'] ) + ) + # --- end of read (...) --- + +# --- end of SrcUriParser ---