From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id B16231381F3 for ; Wed, 10 Jul 2013 16:17:22 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id B2C2AE0A84; Wed, 10 Jul 2013 16:17:15 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id C0F11E0A84 for ; Wed, 10 Jul 2013 16:17:04 +0000 (UTC) Received: from hornbill.gentoo.org (hornbill.gentoo.org [94.100.119.163]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 4A4D233E8AF for ; Wed, 10 Jul 2013 16:16:58 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by hornbill.gentoo.org (Postfix) with ESMTP id BA051E5479 for ; Wed, 10 Jul 2013 16:16:55 +0000 (UTC) From: "André Erdmann" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "André Erdmann" Message-ID: <1373467920.6f31508300364ba709429e5d4e60bc9a05ad7d63.dywi@gentoo> Subject: [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/pkgdir/manifest/, roverlay/config/, roverlay/, ... X-VCS-Repository: proj/R_overlay X-VCS-Files: roverlay/__init__.py roverlay/argutil.py roverlay/config/entrymap.py roverlay/overlay/pkgdir/__init__.py roverlay/overlay/pkgdir/manifest/__init__.py roverlay/overlay/pkgdir/manifest/entry.py roverlay/overlay/pkgdir/manifest/file.py roverlay/overlay/pkgdir/packagedir_newmanifest.py X-VCS-Directories: roverlay/overlay/pkgdir/manifest/ roverlay/config/ roverlay/ roverlay/overlay/pkgdir/ X-VCS-Committer: dywi X-VCS-Committer-Name: André Erdmann X-VCS-Revision: 6f31508300364ba709429e5d4e60bc9a05ad7d63 X-VCS-Branch: master Date: Wed, 10 Jul 2013 16:16:55 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Archives-Salt: d3f59bef-2919-4807-a26f-396b74c4080e X-Archives-Hash: eb8cf19ed504ec749c00cf1c67dcb95c Message-ID: <20130710161655.zk4yyGphZqtLpOrfT6Vl-ThvdnVpWVcmyBsh-Y6cf6o@z> commit: 6f31508300364ba709429e5d4e60bc9a05ad7d63 Author: André Erdmann mailerd de> AuthorDate: Wed Jul 10 14:52:00 2013 +0000 Commit: André Erdmann mailerd de> CommitDate: Wed Jul 10 14:52:00 2013 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=6f315083 pkgdir: newmanifest roverlay already creates checksums for package files since they're required for the distmap. Using ebuildmanifest does not allow to re-use these checksums for faster manifest file creation. In case of portagemanifest, it would (probably) be possible. This commit adds a mostly internal manifest file creation. Compared with ebuildmanifest, it is A LOT faster and threadsafe, and in contrast to portagemanifest, it does not rely on portage libs (which is hard to maintain/verify (for me)) and is slightly faster. Actually, newmanifest is meant to replace portagemanifest and become the default implementation. newmanifest falls back to "ebuild manifest" for imported ebuilds since SRC_URI parsing is not implemented. --- roverlay/__init__.py | 9 +- roverlay/argutil.py | 4 +- roverlay/config/entrymap.py | 1 + roverlay/overlay/pkgdir/__init__.py | 1 + roverlay/overlay/pkgdir/manifest/__init__.py | 0 roverlay/overlay/pkgdir/manifest/entry.py | 187 ++++++++++++++++ roverlay/overlay/pkgdir/manifest/file.py | 261 ++++++++++++++++++++++ roverlay/overlay/pkgdir/packagedir_newmanifest.py | 106 +++++++++ 8 files changed, 562 insertions(+), 7 deletions(-) diff --git a/roverlay/__init__.py b/roverlay/__init__.py index 5b4007e..cd93ba1 100644 --- a/roverlay/__init__.py +++ b/roverlay/__init__.py @@ -1,22 +1,21 @@ # R overlay -- roverlay package (__init__) # -*- coding: utf-8 -*- -# Copyright (C) 2012 André Erdmann +# Copyright (C) 2012, 2013 André Erdmann # Distributed under the terms of the GNU General Public License; # either version 2 of the License, or (at your option) any later version. """R overlay package Provides roverlay initialization helpers (setup_initial_logger, -load_config_file) and some information vars (__version__, name, ...). +load_config_file) and some information vars (version, name, ...). """ __all__ = [ 'setup_initial_logger', 'load_config_file', ] name = "R_overlay" -version = ( 0, 2, 4 ) -__version__ = '.'.join ( str ( i ) for i in version ) +version = "0.2.4" -description_str = "R overlay creation (roverlay) " + __version__ +description_str = "R overlay creation (roverlay) " + version license_str=( 'Copyright (C) 2012, 2013 Andr\xc3\xa9 Erdmann\n' 'Distributed under the terms of the GNU General Public License;\n' diff --git a/roverlay/argutil.py b/roverlay/argutil.py index 869ea8b..2bccfff 100644 --- a/roverlay/argutil.py +++ b/roverlay/argutil.py @@ -107,7 +107,7 @@ def get_parser ( command_map, default_config_file, default_command='create' ): # adding args starts here arg ( - '-V', '--version', action='version', version=roverlay.__version__ + '-V', '--version', action='version', version=roverlay.version ) arg ( @@ -326,7 +326,7 @@ def get_parser ( command_map, default_config_file, default_command='create' ): '--manifest-implementation', '-M', default=argparse.SUPPRESS, help="choose how Manifest files are written (ebuild(1) or portage libs)", metavar="", - choices=frozenset (( 'ebuild', 'e', 'portage', 'p' )), + choices=frozenset (( 'ebuild', 'e', 'portage', 'p', 'next', )), ) # FIXME: description of --no-incremental is not correct, diff --git a/roverlay/config/entrymap.py b/roverlay/config/entrymap.py index 8117585..0768560 100644 --- a/roverlay/config/entrymap.py +++ b/roverlay/config/entrymap.py @@ -255,6 +255,7 @@ CONFIG_ENTRY_MAP = dict ( 'default', 'ebuild', 'portage', + 'next', # 'e', # 'p', )), diff --git a/roverlay/overlay/pkgdir/__init__.py b/roverlay/overlay/pkgdir/__init__.py index d33ced2..7977ca9 100644 --- a/roverlay/overlay/pkgdir/__init__.py +++ b/roverlay/overlay/pkgdir/__init__.py @@ -26,6 +26,7 @@ _PACKAGE_DIR_IMPLEMENTATIONS = { 'ebuild' : 'packagedir_ebuildmanifest', 'p' : 'packagedir_portagemanifest', 'portage' : 'packagedir_portagemanifest', + 'next' : 'packagedir_newmanifest', } def _configure(): diff --git a/roverlay/overlay/pkgdir/manifest/__init__.py b/roverlay/overlay/pkgdir/manifest/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/roverlay/overlay/pkgdir/manifest/entry.py b/roverlay/overlay/pkgdir/manifest/entry.py new file mode 100644 index 0000000..2313f63 --- /dev/null +++ b/roverlay/overlay/pkgdir/manifest/entry.py @@ -0,0 +1,187 @@ +# R overlay -- manifest package, manifest file entries +# -*- coding: utf-8 -*- +# Copyright (C) 2013 André Erdmann +# Distributed under the terms of the GNU General Public License; +# either version 2 of the License, or (at your option) any later version. + +__all__ = [ + 'ManifestEntry', 'PackageFileManifestEntry', + 'ManifestEntryInvalid', 'UnsupportedFileType', +] + +import os.path + +import roverlay.digest +import roverlay.util + + +class ManifestEntryInvalid ( ValueError ): + pass + +class UnsupportedFileType ( ManifestEntryInvalid ): + pass + + +class ManifestEntry ( object ): + """ManifestEntry represents a single line in the Manifest file + ' for K in 1..N', + where filetype is: + + * AUX for files in the files/ directory [not used by roverlay] + * EBUILD for all ebuilds + * MISC for files not directly used by ebuilds (ChangeLog, metadata.xml) + * DIST for release tarballs (SRC_URI) + """ + + # an ordered list of the default digest types to use + # + # GLEP 59: + HASHTYPES = ( 'sha256', 'sha512', 'whirlpool', ) + # + # old and unsupported + ##HASHTYPES = ( 'rmd160', 'sha1', 'sha256', ) + + FILETYPES = frozenset ({ 'AUX', 'EBUILD', 'MISC', 'DIST', }) + + def __init__ ( + self, filetype, filepath, filename=None, filesize=None, hashes=None + ): + """Creates a ManifestEntry. + Some of the data can be calculated using interpolate(). + + arguments: + * filetype -- file type (AUX, EBUILD, MISC, DIST) + * filepath -- path to the file (can be None if interpolation is + not intended) + * filename -- file name + Defaults to None => try auto-detection. + * filesize -- file size, has to be an integer. Defaults to None. + * hashes -- a dict of checksums (sha256, ...) + Defaults to None. + + Required args: + * filetype + * filepath or (filename and filesize and hashes) + """ + self.filetype = filetype + self.filepath = filepath + self.filename = filename + self.filesize = filesize + self.hashes = hashes + # --- end of __init__ (...) --- + + def add_hashes ( self, hashes ): + """Updates the hash dict of this entry. + + arguments: + * hashes -- hash dict that will be used to update the entry's hashdict + """ + if self.hashes is None: + self.hashes = dict() + self.hashes.update ( hashes ) + # --- end of add_hashes (...) --- + + def interpolate ( self, allow_hash_create=True ): + """ + Tries to calculate all missing data (filesize, filename and hashes). + + arguments: + * allow_hash_create -- whether to try hash creation or not + + raises: + * UnsupportedFileType if filetype not valid + """ + if not self.filename: + if ( + self.filetype == 'MISC' or + self.filetype == 'EBUILD' or + self.filetype == 'DIST' + ): + self.filename = os.path.basename ( self.filepath ) + elif self.filetype == 'AUX': + #self.filetype = 'files' + os.path.sep + os.path.basename ( self.filepath ) + self.filetype = 'files/' + os.path.basename ( self.filepath ) + else: + raise UnsupportedFileType ( self.filetype ) + # -- end if filename + + if allow_hash_create: + missing_hashes = self.get_missing_hashes() + if missing_hashes: + self.add_hashes ( + roverlay.digest.multihash_file ( self.filepath, missing_hashes ) + ) + + if not self.filesize and self.filesize != 0: + self.filesize = roverlay.util.getsize ( self.filepath ) + # --- end of interpolate (...) --- + + def get_missing_hashes ( self ): + """Returns an iterable of hashes that need to be calculated.""" + if not self.hashes: + return self.HASHTYPES + else: + missing = set() + for hashtype in self.HASHTYPES: + if hashtype not in self.hashes: + missing.add ( hashtype ) + return missing + # --- end of get_missing_hashes (...) --- + + def __str__ ( self ): + """Returns the string representation of this ManifestEntry + which can directly be used in the Manifest file.""" + return "{ftype} {fname} {fsize} {hashes}".format ( + ftype=self.filetype, fname=self.filename, fsize=self.filesize, + hashes=' '.join ( + h.upper() + ' ' + str ( self.hashes[h] ) for h in self.HASHTYPES + ) + ) + # --- end of __str__ (...) --- + + def __repr__ ( self ): + return "<{} for {}>".format ( + self.__class__.__name__, self.filename or self.filepath + ) + # --- end of __repr__ (...) --- + +# --- end of ManifestEntry --- + +class PackageFileManifestEntry ( ManifestEntry ): + """A ManifestEntry for package files.""" + + def __init__ ( self, p_info ): + """Constructor for PackageFileManifestEntry + + arguments: + * p_info -- package info + """ + pkg_file = p_info ['package_file'] + + super ( PackageFileManifestEntry, self ).__init__ ( + filetype = 'DIST', + filepath = pkg_file, + filename = p_info ['package_src_destpath'], + filesize = roverlay.util.getsize ( pkg_file ), + hashes = None, + ) + + # shared hashdict + # use p_info's hashdict directly + # (as reference, but don't modify it here!) + self.hashes = p_info.make_hashes ( self.HASHTYPES ) + # --- end of __init__ (...) --- + + def add_hashes ( self, *args, **kwargs ): + raise Exception ( + "add_hashes() not supported by {} due to shared hashdict!".format ( + self.__class__.__name__ + ) + ) + # --- end of add_hashes (...) --- + + def interpolate ( self, *args, **kwargs ): + pass + # --- end of interpolate (...) --- + +# --- end of PackageFileManifestEntry --- diff --git a/roverlay/overlay/pkgdir/manifest/file.py b/roverlay/overlay/pkgdir/manifest/file.py new file mode 100644 index 0000000..9cf8425 --- /dev/null +++ b/roverlay/overlay/pkgdir/manifest/file.py @@ -0,0 +1,261 @@ +# R overlay -- manifest package, manifest file creation +# -*- coding: utf-8 -*- +# Copyright (C) 2013 André Erdmann +# Distributed under the terms of the GNU General Public License; +# either version 2 of the License, or (at your option) any later version. + +import os.path +import errno + +import roverlay.overlay.pkgdir.manifest.entry + +from roverlay.overlay.pkgdir.manifest.entry import \ + ManifestEntry, PackageFileManifestEntry + +# TODO: don't write empty file(s) / remove them + +class ManifestFile ( object ): + """ + Internal implementation of the Manifest2 file creation that aims to save + execution time (compared with calling ebuild(1)). + + The Manifest format should be in accordance with GLEP 44, + http://www.gentoo.org/proj/en/glep/glep-0044.html. + + The used digest types are 'SHA256, SHA512, WHIRLPOOL' (GLEP 59), + provided by hashlib (via roverlay.digest). + """ + + def __init__ ( self, root ): + self.root = root + self.filepath = root + os.path.sep + 'Manifest' + self._entries = dict() + self.dirty = False + # --- end of __init__ (...) --- + + def has_entry ( self, filetype, filename ): + return ( filetype, filename ) in self._entries + # --- end of has_entry (...) --- + + def has_entry_search ( self, filename ): + for k in self._entries: + if k[1] == filename: + return k + return None + # --- end of has_entry_search (...) --- + + def _add_entry ( self, new_entry ): + """Adds an entry and marks this file as dirty. + + arguments: + * new_entry -- entry to add + """ + self._entries [ ( new_entry.filetype, new_entry.filename ) ] = new_entry + self.dirty = True + # --- end of _add_entry (...) --- + + def add_entry ( self, filetype, filepath, filename=None ): + """Adds an entry for the given file. + + arguments: + * filetype -- AUX, EBUILD, DIST, MISC + * filepath -- path to the file + * filename -- (optional) name of the file + """ + new_entry = ManifestEntry ( filetype, filepath, filename ) + new_entry.interpolate() + self._add_entry ( new_entry ) + # --- end of add_entry (...) --- + + def add_metadata_entry ( self, ignore_missing=False ): + """Adds an entry for metadata.xml. + + arguments: + * ignore_missing -- check whether metadata.xml actually exists and + don't add an entry if not + + Note: any existing metadata.xml entry will be removed + """ + fname = 'metadata.xml' + fpath = self.root + os.path.sep + fname + if not ignore_missing or os.path.exists ( fpath ): + self.add_entry ( 'MISC', fpath, fname ) + return True + else: + try: + del self._entries [ ( 'MISC', fname ) ] + except KeyError: + pass + else: + self.dirty = True + return False + # --- end of add_metadata_entry (...) --- + + def add_package_entry ( self, p_info, add_ebuild=True ): + """Adds an entry for the package file of a package info object, and + optionally for its ebuild, too. + + arguments: + * p_info -- package info + * add_ebuild -- if True: add an entry for p_info's ebuild + Defaults to True. + + Note: This method can only be used for "new" package infos. + """ + p_entry = PackageFileManifestEntry ( p_info ) + + if add_ebuild: + efile = p_info ['ebuild_file'] + if efile is None: + raise Exception ( "ebuild file must exist." ) + e_entry = ManifestEntry ( 'EBUILD', efile ) + e_entry.interpolate() + + self._add_entry ( e_entry ) + + self._add_entry ( p_entry ) + # --- end of add_package_entry (...) --- + + def remove_entry ( self, filetype, filename ): + """Removes an entry. + + arguments: + * filetype -- the entry's filetype + * filename -- the entry's filename + """ + del self._entries [ ( filetype, filename ) ] + self.dirty = True + # --- end of remove_entry (...) --- + + def remove_metadata_entry ( self ): + """Removes the metadata.xml entry.""" + self.remove_entry ( 'MISC', 'metadata.xml' ) + # --- end of remove_metadata_entry (...) --- + + def remove_package_entry ( self, p_info, with_ebuild=True ): + """Removes a package entry (and optionally its ebuild entry). + + arguments: + * p_info -- package info + * with_ebuild -- whether to remove the ebuild entry (defaults to True) + """ + filename = p_info.get ( 'package_src_destpath', do_fallback=True ) + + if with_ebuild: + efile = p_info.get ( 'ebuild_file' ) + if not efile: + raise Exception ( "package info object has no ebuild file" ) + self.remove_entry ( 'EBUILD', os.path.basename ( efile ) ) + + self.remove_entry ( 'DIST', filename ) + # --- end of remove_package_entry (...) --- + + def read ( self, update=False, ignore_missing=False ): + """Reads and imports the Manifest file. + + arguments: + * update -- if True: add non-existing entries only + Defaults to False. + * ignore_missing -- if True: don't raise an exception if the Manifest + file does not exist. Defaults to False. + """ + FILETYPES = ManifestEntry.FILETYPES + HASHTYPES = ManifestEntry.HASHTYPES + + FH = None + try: + FH = open ( self.filepath, 'rt' ) + + for line in FH.readlines(): + rsline = line.rstrip() + if rsline: + # + lc = rsline.split ( None ) + if lc[0] in FILETYPES: + # expect uneven number of line components (3 + ) + assert len ( lc ) % 2 == 1 and len ( lc ) > 3 + + filetype = lc[0] + filename = lc[1] + key = ( filetype, filename ) + + if not update or key not in self._entries: + filesize = lc[2] + hashes = dict() + hash_name = None + for word in lc[3:]: + if hash_name is None: + hash_name = word.lower() + else: + hashes [hash_name] = word + # ^ = int ( word, 16 ) + hash_name = None + # -- end for; + + self._entries [ ( filetype, filename ) ] = ( + ManifestEntry ( + filetype, None, filename, filesize, hashes + ) + ) + # -- end if not update or new + # else ignore + + except IOError as ioerr: + if ignore_missing and ioerr.errno == errno.ENOENT: + pass + else: + raise + finally: + if FH: + FH.close() + # --- end of read (...) --- + + def gen_lines ( self, do_sort=True ): + """Generates text file lines. + + arguments: + * do_sort -- if True: produce sorted output (defaults to True) + """ + if do_sort: + for item in sorted ( self._entries.items(), key=lambda kv: kv[0] ): + yield str ( item[1] ) + else: + for entry in self._entries.values(): + yield str ( entry ) + # --- end of gen_lines (...) --- + + def __str__ ( self ): + return '\n'.join ( self.gen_lines() ) + # --- end of __str__ (...) --- + + def __repr__ ( self ): + # COULDFIX: very efficient! + return "".format ( + os.path.sep.join ( self.root.rsplit ( os.path.sep, 2 ) [-2:] ) + ) + # --- end of __repr__ (...) --- + + def write ( self, force=False ): + """Writes the Manifest file. + + arguments: + * force -- enforce writing even if no changes made + """ + if force or self.dirty: + with open ( self.filepath, 'wt' ) as FH: + for line in self.gen_lines(): + FH.write ( line ) + FH.write ( '\n' ) + + self.dirty = False + return True + else: + return False + # --- end of write (...) --- + + def exists ( self ): + """Returns True if the Manifest file exists (as file), else False.""" + return os.path.isfile ( self.filepath ) + # --- end of exists (...) --- + +# --- end of ManifestFile --- diff --git a/roverlay/overlay/pkgdir/packagedir_newmanifest.py b/roverlay/overlay/pkgdir/packagedir_newmanifest.py new file mode 100644 index 0000000..04d202d --- /dev/null +++ b/roverlay/overlay/pkgdir/packagedir_newmanifest.py @@ -0,0 +1,106 @@ +# R overlay -- overlay package, package directory ("new" manifest) +# -*- coding: utf-8 -*- +# Copyright (C) 2013 André Erdmann +# Distributed under the terms of the GNU General Public License; +# either version 2 of the License, or (at your option) any later version. + +__all__ = [ 'PackageDir', ] + +import os +import threading + + +import roverlay.config + +import roverlay.tools.ebuild + +import roverlay.overlay.pkgdir.manifest.file +import roverlay.overlay.pkgdir.packagedir_base + +from roverlay.overlay.pkgdir.manifest.file import ManifestFile + + +class PackageDir ( roverlay.overlay.pkgdir.packagedir_base.PackageDirBase ): + """ + PackageDir class that uses an (mostly) internal implementation + for Manifest writing. + """ + + MANIFEST_THREADSAFE = True + + # Manifest entries for imported ebuilds have to be created during import + DOEBUILD_FETCH = roverlay.tools.ebuild.doebuild_fetch_and_manifest + + def _get_manifest ( self ): + """Returns a ManifestFile object.""" + manifest = ManifestFile ( self.physical_location ) + manifest.read ( ignore_missing=True ) + return manifest + # --- end of _get_manifest (...) --- + + def _write_import_manifest ( self, _manifest=None ): + """Verifies that Manifest file entries exist for all imported ebuilds. + + Assumption: ebuild file in Manifest => $SRC_URI, too + + Returns True on success, else False. + + arguments: + * _manifest -- manifest object (defaults to None -> create a new one) + """ + manifest = self._get_manifest() if _manifest is None else _manifest + + self.logger.debug ( "Checking import Manifest entries" ) + + ret = True + ename = None + for p in self._packages.values(): + efile = p.get ( 'ebuild_file' ) + if efile is not None: + ename = os.path.basename ( efile ) + if manifest.has_entry ( 'EBUILD', ename ): + self.logger.debug ( + "manifest entry for {} is ok.".format ( ename ) + ) + else: + ret = False + self.logger.error ( + "manifest entry for imported ebuild {} is missing!".format ( + ename + ) + ) + # -- end for; + return ret + # --- end of _write_import_manifest (...) --- + + def _write_manifest ( self, pkgs_for_manifest ): + """Generates and writes the Manifest file for this package. + + expects: called after writing metadata/ebuilds + + returns: success (True/False) + """ + manifest = self._get_manifest() + + manifest.add_metadata_entry ( ignore_missing=True ) + + # add manifest entries and add hardlinks to DISTROOT + # (replacing existing files/links) + # + distdir = self.DISTROOT.get_distdir ( self.name ) + for p in pkgs_for_manifest: + manifest.add_package_entry ( p ) + distdir.add ( p ['package_file'], p ['package_src_destpath'], p ) + # -- end for; + + #return (...) + if ( + manifest.write ( force=True ) + and self._write_import_manifest ( _manifest=manifest ) + ): + return True + else: + return False + # --- end of write_manifest (...) --- + +# --- end of PackageDir #ebuildmanifest ---