* [gentoo-commits] proj/R_overlay:gsoc13/next commit in: roverlay/overlay/pkgdir/manifest/, roverlay/config/, roverlay/, ...
@ 2013-07-10 15:10 André Erdmann
2013-07-10 16:16 ` [gentoo-commits] proj/R_overlay:master " André Erdmann
0 siblings, 1 reply; 2+ messages in thread
From: André Erdmann @ 2013-07-10 15:10 UTC (permalink / raw
To: gentoo-commits
commit: 6f31508300364ba709429e5d4e60bc9a05ad7d63
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jul 10 14:52:00 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jul 10 14:52:00 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=6f315083
pkgdir: newmanifest
roverlay already creates checksums for package files since they're required for
the distmap. Using ebuildmanifest does not allow to re-use these checksums for
faster manifest file creation. In case of portagemanifest, it would (probably) be
possible.
This commit adds a mostly internal manifest file creation.
Compared with ebuildmanifest, it is A LOT faster and threadsafe, and in contrast
to portagemanifest, it does not rely on portage libs (which is hard to
maintain/verify (for me)) and is slightly faster.
Actually, newmanifest is meant to replace portagemanifest and become the default
implementation.
newmanifest falls back to "ebuild <file> manifest" for imported ebuilds since
SRC_URI parsing is not implemented.
---
roverlay/__init__.py | 9 +-
roverlay/argutil.py | 4 +-
roverlay/config/entrymap.py | 1 +
roverlay/overlay/pkgdir/__init__.py | 1 +
roverlay/overlay/pkgdir/manifest/__init__.py | 0
roverlay/overlay/pkgdir/manifest/entry.py | 187 ++++++++++++++++
roverlay/overlay/pkgdir/manifest/file.py | 261 ++++++++++++++++++++++
roverlay/overlay/pkgdir/packagedir_newmanifest.py | 106 +++++++++
8 files changed, 562 insertions(+), 7 deletions(-)
diff --git a/roverlay/__init__.py b/roverlay/__init__.py
index 5b4007e..cd93ba1 100644
--- a/roverlay/__init__.py
+++ b/roverlay/__init__.py
@@ -1,22 +1,21 @@
# R overlay -- roverlay package (__init__)
# -*- coding: utf-8 -*-
-# Copyright (C) 2012 André Erdmann <dywi@mailerd.de>
+# Copyright (C) 2012, 2013 André Erdmann <dywi@mailerd.de>
# Distributed under the terms of the GNU General Public License;
# either version 2 of the License, or (at your option) any later version.
"""R overlay package
Provides roverlay initialization helpers (setup_initial_logger,
-load_config_file) and some information vars (__version__, name, ...).
+load_config_file) and some information vars (version, name, ...).
"""
__all__ = [ 'setup_initial_logger', 'load_config_file', ]
name = "R_overlay"
-version = ( 0, 2, 4 )
-__version__ = '.'.join ( str ( i ) for i in version )
+version = "0.2.4"
-description_str = "R overlay creation (roverlay) " + __version__
+description_str = "R overlay creation (roverlay) " + version
license_str=(
'Copyright (C) 2012, 2013 Andr\xc3\xa9 Erdmann\n'
'Distributed under the terms of the GNU General Public License;\n'
diff --git a/roverlay/argutil.py b/roverlay/argutil.py
index 869ea8b..2bccfff 100644
--- a/roverlay/argutil.py
+++ b/roverlay/argutil.py
@@ -107,7 +107,7 @@ def get_parser ( command_map, default_config_file, default_command='create' ):
# adding args starts here
arg (
- '-V', '--version', action='version', version=roverlay.__version__
+ '-V', '--version', action='version', version=roverlay.version
)
arg (
@@ -326,7 +326,7 @@ def get_parser ( command_map, default_config_file, default_command='create' ):
'--manifest-implementation', '-M', default=argparse.SUPPRESS,
help="choose how Manifest files are written (ebuild(1) or portage libs)",
metavar="<impl>",
- choices=frozenset (( 'ebuild', 'e', 'portage', 'p' )),
+ choices=frozenset (( 'ebuild', 'e', 'portage', 'p', 'next', )),
)
# FIXME: description of --no-incremental is not correct,
diff --git a/roverlay/config/entrymap.py b/roverlay/config/entrymap.py
index 8117585..0768560 100644
--- a/roverlay/config/entrymap.py
+++ b/roverlay/config/entrymap.py
@@ -255,6 +255,7 @@ CONFIG_ENTRY_MAP = dict (
'default',
'ebuild',
'portage',
+ 'next',
# 'e',
# 'p',
)),
diff --git a/roverlay/overlay/pkgdir/__init__.py b/roverlay/overlay/pkgdir/__init__.py
index d33ced2..7977ca9 100644
--- a/roverlay/overlay/pkgdir/__init__.py
+++ b/roverlay/overlay/pkgdir/__init__.py
@@ -26,6 +26,7 @@ _PACKAGE_DIR_IMPLEMENTATIONS = {
'ebuild' : 'packagedir_ebuildmanifest',
'p' : 'packagedir_portagemanifest',
'portage' : 'packagedir_portagemanifest',
+ 'next' : 'packagedir_newmanifest',
}
def _configure():
diff --git a/roverlay/overlay/pkgdir/manifest/__init__.py b/roverlay/overlay/pkgdir/manifest/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/roverlay/overlay/pkgdir/manifest/entry.py b/roverlay/overlay/pkgdir/manifest/entry.py
new file mode 100644
index 0000000..2313f63
--- /dev/null
+++ b/roverlay/overlay/pkgdir/manifest/entry.py
@@ -0,0 +1,187 @@
+# R overlay -- manifest package, manifest file entries
+# -*- coding: utf-8 -*-
+# Copyright (C) 2013 André Erdmann <dywi@mailerd.de>
+# Distributed under the terms of the GNU General Public License;
+# either version 2 of the License, or (at your option) any later version.
+
+__all__ = [
+ 'ManifestEntry', 'PackageFileManifestEntry',
+ 'ManifestEntryInvalid', 'UnsupportedFileType',
+]
+
+import os.path
+
+import roverlay.digest
+import roverlay.util
+
+
+class ManifestEntryInvalid ( ValueError ):
+ pass
+
+class UnsupportedFileType ( ManifestEntryInvalid ):
+ pass
+
+
+class ManifestEntry ( object ):
+ """ManifestEntry represents a single line in the Manifest file
+ '<filetype> <filename> <filesize> <chksumtypeK> <chksumK> for K in 1..N',
+ where filetype is:
+
+ * AUX for files in the files/ directory [not used by roverlay]
+ * EBUILD for all ebuilds
+ * MISC for files not directly used by ebuilds (ChangeLog, metadata.xml)
+ * DIST for release tarballs (SRC_URI)
+ """
+
+ # an ordered list of the default digest types to use
+ #
+ # GLEP 59:
+ HASHTYPES = ( 'sha256', 'sha512', 'whirlpool', )
+ #
+ # old and unsupported
+ ##HASHTYPES = ( 'rmd160', 'sha1', 'sha256', )
+
+ FILETYPES = frozenset ({ 'AUX', 'EBUILD', 'MISC', 'DIST', })
+
+ def __init__ (
+ self, filetype, filepath, filename=None, filesize=None, hashes=None
+ ):
+ """Creates a ManifestEntry.
+ Some of the data can be calculated using interpolate().
+
+ arguments:
+ * filetype -- file type (AUX, EBUILD, MISC, DIST)
+ * filepath -- path to the file (can be None if interpolation is
+ not intended)
+ * filename -- file name
+ Defaults to None => try auto-detection.
+ * filesize -- file size, has to be an integer. Defaults to None.
+ * hashes -- a dict of checksums (sha256, ...)
+ Defaults to None.
+
+ Required args:
+ * filetype
+ * filepath or (filename and filesize and hashes)
+ """
+ self.filetype = filetype
+ self.filepath = filepath
+ self.filename = filename
+ self.filesize = filesize
+ self.hashes = hashes
+ # --- end of __init__ (...) ---
+
+ def add_hashes ( self, hashes ):
+ """Updates the hash dict of this entry.
+
+ arguments:
+ * hashes -- hash dict that will be used to update the entry's hashdict
+ """
+ if self.hashes is None:
+ self.hashes = dict()
+ self.hashes.update ( hashes )
+ # --- end of add_hashes (...) ---
+
+ def interpolate ( self, allow_hash_create=True ):
+ """
+ Tries to calculate all missing data (filesize, filename and hashes).
+
+ arguments:
+ * allow_hash_create -- whether to try hash creation or not
+
+ raises:
+ * UnsupportedFileType if filetype not valid
+ """
+ if not self.filename:
+ if (
+ self.filetype == 'MISC' or
+ self.filetype == 'EBUILD' or
+ self.filetype == 'DIST'
+ ):
+ self.filename = os.path.basename ( self.filepath )
+ elif self.filetype == 'AUX':
+ #self.filetype = 'files' + os.path.sep + os.path.basename ( self.filepath )
+ self.filetype = 'files/' + os.path.basename ( self.filepath )
+ else:
+ raise UnsupportedFileType ( self.filetype )
+ # -- end if filename
+
+ if allow_hash_create:
+ missing_hashes = self.get_missing_hashes()
+ if missing_hashes:
+ self.add_hashes (
+ roverlay.digest.multihash_file ( self.filepath, missing_hashes )
+ )
+
+ if not self.filesize and self.filesize != 0:
+ self.filesize = roverlay.util.getsize ( self.filepath )
+ # --- end of interpolate (...) ---
+
+ def get_missing_hashes ( self ):
+ """Returns an iterable of hashes that need to be calculated."""
+ if not self.hashes:
+ return self.HASHTYPES
+ else:
+ missing = set()
+ for hashtype in self.HASHTYPES:
+ if hashtype not in self.hashes:
+ missing.add ( hashtype )
+ return missing
+ # --- end of get_missing_hashes (...) ---
+
+ def __str__ ( self ):
+ """Returns the string representation of this ManifestEntry
+ which can directly be used in the Manifest file."""
+ return "{ftype} {fname} {fsize} {hashes}".format (
+ ftype=self.filetype, fname=self.filename, fsize=self.filesize,
+ hashes=' '.join (
+ h.upper() + ' ' + str ( self.hashes[h] ) for h in self.HASHTYPES
+ )
+ )
+ # --- end of __str__ (...) ---
+
+ def __repr__ ( self ):
+ return "<{} for {}>".format (
+ self.__class__.__name__, self.filename or self.filepath
+ )
+ # --- end of __repr__ (...) ---
+
+# --- end of ManifestEntry ---
+
+class PackageFileManifestEntry ( ManifestEntry ):
+ """A ManifestEntry for package files."""
+
+ def __init__ ( self, p_info ):
+ """Constructor for PackageFileManifestEntry
+
+ arguments:
+ * p_info -- package info
+ """
+ pkg_file = p_info ['package_file']
+
+ super ( PackageFileManifestEntry, self ).__init__ (
+ filetype = 'DIST',
+ filepath = pkg_file,
+ filename = p_info ['package_src_destpath'],
+ filesize = roverlay.util.getsize ( pkg_file ),
+ hashes = None,
+ )
+
+ # shared hashdict
+ # use p_info's hashdict directly
+ # (as reference, but don't modify it here!)
+ self.hashes = p_info.make_hashes ( self.HASHTYPES )
+ # --- end of __init__ (...) ---
+
+ def add_hashes ( self, *args, **kwargs ):
+ raise Exception (
+ "add_hashes() not supported by {} due to shared hashdict!".format (
+ self.__class__.__name__
+ )
+ )
+ # --- end of add_hashes (...) ---
+
+ def interpolate ( self, *args, **kwargs ):
+ pass
+ # --- end of interpolate (...) ---
+
+# --- end of PackageFileManifestEntry ---
diff --git a/roverlay/overlay/pkgdir/manifest/file.py b/roverlay/overlay/pkgdir/manifest/file.py
new file mode 100644
index 0000000..9cf8425
--- /dev/null
+++ b/roverlay/overlay/pkgdir/manifest/file.py
@@ -0,0 +1,261 @@
+# R overlay -- manifest package, manifest file creation
+# -*- coding: utf-8 -*-
+# Copyright (C) 2013 André Erdmann <dywi@mailerd.de>
+# Distributed under the terms of the GNU General Public License;
+# either version 2 of the License, or (at your option) any later version.
+
+import os.path
+import errno
+
+import roverlay.overlay.pkgdir.manifest.entry
+
+from roverlay.overlay.pkgdir.manifest.entry import \
+ ManifestEntry, PackageFileManifestEntry
+
+# TODO: don't write empty file(s) / remove them
+
+class ManifestFile ( object ):
+ """
+ Internal implementation of the Manifest2 file creation that aims to save
+ execution time (compared with calling ebuild(1)).
+
+ The Manifest format should be in accordance with GLEP 44,
+ http://www.gentoo.org/proj/en/glep/glep-0044.html.
+
+ The used digest types are 'SHA256, SHA512, WHIRLPOOL' (GLEP 59),
+ provided by hashlib (via roverlay.digest).
+ """
+
+ def __init__ ( self, root ):
+ self.root = root
+ self.filepath = root + os.path.sep + 'Manifest'
+ self._entries = dict()
+ self.dirty = False
+ # --- end of __init__ (...) ---
+
+ def has_entry ( self, filetype, filename ):
+ return ( filetype, filename ) in self._entries
+ # --- end of has_entry (...) ---
+
+ def has_entry_search ( self, filename ):
+ for k in self._entries:
+ if k[1] == filename:
+ return k
+ return None
+ # --- end of has_entry_search (...) ---
+
+ def _add_entry ( self, new_entry ):
+ """Adds an entry and marks this file as dirty.
+
+ arguments:
+ * new_entry -- entry to add
+ """
+ self._entries [ ( new_entry.filetype, new_entry.filename ) ] = new_entry
+ self.dirty = True
+ # --- end of _add_entry (...) ---
+
+ def add_entry ( self, filetype, filepath, filename=None ):
+ """Adds an entry for the given file.
+
+ arguments:
+ * filetype -- AUX, EBUILD, DIST, MISC
+ * filepath -- path to the file
+ * filename -- (optional) name of the file
+ """
+ new_entry = ManifestEntry ( filetype, filepath, filename )
+ new_entry.interpolate()
+ self._add_entry ( new_entry )
+ # --- end of add_entry (...) ---
+
+ def add_metadata_entry ( self, ignore_missing=False ):
+ """Adds an entry for metadata.xml.
+
+ arguments:
+ * ignore_missing -- check whether metadata.xml actually exists and
+ don't add an entry if not
+
+ Note: any existing metadata.xml entry will be removed
+ """
+ fname = 'metadata.xml'
+ fpath = self.root + os.path.sep + fname
+ if not ignore_missing or os.path.exists ( fpath ):
+ self.add_entry ( 'MISC', fpath, fname )
+ return True
+ else:
+ try:
+ del self._entries [ ( 'MISC', fname ) ]
+ except KeyError:
+ pass
+ else:
+ self.dirty = True
+ return False
+ # --- end of add_metadata_entry (...) ---
+
+ def add_package_entry ( self, p_info, add_ebuild=True ):
+ """Adds an entry for the package file of a package info object, and
+ optionally for its ebuild, too.
+
+ arguments:
+ * p_info -- package info
+ * add_ebuild -- if True: add an entry for p_info's ebuild
+ Defaults to True.
+
+ Note: This method can only be used for "new" package infos.
+ """
+ p_entry = PackageFileManifestEntry ( p_info )
+
+ if add_ebuild:
+ efile = p_info ['ebuild_file']
+ if efile is None:
+ raise Exception ( "ebuild file must exist." )
+ e_entry = ManifestEntry ( 'EBUILD', efile )
+ e_entry.interpolate()
+
+ self._add_entry ( e_entry )
+
+ self._add_entry ( p_entry )
+ # --- end of add_package_entry (...) ---
+
+ def remove_entry ( self, filetype, filename ):
+ """Removes an entry.
+
+ arguments:
+ * filetype -- the entry's filetype
+ * filename -- the entry's filename
+ """
+ del self._entries [ ( filetype, filename ) ]
+ self.dirty = True
+ # --- end of remove_entry (...) ---
+
+ def remove_metadata_entry ( self ):
+ """Removes the metadata.xml entry."""
+ self.remove_entry ( 'MISC', 'metadata.xml' )
+ # --- end of remove_metadata_entry (...) ---
+
+ def remove_package_entry ( self, p_info, with_ebuild=True ):
+ """Removes a package entry (and optionally its ebuild entry).
+
+ arguments:
+ * p_info -- package info
+ * with_ebuild -- whether to remove the ebuild entry (defaults to True)
+ """
+ filename = p_info.get ( 'package_src_destpath', do_fallback=True )
+
+ if with_ebuild:
+ efile = p_info.get ( 'ebuild_file' )
+ if not efile:
+ raise Exception ( "package info object has no ebuild file" )
+ self.remove_entry ( 'EBUILD', os.path.basename ( efile ) )
+
+ self.remove_entry ( 'DIST', filename )
+ # --- end of remove_package_entry (...) ---
+
+ def read ( self, update=False, ignore_missing=False ):
+ """Reads and imports the Manifest file.
+
+ arguments:
+ * update -- if True: add non-existing entries only
+ Defaults to False.
+ * ignore_missing -- if True: don't raise an exception if the Manifest
+ file does not exist. Defaults to False.
+ """
+ FILETYPES = ManifestEntry.FILETYPES
+ HASHTYPES = ManifestEntry.HASHTYPES
+
+ FH = None
+ try:
+ FH = open ( self.filepath, 'rt' )
+
+ for line in FH.readlines():
+ rsline = line.rstrip()
+ if rsline:
+ #<filetype> <filename> <filesize> <hashes...>
+ lc = rsline.split ( None )
+ if lc[0] in FILETYPES:
+ # expect uneven number of line components (3 + <hash pairs>)
+ assert len ( lc ) % 2 == 1 and len ( lc ) > 3
+
+ filetype = lc[0]
+ filename = lc[1]
+ key = ( filetype, filename )
+
+ if not update or key not in self._entries:
+ filesize = lc[2]
+ hashes = dict()
+ hash_name = None
+ for word in lc[3:]:
+ if hash_name is None:
+ hash_name = word.lower()
+ else:
+ hashes [hash_name] = word
+ # ^ = int ( word, 16 )
+ hash_name = None
+ # -- end for;
+
+ self._entries [ ( filetype, filename ) ] = (
+ ManifestEntry (
+ filetype, None, filename, filesize, hashes
+ )
+ )
+ # -- end if not update or new
+ # else ignore
+
+ except IOError as ioerr:
+ if ignore_missing and ioerr.errno == errno.ENOENT:
+ pass
+ else:
+ raise
+ finally:
+ if FH:
+ FH.close()
+ # --- end of read (...) ---
+
+ def gen_lines ( self, do_sort=True ):
+ """Generates text file lines.
+
+ arguments:
+ * do_sort -- if True: produce sorted output (defaults to True)
+ """
+ if do_sort:
+ for item in sorted ( self._entries.items(), key=lambda kv: kv[0] ):
+ yield str ( item[1] )
+ else:
+ for entry in self._entries.values():
+ yield str ( entry )
+ # --- end of gen_lines (...) ---
+
+ def __str__ ( self ):
+ return '\n'.join ( self.gen_lines() )
+ # --- end of __str__ (...) ---
+
+ def __repr__ ( self ):
+ # COULDFIX: very efficient!
+ return "<Manifest for {}>".format (
+ os.path.sep.join ( self.root.rsplit ( os.path.sep, 2 ) [-2:] )
+ )
+ # --- end of __repr__ (...) ---
+
+ def write ( self, force=False ):
+ """Writes the Manifest file.
+
+ arguments:
+ * force -- enforce writing even if no changes made
+ """
+ if force or self.dirty:
+ with open ( self.filepath, 'wt' ) as FH:
+ for line in self.gen_lines():
+ FH.write ( line )
+ FH.write ( '\n' )
+
+ self.dirty = False
+ return True
+ else:
+ return False
+ # --- end of write (...) ---
+
+ def exists ( self ):
+ """Returns True if the Manifest file exists (as file), else False."""
+ return os.path.isfile ( self.filepath )
+ # --- end of exists (...) ---
+
+# --- end of ManifestFile ---
diff --git a/roverlay/overlay/pkgdir/packagedir_newmanifest.py b/roverlay/overlay/pkgdir/packagedir_newmanifest.py
new file mode 100644
index 0000000..04d202d
--- /dev/null
+++ b/roverlay/overlay/pkgdir/packagedir_newmanifest.py
@@ -0,0 +1,106 @@
+# R overlay -- overlay package, package directory ("new" manifest)
+# -*- coding: utf-8 -*-
+# Copyright (C) 2013 André Erdmann <dywi@mailerd.de>
+# Distributed under the terms of the GNU General Public License;
+# either version 2 of the License, or (at your option) any later version.
+
+__all__ = [ 'PackageDir', ]
+
+import os
+import threading
+
+
+import roverlay.config
+
+import roverlay.tools.ebuild
+
+import roverlay.overlay.pkgdir.manifest.file
+import roverlay.overlay.pkgdir.packagedir_base
+
+from roverlay.overlay.pkgdir.manifest.file import ManifestFile
+
+
+class PackageDir ( roverlay.overlay.pkgdir.packagedir_base.PackageDirBase ):
+ """
+ PackageDir class that uses an (mostly) internal implementation
+ for Manifest writing.
+ """
+
+ MANIFEST_THREADSAFE = True
+
+ # Manifest entries for imported ebuilds have to be created during import
+ DOEBUILD_FETCH = roverlay.tools.ebuild.doebuild_fetch_and_manifest
+
+ def _get_manifest ( self ):
+ """Returns a ManifestFile object."""
+ manifest = ManifestFile ( self.physical_location )
+ manifest.read ( ignore_missing=True )
+ return manifest
+ # --- end of _get_manifest (...) ---
+
+ def _write_import_manifest ( self, _manifest=None ):
+ """Verifies that Manifest file entries exist for all imported ebuilds.
+
+ Assumption: ebuild file in Manifest => $SRC_URI, too
+
+ Returns True on success, else False.
+
+ arguments:
+ * _manifest -- manifest object (defaults to None -> create a new one)
+ """
+ manifest = self._get_manifest() if _manifest is None else _manifest
+
+ self.logger.debug ( "Checking import Manifest entries" )
+
+ ret = True
+ ename = None
+ for p in self._packages.values():
+ efile = p.get ( 'ebuild_file' )
+ if efile is not None:
+ ename = os.path.basename ( efile )
+ if manifest.has_entry ( 'EBUILD', ename ):
+ self.logger.debug (
+ "manifest entry for {} is ok.".format ( ename )
+ )
+ else:
+ ret = False
+ self.logger.error (
+ "manifest entry for imported ebuild {} is missing!".format (
+ ename
+ )
+ )
+ # -- end for;
+ return ret
+ # --- end of _write_import_manifest (...) ---
+
+ def _write_manifest ( self, pkgs_for_manifest ):
+ """Generates and writes the Manifest file for this package.
+
+ expects: called after writing metadata/ebuilds
+
+ returns: success (True/False)
+ """
+ manifest = self._get_manifest()
+
+ manifest.add_metadata_entry ( ignore_missing=True )
+
+ # add manifest entries and add hardlinks to DISTROOT
+ # (replacing existing files/links)
+ #
+ distdir = self.DISTROOT.get_distdir ( self.name )
+ for p in pkgs_for_manifest:
+ manifest.add_package_entry ( p )
+ distdir.add ( p ['package_file'], p ['package_src_destpath'], p )
+ # -- end for;
+
+ #return (...)
+ if (
+ manifest.write ( force=True )
+ and self._write_import_manifest ( _manifest=manifest )
+ ):
+ return True
+ else:
+ return False
+ # --- end of write_manifest (...) ---
+
+# --- end of PackageDir #ebuildmanifest ---
^ permalink raw reply related [flat|nested] 2+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/overlay/pkgdir/manifest/, roverlay/config/, roverlay/, ...
2013-07-10 15:10 [gentoo-commits] proj/R_overlay:gsoc13/next commit in: roverlay/overlay/pkgdir/manifest/, roverlay/config/, roverlay/, André Erdmann
@ 2013-07-10 16:16 ` André Erdmann
0 siblings, 0 replies; 2+ messages in thread
From: André Erdmann @ 2013-07-10 16:16 UTC (permalink / raw
To: gentoo-commits
commit: 6f31508300364ba709429e5d4e60bc9a05ad7d63
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jul 10 14:52:00 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jul 10 14:52:00 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=6f315083
pkgdir: newmanifest
roverlay already creates checksums for package files since they're required for
the distmap. Using ebuildmanifest does not allow to re-use these checksums for
faster manifest file creation. In case of portagemanifest, it would (probably) be
possible.
This commit adds a mostly internal manifest file creation.
Compared with ebuildmanifest, it is A LOT faster and threadsafe, and in contrast
to portagemanifest, it does not rely on portage libs (which is hard to
maintain/verify (for me)) and is slightly faster.
Actually, newmanifest is meant to replace portagemanifest and become the default
implementation.
newmanifest falls back to "ebuild <file> manifest" for imported ebuilds since
SRC_URI parsing is not implemented.
---
roverlay/__init__.py | 9 +-
roverlay/argutil.py | 4 +-
roverlay/config/entrymap.py | 1 +
roverlay/overlay/pkgdir/__init__.py | 1 +
roverlay/overlay/pkgdir/manifest/__init__.py | 0
roverlay/overlay/pkgdir/manifest/entry.py | 187 ++++++++++++++++
roverlay/overlay/pkgdir/manifest/file.py | 261 ++++++++++++++++++++++
roverlay/overlay/pkgdir/packagedir_newmanifest.py | 106 +++++++++
8 files changed, 562 insertions(+), 7 deletions(-)
diff --git a/roverlay/__init__.py b/roverlay/__init__.py
index 5b4007e..cd93ba1 100644
--- a/roverlay/__init__.py
+++ b/roverlay/__init__.py
@@ -1,22 +1,21 @@
# R overlay -- roverlay package (__init__)
# -*- coding: utf-8 -*-
-# Copyright (C) 2012 André Erdmann <dywi@mailerd.de>
+# Copyright (C) 2012, 2013 André Erdmann <dywi@mailerd.de>
# Distributed under the terms of the GNU General Public License;
# either version 2 of the License, or (at your option) any later version.
"""R overlay package
Provides roverlay initialization helpers (setup_initial_logger,
-load_config_file) and some information vars (__version__, name, ...).
+load_config_file) and some information vars (version, name, ...).
"""
__all__ = [ 'setup_initial_logger', 'load_config_file', ]
name = "R_overlay"
-version = ( 0, 2, 4 )
-__version__ = '.'.join ( str ( i ) for i in version )
+version = "0.2.4"
-description_str = "R overlay creation (roverlay) " + __version__
+description_str = "R overlay creation (roverlay) " + version
license_str=(
'Copyright (C) 2012, 2013 Andr\xc3\xa9 Erdmann\n'
'Distributed under the terms of the GNU General Public License;\n'
diff --git a/roverlay/argutil.py b/roverlay/argutil.py
index 869ea8b..2bccfff 100644
--- a/roverlay/argutil.py
+++ b/roverlay/argutil.py
@@ -107,7 +107,7 @@ def get_parser ( command_map, default_config_file, default_command='create' ):
# adding args starts here
arg (
- '-V', '--version', action='version', version=roverlay.__version__
+ '-V', '--version', action='version', version=roverlay.version
)
arg (
@@ -326,7 +326,7 @@ def get_parser ( command_map, default_config_file, default_command='create' ):
'--manifest-implementation', '-M', default=argparse.SUPPRESS,
help="choose how Manifest files are written (ebuild(1) or portage libs)",
metavar="<impl>",
- choices=frozenset (( 'ebuild', 'e', 'portage', 'p' )),
+ choices=frozenset (( 'ebuild', 'e', 'portage', 'p', 'next', )),
)
# FIXME: description of --no-incremental is not correct,
diff --git a/roverlay/config/entrymap.py b/roverlay/config/entrymap.py
index 8117585..0768560 100644
--- a/roverlay/config/entrymap.py
+++ b/roverlay/config/entrymap.py
@@ -255,6 +255,7 @@ CONFIG_ENTRY_MAP = dict (
'default',
'ebuild',
'portage',
+ 'next',
# 'e',
# 'p',
)),
diff --git a/roverlay/overlay/pkgdir/__init__.py b/roverlay/overlay/pkgdir/__init__.py
index d33ced2..7977ca9 100644
--- a/roverlay/overlay/pkgdir/__init__.py
+++ b/roverlay/overlay/pkgdir/__init__.py
@@ -26,6 +26,7 @@ _PACKAGE_DIR_IMPLEMENTATIONS = {
'ebuild' : 'packagedir_ebuildmanifest',
'p' : 'packagedir_portagemanifest',
'portage' : 'packagedir_portagemanifest',
+ 'next' : 'packagedir_newmanifest',
}
def _configure():
diff --git a/roverlay/overlay/pkgdir/manifest/__init__.py b/roverlay/overlay/pkgdir/manifest/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/roverlay/overlay/pkgdir/manifest/entry.py b/roverlay/overlay/pkgdir/manifest/entry.py
new file mode 100644
index 0000000..2313f63
--- /dev/null
+++ b/roverlay/overlay/pkgdir/manifest/entry.py
@@ -0,0 +1,187 @@
+# R overlay -- manifest package, manifest file entries
+# -*- coding: utf-8 -*-
+# Copyright (C) 2013 André Erdmann <dywi@mailerd.de>
+# Distributed under the terms of the GNU General Public License;
+# either version 2 of the License, or (at your option) any later version.
+
+__all__ = [
+ 'ManifestEntry', 'PackageFileManifestEntry',
+ 'ManifestEntryInvalid', 'UnsupportedFileType',
+]
+
+import os.path
+
+import roverlay.digest
+import roverlay.util
+
+
+class ManifestEntryInvalid ( ValueError ):
+ pass
+
+class UnsupportedFileType ( ManifestEntryInvalid ):
+ pass
+
+
+class ManifestEntry ( object ):
+ """ManifestEntry represents a single line in the Manifest file
+ '<filetype> <filename> <filesize> <chksumtypeK> <chksumK> for K in 1..N',
+ where filetype is:
+
+ * AUX for files in the files/ directory [not used by roverlay]
+ * EBUILD for all ebuilds
+ * MISC for files not directly used by ebuilds (ChangeLog, metadata.xml)
+ * DIST for release tarballs (SRC_URI)
+ """
+
+ # an ordered list of the default digest types to use
+ #
+ # GLEP 59:
+ HASHTYPES = ( 'sha256', 'sha512', 'whirlpool', )
+ #
+ # old and unsupported
+ ##HASHTYPES = ( 'rmd160', 'sha1', 'sha256', )
+
+ FILETYPES = frozenset ({ 'AUX', 'EBUILD', 'MISC', 'DIST', })
+
+ def __init__ (
+ self, filetype, filepath, filename=None, filesize=None, hashes=None
+ ):
+ """Creates a ManifestEntry.
+ Some of the data can be calculated using interpolate().
+
+ arguments:
+ * filetype -- file type (AUX, EBUILD, MISC, DIST)
+ * filepath -- path to the file (can be None if interpolation is
+ not intended)
+ * filename -- file name
+ Defaults to None => try auto-detection.
+ * filesize -- file size, has to be an integer. Defaults to None.
+ * hashes -- a dict of checksums (sha256, ...)
+ Defaults to None.
+
+ Required args:
+ * filetype
+ * filepath or (filename and filesize and hashes)
+ """
+ self.filetype = filetype
+ self.filepath = filepath
+ self.filename = filename
+ self.filesize = filesize
+ self.hashes = hashes
+ # --- end of __init__ (...) ---
+
+ def add_hashes ( self, hashes ):
+ """Updates the hash dict of this entry.
+
+ arguments:
+ * hashes -- hash dict that will be used to update the entry's hashdict
+ """
+ if self.hashes is None:
+ self.hashes = dict()
+ self.hashes.update ( hashes )
+ # --- end of add_hashes (...) ---
+
+ def interpolate ( self, allow_hash_create=True ):
+ """
+ Tries to calculate all missing data (filesize, filename and hashes).
+
+ arguments:
+ * allow_hash_create -- whether to try hash creation or not
+
+ raises:
+ * UnsupportedFileType if filetype not valid
+ """
+ if not self.filename:
+ if (
+ self.filetype == 'MISC' or
+ self.filetype == 'EBUILD' or
+ self.filetype == 'DIST'
+ ):
+ self.filename = os.path.basename ( self.filepath )
+ elif self.filetype == 'AUX':
+ #self.filetype = 'files' + os.path.sep + os.path.basename ( self.filepath )
+ self.filetype = 'files/' + os.path.basename ( self.filepath )
+ else:
+ raise UnsupportedFileType ( self.filetype )
+ # -- end if filename
+
+ if allow_hash_create:
+ missing_hashes = self.get_missing_hashes()
+ if missing_hashes:
+ self.add_hashes (
+ roverlay.digest.multihash_file ( self.filepath, missing_hashes )
+ )
+
+ if not self.filesize and self.filesize != 0:
+ self.filesize = roverlay.util.getsize ( self.filepath )
+ # --- end of interpolate (...) ---
+
+ def get_missing_hashes ( self ):
+ """Returns an iterable of hashes that need to be calculated."""
+ if not self.hashes:
+ return self.HASHTYPES
+ else:
+ missing = set()
+ for hashtype in self.HASHTYPES:
+ if hashtype not in self.hashes:
+ missing.add ( hashtype )
+ return missing
+ # --- end of get_missing_hashes (...) ---
+
+ def __str__ ( self ):
+ """Returns the string representation of this ManifestEntry
+ which can directly be used in the Manifest file."""
+ return "{ftype} {fname} {fsize} {hashes}".format (
+ ftype=self.filetype, fname=self.filename, fsize=self.filesize,
+ hashes=' '.join (
+ h.upper() + ' ' + str ( self.hashes[h] ) for h in self.HASHTYPES
+ )
+ )
+ # --- end of __str__ (...) ---
+
+ def __repr__ ( self ):
+ return "<{} for {}>".format (
+ self.__class__.__name__, self.filename or self.filepath
+ )
+ # --- end of __repr__ (...) ---
+
+# --- end of ManifestEntry ---
+
+class PackageFileManifestEntry ( ManifestEntry ):
+ """A ManifestEntry for package files."""
+
+ def __init__ ( self, p_info ):
+ """Constructor for PackageFileManifestEntry
+
+ arguments:
+ * p_info -- package info
+ """
+ pkg_file = p_info ['package_file']
+
+ super ( PackageFileManifestEntry, self ).__init__ (
+ filetype = 'DIST',
+ filepath = pkg_file,
+ filename = p_info ['package_src_destpath'],
+ filesize = roverlay.util.getsize ( pkg_file ),
+ hashes = None,
+ )
+
+ # shared hashdict
+ # use p_info's hashdict directly
+ # (as reference, but don't modify it here!)
+ self.hashes = p_info.make_hashes ( self.HASHTYPES )
+ # --- end of __init__ (...) ---
+
+ def add_hashes ( self, *args, **kwargs ):
+ raise Exception (
+ "add_hashes() not supported by {} due to shared hashdict!".format (
+ self.__class__.__name__
+ )
+ )
+ # --- end of add_hashes (...) ---
+
+ def interpolate ( self, *args, **kwargs ):
+ pass
+ # --- end of interpolate (...) ---
+
+# --- end of PackageFileManifestEntry ---
diff --git a/roverlay/overlay/pkgdir/manifest/file.py b/roverlay/overlay/pkgdir/manifest/file.py
new file mode 100644
index 0000000..9cf8425
--- /dev/null
+++ b/roverlay/overlay/pkgdir/manifest/file.py
@@ -0,0 +1,261 @@
+# R overlay -- manifest package, manifest file creation
+# -*- coding: utf-8 -*-
+# Copyright (C) 2013 André Erdmann <dywi@mailerd.de>
+# Distributed under the terms of the GNU General Public License;
+# either version 2 of the License, or (at your option) any later version.
+
+import os.path
+import errno
+
+import roverlay.overlay.pkgdir.manifest.entry
+
+from roverlay.overlay.pkgdir.manifest.entry import \
+ ManifestEntry, PackageFileManifestEntry
+
+# TODO: don't write empty file(s) / remove them
+
+class ManifestFile ( object ):
+ """
+ Internal implementation of the Manifest2 file creation that aims to save
+ execution time (compared with calling ebuild(1)).
+
+ The Manifest format should be in accordance with GLEP 44,
+ http://www.gentoo.org/proj/en/glep/glep-0044.html.
+
+ The used digest types are 'SHA256, SHA512, WHIRLPOOL' (GLEP 59),
+ provided by hashlib (via roverlay.digest).
+ """
+
+ def __init__ ( self, root ):
+ self.root = root
+ self.filepath = root + os.path.sep + 'Manifest'
+ self._entries = dict()
+ self.dirty = False
+ # --- end of __init__ (...) ---
+
+ def has_entry ( self, filetype, filename ):
+ return ( filetype, filename ) in self._entries
+ # --- end of has_entry (...) ---
+
+ def has_entry_search ( self, filename ):
+ for k in self._entries:
+ if k[1] == filename:
+ return k
+ return None
+ # --- end of has_entry_search (...) ---
+
+ def _add_entry ( self, new_entry ):
+ """Adds an entry and marks this file as dirty.
+
+ arguments:
+ * new_entry -- entry to add
+ """
+ self._entries [ ( new_entry.filetype, new_entry.filename ) ] = new_entry
+ self.dirty = True
+ # --- end of _add_entry (...) ---
+
+ def add_entry ( self, filetype, filepath, filename=None ):
+ """Adds an entry for the given file.
+
+ arguments:
+ * filetype -- AUX, EBUILD, DIST, MISC
+ * filepath -- path to the file
+ * filename -- (optional) name of the file
+ """
+ new_entry = ManifestEntry ( filetype, filepath, filename )
+ new_entry.interpolate()
+ self._add_entry ( new_entry )
+ # --- end of add_entry (...) ---
+
+ def add_metadata_entry ( self, ignore_missing=False ):
+ """Adds an entry for metadata.xml.
+
+ arguments:
+ * ignore_missing -- check whether metadata.xml actually exists and
+ don't add an entry if not
+
+ Note: any existing metadata.xml entry will be removed
+ """
+ fname = 'metadata.xml'
+ fpath = self.root + os.path.sep + fname
+ if not ignore_missing or os.path.exists ( fpath ):
+ self.add_entry ( 'MISC', fpath, fname )
+ return True
+ else:
+ try:
+ del self._entries [ ( 'MISC', fname ) ]
+ except KeyError:
+ pass
+ else:
+ self.dirty = True
+ return False
+ # --- end of add_metadata_entry (...) ---
+
+ def add_package_entry ( self, p_info, add_ebuild=True ):
+ """Adds an entry for the package file of a package info object, and
+ optionally for its ebuild, too.
+
+ arguments:
+ * p_info -- package info
+ * add_ebuild -- if True: add an entry for p_info's ebuild
+ Defaults to True.
+
+ Note: This method can only be used for "new" package infos.
+ """
+ p_entry = PackageFileManifestEntry ( p_info )
+
+ if add_ebuild:
+ efile = p_info ['ebuild_file']
+ if efile is None:
+ raise Exception ( "ebuild file must exist." )
+ e_entry = ManifestEntry ( 'EBUILD', efile )
+ e_entry.interpolate()
+
+ self._add_entry ( e_entry )
+
+ self._add_entry ( p_entry )
+ # --- end of add_package_entry (...) ---
+
+ def remove_entry ( self, filetype, filename ):
+ """Removes an entry.
+
+ arguments:
+ * filetype -- the entry's filetype
+ * filename -- the entry's filename
+ """
+ del self._entries [ ( filetype, filename ) ]
+ self.dirty = True
+ # --- end of remove_entry (...) ---
+
+ def remove_metadata_entry ( self ):
+ """Removes the metadata.xml entry."""
+ self.remove_entry ( 'MISC', 'metadata.xml' )
+ # --- end of remove_metadata_entry (...) ---
+
+ def remove_package_entry ( self, p_info, with_ebuild=True ):
+ """Removes a package entry (and optionally its ebuild entry).
+
+ arguments:
+ * p_info -- package info
+ * with_ebuild -- whether to remove the ebuild entry (defaults to True)
+ """
+ filename = p_info.get ( 'package_src_destpath', do_fallback=True )
+
+ if with_ebuild:
+ efile = p_info.get ( 'ebuild_file' )
+ if not efile:
+ raise Exception ( "package info object has no ebuild file" )
+ self.remove_entry ( 'EBUILD', os.path.basename ( efile ) )
+
+ self.remove_entry ( 'DIST', filename )
+ # --- end of remove_package_entry (...) ---
+
+ def read ( self, update=False, ignore_missing=False ):
+ """Reads and imports the Manifest file.
+
+ arguments:
+ * update -- if True: add non-existing entries only
+ Defaults to False.
+ * ignore_missing -- if True: don't raise an exception if the Manifest
+ file does not exist. Defaults to False.
+ """
+ FILETYPES = ManifestEntry.FILETYPES
+ HASHTYPES = ManifestEntry.HASHTYPES
+
+ FH = None
+ try:
+ FH = open ( self.filepath, 'rt' )
+
+ for line in FH.readlines():
+ rsline = line.rstrip()
+ if rsline:
+ #<filetype> <filename> <filesize> <hashes...>
+ lc = rsline.split ( None )
+ if lc[0] in FILETYPES:
+ # expect uneven number of line components (3 + <hash pairs>)
+ assert len ( lc ) % 2 == 1 and len ( lc ) > 3
+
+ filetype = lc[0]
+ filename = lc[1]
+ key = ( filetype, filename )
+
+ if not update or key not in self._entries:
+ filesize = lc[2]
+ hashes = dict()
+ hash_name = None
+ for word in lc[3:]:
+ if hash_name is None:
+ hash_name = word.lower()
+ else:
+ hashes [hash_name] = word
+ # ^ = int ( word, 16 )
+ hash_name = None
+ # -- end for;
+
+ self._entries [ ( filetype, filename ) ] = (
+ ManifestEntry (
+ filetype, None, filename, filesize, hashes
+ )
+ )
+ # -- end if not update or new
+ # else ignore
+
+ except IOError as ioerr:
+ if ignore_missing and ioerr.errno == errno.ENOENT:
+ pass
+ else:
+ raise
+ finally:
+ if FH:
+ FH.close()
+ # --- end of read (...) ---
+
+ def gen_lines ( self, do_sort=True ):
+ """Generates text file lines.
+
+ arguments:
+ * do_sort -- if True: produce sorted output (defaults to True)
+ """
+ if do_sort:
+ for item in sorted ( self._entries.items(), key=lambda kv: kv[0] ):
+ yield str ( item[1] )
+ else:
+ for entry in self._entries.values():
+ yield str ( entry )
+ # --- end of gen_lines (...) ---
+
+ def __str__ ( self ):
+ return '\n'.join ( self.gen_lines() )
+ # --- end of __str__ (...) ---
+
+ def __repr__ ( self ):
+ # COULDFIX: very efficient!
+ return "<Manifest for {}>".format (
+ os.path.sep.join ( self.root.rsplit ( os.path.sep, 2 ) [-2:] )
+ )
+ # --- end of __repr__ (...) ---
+
+ def write ( self, force=False ):
+ """Writes the Manifest file.
+
+ arguments:
+ * force -- enforce writing even if no changes made
+ """
+ if force or self.dirty:
+ with open ( self.filepath, 'wt' ) as FH:
+ for line in self.gen_lines():
+ FH.write ( line )
+ FH.write ( '\n' )
+
+ self.dirty = False
+ return True
+ else:
+ return False
+ # --- end of write (...) ---
+
+ def exists ( self ):
+ """Returns True if the Manifest file exists (as file), else False."""
+ return os.path.isfile ( self.filepath )
+ # --- end of exists (...) ---
+
+# --- end of ManifestFile ---
diff --git a/roverlay/overlay/pkgdir/packagedir_newmanifest.py b/roverlay/overlay/pkgdir/packagedir_newmanifest.py
new file mode 100644
index 0000000..04d202d
--- /dev/null
+++ b/roverlay/overlay/pkgdir/packagedir_newmanifest.py
@@ -0,0 +1,106 @@
+# R overlay -- overlay package, package directory ("new" manifest)
+# -*- coding: utf-8 -*-
+# Copyright (C) 2013 André Erdmann <dywi@mailerd.de>
+# Distributed under the terms of the GNU General Public License;
+# either version 2 of the License, or (at your option) any later version.
+
+__all__ = [ 'PackageDir', ]
+
+import os
+import threading
+
+
+import roverlay.config
+
+import roverlay.tools.ebuild
+
+import roverlay.overlay.pkgdir.manifest.file
+import roverlay.overlay.pkgdir.packagedir_base
+
+from roverlay.overlay.pkgdir.manifest.file import ManifestFile
+
+
+class PackageDir ( roverlay.overlay.pkgdir.packagedir_base.PackageDirBase ):
+ """
+ PackageDir class that uses an (mostly) internal implementation
+ for Manifest writing.
+ """
+
+ MANIFEST_THREADSAFE = True
+
+ # Manifest entries for imported ebuilds have to be created during import
+ DOEBUILD_FETCH = roverlay.tools.ebuild.doebuild_fetch_and_manifest
+
+ def _get_manifest ( self ):
+ """Returns a ManifestFile object."""
+ manifest = ManifestFile ( self.physical_location )
+ manifest.read ( ignore_missing=True )
+ return manifest
+ # --- end of _get_manifest (...) ---
+
+ def _write_import_manifest ( self, _manifest=None ):
+ """Verifies that Manifest file entries exist for all imported ebuilds.
+
+ Assumption: ebuild file in Manifest => $SRC_URI, too
+
+ Returns True on success, else False.
+
+ arguments:
+ * _manifest -- manifest object (defaults to None -> create a new one)
+ """
+ manifest = self._get_manifest() if _manifest is None else _manifest
+
+ self.logger.debug ( "Checking import Manifest entries" )
+
+ ret = True
+ ename = None
+ for p in self._packages.values():
+ efile = p.get ( 'ebuild_file' )
+ if efile is not None:
+ ename = os.path.basename ( efile )
+ if manifest.has_entry ( 'EBUILD', ename ):
+ self.logger.debug (
+ "manifest entry for {} is ok.".format ( ename )
+ )
+ else:
+ ret = False
+ self.logger.error (
+ "manifest entry for imported ebuild {} is missing!".format (
+ ename
+ )
+ )
+ # -- end for;
+ return ret
+ # --- end of _write_import_manifest (...) ---
+
+ def _write_manifest ( self, pkgs_for_manifest ):
+ """Generates and writes the Manifest file for this package.
+
+ expects: called after writing metadata/ebuilds
+
+ returns: success (True/False)
+ """
+ manifest = self._get_manifest()
+
+ manifest.add_metadata_entry ( ignore_missing=True )
+
+ # add manifest entries and add hardlinks to DISTROOT
+ # (replacing existing files/links)
+ #
+ distdir = self.DISTROOT.get_distdir ( self.name )
+ for p in pkgs_for_manifest:
+ manifest.add_package_entry ( p )
+ distdir.add ( p ['package_file'], p ['package_src_destpath'], p )
+ # -- end for;
+
+ #return (...)
+ if (
+ manifest.write ( force=True )
+ and self._write_import_manifest ( _manifest=manifest )
+ ):
+ return True
+ else:
+ return False
+ # --- end of write_manifest (...) ---
+
+# --- end of PackageDir #ebuildmanifest ---
^ permalink raw reply related [flat|nested] 2+ messages in thread
end of thread, other threads:[~2013-07-10 16:17 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2013-07-10 15:10 [gentoo-commits] proj/R_overlay:gsoc13/next commit in: roverlay/overlay/pkgdir/manifest/, roverlay/config/, roverlay/, André Erdmann
2013-07-10 16:16 ` [gentoo-commits] proj/R_overlay:master " André Erdmann
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox