From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from pigeon.gentoo.org ([208.92.234.80] helo=lists.gentoo.org) by finch.gentoo.org with esmtp (Exim 4.60) (envelope-from ) id 1SjDtY-0001VJ-EE for garchives@archives.gentoo.org; Mon, 25 Jun 2012 18:20:16 +0000 Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 6CB98E0B23; Mon, 25 Jun 2012 18:19:54 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) by pigeon.gentoo.org (Postfix) with ESMTP id 2CDBFE0B23 for ; Mon, 25 Jun 2012 18:19:54 +0000 (UTC) Received: from hornbill.gentoo.org (hornbill.gentoo.org [94.100.119.163]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 5C8E41B4452 for ; Mon, 25 Jun 2012 18:19:53 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by hornbill.gentoo.org (Postfix) with ESMTP id 4FF6FE5445 for ; Mon, 25 Jun 2012 18:19:50 +0000 (UTC) From: "André Erdmann" To: gentoo-commits@lists.gentoo.org Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "André Erdmann" Message-ID: <1340648138.9a0b0c9e1740e88ee2426fefa2c97fb7e629bdb1.dywi@gentoo> Subject: [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/ X-VCS-Repository: proj/R_overlay X-VCS-Files: roverlay/remote/__init__.py roverlay/remote/basicrepo.py roverlay/remote/repo.py roverlay/remote/repolist.py roverlay/remote/repoloader.py roverlay/remote/rsync.py X-VCS-Directories: roverlay/remote/ X-VCS-Committer: dywi X-VCS-Committer-Name: André Erdmann X-VCS-Revision: 9a0b0c9e1740e88ee2426fefa2c97fb7e629bdb1 X-VCS-Branch: master Date: Mon, 25 Jun 2012 18:19:50 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: quoted-printable X-Archives-Salt: 20defb20-85c1-45d9-9cd2-c51c5cf2006d X-Archives-Hash: 8d8996c6515e0e99fbcecfa0afda760b commit: 9a0b0c9e1740e88ee2426fefa2c97fb7e629bdb1 Author: Andr=C3=A9 Erdmann mailerd de> AuthorDate: Mon Jun 25 18:15:38 2012 +0000 Commit: Andr=C3=A9 Erdmann mailerd de> CommitDate: Mon Jun 25 18:15:38 2012 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=3Dproj/R_overlay.git= ;a=3Dcommit;h=3D9a0b0c9e the remote module (used for repo syncing) * this modules handles repositories, both local (directory) and remote (currently only rsync) * some parts, mainly integration into roverlay, are todo new file: roverlay/remote/__init__.py new file: roverlay/remote/basicrepo.py new file: roverlay/remote/repo.py new file: roverlay/remote/repolist.py new file: roverlay/remote/repoloader.py new file: roverlay/remote/rsync.py --- roverlay/remote/__init__.py | 1 + roverlay/remote/basicrepo.py | 245 +++++++++++++++++++++++++++++++++++= ++++++ roverlay/remote/repo.py | 54 +++++++++ roverlay/remote/repolist.py | 55 +++++++++ roverlay/remote/repoloader.py | 66 +++++++++++ roverlay/remote/rsync.py | 86 ++++++++++++++ 6 files changed, 507 insertions(+), 0 deletions(-) diff --git a/roverlay/remote/__init__.py b/roverlay/remote/__init__.py new file mode 100644 index 0000000..e7521be --- /dev/null +++ b/roverlay/remote/__init__.py @@ -0,0 +1 @@ +from roverlay.remote.repolist import RepoList diff --git a/roverlay/remote/basicrepo.py b/roverlay/remote/basicrepo.py new file mode 100644 index 0000000..9ade3a2 --- /dev/null +++ b/roverlay/remote/basicrepo.py @@ -0,0 +1,245 @@ +import os.path + +from roverlay import config +from roverlay.packageinfo import PackageInfo + +URI_SEPARATOR =3D '://' +DEFAULT_PROTOCOL =3D 'http' + +LOCALREPO_SRC_URI =3D 'http://localhost/R-Packages' + +def normalize_uri ( uri, protocol, force_protocol=3DFalse ): + + if not protocol: + return uri + + proto, sep, base_uri =3D uri.partition ( URI_SEPARATOR ) + if sep !=3D URI_SEPARATOR: + return URI_SEPARATOR.join ( ( protocol, uri ) ) + elif force_protocol: + return URI_SEPARATOR.join ( ( protocol, base_uri ) ) + else: + return uri +# --- end of normalize_uri (...) --- + +class LocalRepo ( object ): + """ + This class represents a local repository - all packages are assumed + to exist in its distfiles dir and no remote syncing will occur. + It's the base class for remote repos. + """ + + def __init__ ( self, name, directory=3DNone, src_uri=3DNone ): + """Initializes a LocalRepo. + + arguments: + * name -- + * directory -- distfiles dir, defaults to / + * src_uri -- SRC_URI, defaults to http://localhost/R-Packages/ + """ + self.name =3D name + if directory is None: + self.distdir =3D os.path.join ( + config.get_or_fail ( [ 'DISTFILES', 'root' ] ), + # subdir repo names like CRAN/contrib are ok, + # but make sure to use the correct path separator + self.name.replace ( '/', os.path.sep ), + ) + else: + self.distdir =3D directory + + if src_uri is None: + self.src_uri =3D '/'.join ( ( LOCALREPO_SRC_URI, self.name ) ) + else: + self.src_uri =3D src_uri + + # --- end of __init__ (...) --- + + def __str__ ( self ): + return "repo '%s': DISTDIR '%s', SRC_URI '%s'" % ( + self.name, self.distdir, self.src_uri + ) + + def get_name ( self ): + """Returns the name of this repository.""" + return self.name + # --- end of get_name (...) --- + + def get_distdir ( self ): + """Returns the distfiles directory of this repository.""" + return self.distdir + # --- end of get_distdir (...) --- + + def get_src_uri ( self, package_file=3DNone ): + """Returns the SRC_URI of this repository. + + arguments: + * package_file -- if set and not None: returns a SRC_URI for this pkg + """ + if package_file is None: + return self.src_uri + else: + return '/'.join ( self.src_uri, package_file ) + # --- end of get_src_uri (...) --- + + # get_src(...) -> get_src_uri(...) + get_src =3D get_src_uri + + def exists ( self ): + """Returns True if this repo locally exists.""" + return os.path.isdir ( self.distdir ) + # --- end of exists (...) --- + + def nosync ( self ): + """Returns True if the repo is ready for overlay creation, else False. + Useful for basic local distfiles verification without downloading + anything. + """ + return self.exists() + + # --- end of nosync (...) --- + + # sync() -> nosync(), LocalRepos don't have anything to sync + sync =3D nosync + + def scan_distdir ( self, is_package=3DNone ): + """Generator that scans the local distfiles dir of this repo and + yields PackageInfo instances. + + arguments: + * is_package -- function returning True if the given file is a package + or None which means that all files are packages. + Defaults to None. + """ + if is_package is None: + # unfiltered variant + + for dirpath, dirnames, filenames in os.walk ( self.distdir ): + for pkg in filenames: + yield PackageInfo ( filename=3Dpkg, origin=3Dself ) + + elif hasattr ( is_package, '__call__' ): + # filtered variant (adds an if is_package... before yield) + for dirpath, dirnames, filenames in os.walk ( self.distdir ): + for pkg in filenames: + if is_package ( os.path.join ( dirpath, pkg ) ): + yield PackageInfo ( filename=3Dpkg, origin=3Dself ) + + + else: + # faulty variant, raises Exception + raise Exception ( "is_package should either be None or a function." ) + #yield None + + # --- end of scan_distdir (...) --- + +# --- end of LocalRepo --- + + +class RemoteRepo ( LocalRepo ): + """A template for remote repositories.""" + + def __init__ ( + self, name, sync_proto, + directory=3DNone, + src_uri=3DNone, remote_uri=3DNone, base_uri=3DNone + ): + """Initializes a RemoteRepo. + Mainly consists of URI calculation that derived classes may find usefu= l. + + arguments: + * name -- + * sync_proto -- protocol used for syncing (e.g. 'rsync') + * directory -- + * src_uri -- src uri, if set, else calculated using base/remote uri= , + the leading :// can be left out in which case + http is assumed + * remote_uri -- uri used for syncing, if set, else calculated using + base/src uri, the leading :// can be left out + * base_uri -- used to calculate remote/src uri, + example: localhost/R-packages/something + + keyword condition: + * | { x : x in union(src,remote,base) and x not None } | >=3D 1 + ^=3D at least one out of src/remote/base uri is not None + """ + super ( RemoteRepo, self ) . __init__ ( name, directory, src_uri=3D'' = ) + + self.sync_proto =3D sync_proto + + # detemerine uris + if src_uri is None and remote_uri is None: + if base_uri is None: + # keyword condition not met + raise Exception ( "Bad initialization of RemoteRepo!" ) + + else: + # using base_uri for src,remote + self.src_uri =3D URI_SEPARATOR.join ( + ( DEFAULT_PROTOCOL, base_uri ) + ) + + self.remote_uri =3D URI_SEPARATOR.join ( + ( sync_proto, base_uri ) + ) + + elif src_uri is None: + # remote_uri is not None + self.remote_uri =3D normalize_uri ( remote_uri, self.sync_proto ) + + if base_uri is not None: + # using base_uri for src_uri + self.src_uri =3D URI_SEPARATOR.join ( + ( DEFAULT_PROTOCOL, base_uri ) + ) + else: + # using remote_uri for src_uri + self.src_uri =3D normalize_uri ( + self.remote_uri, DEFAULT_PROTOCOL, force_protocol=3DTrue + ) + + elif remote_uri is None: + # src_uri is not None + self.src_uri =3D normalize_uri ( src_uri, DEFAULT_PROTOCOL ) + + if base_uri is not None: + # using base_uri for remote_uri + self.remote_uri =3D URI_SEPARATOR.join ( + ( self.sync_proto, base_uri ) + ) + else: + # using src_uri for remote_uri + self.remote_uri =3D normalize_uri ( + self.src_uri, self.sync_proto, force_protocol=3DTrue + ) + else: + # remote and src not None + self.remote_uri =3D normalize_uri ( remote_uri, self.sync_proto ) + self.src_uri =3D normalize_uri ( src_uri, DEFAULT_PROTOCOL ) + + # --- end of __init__ (...) --- + + def get_remote_uri ( self ): + """Returns the remote uri of this RemoteRepo which used for syncing.""= " + return self.remote_uri + # --- end of get_remote_uri (...) --- + + # get_remote(...) -> get_remote_uri(...) + get_remote =3D get_remote_uri + + def sync ( self ): + """Gets packages from remote(s) and returns True if the repo is ready + for overlay creation, else False. + + Derived classes have to implement this method. + """ + raise Exception ( "RemoteRepo does not implement sync()." ) + # --- end of sync (...) --- + + def __str__ ( self ): + return "repo '%s': DISTDIR '%s', SRC_URI '%s', REMOTE_URI '%s'" % ( + self.name, self.distdir, self.src_uri, self.remote_uri + ) + +# --- end of RemoteRepo --- + diff --git a/roverlay/remote/repo.py b/roverlay/remote/repo.py new file mode 100644 index 0000000..f54f448 --- /dev/null +++ b/roverlay/remote/repo.py @@ -0,0 +1,54 @@ + +import logging + +#from roverlay.remote.basicrepo import LocalRepo, RemoteRepo +from roverlay.remote.basicrepo import RemoteRepo + +from roverlay.remote.rsync import RsyncJob + +class RsyncRepo ( RemoteRepo ): + + def __init__ ( + self, name, + directory=3DNone, src_uri=3DNone, rsync_uri=3DNone, base_uri=3DNone, + extra_rsync_opts=3DNone + ): + # super's init: name, remote protocol, directory_kw, **uri_kw + # using '' as remote protocol which leaves uris unchanged when + # normalizing them for rsync usage + super ( RsyncRepo, self ) . __init__ ( + name, '', directory=3Ddirectory, + src_uri=3Dsrc_uri, remote_uri=3Drsync_uri, base_uri=3Dbase_uri + ) + self.extra_rsync_opts =3D extra_rsync_opts + # --- end of __init__ (...) --- + + + def sync ( self ): + retcode =3D None + try: + job =3D RsyncJob ( + remote=3Dself.remote_uri, distdir=3Dself.distdir, + run_now=3DTrue, + extra_opts=3Dself.extra_rsync_opts + ) + if job.returncode =3D=3D 0: return True + + retcode =3D job.returncode + except Exception as e: + # catch exceptions, log them and return False + ## TODO: which exceptions to catch||pass? + logging.exception ( e ) + retcode =3D '' + + + logging.error ( + 'Repo %s cannot be used for ebuild creation due to errors ' + 'while running rsync (return code was %s).' % ( self.name, retcode ) + ) + return False + # --- end of sync (...) --- + + def __str__ ( self ): + return "rsync repo '%s': DISTDIR '%s', SRC_URI '%s', RSYNC_URI '%s'" \ + % ( self.name, self.distdir, self.src_uri, self.remote_uri ) diff --git a/roverlay/remote/repolist.py b/roverlay/remote/repolist.py new file mode 100644 index 0000000..4617057 --- /dev/null +++ b/roverlay/remote/repolist.py @@ -0,0 +1,55 @@ + +from roverlay import config + +from roverlay.remote.repoloader import read_repofile + +class RepoList ( object ): + + def __init__ ( self ): + self.repos =3D list() + self.sync_enabled =3D True + self.use_broken_repos =3D False + + def sort ( self ): + raise Exception ( "method stub." ) + + def load_file ( self, _file ): + new_repos =3D read_repofile ( _file ) + if new_repos: + self.repos.extend ( new_repos ) + # --- end of load_file (...) --- + + def load ( self ): + files =3D config.get_or_fail ( 'REPO.config_files' ) + for f in files: + self.load_file ( f ) + # --- end of load (...) --- + + def sync_all ( self, package_queue=3DNone ): + q =3D None + if package_queue is None: + q =3D list() + add =3D q.append + else: + # TODO: _nowait? raises Exception when queue is full which is + # good in non-threaded execution + # -> timeout,.. + add =3D q.put + + + # !! TODO resume here. + + for repo in self.repos: + if repo.sync() if self.sync_enabled else repo.nosync(): + # scan repo and create package infos + for p in repo.scan_distdir(): add ( p ) + elif self.use_broken_repos: + # warn and scan repo + ## .. + for p in repo.scan_distdir(): add ( p ) + + # --- end of sync_all (...) --- + + def __str__ ( self ): + return '\n'.join ( ( str ( x ) for x in self.repos ) ) + diff --git a/roverlay/remote/repoloader.py b/roverlay/remote/repoloader.p= y new file mode 100644 index 0000000..eae35c5 --- /dev/null +++ b/roverlay/remote/repoloader.py @@ -0,0 +1,66 @@ + +import logging + +try: + import configparser +except ImportError as running_python2: + # configparser is named ConfigParser in python2 + import ConfigParser as configparser + + +from roverlay import config + +from roverlay.remote.basicrepo import LocalRepo +from roverlay.remote.repo import RsyncRepo + +LOGGER =3D logging.getLogger ( 'repoloader' ) + +def read_repofile ( repo_file, lenient=3DFalse ): + + parser =3D configparser.SafeConfigParser ( allow_no_value=3DFalse ) + + if lenient: + parser.read ( repo_file ) + else: + fh =3D None + try: + fh =3D open ( repo_file, 'r' ) + parser.readfp ( fh ) + finally: + if fh: fh.close() + + repos =3D list() + + for name in parser.sections(): + + get =3D lambda a, b=3DNone : parser.get ( name, a, raw=3DTrue, fallbac= k=3Db ) + + repo_type =3D get ( 'type', 'rsync' ).lower() + + if repo_type =3D=3D 'local': + repo =3D LocalRepo ( + name =3D get ( 'name', name ), + directory =3D get ( 'directory' ), + src_uri =3D get ( 'src_uri' ) + ) + elif repo_type =3D=3D 'rsync': + repo =3D RsyncRepo ( + name =3D get ( 'name', name ), + directory =3D get ( 'directory' ), + src_uri =3D get ( 'src_uri' ), + rsync_uri =3D get ( 'rsync_uri' ), + base_uri =3D get ( 'base_uri' ), + extra_rsync_opts =3D get ( 'extra_rsync_opts' ) + ) + else: + LOGGER.error ( "Unknown repo type %s for %s" % ( repo_type, name ) ) + continue + + LOGGER.debug ( 'New entry, ' + str ( repo ) ) + + repos.append ( repo ) + repo =3D None + + + return repos +# --- end of read_repofile (...) --- diff --git a/roverlay/remote/rsync.py b/roverlay/remote/rsync.py new file mode 100644 index 0000000..e46d1db --- /dev/null +++ b/roverlay/remote/rsync.py @@ -0,0 +1,86 @@ +import os +import subprocess + +from roverlay import config +from roverlay.util import keepenv + + +RSYNC_ENV =3D keepenv ( + 'PATH', + 'USER', + 'LOGNAME', + 'RSYNC_PROXY', + 'RSYNC_PASSWORD', +) + + +# --recursive is not in the default opts, subdirs in CRAN/contrib are +# either R release (2.xx.x[-patches] or the package archive) +DEFAULT_RSYNC_OPTS =3D ( + '--links', # copy symlinks as symlinks, + '--safe-links', # but ignore links outside of tree + '--times', # + '--compress', # FIXME: add lzo if necessary + '--delete', # + '--force', # allow deletion of non-empty dirs + '--human-readable', # + '--stats', # + '--chmod=3Dugo=3Dr,u+w,Dugo+x', # 0755 for transferred dirs, 0644 for f= iles +) + +class RsyncJob ( object ): + def __init__ ( + self, remote=3DNone, distdir=3DNone, run_now=3DTrue, extra_opts=3DNone + ): + self.remote =3D remote + self.distdir =3D distdir + self.extra_opts =3D None + + if run_now: self.run() + # --- end of __init__ (...) --- + + def _rsync_argv ( self ): + if self.remote is None or self.distdir is None: + raise Exception ( "None in (remote,distdir)." ) + + argv =3D [ 'rsync' ] + + argv.extend ( DEFAULT_RSYNC_OPTS ) + + max_bw =3D config.get ( 'RSYNC_BWLIMIT', None ) + if max_bw is not None: + argv.append ( '--bwlimit=3D%i' % max_bw ) + + if self.extra_opts is not None: + if isinstance ( self.extra_opts, str ) or \ + not hasattr ( self.extra_opts, '__iter__' )\ + : + argv.append ( self.extra_opts ) + else: + argv.extend ( self.extra_opts ) + + argv.extend ( ( self.remote, self.distdir ) ) + + return argv + # --- end of _rsync_argv (...) --- + + def run ( self ): + + rsync_cmd =3D self._rsync_argv() + + os.makedirs ( self.distdir, exist_ok=3DTrue ) + + # TODO pipe/log/.., running this in blocking mode until implemented + + proc =3D subprocess.Popen ( + rsync_cmd, + stdin=3DNone, stdout=3DNone, stderr=3DNone, + env=3DRSYNC_ENV + ) + + if proc.communicate() !=3D ( None, None ): + raise AssertionError ( "expected None,None from communicate!" ) + + self.returncode =3D proc.returncode + + # --- end of start (...) ---