* [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
@ 2013-07-24 9:54 André Erdmann
0 siblings, 0 replies; 35+ messages in thread
From: André Erdmann @ 2013-07-24 9:54 UTC (permalink / raw
To: gentoo-commits
commit: 789651939ae2187e89403e6c09fa30a43026e126
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jul 24 09:50:13 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jul 24 09:50:13 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=78965193
roverlay/remote: more accurate (no)sync logging
---
roverlay/remote/basicrepo.py | 37 +++++++++++++++++++++++++++++--------
roverlay/remote/status.py | 10 ++++++++++
2 files changed, 39 insertions(+), 8 deletions(-)
diff --git a/roverlay/remote/basicrepo.py b/roverlay/remote/basicrepo.py
index 33d8f30..b126f9b 100644
--- a/roverlay/remote/basicrepo.py
+++ b/roverlay/remote/basicrepo.py
@@ -13,15 +13,18 @@ import logging
from roverlay.packageinfo import PackageInfo
+from . import status
+
+
URI_SEPARATOR = '://'
DEFAULT_PROTOCOL = 'http'
LOCALREPO_SRC_URI = 'http://localhost/R-Packages'
-SYNC_SUCCESS = 1
-SYNC_FAIL = 2
-SYNC_DONE = SYNC_SUCCESS | SYNC_FAIL
-REPO_READY = 4
+SYNC_SUCCESS = status.SYNC_SUCCESS
+SYNC_FAIL = status.SYNC_FAIL
+SYNC_DONE = status.SYNC_DONE
+REPO_READY = status.REPO_READY
def normalize_uri ( uri, protocol, force_protocol=False ):
"""Returns an uri that is prefixed by its protocol ('http://', ...).
@@ -84,15 +87,20 @@ class BasicRepo ( object ):
else:
self.src_uri = src_uri
- self.sync_status = 0
-
if remote_uri is not None:
self.is_remote = True
self.remote_uri = remote_uri
else:
self.is_remote = is_remote
+
+
+ self.sync_status = 0
# --- end of __init__ (...) ---
+ def reset ( self ):
+ self.sync_status = 0
+ # --- end of reset (...) ---
+
def ready ( self ):
"""Returns True if this repo is ready (for package scanning using
scan_distdir).
@@ -192,20 +200,33 @@ class BasicRepo ( object ):
"""Syncs this repo."""
status = False
- print ( "Syncing {!r} ...".format ( self.name ) )
if sync_enabled and hasattr ( self, '_dosync' ):
+ print ( "Syncing {!r} ...".format ( self.name ) )
status = self._dosync()
elif hasattr ( self, '_nosync'):
+ self.logger.debug ( "calling repo-specific nosync() method." )
status = self._nosync()
+ elif self.exists():
+ self.logger.info ( "directory {!r} exists".format ( self.distdir ) )
+ status = True
else:
- status = self.exists()
+ self.logger.warning (
+ "directory {!r} does not exist".format ( self.distdir )
+ )
+ status = False
if status:
+ self.logger.debug (
+ "sync(online={}) succeeded.".format ( sync_enabled )
+ )
self._set_ready ( is_synced=sync_enabled )
else:
+ self.logger.info (
+ "sync(online{}) failed.".format ( sync_enabled )
+ )
self._set_fail()
return status
diff --git a/roverlay/remote/status.py b/roverlay/remote/status.py
new file mode 100644
index 0000000..7caa896
--- /dev/null
+++ b/roverlay/remote/status.py
@@ -0,0 +1,10 @@
+# R overlay -- remote, status codes
+# -*- coding: utf-8 -*-
+# Copyright (C) 2013 André Erdmann <dywi@mailerd.de>
+# Distributed under the terms of the GNU General Public License;
+# either version 2 of the License, or (at your option) any later version.
+
+SYNC_SUCCESS = 1
+SYNC_FAIL = 2
+SYNC_DONE = SYNC_SUCCESS | SYNC_FAIL
+REPO_READY = 4
^ permalink raw reply related [flat|nested] 35+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
@ 2015-01-26 17:41 André Erdmann
0 siblings, 0 replies; 35+ messages in thread
From: André Erdmann @ 2015-01-26 17:41 UTC (permalink / raw
To: gentoo-commits
commit: f289f88472e6b73579906f520d6ca77e633e061b
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Mon Dec 15 23:04:38 2014 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Mon Dec 15 23:04:38 2014 +0000
URL: http://sources.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=f289f884
run rsync with /dev/null as stdin
---
roverlay/remote/rsync.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/roverlay/remote/rsync.py b/roverlay/remote/rsync.py
index 24e25f8..402d68f 100644
--- a/roverlay/remote/rsync.py
+++ b/roverlay/remote/rsync.py
@@ -82,7 +82,7 @@ def run_rsync ( cmdv, env=RSYNC_ENV ):
)
# send SIGTERM and wait,
# fall back to _stop_subprocess() if another exception is hit
- _gracefully_stop_subprocess ( proc, kill_timeout_cs=40 )
+ _gracefully_stop_subprocess ( proc, stdin=False, kill_timeout_cs=40 )
raise
except Exception:
^ permalink raw reply related [flat|nested] 35+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
@ 2014-02-16 16:45 André Erdmann
0 siblings, 0 replies; 35+ messages in thread
From: André Erdmann @ 2014-02-16 16:45 UTC (permalink / raw
To: gentoo-commits
commit: 0ac703da27fb19b6cd9607284607463369f43713
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Sun Feb 16 16:37:01 2014 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Sun Feb 16 16:37:01 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=0ac703da
roverlay/remote/websync: cleanup
* create distdir with mkdir_p=True (similar to rsync repo)
* print total number of files that will be synced
* _dosync(): check want_retry before retval
* _fetch_package_list(): add timeout to urlopen
---
roverlay/remote/websync.py | 39 +++++++++++++++++++++++++--------------
1 file changed, 25 insertions(+), 14 deletions(-)
diff --git a/roverlay/remote/websync.py b/roverlay/remote/websync.py
index dbdf0d4..24f5cd5 100644
--- a/roverlay/remote/websync.py
+++ b/roverlay/remote/websync.py
@@ -1,6 +1,6 @@
# R overlay -- remote, websync
# -*- coding: utf-8 -*-
-# Copyright (C) 2012, 2013 André Erdmann <dywi@mailerd.de>
+# Copyright (C) 2012-2014 André Erdmann <dywi@mailerd.de>
# Distributed under the terms of the GNU General Public License;
# either version 2 of the License, or (at your option) any later version.
@@ -166,7 +166,9 @@ class WebsyncBase ( BasicRepo ):
fetch_required = True
if fetch_required:
+ blocksize = self.transfer_blocksize
bytes_fetched = 0
+ assert blocksize
# FIXME: debug print (?)
if VERBOSE:
@@ -175,10 +177,11 @@ class WebsyncBase ( BasicRepo ):
)
# unlink the existing file first (if it exists)
+ # this is necessary for keeping hardlinks intact (-> package mirror)
util.try_unlink ( distfile )
with open ( distfile, mode='wb' ) as fh:
- block = webh.read ( self.transfer_blocksize )
+ block = webh.read ( blocksize )
while block:
# write block to file
fh.write ( block )
@@ -186,7 +189,8 @@ class WebsyncBase ( BasicRepo ):
bytes_fetched += len ( block )
# get the next block
- block = webh.read ( self.transfer_blocksize )
+ block = webh.read ( blocksize )
+ # -- end while
# -- with
if bytes_fetched == expected_filesize:
@@ -244,7 +248,10 @@ class WebsyncBase ( BasicRepo ):
)
return True
- util.dodir ( self.distdir )
+ util.dodir ( self.distdir, mkdir_p=True )
+
+ if VERBOSE:
+ print ( "{:d} files to consider.".format ( len(package_list) ) )
success = True
@@ -359,19 +366,19 @@ class WebsyncBase ( BasicRepo ):
retval = retval_tmp
# -- end while
- if retval is None:
+ if want_retry:
self.logger.error (
'Repo {name} cannot be used for ebuild creation: '
- 'did not try to sync (max_retry={max_retry:d})'.format (
- name=self.name, max_retry=max_retry
- )
+ 'retry count exhausted.'.format ( name=self.name )
)
return False
- elif want_retry:
+ elif retval is None:
self.logger.error (
'Repo {name} cannot be used for ebuild creation: '
- 'retry count exhausted.'.format ( name=self.name )
+ 'did not try to sync (max_retry={max_retry:d})'.format (
+ name=self.name, max_retry=max_retry
+ )
)
return False
@@ -479,7 +486,7 @@ class WebsyncRepo ( WebsyncBase ):
name, value = match.group ( 'name', 'value' )
info [name.lower()] = value
- if len ( info.keys() ) == max_info_len:
+ if len ( info ) == max_info_len:
pkgfile = '{name}_{version}.tar.gz'.format (
name=info ['package'], version=info ['version']
@@ -495,8 +502,9 @@ class WebsyncRepo ( WebsyncBase ):
# --- end of generate_pkglist (...) ---
package_list = ()
- with contextlib.closing ( urlopen ( self.pkglist_uri ) ) as webh:
-
+ with contextlib.closing (
+ urlopen ( self.pkglist_uri, None, self.timeout )
+ ) as webh:
content_type = webh.info().get ( 'content-type', None )
if content_type != 'text/plain':
@@ -643,10 +651,13 @@ class WebsyncPackageList ( WebsyncBase ):
)
return True
- util.dodir ( self.distdir )
+ util.dodir ( self.distdir, mkdir_p=True )
success = True
+ if VERBOSE:
+ print ( "{:d} files to consider.".format ( len(package_list) ) )
+
for package_file, src_uri in package_list:
if not self._get_package (
package_file, src_uri, expected_digest=None
^ permalink raw reply related [flat|nested] 35+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
@ 2014-02-15 19:49 André Erdmann
0 siblings, 0 replies; 35+ messages in thread
From: André Erdmann @ 2014-02-15 19:49 UTC (permalink / raw
To: gentoo-commits
commit: 1534865a3a5af437af52ec3e7d5d9dc3897a4197
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Sat Feb 15 19:02:47 2014 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Sat Feb 15 19:05:56 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=1534865a
roverlay/remote/websync: catch exceptions
Catch "unknown"/unhandled exceptions from urllib et al and set the repo's status
accordingly (similar to RsyncRepo).
Additionally, be a bit more specific about sync errors.
---
roverlay/remote/websync.py | 74 +++++++++++++++++++++++++++++++++++++---------
1 file changed, 60 insertions(+), 14 deletions(-)
diff --git a/roverlay/remote/websync.py b/roverlay/remote/websync.py
index 833633f..86f4d48 100644
--- a/roverlay/remote/websync.py
+++ b/roverlay/remote/websync.py
@@ -27,14 +27,16 @@ else:
urlopen = _urllib.urlopen
URLError = _urllib_error.URLError
HTTPError = _urllib_error.HTTPError
-del sys
from roverlay import digest, util
from roverlay.remote.basicrepo import BasicRepo
-# this count includes the first run
-# (in contrast to rsync!)
-MAX_WEBSYNC_RETRY = 4
+# number of sync retries
+# changed 2014-02-15: does no longer include the first run
+#
+# total number of sync tries := 1 + max ( MAX_WEBSYNC_RETRY, 0 )
+#
+MAX_WEBSYNC_RETRY = 3
VERBOSE = True
@@ -228,7 +230,13 @@ class WebsyncBase ( BasicRepo ):
package_list = self._fetch_package_list()
# empty/unset package list
- if not package_list: return True
+ if not package_list:
+ self.logger.info (
+ 'Repo {name}: nothing to sync - package list is empty.'.format (
+ name=self.name
+ )
+ )
+ return True
util.dodir ( self.distdir )
@@ -256,12 +264,13 @@ class WebsyncBase ( BasicRepo ):
return success
# --- end of _sync_packages (...) ---
- def _dosync ( self, max_retry=MAX_WEBSYNC_RETRY ):
+ def _dosync ( self ):
"""Syncs this repo."""
retry_count = 0
want_retry = True
retval_tmp = None
retval = None
+ max_retry = max ( MAX_WEBSYNC_RETRY, 0 ) + 1
while want_retry and retry_count < max_retry:
retry_count += 1
@@ -279,11 +288,12 @@ class WebsyncBase ( BasicRepo ):
)
want_retry = True
else:
- self.logger.critical (
+ self.logger.error (
"got an unexpected http error code: {:d}".format ( err.code )
)
self.logger.exception ( err )
- raise
+ retval = False
+ #break
except URLError as err:
if err.reason.errno in self.URL_ERROR_RETRY_CODES:
@@ -293,22 +303,52 @@ class WebsyncBase ( BasicRepo ):
)
want_retry = True
else:
- self.logger.critical (
+ self.logger.error (
"got an unexpected url error code: {:d}".format (
err.reason.errno
)
)
self.logger.exception ( err )
- raise
+ retval = False
+ #break
+
+ except KeyboardInterrupt:
+ #sys.stderr.write ( "\nKeyboard Interrupt\n" )
+ #if RERAISE_INTERRUPT ...
+ raise
+
+ except Exception as err:
+ self.logger.exception ( err )
+ retval = False
+ #break
+
else:
retval = retval_tmp
# -- end while
- if want_retry:
- self.logger.error ( "retry count exhausted - sync finally failed" )
+ if retval is None:
+ assert max_retry < 0
+ self.logger.error (
+ 'Repo {name} cannot be used for ebuild creation: '
+ 'did not try to sync (max_retry={max_retry:d})'.format (
+ name=self.name, max_retry=max_retry
+ )
+ )
+
+ elif want_retry:
+ self.logger.error (
+ 'Repo {name} cannot be used for ebuild creation: '
+ 'retry count exhausted.'.format ( name=self.name )
+ )
return False
+ elif retval:
+ return True
else:
- return retval
+ self.logger.error (
+ 'Repo {name} cannot be used for ebuild creation due to errors '
+ 'while syncing.'.format ( name=self.name )
+ )
+ return False
# --- end of _dosync (...) ---
def skip_fetch ( self, package_filename, distfile, src_uri ):
@@ -560,7 +600,13 @@ class WebsyncPackageList ( WebsyncBase ):
package_list = self._fetch_package_list()
# empty/unset package list
- if not package_list: return True
+ if not package_list:
+ self.logger.info (
+ 'Repo {name}: nothing to sync - package list is empty.'.format (
+ name=self.name
+ )
+ )
+ return True
util.dodir ( self.distdir )
^ permalink raw reply related [flat|nested] 35+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
@ 2014-02-15 19:49 André Erdmann
0 siblings, 0 replies; 35+ messages in thread
From: André Erdmann @ 2014-02-15 19:49 UTC (permalink / raw
To: gentoo-commits
commit: 090836adcae4aafa03baa199c44e06c5691bf647
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Sat Feb 15 19:23:30 2014 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Sat Feb 15 19:32:40 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=090836ad
roverlay/remote/websync: handle connection timeouts
Wait up to URLOPEN_TIMEOUT(=10) seconds before giving up (and setting
want_retry). The timeout value cannot be changed in the config file.
---
roverlay/remote/websync.py | 60 ++++++++++++++++++++++++++++++++++++----------
1 file changed, 48 insertions(+), 12 deletions(-)
diff --git a/roverlay/remote/websync.py b/roverlay/remote/websync.py
index 86f4d48..cbad78d 100644
--- a/roverlay/remote/websync.py
+++ b/roverlay/remote/websync.py
@@ -14,6 +14,7 @@ import errno
import contextlib
import re
import os
+import socket
import sys
# py2 urllib2 vs py3 urllib.request
@@ -38,6 +39,9 @@ from roverlay.remote.basicrepo import BasicRepo
#
MAX_WEBSYNC_RETRY = 3
+# timeout for urlopen, in seconds
+URLOPEN_TIMEOUT = 10
+
VERBOSE = True
# FIXME: websync does not support package deletion
@@ -48,6 +52,7 @@ class WebsyncBase ( BasicRepo ):
HTTP_ERROR_RETRY_CODES = frozenset ({ 404, 410, 500, 503 })
URL_ERROR_RETRY_CODES = frozenset ({ errno.ETIMEDOUT, })
+ RETRY_ON_TIMEOUT = True
def __init__ ( self,
name,
@@ -118,7 +123,9 @@ class WebsyncBase ( BasicRepo ):
return True
- with contextlib.closing ( urlopen ( src_uri ) ) as webh:
+ with contextlib.closing (
+ urlopen ( src_uri, None, URLOPEN_TIMEOUT )
+ ) as webh:
#web_info = webh.info()
expected_filesize = int ( webh.info().get ( 'content-length', -1 ) )
@@ -296,18 +303,44 @@ class WebsyncBase ( BasicRepo ):
#break
except URLError as err:
- if err.reason.errno in self.URL_ERROR_RETRY_CODES:
- self.logger.info (
- 'sync failed with an url error (errno {:d}). '
- 'Retrying...'.format ( err.reason.errno )
- )
+ if isinstance ( err.reason, socket.timeout ):
+ if self.RETRY_ON_TIMEOUT:
+ self.logger.info ( 'Connection timed out (#1). Retrying...' )
+ want_retry = True
+ else:
+ self.logger.error ( 'Connection timed out (#1).' )
+ self.logger.exception ( err )
+ retval = False
+ #break
+
+ elif hasattr ( err.reason, 'errno' ):
+ if err.reason.errno in self.URL_ERROR_RETRY_CODES:
+ self.logger.info (
+ 'sync failed with an url error (errno {:d}). '
+ 'Retrying...'.format ( err.reason.errno )
+ )
+ want_retry = True
+ else:
+ self.logger.error (
+ "got an unexpected url error code: {:d}".format (
+ err.reason.errno
+ )
+ )
+ self.logger.exception ( err )
+ retval = False
+ #break
+ else:
+ self.logger.error ( "got an unexpected url error." )
+ self.logger.exception ( err )
+ retval = False
+ #break
+
+ except socket.timeout as err:
+ if self.RETRY_ON_TIMEOUT:
+ self.logger.info ( 'Connection timed out (#2). Retrying...' )
want_retry = True
else:
- self.logger.error (
- "got an unexpected url error code: {:d}".format (
- err.reason.errno
- )
- )
+ self.logger.error ( 'Connection timed out (#2).' )
self.logger.exception ( err )
retval = False
#break
@@ -315,6 +348,7 @@ class WebsyncBase ( BasicRepo ):
except KeyboardInterrupt:
#sys.stderr.write ( "\nKeyboard Interrupt\n" )
#if RERAISE_INTERRUPT ...
+ #retval = False
raise
except Exception as err:
@@ -327,13 +361,13 @@ class WebsyncBase ( BasicRepo ):
# -- end while
if retval is None:
- assert max_retry < 0
self.logger.error (
'Repo {name} cannot be used for ebuild creation: '
'did not try to sync (max_retry={max_retry:d})'.format (
name=self.name, max_retry=max_retry
)
)
+ return False
elif want_retry:
self.logger.error (
@@ -341,8 +375,10 @@ class WebsyncBase ( BasicRepo ):
'retry count exhausted.'.format ( name=self.name )
)
return False
+
elif retval:
return True
+
else:
self.logger.error (
'Repo {name} cannot be used for ebuild creation due to errors '
^ permalink raw reply related [flat|nested] 35+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
@ 2014-02-15 19:49 André Erdmann
0 siblings, 0 replies; 35+ messages in thread
From: André Erdmann @ 2014-02-15 19:49 UTC (permalink / raw
To: gentoo-commits
commit: 668d08e8d76e332e63884b1e10ebc26faa2eb358
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Sat Feb 15 18:58:56 2014 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Sat Feb 15 18:58:56 2014 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=668d08e8
roverlay/remote/rsync: remote redundant _set_status
RsyncRepo->_dosync(): don't call _set_ready()/_set_fail()
This is done by BasicRepo->sync() anyway.
---
roverlay/remote/rsync.py | 25 +++++++++++++------------
1 file changed, 13 insertions(+), 12 deletions(-)
diff --git a/roverlay/remote/rsync.py b/roverlay/remote/rsync.py
index 85cbde7..925dc87 100644
--- a/roverlay/remote/rsync.py
+++ b/roverlay/remote/rsync.py
@@ -8,6 +8,7 @@
__all__ = [ 'RsyncRepo', ]
+import os
import sys
import subprocess
@@ -121,6 +122,7 @@ class RsyncRepo ( BasicRepo ):
"""Syncs this repo. Returns True if sync succeeded, else False.
All exceptions(?) are catched and interpreted as sync failure.
"""
+ assert os.EX_OK not in RETRY_ON_RETCODE
def waitfor ( p ):
if p.communicate() != ( None, None ):
@@ -160,11 +162,7 @@ class RsyncRepo ( BasicRepo ):
proc = subprocess.Popen ( rsync_cmd, env=RSYNC_ENV )
retcode = waitfor ( proc )
proc = None
-
- if retcode == 0:
- self._set_ready ( is_synced=True )
- return True
-
+ # -- end while
except KeyboardInterrupt:
# maybe add terminate/kill code here,
@@ -186,11 +184,14 @@ class RsyncRepo ( BasicRepo ):
# catch exceptions, log them and return False
self.logger.exception ( e )
- self.logger.error (
- 'Repo {name} cannot be used for ebuild creation due to errors '
- 'while running rsync (return code was {ret}).'.format (
- name=self.name, ret=retcode
- ) )
- self._set_fail()
- return False
+
+ if retcode == os.EX_OK:
+ return True
+ else:
+ self.logger.error (
+ 'Repo {name} cannot be used for ebuild creation due to errors '
+ 'while running rsync (return code was {ret}).'.format (
+ name=self.name, ret=retcode
+ ) )
+ return False
# --- end of _dosync (...) ---
^ permalink raw reply related [flat|nested] 35+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
@ 2013-09-03 13:15 André Erdmann
0 siblings, 0 replies; 35+ messages in thread
From: André Erdmann @ 2013-09-03 13:15 UTC (permalink / raw
To: gentoo-commits
commit: 049e688336b6f1880cb896e35bdde57f1262676f
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Sep 3 13:07:54 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Sep 3 13:07:54 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=049e6883
repolist: create_repo_identifier_map()
returns a dict< repo_name->repo_id >
---
roverlay/remote/repolist.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/roverlay/remote/repolist.py b/roverlay/remote/repolist.py
index f9b288a..b75d5e1 100644
--- a/roverlay/remote/repolist.py
+++ b/roverlay/remote/repolist.py
@@ -101,6 +101,10 @@ class RepoList ( object ):
self.repos.extend ( gen_repos() )
# --- end of add_distdirs (...) ---
+ def create_repo_identifier_map ( self ):
+ return { repo.name: repo.get_identifier() for repo in self.repos }
+ # --- end of create_repo_identifier_map (...) ---
+
def load_file ( self, _file ):
"""Loads a repo config file and adds the repos to this RepoList.
^ permalink raw reply related [flat|nested] 35+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
@ 2013-09-03 8:35 André Erdmann
0 siblings, 0 replies; 35+ messages in thread
From: André Erdmann @ 2013-09-03 8:35 UTC (permalink / raw
To: gentoo-commits
commit: 631ca545be9c7471542f8e993850b7495f544348
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Sep 3 08:31:05 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Sep 3 08:31:05 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=631ca545
roverlay/remote/basicrepo: use counter as ident
this allows to sort repos based on their ids (where lower number means that the
repo has been read earlier from the repo config file(s)).
---
roverlay/remote/basicrepo.py | 13 ++++++++++---
1 file changed, 10 insertions(+), 3 deletions(-)
diff --git a/roverlay/remote/basicrepo.py b/roverlay/remote/basicrepo.py
index 9aebfac..5ca9f51 100644
--- a/roverlay/remote/basicrepo.py
+++ b/roverlay/remote/basicrepo.py
@@ -11,6 +11,8 @@ __all__ = [ 'BasicRepo', ]
import os.path
import logging
+import roverlay.util.counter
+
from roverlay.packageinfo import PackageInfo
from . import status
@@ -55,6 +57,8 @@ class BasicRepo ( object ):
It's the base class for remote repos.
"""
+ ID_GENERATOR = roverlay.util.counter.IDGenerator()
+
def __init__ ( self,
name, distroot,
directory=None, src_uri=None, is_remote=False, remote_uri=None
@@ -66,8 +70,11 @@ class BasicRepo ( object ):
* directory -- distfiles dir, defaults to <DISTFILES root>/<name>
* src_uri -- SRC_URI, defaults to http://localhost/R-Packages/<name>
"""
- self.name = name
- self.logger = logging.getLogger (
+ super ( BasicRepo, self ).__init__()
+
+ self._identifier = next ( self.__class__.ID_GENERATOR )
+ self.name = name
+ self.logger = logging.getLogger (
self.__class__.__name__ + ':' + self.name
)
@@ -98,7 +105,7 @@ class BasicRepo ( object ):
# --- end of __init__ (...) ---
def get_identifier ( self ):
- return id ( self )
+ return self._identifier
# --- end of get_identifier (...) ---
def reset ( self ):
^ permalink raw reply related [flat|nested] 35+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
@ 2013-09-02 16:21 André Erdmann
0 siblings, 0 replies; 35+ messages in thread
From: André Erdmann @ 2013-09-02 16:21 UTC (permalink / raw
To: gentoo-commits
commit: 9c05fda7d8e19f02f833d115f149bf61ce0cbdae
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Mon Sep 2 16:17:19 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Mon Sep 2 16:17:19 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=9c05fda7
roverlay/remote/basicrepo: get_identifier()
---
roverlay/remote/basicrepo.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/roverlay/remote/basicrepo.py b/roverlay/remote/basicrepo.py
index e14d317..9aebfac 100644
--- a/roverlay/remote/basicrepo.py
+++ b/roverlay/remote/basicrepo.py
@@ -97,6 +97,10 @@ class BasicRepo ( object ):
self.sync_status = 0
# --- end of __init__ (...) ---
+ def get_identifier ( self ):
+ return id ( self )
+ # --- end of get_identifier (...) ---
+
def reset ( self ):
self.sync_status = 0
# --- end of reset (...) ---
^ permalink raw reply related [flat|nested] 35+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
@ 2013-08-29 15:08 André Erdmann
0 siblings, 0 replies; 35+ messages in thread
From: André Erdmann @ 2013-08-29 15:08 UTC (permalink / raw
To: gentoo-commits
commit: 4af1090c298e085981877e43a71c624bd5b3afe1
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Aug 29 15:05:54 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Aug 29 15:05:54 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=4af1090c
roverlay/remote: fix typo
s/repo_name/repo.name
---
roverlay/remote/repolist.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/roverlay/remote/repolist.py b/roverlay/remote/repolist.py
index f2eb6e7..f9b288a 100644
--- a/roverlay/remote/repolist.py
+++ b/roverlay/remote/repolist.py
@@ -186,7 +186,7 @@ class RepoList ( object ):
self.logger.debug ( "Syncing repos ..." )
for repo in self.repos:
- self.repo_stats.sync_time.begin ( repo_name )
+ self.repo_stats.sync_time.begin ( repo.name )
if repo.sync ( sync_enabled=self.sync_enabled ):
self.repo_stats.sync_time.end ( repo.name )
^ permalink raw reply related [flat|nested] 35+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
@ 2013-08-07 16:10 André Erdmann
0 siblings, 0 replies; 35+ messages in thread
From: André Erdmann @ 2013-08-07 16:10 UTC (permalink / raw
To: gentoo-commits
commit: 813ff2036cfba0ec4f0f69fec59ec5e568c9ba1a
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Aug 7 15:37:18 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Aug 7 15:37:21 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=813ff203
roverlay/remote/basicrepo: fix minor typo
---
roverlay/remote/basicrepo.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/roverlay/remote/basicrepo.py b/roverlay/remote/basicrepo.py
index b126f9b..e14d317 100644
--- a/roverlay/remote/basicrepo.py
+++ b/roverlay/remote/basicrepo.py
@@ -225,7 +225,7 @@ class BasicRepo ( object ):
self._set_ready ( is_synced=sync_enabled )
else:
self.logger.info (
- "sync(online{}) failed.".format ( sync_enabled )
+ "sync(online={}) failed.".format ( sync_enabled )
)
self._set_fail()
^ permalink raw reply related [flat|nested] 35+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
@ 2013-08-07 16:10 André Erdmann
0 siblings, 0 replies; 35+ messages in thread
From: André Erdmann @ 2013-08-07 16:10 UTC (permalink / raw
To: gentoo-commits
commit: 2bca3d3b9c5e2acc9aa9915bf57ccb02f7716588
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Aug 7 15:37:46 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Aug 7 15:37:46 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=2bca3d3b
roverlay/remote/repolist, sync(): greedy on demand
RepoList.sync() returns success(True/False) now.
Additionally, it accepts a fail_greedy arg, which causes sync to immediately
abort on failure. This behavior is disabled by default (as it used to be prior
to this commit).
---
roverlay/remote/repolist.py | 25 +++++++++++++++++++++----
1 file changed, 21 insertions(+), 4 deletions(-)
diff --git a/roverlay/remote/repolist.py b/roverlay/remote/repolist.py
index fac2ea5..f2eb6e7 100644
--- a/roverlay/remote/repolist.py
+++ b/roverlay/remote/repolist.py
@@ -203,13 +203,30 @@ class RepoList ( object ):
try_call ( when_all_done )
# --- end of _sync_all_repos_and_run (...) ---
- def sync ( self ):
- """Syncs all repos."""
+ def sync ( self, fail_greedy=False ):
+ """Syncs all repos.
+
+ Returns True if sync was successful for all repos, else False.
+
+ arguments:
+ * fail_greedy -- abort on first sync failure (defaults to False)
+ "screws" up time stats since sync_time should
+ include all repos
+ """
+ all_success = True
self.logger.debug ( "Syncing repos ..." )
for repo in self.repos:
self.repo_stats.sync_time.begin ( repo.name )
- repo.sync ( sync_enabled=self.sync_enabled )
- self.repo_stats.sync_time.end ( repo.name )
+ if repo.sync ( sync_enabled=self.sync_enabled ):
+ self.repo_stats.sync_time.end ( repo.name )
+ elif fail_greedy:
+ self.repo_stats.sync_time.end ( repo.name )
+ return False
+ else:
+ self.repo_stats.sync_time.end ( repo.name )
+ all_success = False
+ # -- end for
+ return all_success
# --- end of sync_all (...) ---
def sync_and_add ( self, add_method ):
^ permalink raw reply related [flat|nested] 35+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
@ 2013-07-24 9:54 André Erdmann
0 siblings, 0 replies; 35+ messages in thread
From: André Erdmann @ 2013-07-24 9:54 UTC (permalink / raw
To: gentoo-commits
commit: 25cc7b92fb635b025323ccfaa35ef7ae7aa2dfbc
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jul 24 09:49:47 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jul 24 09:49:47 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=25cc7b92
roverlay/remote/websync: use sys.hexversion
---
roverlay/remote/websync.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/roverlay/remote/websync.py b/roverlay/remote/websync.py
index a7250b6..f6dd948 100644
--- a/roverlay/remote/websync.py
+++ b/roverlay/remote/websync.py
@@ -17,7 +17,7 @@ import os
import sys
# py2 urllib2 vs py3 urllib.request
-if sys.version_info >= ( 3, ):
+if sys.hexversion >= 0x3000000:
import urllib.request as _urllib
import urllib.error as _urllib_error
else:
^ permalink raw reply related [flat|nested] 35+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
@ 2013-07-24 9:54 André Erdmann
0 siblings, 0 replies; 35+ messages in thread
From: André Erdmann @ 2013-07-24 9:54 UTC (permalink / raw
To: gentoo-commits
commit: 9c70c674debb3c3273b3762151140be4431d33b5
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jul 24 09:48:54 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jul 24 09:48:54 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=9c70c674
repolist: log more details about ignored repos
---
roverlay/remote/repolist.py | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/roverlay/remote/repolist.py b/roverlay/remote/repolist.py
index 5966086..3b403ed 100644
--- a/roverlay/remote/repolist.py
+++ b/roverlay/remote/repolist.py
@@ -141,11 +141,17 @@ class RepoList ( object ):
pass
else:
# repo cannot be used
- self.logger.warning ( "ignoring repo %s." % repo.name )
+ self.logger.warning (
+ "ignoring repo {!r} (sync_status={:d}).".format (
+ repo.name, repo.sync_status
+ )
+ )
return False
for p in repo.scan_distdir ( is_package=self._pkg_filter ):
- self.logger.debug ( "adding package %s from repo %s" % ( p, repo ) )
+ self.logger.debug (
+ "adding package {p} from repo {r}".format ( p=p, r=repo )
+ )
add_method ( p )
# --- end of _queue_packages_from_repo (...) ---
^ permalink raw reply related [flat|nested] 35+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
@ 2013-07-23 14:57 André Erdmann
0 siblings, 0 replies; 35+ messages in thread
From: André Erdmann @ 2013-07-23 14:57 UTC (permalink / raw
To: gentoo-commits
commit: a6aae1ff02d7cfa28cf6cf9025eccedbf71aab08
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jul 23 13:32:46 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jul 23 13:32:46 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=a6aae1ff
roverlay/remote/websync, retry: skip fetched files
Entirely skip files that have already been downloaded in "this" roverlay run.
roverlay would've already skipped the actual download of existing files (if
filesize,.. match), but this commit adds functionality to skip fetching _before_
opening a connection.
---
roverlay/remote/websync.py | 221 +++++++++++++++++++++++++++++----------------
1 file changed, 141 insertions(+), 80 deletions(-)
diff --git a/roverlay/remote/websync.py b/roverlay/remote/websync.py
index 87abab8..a7250b6 100644
--- a/roverlay/remote/websync.py
+++ b/roverlay/remote/websync.py
@@ -1,6 +1,6 @@
# R overlay -- remote, websync
# -*- coding: utf-8 -*-
-# Copyright (C) 2012 André Erdmann <dywi@mailerd.de>
+# Copyright (C) 2012, 2013 André Erdmann <dywi@mailerd.de>
# Distributed under the terms of the GNU General Public License;
# either version 2 of the License, or (at your option) any later version.
@@ -33,7 +33,9 @@ from roverlay import digest, util
from roverlay.packageinfo import PackageInfo
from roverlay.remote.basicrepo import BasicRepo
-MAX_WEBSYNC_RETRY = 3
+# this count includes the first run
+# (in contrast to rsync!)
+MAX_WEBSYNC_RETRY = 4
VERBOSE = True
@@ -108,98 +110,104 @@ class WebsyncBase ( BasicRepo ):
* expected_digest -- expected digest for package_file or None (^=disable)
"""
distfile = self.distdir + os.sep + package_file
- webh = urlopen ( src_uri )
- #web_info = webh.info()
-
- expected_filesize = int ( webh.info().get ( 'content-length', -1 ) )
-
- if os.access ( distfile, os.F_OK ):
- # package exists locally, verify it (size, digest)
- fetch_required = False
- localsize = os.path.getsize ( distfile )
-
- if localsize != expected_filesize:
- # size mismatch
- self.logger.info (
- 'size mismatch for {f!r}: expected {websize} bytes '
- 'but got {localsize}!'.format (
- f = package_file,
- websize = expected_filesize,
- localsize = localsize
- )
- )
- fetch_required = True
- elif expected_digest is not None:
- our_digest = digest.dodigest_file ( distfile, self._digest_type )
-
- if our_digest != expected_digest:
- # digest mismatch
- self.logger.warning (
- '{dtype} mismatch for {f!r}: '
- 'expected {theirs} but got {ours} - refetching.'.format (
- dtype = self._digest_type,
- f = distfile,
- theirs = expected_digest,
- ours = our_digest
- )
- )
- fetch_required = True
+ if self.skip_fetch ( package_file, distfile, src_uri ):
+ if VERBOSE:
+ print ( "Skipping fetch (early) for {f!r}".format ( f=distfile ) )
+ return True
- else:
- fetch_required = True
- if fetch_required:
- bytes_fetched = 0
+ with contextlib.closing ( urlopen ( src_uri ) ) as webh:
+ #web_info = webh.info()
- # FIXME: debug print (?)
- if VERBOSE:
- print (
- "Fetching {f} from {u} ...".format ( f=package_file, u=src_uri )
- )
+ expected_filesize = int ( webh.info().get ( 'content-length', -1 ) )
- # unlink the existing file first (if it exists)
- util.try_unlink ( distfile )
-
- with open ( distfile, mode='wb' ) as fh:
- block = webh.read ( self.transfer_blocksize )
- while block:
- # write block to file
- fh.write ( block )
- # ? bytelen
- bytes_fetched += len ( block )
+ if os.access ( distfile, os.F_OK ):
+ # package exists locally, verify it (size, digest)
+ fetch_required = False
+ localsize = os.path.getsize ( distfile )
- # get the next block
- block = webh.read ( self.transfer_blocksize )
- # -- with
+ if localsize != expected_filesize:
+ # size mismatch
+ self.logger.info (
+ 'size mismatch for {f!r}: expected {websize} bytes '
+ 'but got {localsize}!'.format (
+ f = package_file,
+ websize = expected_filesize,
+ localsize = localsize
+ )
+ )
+ fetch_required = True
- if bytes_fetched == expected_filesize:
- if expected_digest is not None:
+ elif expected_digest is not None:
our_digest = digest.dodigest_file ( distfile, self._digest_type )
if our_digest != expected_digest:
- # fetched package's digest does not match the expected one,
- # refuse to use it
+ # digest mismatch
self.logger.warning (
- 'bad {dtype} digest for {f!r}, expected {theirs} but '
- 'got {ours} - removing this package.'.format (
+ '{dtype} mismatch for {f!r}: '
+ 'expected {theirs} but got {ours} - refetching.'.format (
dtype = self._digest_type,
f = distfile,
theirs = expected_digest,
ours = our_digest
)
)
- os.remove ( distfile )
-
- # package removed -> return success
- return True
- # -- if
- # -- if
+ fetch_required = True
else:
- return False
- elif VERBOSE:
- print ( "Skipping fetch for {f!r}".format ( f=distfile ) )
+ fetch_required = True
+
+ if fetch_required:
+ bytes_fetched = 0
+
+ # FIXME: debug print (?)
+ if VERBOSE:
+ print (
+ "Fetching {f} from {u} ...".format ( f=package_file, u=src_uri )
+ )
+
+ # unlink the existing file first (if it exists)
+ util.try_unlink ( distfile )
+
+ with open ( distfile, mode='wb' ) as fh:
+ block = webh.read ( self.transfer_blocksize )
+ while block:
+ # write block to file
+ fh.write ( block )
+ # ? bytelen
+ bytes_fetched += len ( block )
+
+ # get the next block
+ block = webh.read ( self.transfer_blocksize )
+ # -- with
+
+ if bytes_fetched == expected_filesize:
+ if expected_digest is not None:
+ our_digest = digest.dodigest_file ( distfile, self._digest_type )
+
+ if our_digest != expected_digest:
+ # fetched package's digest does not match the expected one,
+ # refuse to use it
+ self.logger.warning (
+ 'bad {dtype} digest for {f!r}, expected {theirs} but '
+ 'got {ours} - removing this package.'.format (
+ dtype = self._digest_type,
+ f = distfile,
+ theirs = expected_digest,
+ ours = our_digest
+ )
+ )
+ # package removed? -> return success (True/False)
+ return util.try_unlink ( distfile )
+ # -- end if <compare digest>
+ # -- end if <have digest?>
+
+ else:
+ return False
+ # -- end if <enough bytes fetched?>
+ elif VERBOSE:
+ print ( "Skipping fetch for {f!r}".format ( f=distfile ) )
return self._package_synced ( package_file, distfile, src_uri )
# --- end of get_package (...) ---
@@ -281,7 +289,7 @@ class WebsyncBase ( BasicRepo ):
except URLError as err:
if err.reason.errno in self.URL_ERROR_RETRY_CODES:
self.logger.info (
- 'sync failed with an url error (errno {:d}. '
+ 'sync failed with an url error (errno {:d}). '
'Retrying...'.format ( err.reason.errno )
)
want_retry = True
@@ -304,6 +312,18 @@ class WebsyncBase ( BasicRepo ):
return retval
# --- end of _dosync (...) ---
+ def skip_fetch ( self, package_filename, distfile, src_uri ):
+ """Returns True if downloading of a package file should be skipped,
+ else False. Called _before_ opening a web handle (urlopen()).
+
+ arguments:
+ * package_filename --
+ * distfile --
+ * src_uri --
+ """
+ return False
+ # --- end of skip_fetch (...) ---
+
# --- end of WebsyncBase ---
@@ -348,6 +368,8 @@ class WebsyncRepo ( WebsyncBase ):
self.pkglist_uri = pkglist_uri or self.get_src_uri ( pkglist_file )
if not self.pkglist_uri:
raise Exception ( "pkglist_uri is unset!" )
+
+ self._synced_packages = set()
# --- end of __init__ (...) ---
def _fetch_package_list ( self ):
@@ -414,6 +436,35 @@ class WebsyncRepo ( WebsyncBase ):
return package_list
# --- end fetch_pkglist (...) ---
+ def skip_fetch ( self, package_filename, distfile, src_uri ):
+ """Returns True if downloading of a package file should be skipped,
+ else False. Called _before_ opening a web handle (urlopen()).
+
+ arguments:
+ * package_filename --
+ * distfile --
+ * src_uri --
+ """
+ return distfile in self._synced_packages
+ # --- end of skip_fetch (...) ---
+
+
+ def _package_synced ( self, package_filename, distfile, src_uri ):
+ """Called when a package has been synced (=exists locally when
+ _get_package() is done).
+
+ arguments:
+ * package_filename --
+ * distfile --
+ * src_uri --
+ """
+ self._synced_packages.add ( distfile )
+ return True
+ # --- end of _package_synced (...) ---
+
+# --- end of WebsyncRepo ---
+
+
class WebsyncPackageList ( WebsyncBase ):
"""Sync packages from multiple remotes via http. Packages uris are read
from a file."""
@@ -444,7 +495,7 @@ class WebsyncPackageList ( WebsyncBase ):
del self.src_uri
- self._synced_packages = list()
+ self._synced_packages = set()
# --- end of __init__ (...) ---
@@ -467,12 +518,22 @@ class WebsyncPackageList ( WebsyncBase ):
# --- end of _fetch_package_list (...) ---
def _package_synced ( self, package_filename, distfile, src_uri ):
- self._synced_packages.append (
- ( package_filename, src_uri )
- )
+ self._synced_packages.add ( ( package_filename, src_uri ) )
return True
# --- end of _package_synced (...) ---
+ def skip_fetch ( self, package_filename, distfile, src_uri ):
+ """Returns True if downloading of a package file should be skipped,
+ else False. Called _before_ opening a web handle (urlopen()).
+
+ arguments:
+ * package_filename --
+ * distfile --
+ * src_uri --
+ """
+ return ( package_filename, distfile ) in self._synced_packages
+ # --- end of skip_fetch (...) ---
+
def scan_distdir ( self, log_bad=True, **kwargs_ignored ):
for package_filename, src_uri in self._synced_packages:
pkg = self._package_nofail (
^ permalink raw reply related [flat|nested] 35+ messages in thread* [gentoo-commits] proj/R_overlay:gsoc13/next commit in: roverlay/remote/
@ 2013-07-23 9:38 André Erdmann
2013-07-23 14:57 ` [gentoo-commits] proj/R_overlay:master " André Erdmann
0 siblings, 1 reply; 35+ messages in thread
From: André Erdmann @ 2013-07-23 9:38 UTC (permalink / raw
To: gentoo-commits
commit: 92d373d921c730123e5728e62c54328ff2baed9a
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jul 23 09:30:50 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jul 23 09:30:50 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=92d373d9
roverlay/remote/rsync: minor fixup
don't use a string for the undefined value if "None" does it as well.
---
roverlay/remote/rsync.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/roverlay/remote/rsync.py b/roverlay/remote/rsync.py
index f60b032..8102417 100644
--- a/roverlay/remote/rsync.py
+++ b/roverlay/remote/rsync.py
@@ -132,7 +132,7 @@ class RsyncRepo ( BasicRepo ):
return p.returncode
# --- end of waitfor (...) ---
- retcode = '<undef>'
+ retcode = None
try:
^ permalink raw reply related [flat|nested] 35+ messages in thread* [gentoo-commits] proj/R_overlay:gsoc13/next commit in: roverlay/remote/
@ 2013-07-23 9:38 André Erdmann
2013-07-23 14:57 ` [gentoo-commits] proj/R_overlay:master " André Erdmann
0 siblings, 1 reply; 35+ messages in thread
From: André Erdmann @ 2013-07-23 9:38 UTC (permalink / raw
To: gentoo-commits
commit: 623d65c8a41808bd2962512f70a5e0db5bdf9f9d
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jul 23 09:32:27 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jul 23 09:32:27 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=623d65c8
roverlay/remote/websync: retry on sync error
Renamed _dosync() to _sync_packages().
_dosync() calls _sync_packages() and retries that up to MAX_WEBSYNC_RETRY
times if a "known" url/http exception is caught (known := known and it makes
sense to retry).
Retry behavior needs some fine-tuning (e.g. don't try to refetch already
downloaded packages - this (sooner or later) causes a connection timeout
for me).
---
roverlay/remote/websync.py | 113 +++++++++++++++++++++++++++++++++++++--------
1 file changed, 94 insertions(+), 19 deletions(-)
diff --git a/roverlay/remote/websync.py b/roverlay/remote/websync.py
index fa0a555..87abab8 100644
--- a/roverlay/remote/websync.py
+++ b/roverlay/remote/websync.py
@@ -4,10 +4,14 @@
# Distributed under the terms of the GNU General Public License;
# either version 2 of the License, or (at your option) any later version.
+from __future__ import print_function
+
"""websync, sync packages via http"""
__all__ = [ 'WebsyncPackageList', 'WebsyncRepo', ]
+import errno
+import contextlib
import re
import os
import sys
@@ -15,22 +19,33 @@ import sys
# py2 urllib2 vs py3 urllib.request
if sys.version_info >= ( 3, ):
import urllib.request as _urllib
+ import urllib.error as _urllib_error
else:
import urllib2 as _urllib
+ import urllib2 as _urllib_error
-urlopen = _urllib.urlopen
+urlopen = _urllib.urlopen
+URLError = _urllib_error.URLError
+HTTPError = _urllib_error.HTTPError
del sys
from roverlay import digest, util
from roverlay.packageinfo import PackageInfo
from roverlay.remote.basicrepo import BasicRepo
+MAX_WEBSYNC_RETRY = 3
+
+VERBOSE = True
+
# FIXME: websync does not support package deletion
class WebsyncBase ( BasicRepo ):
"""Provides functionality for retrieving R packages via http.
Not meant for direct usage."""
+ HTTP_ERROR_RETRY_CODES = frozenset ({ 404, 410, 500, 503 })
+ URL_ERROR_RETRY_CODES = frozenset ({ errno.ETIMEDOUT, })
+
def __init__ ( self,
name,
distroot,
@@ -138,9 +153,13 @@ class WebsyncBase ( BasicRepo ):
bytes_fetched = 0
# FIXME: debug print (?)
- print (
- "Fetching {f} from {u} ...".format ( f=package_file, u=src_uri )
- )
+ if VERBOSE:
+ print (
+ "Fetching {f} from {u} ...".format ( f=package_file, u=src_uri )
+ )
+
+ # unlink the existing file first (if it exists)
+ util.try_unlink ( distfile )
with open ( distfile, mode='wb' ) as fh:
block = webh.read ( self.transfer_blocksize )
@@ -179,8 +198,7 @@ class WebsyncBase ( BasicRepo ):
else:
return False
- else:
- # FIXME: debug print
+ elif VERBOSE:
print ( "Skipping fetch for {f!r}".format ( f=distfile ) )
return self._package_synced ( package_file, distfile, src_uri )
@@ -198,8 +216,8 @@ class WebsyncBase ( BasicRepo ):
return True
# --- end of _package_synced (...) ---
- def _dosync ( self ):
- """Syncs this repo."""
+ def _sync_packages ( self ):
+ """Fetches the package list and downloads the packages."""
package_list = self._fetch_package_list()
# empty/unset package list
@@ -229,8 +247,66 @@ class WebsyncBase ( BasicRepo ):
break
return success
+ # --- end of _sync_packages (...) ---
+
+ def _dosync ( self, max_retry=MAX_WEBSYNC_RETRY ):
+ """Syncs this repo."""
+ retry_count = 0
+ want_retry = True
+ retval_tmp = None
+ retval = None
+
+ while want_retry and retry_count < max_retry:
+ retry_count += 1
+ want_retry = False
+
+ try:
+ retval_tmp = self._sync_packages()
+
+ except HTTPError as err:
+ # catch some error codes that are worth a retry
+ if err.code in self.HTTP_ERROR_RETRY_CODES:
+ self.logger.info (
+ 'sync failed with http error code {:d}. '
+ 'Retrying...'.format ( err.code )
+ )
+ want_retry = True
+ else:
+ self.logger.critical (
+ "got an unexpected http error code: {:d}".format ( err.code )
+ )
+ self.logger.exception ( err )
+ raise
+
+ except URLError as err:
+ if err.reason.errno in self.URL_ERROR_RETRY_CODES:
+ self.logger.info (
+ 'sync failed with an url error (errno {:d}. '
+ 'Retrying...'.format ( err.reason.errno )
+ )
+ want_retry = True
+ else:
+ self.logger.critical (
+ "got an unexpected url error code: {:d}".format (
+ err.reason.errno
+ )
+ )
+ self.logger.exception ( err )
+ raise
+ else:
+ retval = retval_tmp
+ # -- end while
+
+ if want_retry:
+ self.logger.error ( "retry count exhausted - sync finally failed" )
+ return False
+ else:
+ return retval
# --- end of _dosync (...) ---
+# --- end of WebsyncBase ---
+
+
class WebsyncRepo ( WebsyncBase ):
"""Sync a http repo using its PACKAGES file."""
@@ -323,8 +399,7 @@ class WebsyncRepo ( WebsyncBase ):
# --- end of generate_pkglist (...) ---
package_list = ()
- try:
- webh = urlopen ( self.pkglist_uri )
+ with contextlib.closing ( urlopen ( self.pkglist_uri ) ) as webh:
content_type = webh.info().get ( 'content-type', None )
@@ -333,12 +408,8 @@ class WebsyncRepo ( WebsyncBase ):
"content type {!r} is not supported!".format ( content_type )
)
else:
- package_list = tuple ( generate_pkglist ( webh ) )
-
- webh.close()
-
- finally:
- if 'webh' in locals() and webh: webh.close()
+ package_list = list ( generate_pkglist ( webh ) )
+ # -- end with
return package_list
# --- end fetch_pkglist (...) ---
@@ -347,6 +418,10 @@ class WebsyncPackageList ( WebsyncBase ):
"""Sync packages from multiple remotes via http. Packages uris are read
from a file."""
+ # retry on 404 makes no sense for this sync type since a local package list
+ # is used
+ HTTP_ERROR_RETRY_CODES = frozenset ({ 410, 500, 503 })
+
def __init__ ( self, pkglist_file, *args, **kwargs ):
"""Initializes a WebsyncPackageList instance.
@@ -420,8 +495,8 @@ class WebsyncPackageList ( WebsyncBase ):
return True
# --- end of _nosync (...) ---
- def _dosync ( self ):
- """Sync packages."""
+ def _sync_packages ( self ):
+ """Fetches package files."""
package_list = self._fetch_package_list()
# empty/unset package list
@@ -439,4 +514,4 @@ class WebsyncPackageList ( WebsyncBase ):
break
return success
- # --- end of _dosync (...) ---
+ # --- end of _sync_packages (...) ---
^ permalink raw reply related [flat|nested] 35+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
2013-07-23 9:38 [gentoo-commits] proj/R_overlay:gsoc13/next " André Erdmann
@ 2013-07-23 14:57 ` André Erdmann
0 siblings, 0 replies; 35+ messages in thread
From: André Erdmann @ 2013-07-23 14:57 UTC (permalink / raw
To: gentoo-commits
commit: 623d65c8a41808bd2962512f70a5e0db5bdf9f9d
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jul 23 09:32:27 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jul 23 09:32:27 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=623d65c8
roverlay/remote/websync: retry on sync error
Renamed _dosync() to _sync_packages().
_dosync() calls _sync_packages() and retries that up to MAX_WEBSYNC_RETRY
times if a "known" url/http exception is caught (known := known and it makes
sense to retry).
Retry behavior needs some fine-tuning (e.g. don't try to refetch already
downloaded packages - this (sooner or later) causes a connection timeout
for me).
---
roverlay/remote/websync.py | 113 +++++++++++++++++++++++++++++++++++++--------
1 file changed, 94 insertions(+), 19 deletions(-)
diff --git a/roverlay/remote/websync.py b/roverlay/remote/websync.py
index fa0a555..87abab8 100644
--- a/roverlay/remote/websync.py
+++ b/roverlay/remote/websync.py
@@ -4,10 +4,14 @@
# Distributed under the terms of the GNU General Public License;
# either version 2 of the License, or (at your option) any later version.
+from __future__ import print_function
+
"""websync, sync packages via http"""
__all__ = [ 'WebsyncPackageList', 'WebsyncRepo', ]
+import errno
+import contextlib
import re
import os
import sys
@@ -15,22 +19,33 @@ import sys
# py2 urllib2 vs py3 urllib.request
if sys.version_info >= ( 3, ):
import urllib.request as _urllib
+ import urllib.error as _urllib_error
else:
import urllib2 as _urllib
+ import urllib2 as _urllib_error
-urlopen = _urllib.urlopen
+urlopen = _urllib.urlopen
+URLError = _urllib_error.URLError
+HTTPError = _urllib_error.HTTPError
del sys
from roverlay import digest, util
from roverlay.packageinfo import PackageInfo
from roverlay.remote.basicrepo import BasicRepo
+MAX_WEBSYNC_RETRY = 3
+
+VERBOSE = True
+
# FIXME: websync does not support package deletion
class WebsyncBase ( BasicRepo ):
"""Provides functionality for retrieving R packages via http.
Not meant for direct usage."""
+ HTTP_ERROR_RETRY_CODES = frozenset ({ 404, 410, 500, 503 })
+ URL_ERROR_RETRY_CODES = frozenset ({ errno.ETIMEDOUT, })
+
def __init__ ( self,
name,
distroot,
@@ -138,9 +153,13 @@ class WebsyncBase ( BasicRepo ):
bytes_fetched = 0
# FIXME: debug print (?)
- print (
- "Fetching {f} from {u} ...".format ( f=package_file, u=src_uri )
- )
+ if VERBOSE:
+ print (
+ "Fetching {f} from {u} ...".format ( f=package_file, u=src_uri )
+ )
+
+ # unlink the existing file first (if it exists)
+ util.try_unlink ( distfile )
with open ( distfile, mode='wb' ) as fh:
block = webh.read ( self.transfer_blocksize )
@@ -179,8 +198,7 @@ class WebsyncBase ( BasicRepo ):
else:
return False
- else:
- # FIXME: debug print
+ elif VERBOSE:
print ( "Skipping fetch for {f!r}".format ( f=distfile ) )
return self._package_synced ( package_file, distfile, src_uri )
@@ -198,8 +216,8 @@ class WebsyncBase ( BasicRepo ):
return True
# --- end of _package_synced (...) ---
- def _dosync ( self ):
- """Syncs this repo."""
+ def _sync_packages ( self ):
+ """Fetches the package list and downloads the packages."""
package_list = self._fetch_package_list()
# empty/unset package list
@@ -229,8 +247,66 @@ class WebsyncBase ( BasicRepo ):
break
return success
+ # --- end of _sync_packages (...) ---
+
+ def _dosync ( self, max_retry=MAX_WEBSYNC_RETRY ):
+ """Syncs this repo."""
+ retry_count = 0
+ want_retry = True
+ retval_tmp = None
+ retval = None
+
+ while want_retry and retry_count < max_retry:
+ retry_count += 1
+ want_retry = False
+
+ try:
+ retval_tmp = self._sync_packages()
+
+ except HTTPError as err:
+ # catch some error codes that are worth a retry
+ if err.code in self.HTTP_ERROR_RETRY_CODES:
+ self.logger.info (
+ 'sync failed with http error code {:d}. '
+ 'Retrying...'.format ( err.code )
+ )
+ want_retry = True
+ else:
+ self.logger.critical (
+ "got an unexpected http error code: {:d}".format ( err.code )
+ )
+ self.logger.exception ( err )
+ raise
+
+ except URLError as err:
+ if err.reason.errno in self.URL_ERROR_RETRY_CODES:
+ self.logger.info (
+ 'sync failed with an url error (errno {:d}. '
+ 'Retrying...'.format ( err.reason.errno )
+ )
+ want_retry = True
+ else:
+ self.logger.critical (
+ "got an unexpected url error code: {:d}".format (
+ err.reason.errno
+ )
+ )
+ self.logger.exception ( err )
+ raise
+ else:
+ retval = retval_tmp
+ # -- end while
+
+ if want_retry:
+ self.logger.error ( "retry count exhausted - sync finally failed" )
+ return False
+ else:
+ return retval
# --- end of _dosync (...) ---
+# --- end of WebsyncBase ---
+
+
class WebsyncRepo ( WebsyncBase ):
"""Sync a http repo using its PACKAGES file."""
@@ -323,8 +399,7 @@ class WebsyncRepo ( WebsyncBase ):
# --- end of generate_pkglist (...) ---
package_list = ()
- try:
- webh = urlopen ( self.pkglist_uri )
+ with contextlib.closing ( urlopen ( self.pkglist_uri ) ) as webh:
content_type = webh.info().get ( 'content-type', None )
@@ -333,12 +408,8 @@ class WebsyncRepo ( WebsyncBase ):
"content type {!r} is not supported!".format ( content_type )
)
else:
- package_list = tuple ( generate_pkglist ( webh ) )
-
- webh.close()
-
- finally:
- if 'webh' in locals() and webh: webh.close()
+ package_list = list ( generate_pkglist ( webh ) )
+ # -- end with
return package_list
# --- end fetch_pkglist (...) ---
@@ -347,6 +418,10 @@ class WebsyncPackageList ( WebsyncBase ):
"""Sync packages from multiple remotes via http. Packages uris are read
from a file."""
+ # retry on 404 makes no sense for this sync type since a local package list
+ # is used
+ HTTP_ERROR_RETRY_CODES = frozenset ({ 410, 500, 503 })
+
def __init__ ( self, pkglist_file, *args, **kwargs ):
"""Initializes a WebsyncPackageList instance.
@@ -420,8 +495,8 @@ class WebsyncPackageList ( WebsyncBase ):
return True
# --- end of _nosync (...) ---
- def _dosync ( self ):
- """Sync packages."""
+ def _sync_packages ( self ):
+ """Fetches package files."""
package_list = self._fetch_package_list()
# empty/unset package list
@@ -439,4 +514,4 @@ class WebsyncPackageList ( WebsyncBase ):
break
return success
- # --- end of _dosync (...) ---
+ # --- end of _sync_packages (...) ---
^ permalink raw reply related [flat|nested] 35+ messages in thread
* [gentoo-commits] proj/R_overlay:gsoc13/next commit in: roverlay/remote/
@ 2013-07-16 16:35 André Erdmann
2013-07-16 16:36 ` [gentoo-commits] proj/R_overlay:master " André Erdmann
0 siblings, 1 reply; 35+ messages in thread
From: André Erdmann @ 2013-07-16 16:35 UTC (permalink / raw
To: gentoo-commits
commit: 0d67413e156934952f236ae8161e8536bad03293
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jul 16 16:34:51 2013 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jul 16 16:34:51 2013 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=0d67413e
roverlay/remote: print repo name while syncing
---
roverlay/remote/basicrepo.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/roverlay/remote/basicrepo.py b/roverlay/remote/basicrepo.py
index 9ce512c..33d8f30 100644
--- a/roverlay/remote/basicrepo.py
+++ b/roverlay/remote/basicrepo.py
@@ -192,6 +192,8 @@ class BasicRepo ( object ):
"""Syncs this repo."""
status = False
+ print ( "Syncing {!r} ...".format ( self.name ) )
+
if sync_enabled and hasattr ( self, '_dosync' ):
status = self._dosync()
^ permalink raw reply related [flat|nested] 35+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
@ 2012-08-13 18:07 André Erdmann
0 siblings, 0 replies; 35+ messages in thread
From: André Erdmann @ 2012-08-13 18:07 UTC (permalink / raw
To: gentoo-commits
commit: 5b031e5e5bd2776f04d41d025043e8dbc8a662e3
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Mon Aug 13 18:06:46 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Mon Aug 13 18:06:46 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=5b031e5e
remote, package_nofail: call update() once
---
roverlay/remote/basicrepo.py | 13 ++++++-------
1 files changed, 6 insertions(+), 7 deletions(-)
diff --git a/roverlay/remote/basicrepo.py b/roverlay/remote/basicrepo.py
index a93bc5b..bab83db 100644
--- a/roverlay/remote/basicrepo.py
+++ b/roverlay/remote/basicrepo.py
@@ -250,14 +250,13 @@ class BasicRepo ( object ):
"""
def package_nofail ( filename, distdir, src_uri_base ):
- p = self._package_nofail (
- log_bad=log_bad, filename=filename, distdir=distdir, origin=self
+ return self._package_nofail (
+ log_bad=log_bad,
+ filename=filename,
+ origin=self,
+ distdir=distdir,
+ src_uri_base=src_uri_base
)
- if p is not None:
- p.update ( src_uri_base=src_uri_base )
- return p
- else:
- return None
# --- end of package_nofail (...) ---
def get_distdir_and_srcuri_base ( dirpath ):
^ permalink raw reply related [flat|nested] 35+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
@ 2012-08-11 0:01 André Erdmann
0 siblings, 0 replies; 35+ messages in thread
From: André Erdmann @ 2012-08-11 0:01 UTC (permalink / raw
To: gentoo-commits
commit: 140ca1692ef8352466b7c3e95d62a541c5203592
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Sat Aug 11 00:01:02 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Sat Aug 11 00:01:02 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=140ca169
package_nofail: add src_uri_base to successful PackageInfos only
---
roverlay/remote/basicrepo.py | 7 +++++--
1 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/roverlay/remote/basicrepo.py b/roverlay/remote/basicrepo.py
index ebde867..a93bc5b 100644
--- a/roverlay/remote/basicrepo.py
+++ b/roverlay/remote/basicrepo.py
@@ -253,8 +253,11 @@ class BasicRepo ( object ):
p = self._package_nofail (
log_bad=log_bad, filename=filename, distdir=distdir, origin=self
)
- p.update ( src_uri_base=src_uri_base )
- return p
+ if p is not None:
+ p.update ( src_uri_base=src_uri_base )
+ return p
+ else:
+ return None
# --- end of package_nofail (...) ---
def get_distdir_and_srcuri_base ( dirpath ):
^ permalink raw reply related [flat|nested] 35+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
@ 2012-08-10 15:16 André Erdmann
0 siblings, 0 replies; 35+ messages in thread
From: André Erdmann @ 2012-08-10 15:16 UTC (permalink / raw
To: gentoo-commits
commit: 4c7fe99c23a3ee92cbdc90116deea623cf21c0a5
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Fri Aug 10 15:15:22 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Fri Aug 10 15:15:22 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=4c7fe99c
pkglist websync: expand ~ in paths
---
roverlay/remote/websync.py | 10 +++++++++-
1 files changed, 9 insertions(+), 1 deletions(-)
diff --git a/roverlay/remote/websync.py b/roverlay/remote/websync.py
index c8548d1..bd53f3e 100644
--- a/roverlay/remote/websync.py
+++ b/roverlay/remote/websync.py
@@ -357,7 +357,15 @@ class WebsyncPackageList ( WebsyncBase ):
"""
super ( WebsyncPackageList, self ) . __init__ ( *args, **kwargs )
- self._pkglist_file = os.path.abspath ( pkglist_file )
+ # len (pkglist_file) == 0: raise implicit Exception since
+ # pkglist_file is not set
+
+ if pkglist_file [0] == '~':
+ self._pkglist_file = os.path.abspath (
+ os.path.expanduser ( pkglist_file )
+ )
+ else:
+ self._pkglist_file = os.path.abspath ( pkglist_file )
del self.src_uri
^ permalink raw reply related [flat|nested] 35+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
@ 2012-08-10 15:16 André Erdmann
0 siblings, 0 replies; 35+ messages in thread
From: André Erdmann @ 2012-08-10 15:16 UTC (permalink / raw
To: gentoo-commits
commit: 8cdb033dc74b39fd596eb29a174abea1e2bf8e20
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Fri Aug 10 15:14:41 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Fri Aug 10 15:14:41 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=8cdb033d
remove unused import
---
roverlay/remote/repoloader.py | 1 -
1 files changed, 0 insertions(+), 1 deletions(-)
diff --git a/roverlay/remote/repoloader.py b/roverlay/remote/repoloader.py
index 30d0971..3f47fd4 100644
--- a/roverlay/remote/repoloader.py
+++ b/roverlay/remote/repoloader.py
@@ -9,7 +9,6 @@
__all__ = [ 'read_repofile', ]
import sys
-import os.path
import logging
try:
^ permalink raw reply related [flat|nested] 35+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
@ 2012-08-09 9:26 André Erdmann
0 siblings, 0 replies; 35+ messages in thread
From: André Erdmann @ 2012-08-09 9:26 UTC (permalink / raw
To: gentoo-commits
commit: 068a192c7074135b69eff73058ad762bcacd6af5
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Aug 9 08:37:09 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Aug 9 08:37:09 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=068a192c
fix SRC_URI for packages from subdirectories
---
roverlay/remote/basicrepo.py | 39 ++++++++++++++++++++++++++++++---------
1 files changed, 30 insertions(+), 9 deletions(-)
diff --git a/roverlay/remote/basicrepo.py b/roverlay/remote/basicrepo.py
index 7f2f0b4..ebde867 100644
--- a/roverlay/remote/basicrepo.py
+++ b/roverlay/remote/basicrepo.py
@@ -248,34 +248,55 @@ class BasicRepo ( object ):
raises: AssertionError if is_package is neither None nor a callable.
"""
- package_nofail = lambda filename, distdir : self._package_nofail (
- log_bad=log_bad, filename=filename, distdir=distdir, origin=self
- )
+
+ def package_nofail ( filename, distdir, src_uri_base ):
+ p = self._package_nofail (
+ log_bad=log_bad, filename=filename, distdir=distdir, origin=self
+ )
+ p.update ( src_uri_base=src_uri_base )
+ return p
+ # --- end of package_nofail (...) ---
+
+ def get_distdir_and_srcuri_base ( dirpath ):
+ if len ( dirpath ) > len ( self.distdir ):
+ # package is in a subdirectory,
+ # get the relative path which is required for valid SRC_URIs
+ if os.sep == '/':
+ # a simple array slice does the job if os.sep is '/'
+ subdir = dirpath [ len ( self.distdir ) + 1 : ]
+ else:
+ subdir = os.path.relpath ( dirpath, self.distdir ).replace (
+ os.sep, '/'
+ )
+
+ return ( dirpath, self.src_uri + '/' + subdir )
+ else:
+ return ( None, None )
+ # --- end of get_distdir_and_srcuri_base (...) ---
if is_package is None:
# unfiltered variant
for dirpath, dirnames, filenames in os.walk ( self.distdir ):
- distdir = dirpath if dirpath != self.distdir else None
-
+ distdir, srcuri_base = get_distdir_and_srcuri_base ( dirpath )
for filename in filenames:
- pkg = package_nofail ( filename, distdir )
+ pkg = package_nofail ( filename, distdir, srcuri_base )
if pkg is not None:
yield pkg
else:
# filtered variant (adds an if is_package... before yield)
for dirpath, dirnames, filenames in os.walk ( self.distdir ):
- distdir = dirpath if dirpath != self.distdir else None
+ distdir, srcuri_base = get_distdir_and_srcuri_base ( dirpath )
for filename in filenames:
if is_package ( os.path.join ( dirpath, filename ) ):
- pkg = package_nofail ( filename, distdir )
+ pkg = package_nofail ( filename, distdir, srcuri_base )
if pkg is not None:
yield pkg
elif log_filtered:
self.logger.debug (
- "filtered %r: not a package" % filename
+ "filtered {f!r}: not a package".format ( f=filename )
)
# --- end of scan_distdir (...) ---
^ permalink raw reply related [flat|nested] 35+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
@ 2012-08-02 15:14 André Erdmann
0 siblings, 0 replies; 35+ messages in thread
From: André Erdmann @ 2012-08-02 15:14 UTC (permalink / raw
To: gentoo-commits
commit: 6ce0ea0ace5aa072f1f09040f277940e2d5fd729
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Thu Aug 2 10:15:49 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Thu Aug 2 10:15:49 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=6ce0ea0a
websync: py2, urllib2 <> py3, urllib.request
---
roverlay/remote/websync.py | 32 +++++++++++++++++++++++---------
1 files changed, 23 insertions(+), 9 deletions(-)
diff --git a/roverlay/remote/websync.py b/roverlay/remote/websync.py
index a0ded6c..40ebc0e 100644
--- a/roverlay/remote/websync.py
+++ b/roverlay/remote/websync.py
@@ -1,12 +1,22 @@
-
import re
import os
-import urllib2
+import sys
+
+# py2 urllib2 vs py3 urllib.request
+if sys.version_info >= ( 3, ):
+ import urllib.request as _urllib
+else:
+ import urllib2 as _urllib
+
+urlopen = _urllib.urlopen
+del sys
from roverlay import digest, util
from roverlay.packageinfo import PackageInfo
from roverlay.remote.basicrepo import BasicRepo
+# FIXME: websync does not support package deletion
+
class WebsyncBase ( BasicRepo ):
"""Provides functionality for retrieving R packages via http.
Not meant for direct usage."""
@@ -73,7 +83,7 @@ class WebsyncBase ( BasicRepo ):
* expected_digest -- expected digest for package_file or None (^=disable)
"""
distfile = self.distdir + os.sep + package_file
- webh = urllib2.urlopen ( src_uri )
+ webh = urlopen ( src_uri )
#web_info = webh.info()
expected_filesize = int ( webh.info().get ( 'content-length', -1 ) )
@@ -274,9 +284,15 @@ class WebsyncRepo ( WebsyncBase ):
max_info_len = 3 if self._digest_type is not None else 2
for match in (
- filter ( None, (
- self.FIELDREGEX.match ( l ) for l in fh.readlines()
- ) )
+ filter (
+ None,
+ (
+ self.FIELDREGEX.match (
+ l if isinstance ( l, str ) else l.decode()
+ )
+ for l in fh.readlines()
+ )
+ )
):
name, value = match.group ( 'name', 'value' )
info [name.lower()] = value
@@ -298,7 +314,7 @@ class WebsyncRepo ( WebsyncBase ):
package_list = ()
try:
- webh = urllib2.urlopen ( self.pkglist_uri )
+ webh = urlopen ( self.pkglist_uri )
content_type = webh.info().get ( 'content-type', None )
@@ -321,8 +337,6 @@ class WebsyncPackageList ( WebsyncBase ):
"""Sync packages from multiple remotes via http. Packages uris are read
from a file."""
- # FIXME: does not support --nosync
-
def __init__ ( self, pkglist_file, *args, **kwargs ):
"""Initializes a WebsyncPackageList instance.
^ permalink raw reply related [flat|nested] 35+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
@ 2012-08-01 7:33 André Erdmann
0 siblings, 0 replies; 35+ messages in thread
From: André Erdmann @ 2012-08-01 7:33 UTC (permalink / raw
To: gentoo-commits
commit: f0b0f781b5843d9a782596a43c4b98e21272e5ac
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Aug 1 07:33:40 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Aug 1 07:33:40 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=f0b0f781
fix logger message in BasicRepo._package_nofail()
---
roverlay/remote/basicrepo.py | 4 ++--
1 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/roverlay/remote/basicrepo.py b/roverlay/remote/basicrepo.py
index 65b07eb..58a0419 100644
--- a/roverlay/remote/basicrepo.py
+++ b/roverlay/remote/basicrepo.py
@@ -199,7 +199,7 @@ class BasicRepo ( object ):
return status
# --- end of sync (...) ---
- def _package_nofail ( self, log_bad, **data ):
+ def _package_nofail ( self, log_bad, filename, **data ):
"""Tries to create a PackageInfo.
Logs failure if log_bad is True.
@@ -210,7 +210,7 @@ class BasicRepo ( object ):
returns: PackageInfo on success, else None.
"""
try:
- return PackageInfo ( **data )
+ return PackageInfo ( filename=filename, **data )
except ValueError as expected:
if log_bad:
#self.logger.exception ( expected )
^ permalink raw reply related [flat|nested] 35+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
@ 2012-07-31 17:51 André Erdmann
0 siblings, 0 replies; 35+ messages in thread
From: André Erdmann @ 2012-07-31 17:51 UTC (permalink / raw
To: gentoo-commits
commit: 2b49ac8b4752fa1e5efd3f51f15720e7d70f12a9
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jul 31 17:51:02 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jul 31 17:51:02 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=2b49ac8b
remote: get packages via http
---
roverlay/remote/basicrepo.py | 241 ++++++++----------------
roverlay/remote/repolist.py | 10 +-
roverlay/remote/repoloader.py | 49 ++++--
roverlay/remote/rsync.py | 23 ++-
roverlay/remote/websync.py | 410 +++++++++++++++++++++++++++++++++++++++++
5 files changed, 536 insertions(+), 197 deletions(-)
diff --git a/roverlay/remote/basicrepo.py b/roverlay/remote/basicrepo.py
index 3dd09de..65b07eb 100644
--- a/roverlay/remote/basicrepo.py
+++ b/roverlay/remote/basicrepo.py
@@ -35,14 +35,17 @@ def normalize_uri ( uri, protocol, force_protocol=False ):
return uri
# --- end of normalize_uri (...) ---
-class LocalRepo ( object ):
+class BasicRepo ( object ):
"""
This class represents a local repository - all packages are assumed
to exist in its distfiles dir and no remote syncing will occur.
It's the base class for remote repos.
"""
- def __init__ ( self, name, distroot, directory=None, src_uri=None ):
+ def __init__ ( self,
+ name, distroot,
+ directory=None, src_uri=None, is_remote=False, remote_uri=None
+ ):
"""Initializes a LocalRepo.
arguments:
@@ -50,29 +53,34 @@ class LocalRepo ( object ):
* directory -- distfiles dir, defaults to <DISTFILES root>/<name>
* src_uri -- SRC_URI, defaults to http://localhost/R-Packages/<name>
"""
- self.name = name
-
+ self.name = name
self.logger = logging.getLogger (
self.__class__.__name__ + ':' + self.name
)
if directory is None:
- self.distdir = os.path.join (
- distroot,
- # subdir repo names like CRAN/contrib are ok,
- # but make sure to use the correct path separator
- self.name.replace ( '/', os.path.sep ),
- )
+ # subdir repo names like CRAN/contrib are ok,
+ # but make sure to use the correct path separator
+ self.distdir = \
+ distroot + os.path.sep + self.name.replace ( '/', os.path.sep )
+
else:
self.distdir = directory
if src_uri is None:
- self.src_uri = '/'.join ( ( LOCALREPO_SRC_URI, self.name ) )
+ self.src_uri = LOCALREPO_SRC_URI + '/' + self.name
+ elif len ( src_uri ) > 0 and src_uri [-1] == '/':
+ self.src_uri = src_uri [:-1]
else:
self.src_uri = src_uri
self.sync_status = 0
+ if remote_uri is not None:
+ self.is_remote = True
+ self.remote_uri = remote_uri
+ else:
+ self.is_remote = is_remote
# --- end of __init__ (...) ---
def ready ( self ):
@@ -110,9 +118,26 @@ class LocalRepo ( object ):
# --- end of _set_fail (...) ---
def __str__ ( self ):
- return "repo '%s': DISTDIR '%s', SRC_URI '%s'" % (
- self.name, self.distdir, self.src_uri
- )
+ if hasattr ( self, 'remote_uri' ):
+ return \
+ '{cls} {name}: DISTDIR {distdir!r}, SRC_URI {src_uri!r}, '\
+ 'REMOTE_URI {remote_uri!r}.'.format (
+ cls = self.__class__.__name__,
+ name = self.name,
+ distdir = self.distdir,
+ src_uri = self.src_uri \
+ if hasattr ( self, 'src_uri' ) else '[none]',
+ remote_uri = self.remote_uri
+ )
+ else:
+ return '{cls} {name}: DISTDIR {distdir!r}, SRC_URI {src_uri!r}.'.\
+ format (
+ cls = self.__class__.__name__,
+ name = self.name,
+ distdir = self.distdir,
+ src_uri = self.src_uri \
+ if hasattr ( self, 'src_uri' ) else '[none]'
+ )
# --- end of __str__ (...) ---
def get_name ( self ):
@@ -125,16 +150,24 @@ class LocalRepo ( object ):
return self.distdir
# --- end of get_distdir (...) ---
+ def get_remote_uri ( self ):
+ """Returns the remote uri of this RemoteRepo which used for syncing."""
+ return self.remote_uri if hasattr ( self, 'remote_uri' ) else None
+ # --- end of get_remote_uri (...) ---
+
+ # get_remote(...) -> get_remote_uri(...)
+ get_remote = get_remote_uri
+
def get_src_uri ( self, package_file=None ):
"""Returns the SRC_URI of this repository.
arguments:
* package_file -- if set and not None: returns a SRC_URI for this pkg
"""
- if package_file is None:
- return self.src_uri
+ if package_file is not None:
+ return self.src_uri + '/' + package_file
else:
- return '/'.join ( ( self.src_uri, package_file ) )
+ return self.src_uri
# --- end of get_src_uri (...) ---
# get_src(...) -> get_src_uri(...)
@@ -166,6 +199,28 @@ class LocalRepo ( object ):
return status
# --- end of sync (...) ---
+ def _package_nofail ( self, log_bad, **data ):
+ """Tries to create a PackageInfo.
+ Logs failure if log_bad is True.
+
+ arguments:
+ * log_bad --
+ * data -- PackageInfo data
+
+ returns: PackageInfo on success, else None.
+ """
+ try:
+ return PackageInfo ( **data )
+ except ValueError as expected:
+ if log_bad:
+ #self.logger.exception ( expected )
+ self.logger.info (
+ "filtered {f!r}: bad package".format ( f=filename )
+ )
+ return None
+
+ # --- end of _package_nofail (...) ---
+
def scan_distdir ( self,
is_package=None, log_filtered=False, log_bad=True
):
@@ -183,30 +238,9 @@ class LocalRepo ( object ):
raises: AssertionError if is_package is neither None nor a callable.
"""
-
- def package_nofail ( filename, distdir ):
- """Tries to create a PackageInfo.
- Logs failure if log_bad is True.
-
- arguments:
- * filename -- name of the package file (including .tar* suffix)
- * distdir -- filename's directory
-
- returns: PackageInfo on success, else None.
- """
- try:
- return PackageInfo (
- filename=filename, origin=self, distdir=distdir
- )
- except ( ValueError, ) as expected:
- if log_bad:
- #self.logger.exception ( expected )
- self.logger.info (
- "filtered %r: bad package" % filename
- )
- return None
-
- # --- end of package_nofail (...) ---
+ package_nofail = lambda filename, distdir : self._package_nofail (
+ log_bad=log_bad, filename=filename, distdir=distdir, origin=self
+ )
if is_package is None:
# unfiltered variant
@@ -219,7 +253,7 @@ class LocalRepo ( object ):
if pkg is not None:
yield pkg
- elif hasattr ( is_package, '__call__' ):
+ else:
# filtered variant (adds an if is_package... before yield)
for dirpath, dirnames, filenames in os.walk ( self.distdir ):
distdir = dirpath if dirpath != self.distdir else None
@@ -233,127 +267,6 @@ class LocalRepo ( object ):
self.logger.debug (
"filtered %r: not a package" % filename
)
-
-
- else:
- # faulty variant, raises Exception
- raise AssertionError (
- "is_package should either be None or a function."
- )
- #yield None
-
# --- end of scan_distdir (...) ---
-# --- end of LocalRepo ---
-
-
-class RemoteRepo ( LocalRepo ):
- """A template for remote repositories."""
-
- def __init__ (
- self, name, distroot, sync_proto,
- directory=None,
- src_uri=None, remote_uri=None, base_uri=None
- ):
- """Initializes a RemoteRepo.
- Mainly consists of URI calculation that derived classes may find useful.
-
- arguments:
- * name --
- * sync_proto -- protocol used for syncing (e.g. 'rsync')
- * directory --
- * src_uri -- src uri, if set, else calculated using base/remote uri,
- the leading <proto>:// can be left out in which case
- http is assumed
- * remote_uri -- uri used for syncing, if set, else calculated using
- base/src uri, the leading <proto>:// can be left out
- * base_uri -- used to calculate remote/src uri,
- example: localhost/R-packages/something
-
- keyword condition:
- * | { x : x in union(src,remote,base) and x not None } | >= 1
- ^= at least one out of src/remote/base uri is not None
- """
- super ( RemoteRepo, self ) . __init__ (
- name, distroot, directory, src_uri=''
- )
-
- self.sync_proto = sync_proto
-
- # detemerine uris
- if src_uri is None and remote_uri is None:
- if base_uri is None:
- # keyword condition not met
- raise Exception ( "Bad initialization of RemoteRepo!" )
-
- else:
- # using base_uri for src,remote
- self.src_uri = URI_SEPARATOR.join (
- ( DEFAULT_PROTOCOL, base_uri )
- )
-
- self.remote_uri = URI_SEPARATOR.join (
- ( sync_proto, base_uri )
- )
-
- elif src_uri is None:
- # remote_uri is not None
- self.remote_uri = normalize_uri ( remote_uri, self.sync_proto )
-
- if base_uri is not None:
- # using base_uri for src_uri
- self.src_uri = URI_SEPARATOR.join (
- ( DEFAULT_PROTOCOL, base_uri )
- )
- else:
- # using remote_uri for src_uri
- self.src_uri = normalize_uri (
- self.remote_uri, DEFAULT_PROTOCOL, force_protocol=True
- )
-
- elif remote_uri is None:
- # src_uri is not None
- self.src_uri = normalize_uri ( src_uri, DEFAULT_PROTOCOL )
-
- if base_uri is not None:
- # using base_uri for remote_uri
- self.remote_uri = URI_SEPARATOR.join (
- ( self.sync_proto, base_uri )
- )
- else:
- # using src_uri for remote_uri
- self.remote_uri = normalize_uri (
- self.src_uri, self.sync_proto, force_protocol=True
- )
- else:
- # remote and src not None
- self.remote_uri = normalize_uri ( remote_uri, self.sync_proto )
- self.src_uri = normalize_uri ( src_uri, DEFAULT_PROTOCOL )
-
- # --- end of __init__ (...) ---
-
- def get_remote_uri ( self ):
- """Returns the remote uri of this RemoteRepo which used for syncing."""
- return self.remote_uri
- # --- end of get_remote_uri (...) ---
-
- # get_remote(...) -> get_remote_uri(...)
- get_remote = get_remote_uri
-
- def _dosync ( self ):
- """Gets packages from remote(s) and returns True if the repo is ready
- for overlay creation, else False.
-
- Derived classes have to implement this method.
- """
- raise Exception ( "RemoteRepo does not implement sync()." )
- # --- end of _dosync (...) ---
-
- def __str__ ( self ):
- return "repo '%s': DISTDIR '%s', SRC_URI '%s', REMOTE_URI '%s'" % (
- self.name, self.distdir, self.src_uri, self.remote_uri
- )
- # --- end of __str__ (...) ---
-
-# --- end of RemoteRepo ---
-
+# --- end of BasicRepo ---
diff --git a/roverlay/remote/repolist.py b/roverlay/remote/repolist.py
index cc673e6..a623db0 100644
--- a/roverlay/remote/repolist.py
+++ b/roverlay/remote/repolist.py
@@ -4,7 +4,7 @@ import os.path
from roverlay import config
from roverlay.remote.repoloader import read_repofile
-from roverlay.remote.basicrepo import LocalRepo
+from roverlay.remote.basicrepo import BasicRepo
class RepoList ( object ):
"""Controls several Repo objects."""
@@ -55,7 +55,7 @@ class RepoList ( object ):
# --- end of _pkg_filter (...) ---
def add_distdir ( self, distdir, src_uri=None, name=None ):
- """Adds a local package directory as LocalRepo.
+ """Adds a local package directory as BasicRepo.
arguments:
* distdir --
@@ -64,7 +64,7 @@ class RepoList ( object ):
(FIXME: could add RESTRICT="fetch" to those ebuilds)
* name -- name of the repo, defaults to os.path.basename (distdir)
"""
- self.repos.append ( LocalRepo (
+ self.repos.append ( BasicRepo (
name=os.path.basename ( distdir ) if name is None else name,
directory=distdir,
src_uri=src_uri
@@ -72,7 +72,7 @@ class RepoList ( object ):
# --- end of add_distdir (...) ---
def add_distdirs ( self, distdirs ):
- """Adds several distdirs as LocalRepos.
+ """Adds several distdirs as BasicRepos.
All distdirs will have an invalid SRC_URI and a default name,
use add_distdir() if you want usable ebuilds.
@@ -81,7 +81,7 @@ class RepoList ( object ):
"""
def gen_repos():
for d in distdirs:
- repo = LocalRepo (
+ repo = BasicRepo (
name=os.path.basename ( d ),
# FIXME: --force_distroot should block --distdir
directory=d,
diff --git a/roverlay/remote/repoloader.py b/roverlay/remote/repoloader.py
index 2a92526..ba49d0a 100644
--- a/roverlay/remote/repoloader.py
+++ b/roverlay/remote/repoloader.py
@@ -11,8 +11,9 @@ except ImportError:
from roverlay import config
-from roverlay.remote.basicrepo import LocalRepo
-from roverlay.remote.rsync import RsyncRepo
+from roverlay.remote import basicrepo
+from roverlay.remote import rsync
+from roverlay.remote import websync
LOGGER = logging.getLogger ( 'repoloader' )
@@ -51,35 +52,49 @@ def read_repofile ( repo_file, distroot, lenient=False, force_distroot=False ):
repo_type = get ( 'type', 'rsync' ).lower()
- repo_name = get ( 'name', name )
+ common_kwargs = dict (
+ name = get ( 'name', name ),
+ directory = None if force_distroot else get ( 'directory' ),
+ distroot = distroot,
+ src_uri = get ( 'src_uri' )
+ )
- repo_distdir = None if force_distroot else get ( 'directory' )
if repo_type == 'local':
- repo = LocalRepo (
- name = repo_name,
- distroot = distroot,
- directory = repo_distdir,
- src_uri = get ( 'src_uri' )
- )
+ repo = basicrepo.BasicRepo ( **common_kwargs )
+
elif repo_type == 'rsync':
extra_opts = get ( 'extra_rsync_opts' )
if extra_opts:
extra_opts = extra_opts.split ( ' ' )
- repo = RsyncRepo (
- name = repo_name,
- distroot = distroot,
- directory = repo_distdir,
- src_uri = get ( 'src_uri' ),
+ repo = rsync.RsyncRepo (
rsync_uri = get ( 'rsync_uri' ),
- base_uri = get ( 'base_uri' ),
extra_opts = extra_opts,
recursive = get ( 'recursive', False ) == 'yes',
+ **common_kwargs
+ )
+
+ elif repo_type == 'websync_repo':
+ repo = websync.WebsyncRepo (
+ pkglist_file = get ( 'pkglist_file', 'PACKAGES' ),
+ pkglist_uri = get ( 'pkglist_uri' ),
+ digest_type = get ( 'digest_type' ) or get ( 'digest' ),
+ **common_kwargs
)
+
+ elif repo_type in ( 'websync_pkglist', 'websync_package_list' ):
+ repo = websync.WebsyncPackageList (
+ pkglist_file = get ( 'pkglist_file' ) or get ( 'pkglist' ),
+ #digest_type = get ( 'digest_type' ) or get ( 'digest' ),
+ **common_kwargs
+ )
+
else:
- LOGGER.error ( "Unknown repo type %s for %s" % ( repo_type, name ) )
+ LOGGER.error ( "Unknown repo type {} for {}!".format (
+ repo_type, name
+ ) )
if repo is not None:
diff --git a/roverlay/remote/rsync.py b/roverlay/remote/rsync.py
index 11efd27..90f21f5 100644
--- a/roverlay/remote/rsync.py
+++ b/roverlay/remote/rsync.py
@@ -4,8 +4,7 @@ import subprocess
from roverlay import config, util
-#from roverlay.remote.basicrepo import LocalRepo, RemoteRepo
-from roverlay.remote.basicrepo import RemoteRepo
+from roverlay.remote.basicrepo import BasicRepo
RSYNC_ENV = util.keepenv (
'PATH',
@@ -46,12 +45,16 @@ DEFAULT_RSYNC_OPTS = (
'--chmod=ugo=r,u+w,Dugo+x', # 0755 for transferred dirs, 0644 for files
)
-class RsyncRepo ( RemoteRepo ):
+class RsyncRepo ( BasicRepo ):
- def __init__ (
- self, name, distroot,
- directory=None, src_uri=None, rsync_uri=None, base_uri=None,
- recursive=False, extra_opts=None
+ def __init__ ( self,
+ name,
+ distroot,
+ src_uri,
+ rsync_uri,
+ directory=None,
+ recursive=False,
+ extra_opts=None
):
"""Initializes an RsyncRepo.
@@ -68,8 +71,8 @@ class RsyncRepo ( RemoteRepo ):
# using '' as remote protocol which leaves uris unchanged when
# normalizing them for rsync usage
super ( RsyncRepo, self ) . __init__ (
- name, distroot=distroot, sync_proto='', directory=directory,
- src_uri=src_uri, remote_uri=rsync_uri, base_uri=base_uri
+ name=name, distroot=distroot, directory=directory,
+ src_uri=src_uri, remote_uri=rsync_uri
)
# syncing directories, not files - always appending a slash at the end
@@ -83,8 +86,6 @@ class RsyncRepo ( RemoteRepo ):
self.extra_opts.extend ( extra_opts )
else:
self.extra_opts = extra_opts
-
- self.sync_protocol = 'rsync'
# --- end of __init__ (...) ---
def _rsync_argv ( self ):
diff --git a/roverlay/remote/websync.py b/roverlay/remote/websync.py
new file mode 100644
index 0000000..a0ded6c
--- /dev/null
+++ b/roverlay/remote/websync.py
@@ -0,0 +1,410 @@
+
+import re
+import os
+import urllib2
+
+from roverlay import digest, util
+from roverlay.packageinfo import PackageInfo
+from roverlay.remote.basicrepo import BasicRepo
+
+class WebsyncBase ( BasicRepo ):
+ """Provides functionality for retrieving R packages via http.
+ Not meant for direct usage."""
+
+ def __init__ ( self,
+ name,
+ distroot,
+ src_uri,
+ directory=None,
+ digest_type=None
+ ):
+ """Initializes a WebsyncBase instance.
+
+ arguments:
+ * name -- see BasicRepo
+ * distroot -- ^
+ * src_uri -- ^
+ * directory -- ^
+ * digest_type -- if set and not None/"None":
+ verify packages using the given digest type
+ Supported digest types: 'md5'.
+ """
+ super ( WebsyncBase, self ) . __init__ (
+ name=name,
+ distroot=distroot,
+ src_uri=src_uri,
+ remote_uri=src_uri,
+ directory=directory
+ )
+
+ if digest_type is None:
+ self._digest_type = None
+
+ elif str ( digest_type ).lower() in ( 'none', 'disabled', 'off' ):
+ self._digest_type = None
+
+ elif digest.digest_supported ( digest_type ):
+ # setting a digest_type (other than None) expects package_list
+ # to be a 2-tuple <package_file, digest sum> list,
+ # else a list of package_files is expected.
+ self._digest_type = digest_type
+
+ else:
+ raise Exception (
+ "Unknown/unsupported digest type {}!".format ( digest_type )
+ )
+
+ # download 8KiB per block
+ self.transfer_blocksize = 8192
+ # --- end of __init__ (...) ---
+
+ def _fetch_package_list ( self ):
+ """This function returns a list of packages to download."""
+ raise Exception ( "method stub" )
+ # --- end of _fetch_package_list (...) ---
+
+ def _get_package ( self, package_file, src_uri, expected_digest ):
+ """Gets a packages, i.e. downloads if it doesn't exist locally
+ or fails verification (size, digest).
+
+ arguments:
+ * package_file -- package file name
+ * src_uri -- uri for package_file
+ * expected_digest -- expected digest for package_file or None (^=disable)
+ """
+ distfile = self.distdir + os.sep + package_file
+ webh = urllib2.urlopen ( src_uri )
+ #web_info = webh.info()
+
+ expected_filesize = int ( webh.info().get ( 'content-length', -1 ) )
+
+ if os.access ( distfile, os.F_OK ):
+ # package exists locally, verify it (size, digest)
+ fetch_required = False
+ localsize = os.path.getsize ( distfile )
+
+ if localsize != expected_filesize:
+ # size mismatch
+ self.logger.info (
+ 'size mismatch for {f!r}: expected {websize} bytes '
+ 'but got {localsize}!'.format (
+ f = package_file,
+ websize = expected_filesize,
+ localsize = localsize
+ )
+ )
+ fetch_required = True
+
+ elif expected_digest is not None:
+ our_digest = digest.dodigest_file ( distfile, self._digest_type )
+
+ if our_digest != expected_digest:
+ # digest mismatch
+ self.logger.warning (
+ '{dtype} mismatch for {f!r}: '
+ 'expected {theirs} but got {ours} - refetching.'.format (
+ dtype = self._digest_type,
+ f = distfile,
+ theirs = expected_digest,
+ ours = our_digest
+ )
+ )
+ fetch_required = True
+
+ else:
+ fetch_required = True
+
+ if fetch_required:
+ bytes_fetched = 0
+
+ # FIXME: debug print (?)
+ print (
+ "Fetching {f} from {u} ...".format ( f=package_file, u=src_uri )
+ )
+
+ with open ( distfile, mode='wb' ) as fh:
+ block = webh.read ( self.transfer_blocksize )
+ while block:
+ # write block to file
+ fh.write ( block )
+ # ? bytelen
+ bytes_fetched += len ( block )
+
+ # get the next block
+ block = webh.read ( self.transfer_blocksize )
+ # -- with
+
+ if bytes_fetched == expected_filesize:
+ if expected_digest is not None:
+ our_digest = digest.dodigest_file ( distfile, self._digest_type )
+
+ if our_digest != expected_digest:
+ # fetched package's digest does not match the expected one,
+ # refuse to use it
+ self.logger.warning (
+ 'bad {dtype} digest for {f!r}, expected {theirs} but '
+ 'got {ours} - removing this package.'.format (
+ dtype = self._digest_type,
+ f = distfile,
+ theirs = expected_digest,
+ ours = our_digest
+ )
+ )
+ os.remove ( distfile )
+
+ # package removed -> return success
+ return True
+ # -- if
+ # -- if
+
+ else:
+ return False
+ else:
+ # FIXME: debug print
+ print ( "Skipping fetch for {f!r}".format ( f=distfile ) )
+
+ return self._package_synced ( package_file, distfile, src_uri )
+ # --- end of get_package (...) ---
+
+ def _package_synced ( self, package_filename, distfile, src_uri ):
+ """Called when a package has been synced (=exists locally when
+ _get_package() is done).
+
+ arguments:
+ * package_filename --
+ * distfile --
+ * src_uri --
+ """
+ return True
+ # --- end of _package_synced (...) ---
+
+ def _dosync ( self ):
+ """Syncs this repo."""
+ package_list = self._fetch_package_list()
+
+ # empty/unset package list
+ if not package_list: return True
+
+ util.dodir ( self.distdir )
+
+ success = True
+
+ if self._digest_type is not None:
+ for package_file, expected_digest in package_list:
+ src_uri = self.get_src_uri ( package_file )
+
+ if not self._get_package (
+ package_file, src_uri, expected_digest
+ ):
+ success = False
+ break
+ else:
+ for package_file in package_list:
+ src_uri = self.get_src_uri ( package_file )
+
+ if not self._get_package (
+ package_file, src_uri, expected_digest=None
+ ):
+ success = False
+ break
+
+ return success
+ # --- end of _dosync (...) ---
+
+
+class WebsyncRepo ( WebsyncBase ):
+ """Sync a http repo using its PACKAGES file."""
+ # FIXME: hardcoded for md5
+
+ def __init__ ( self,
+ pkglist_uri=None,
+ pkglist_file=None,
+ *args,
+ **kwargs
+ ):
+ """Initializes a WebsyncRepo instance.
+
+ arguments:
+ * pkglist_uri -- if set and not None: uri of the package list file
+ * pkglist_file -- if set and not None: name of the package list file,
+ this is used to calculate the pkglist_uri
+ pkglist_uri = <src_uri>/<pkglist_file>
+ * *args / **kwargs -- see WebsyncBase / BasicRepo
+
+ pkglist file: this is a file with debian control file-like syntax
+ listing all packages.
+ Example: http://www.omegahat.org/R/src/contrib/PACKAGES (2012-07-31)
+ """
+ super ( WebsyncRepo, self ) . __init__ ( *args, **kwargs )
+
+ if self._digest_type is None:
+ self.FIELDREGEX = re.compile (
+ '^\s*(?P<name>(package|version))[:]\s*(?P<value>.+)',
+ re.IGNORECASE
+ )
+ else:
+ # used to filter field names (package,version,md5sum)
+ self.FIELDREGEX = re.compile (
+ '^\s*(?P<name>(package|version|md5sum))[:]\s*(?P<value>.+)',
+ re.IGNORECASE
+ )
+
+ self.pkglist_uri = pkglist_uri or self.get_src_uri ( pkglist_file )
+ if not self.pkglist_uri:
+ raise Exception ( "pkglist_uri is unset!" )
+ # --- end of __init__ (...) ---
+
+ def _fetch_package_list ( self ):
+ """Returns the list of packages to be downloaded.
+ List format:
+ * if digest verification is enabled:
+ List ::= [ ( package_file, digest ), ... ]
+ * else
+ List ::= [ package_file, ... ]
+ """
+
+ def generate_pkglist ( fh ):
+ """Generates the package list using the given file handle.
+
+ arguments:
+ * fh -- file handle to read from
+ """
+ info = dict()
+
+ max_info_len = 3 if self._digest_type is not None else 2
+
+ for match in (
+ filter ( None, (
+ self.FIELDREGEX.match ( l ) for l in fh.readlines()
+ ) )
+ ):
+ name, value = match.group ( 'name', 'value' )
+ info [name.lower()] = value
+
+ if len ( info.keys() ) == max_info_len:
+
+ pkgfile = '{name}_{version}.tar.gz'.format (
+ name=info ['package'], version=info ['version']
+ )
+
+ if self._digest_type is not None:
+ yield ( pkgfile, info ['md5sum'] )
+ #yield ( pkgfile, ( 'md5', info ['md5sum'] ) )
+ else:
+ yield pkgfile
+
+ info.clear()
+ # --- end of generate_pkglist (...) ---
+
+ package_list = ()
+ try:
+ webh = urllib2.urlopen ( self.pkglist_uri )
+
+ content_type = webh.info().get ( 'content-type', None )
+
+ if content_type != 'text/plain':
+ print (
+ "content type {!r} is not supported!".format ( content_type )
+ )
+ else:
+ package_list = tuple ( generate_pkglist ( webh ) )
+
+ webh.close()
+
+ finally:
+ if 'webh' in locals() and webh: webh.close()
+
+ return package_list
+ # --- end fetch_pkglist (...) ---
+
+class WebsyncPackageList ( WebsyncBase ):
+ """Sync packages from multiple remotes via http. Packages uris are read
+ from a file."""
+
+ # FIXME: does not support --nosync
+
+ def __init__ ( self, pkglist_file, *args, **kwargs ):
+ """Initializes a WebsyncPackageList instance.
+
+ arguments:
+ * pkglist_file -- path to the package list file that lists
+ one package http uri per line
+ * *args / **kwargs -- see WebsyncBase, BasicRepo
+ """
+ super ( WebsyncPackageList, self ) . __init__ ( *args, **kwargs )
+
+ self._pkglist_file = os.path.abspath ( pkglist_file )
+
+ del self.src_uri
+
+ self._synced_packages = list()
+
+ # --- end of __init__ (...) ---
+
+ def _fetch_package_list ( self ):
+ """Returns the package list.
+ Format:
+ pkglist ::= [ ( package_file, src_uri ), ... ]
+ """
+ pkglist = list()
+ with open ( self._pkglist_file, mode='r' ) as fh:
+ for line in fh.readlines():
+ src_uri = line.strip()
+ if src_uri:
+ pkglist.append ( (
+ src_uri.rpartition ( '/' ) [-1],
+ src_uri
+ ) )
+
+ return pkglist
+ # --- end of _fetch_package_list (...) ---
+
+ def _package_synced ( self, package_filename, distfile, src_uri ):
+ self._synced_packages.append (
+ ( package_filename, src_uri )
+ )
+ return True
+ # --- end of _package_synced (...) ---
+
+ def scan_distdir ( self, log_bad=True, **kwargs_ignored ):
+ for package_filename, src_uri in self._synced_packages:
+ pkg = self._package_nofail (
+ log_bad,
+ filename = package_filename,
+ origin = self,
+ src_uri = src_uri
+ )
+ if pkg is not None:
+ yield pkg
+ # --- end of scan_distdir (...) ---
+
+ def _nosync ( self ):
+ """nosync - report existing packages"""
+ for package_file, src_uri in self._fetch_package_list():
+ distfile = self.distdir + os.sep + package_file
+ if os.access ( distfile, os.F_OK ):
+ self._package_synced ( package_file, distfile, src_uri )
+
+ return True
+ # --- end of _nosync (...) ---
+
+ def _dosync ( self ):
+ """Sync packages."""
+ package_list = self._fetch_package_list()
+
+ # empty/unset package list
+ if not package_list: return True
+
+ util.dodir ( self.distdir )
+
+ success = True
+
+ for package_file, src_uri in package_list:
+ if not self._get_package (
+ package_file, src_uri, expected_digest=None
+ ):
+ success = False
+ break
+
+ return success
+ # --- end of _dosync (...) ---
^ permalink raw reply related [flat|nested] 35+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
@ 2012-07-09 17:25 André Erdmann
0 siblings, 0 replies; 35+ messages in thread
From: André Erdmann @ 2012-07-09 17:25 UTC (permalink / raw
To: gentoo-commits
commit: f7d69f6c4c09d370b1b0c8a0dcc39395202bfbf5
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Mon Jul 9 17:26:10 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Mon Jul 9 17:26:10 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=f7d69f6c
typo
---
roverlay/remote/rsync.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/roverlay/remote/rsync.py b/roverlay/remote/rsync.py
index 9fcc348..11efd27 100644
--- a/roverlay/remote/rsync.py
+++ b/roverlay/remote/rsync.py
@@ -145,7 +145,7 @@ class RsyncRepo ( RemoteRepo ):
retry_count += 1
self.logger.warning (
- "rsync returned {!r}, retrying ((}/{})".format (
+ "rsync returned {!r}, retrying ({}/{})".format (
retcode, retry_count, MAX_RSYNC_RETRY
)
)
^ permalink raw reply related [flat|nested] 35+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
@ 2012-07-06 8:15 André Erdmann
0 siblings, 0 replies; 35+ messages in thread
From: André Erdmann @ 2012-07-06 8:15 UTC (permalink / raw
To: gentoo-commits
commit: cf8a0d0dd57f04d3b9da37f091a2f401c16e5b3c
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Fri Jul 6 08:14:14 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Fri Jul 6 08:14:14 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=cf8a0d0d
apply distroot changes to RemoteRepo/RsyncRepo
* also added the possibility to retry rsync transfers on certain error codes
modified: roverlay/remote/basicrepo.py
modified: roverlay/remote/rsync.py
---
roverlay/remote/basicrepo.py | 6 +++-
roverlay/remote/rsync.py | 56 ++++++++++++++++++++++++++++++++---------
2 files changed, 47 insertions(+), 15 deletions(-)
diff --git a/roverlay/remote/basicrepo.py b/roverlay/remote/basicrepo.py
index 0248413..3dd09de 100644
--- a/roverlay/remote/basicrepo.py
+++ b/roverlay/remote/basicrepo.py
@@ -251,7 +251,7 @@ class RemoteRepo ( LocalRepo ):
"""A template for remote repositories."""
def __init__ (
- self, name, sync_proto,
+ self, name, distroot, sync_proto,
directory=None,
src_uri=None, remote_uri=None, base_uri=None
):
@@ -274,7 +274,9 @@ class RemoteRepo ( LocalRepo ):
* | { x : x in union(src,remote,base) and x not None } | >= 1
^= at least one out of src/remote/base uri is not None
"""
- super ( RemoteRepo, self ) . __init__ ( name, directory, src_uri='' )
+ super ( RemoteRepo, self ) . __init__ (
+ name, distroot, directory, src_uri=''
+ )
self.sync_proto = sync_proto
diff --git a/roverlay/remote/rsync.py b/roverlay/remote/rsync.py
index 4744118..9fcc348 100644
--- a/roverlay/remote/rsync.py
+++ b/roverlay/remote/rsync.py
@@ -15,6 +15,15 @@ RSYNC_ENV = util.keepenv (
'RSYNC_PASSWORD',
)
+MAX_RSYNC_RETRY = 3
+
+RSYNC_SIGINT = 20
+
+RETRY_ON_RETCODE = frozenset ((
+ 23, # "Partial transfer due to error"
+ 24, # "Partial transfer due to vanished source files"
+))
+
# TODO:
# either reraise an KeyboardInterrupt while running rsync (which stops script
# execution unless the interrupt is catched elsewhere) or just set a
@@ -40,7 +49,7 @@ DEFAULT_RSYNC_OPTS = (
class RsyncRepo ( RemoteRepo ):
def __init__ (
- self, name,
+ self, name, distroot,
directory=None, src_uri=None, rsync_uri=None, base_uri=None,
recursive=False, extra_opts=None
):
@@ -59,7 +68,7 @@ class RsyncRepo ( RemoteRepo ):
# using '' as remote protocol which leaves uris unchanged when
# normalizing them for rsync usage
super ( RsyncRepo, self ) . __init__ (
- name, '', directory=directory,
+ name, distroot=distroot, sync_proto='', directory=directory,
src_uri=src_uri, remote_uri=rsync_uri, base_uri=base_uri
)
@@ -103,6 +112,15 @@ class RsyncRepo ( RemoteRepo ):
All exceptions(?) are catched and interpreted as sync failure.
"""
+ def waitfor ( p ):
+ if p.communicate() != ( None, None ):
+ raise AssertionError ( "expected None,None from communicate!" )
+ if p.returncode == RSYNC_SIGINT:
+ raise KeyboardInterrupt ( "propagated from rsync" )
+
+ return p.returncode
+ # --- end of waitfor (...) ---
+
retcode = '<undef>'
try:
@@ -113,31 +131,43 @@ class RsyncRepo ( RemoteRepo ):
self.logger.debug ( 'running rsync cmd: ' + ' '.join ( rsync_cmd ) )
+ retry_count = 0
- proc = subprocess.Popen (
- rsync_cmd,
- stdin=None, stdout=None, stderr=None,
- env=RSYNC_ENV
- )
+ proc = subprocess.Popen ( rsync_cmd, env=RSYNC_ENV )
+ retcode = waitfor ( proc )
+ del proc
- if proc.communicate() != ( None, None ):
- raise AssertionError ( "expected None,None from communicate!" )
+ while retcode in RETRY_ON_RETCODE and retry_count < MAX_RSYNC_RETRY:
+ # this handles retcodes like
+ # * 24: "Partial transfer due to vanished source files"
+
+ # FIXME replace loop condition "retcode != 0"
+ retry_count += 1
+
+ self.logger.warning (
+ "rsync returned {!r}, retrying ((}/{})".format (
+ retcode, retry_count, MAX_RSYNC_RETRY
+ )
+ )
+
+ proc = subprocess.Popen ( rsync_cmd, env=RSYNC_ENV )
+ retcode = waitfor ( proc )
+ del proc
- if proc.returncode == 0:
+ if retcode == 0:
self._set_ready ( is_synced=True )
return True
- retcode = proc.returncode
except KeyboardInterrupt:
sys.stderr.write (
"\nKeyboard interrupt - waiting for rsync to exit...\n"
)
- if 'proc' in locals():
+ if 'proc' in locals() and proc is not None:
proc.communicate()
retcode = proc.returncode
else:
- retcode = 130
+ retcode = RSYNC_SIGINT
if RERAISE_INTERRUPT:
raise
^ permalink raw reply related [flat|nested] 35+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
@ 2012-07-04 18:21 André Erdmann
0 siblings, 0 replies; 35+ messages in thread
From: André Erdmann @ 2012-07-04 18:21 UTC (permalink / raw
To: gentoo-commits
commit: ff1c2717d72ca43ab843a857ccb82c1791f3094f
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jul 4 18:15:29 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jul 4 18:15:29 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=ff1c2717
remote: add force_distroot
also added a per RepoList DISTROOT variable
modified: roverlay/remote/repolist.py
modified: roverlay/remote/repoloader.py
---
roverlay/remote/repolist.py | 65 +++++++++++++++++++++++++++++++++++++++-
roverlay/remote/repoloader.py | 51 ++++++++++++++++++--------------
2 files changed, 92 insertions(+), 24 deletions(-)
diff --git a/roverlay/remote/repolist.py b/roverlay/remote/repolist.py
index a32f53d..cc673e6 100644
--- a/roverlay/remote/repolist.py
+++ b/roverlay/remote/repolist.py
@@ -1,13 +1,25 @@
import re
import logging
+import os.path
from roverlay import config
from roverlay.remote.repoloader import read_repofile
+from roverlay.remote.basicrepo import LocalRepo
class RepoList ( object ):
"""Controls several Repo objects."""
- def __init__ ( self, sync_enabled=True ):
+ def __init__ ( self,
+ sync_enabled=True, force_distroot=False, distroot=None
+ ):
+ """Initializes a RepoList.
+
+ arguments:
+ * sync_enabled -- whether sync is enabled, defaults to True
+ * force_distdir -- if set and True: put all distdirs into distroot,
+ ignoring repo-specific dirs
+ * distroot --
+ """
self.repos = list()
self.sync_enabled = sync_enabled
@@ -18,6 +30,13 @@ class RepoList ( object ):
# could not be synced
self.use_broken_repos = False
+ self.force_distroot = force_distroot
+
+ if distroot is None:
+ self.distroot = config.get_or_fail ( "DISTFILES.root" )
+ else:
+ self.distroot = distroot
+
# <name>_<version>.<tar suffix>
# '^..*_[0-9.]{1,}%s$' or '^[^_]{1,}_[0-9._-]{1,}%s$'
@@ -35,13 +54,55 @@ class RepoList ( object ):
return self.pkg_regex.match ( pkg_filename ) is not None
# --- end of _pkg_filter (...) ---
+ def add_distdir ( self, distdir, src_uri=None, name=None ):
+ """Adds a local package directory as LocalRepo.
+
+ arguments:
+ * distdir --
+ * src_uri -- SRC_URI used in created ebuilds,
+ defaults to None which results in non-fetchable ebuilds
+ (FIXME: could add RESTRICT="fetch" to those ebuilds)
+ * name -- name of the repo, defaults to os.path.basename (distdir)
+ """
+ self.repos.append ( LocalRepo (
+ name=os.path.basename ( distdir ) if name is None else name,
+ directory=distdir,
+ src_uri=src_uri
+ ) )
+ # --- end of add_distdir (...) ---
+
+ def add_distdirs ( self, distdirs ):
+ """Adds several distdirs as LocalRepos.
+ All distdirs will have an invalid SRC_URI and a default name,
+ use add_distdir() if you want usable ebuilds.
+
+ arguments:
+ * distdirs -- distdirs to add (must be an iterable non-str type)
+ """
+ def gen_repos():
+ for d in distdirs:
+ repo = LocalRepo (
+ name=os.path.basename ( d ),
+ # FIXME: --force_distroot should block --distdir
+ directory=d,
+ distroot=self.distroot
+ )
+ self.logger.debug ( 'New entry, ' + str ( repo ) )
+ yield repo
+ # --- end of gen_repos() ---
+ self.repos.extend ( gen_repos() )
+ # --- end of add_distdirs (...) ---
+
def load_file ( self, _file ):
"""Loads a repo config file and adds the repos to this RepoList.
arguments:
* _file --
"""
- new_repos = read_repofile ( _file )
+ new_repos = read_repofile ( _file,
+ distroot=self.distroot,
+ force_distroot=self.force_distroot
+ )
if new_repos:
self.repos.extend ( new_repos )
# --- end of load_file (...) ---
diff --git a/roverlay/remote/repoloader.py b/roverlay/remote/repoloader.py
index 4ffaf41..2a92526 100644
--- a/roverlay/remote/repoloader.py
+++ b/roverlay/remote/repoloader.py
@@ -1,4 +1,5 @@
import sys
+import os.path
import logging
try:
@@ -15,13 +16,16 @@ from roverlay.remote.rsync import RsyncRepo
LOGGER = logging.getLogger ( 'repoloader' )
-def read_repofile ( repo_file, lenient=False ):
- """Reads a repo config file. Returns the list of created repos.
+def read_repofile ( repo_file, distroot, lenient=False, force_distroot=False ):
+ """Reads a repo config file. Yields created repos.
arguments:
- * repo_file --
- * lenient -- if True: do not fail if file is missing, allows reading
- of more than one file at once
+ * repo_file --
+ * distroot --
+ * lenient -- if set and True: do not fail if file is missing,
+ allows reading of more than one file at once
+ * force_distdir -- if set and True: use <DISTFILES.root>/<repo name>
+ as distdir for repos
"""
parser = configparser.SafeConfigParser ( allow_no_value=False )
@@ -35,9 +39,8 @@ def read_repofile ( repo_file, lenient=False ):
finally:
if fh: fh.close()
- repos = list()
-
for name in parser.sections():
+ repo = None
if sys.version_info < ( 3, 2 ):
# FIXME replace this and use more accurate version condition
@@ -48,10 +51,16 @@ def read_repofile ( repo_file, lenient=False ):
repo_type = get ( 'type', 'rsync' ).lower()
+ repo_name = get ( 'name', name )
+
+ repo_distdir = None if force_distroot else get ( 'directory' )
+
+
if repo_type == 'local':
repo = LocalRepo (
- name = get ( 'name', name ),
- directory = get ( 'directory' ),
+ name = repo_name,
+ distroot = distroot,
+ directory = repo_distdir,
src_uri = get ( 'src_uri' )
)
elif repo_type == 'rsync':
@@ -60,23 +69,21 @@ def read_repofile ( repo_file, lenient=False ):
extra_opts = extra_opts.split ( ' ' )
repo = RsyncRepo (
- name = get ( 'name', name ),
- directory = get ( 'directory' ),
- src_uri = get ( 'src_uri' ),
- rsync_uri = get ( 'rsync_uri' ),
- base_uri = get ( 'base_uri' ),
- extra_opts = extra_opts,
- recursive = get ( 'recursive', False ) == 'yes',
+ name = repo_name,
+ distroot = distroot,
+ directory = repo_distdir,
+ src_uri = get ( 'src_uri' ),
+ rsync_uri = get ( 'rsync_uri' ),
+ base_uri = get ( 'base_uri' ),
+ extra_opts = extra_opts,
+ recursive = get ( 'recursive', False ) == 'yes',
)
else:
LOGGER.error ( "Unknown repo type %s for %s" % ( repo_type, name ) )
- continue
-
- LOGGER.debug ( 'New entry, ' + str ( repo ) )
- repos.append ( repo )
- repo = None
+ if repo is not None:
+ LOGGER.debug ( 'New entry, ' + str ( repo ) )
+ yield repo
- return repos
# --- end of read_repofile (...) ---
^ permalink raw reply related [flat|nested] 35+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
@ 2012-07-04 18:21 André Erdmann
0 siblings, 0 replies; 35+ messages in thread
From: André Erdmann @ 2012-07-04 18:21 UTC (permalink / raw
To: gentoo-commits
commit: 3f50626f88d91adb2dca9911c17c2c69db07f261
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jul 4 18:12:46 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jul 4 18:12:46 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=3f50626f
remote/basicrepo
* scan_distdir(): don't fail during PackageInfo creation
* added distroot to LocalRepo initialization
modified: roverlay/remote/basicrepo.py
---
roverlay/remote/basicrepo.py | 71 ++++++++++++++++++++++++++++++++----------
1 files changed, 54 insertions(+), 17 deletions(-)
diff --git a/roverlay/remote/basicrepo.py b/roverlay/remote/basicrepo.py
index fa6c5da..0248413 100644
--- a/roverlay/remote/basicrepo.py
+++ b/roverlay/remote/basicrepo.py
@@ -1,7 +1,6 @@
import os.path
import logging
-from roverlay import config
from roverlay.packageinfo import PackageInfo
URI_SEPARATOR = '://'
@@ -43,7 +42,7 @@ class LocalRepo ( object ):
It's the base class for remote repos.
"""
- def __init__ ( self, name, directory=None, src_uri=None ):
+ def __init__ ( self, name, distroot, directory=None, src_uri=None ):
"""Initializes a LocalRepo.
arguments:
@@ -59,7 +58,7 @@ class LocalRepo ( object ):
if directory is None:
self.distdir = os.path.join (
- config.get_or_fail ( [ 'DISTFILES', 'root' ] ),
+ distroot,
# subdir repo names like CRAN/contrib are ok,
# but make sure to use the correct path separator
self.name.replace ( '/', os.path.sep ),
@@ -167,42 +166,80 @@ class LocalRepo ( object ):
return status
# --- end of sync (...) ---
- def scan_distdir ( self, is_package=None, log_filtered=True ):
+ def scan_distdir ( self,
+ is_package=None, log_filtered=False, log_bad=True
+ ):
"""Generator that scans the local distfiles dir of this repo and
yields PackageInfo instances.
arguments:
- * is_package -- function returning True if the given file is a package
- or None which means that all files are packages.
- Defaults to None.
+ * is_package -- function returning True if the given file is a package
+ or None which means that all files are packages.
+ Defaults to None.
+ * log_filtered -- log files that did not pass is_package().
+ Defaults to False; no effect if is_package is None.
+ * log_bad -- log files that failed the PackageInfo creation step
+ Defaults to True.
+
+ raises: AssertionError if is_package is neither None nor a callable.
"""
- kw = { 'origin' : self }
+ def package_nofail ( filename, distdir ):
+ """Tries to create a PackageInfo.
+ Logs failure if log_bad is True.
+
+ arguments:
+ * filename -- name of the package file (including .tar* suffix)
+ * distdir -- filename's directory
+
+ returns: PackageInfo on success, else None.
+ """
+ try:
+ return PackageInfo (
+ filename=filename, origin=self, distdir=distdir
+ )
+ except ( ValueError, ) as expected:
+ if log_bad:
+ #self.logger.exception ( expected )
+ self.logger.info (
+ "filtered %r: bad package" % filename
+ )
+ return None
+
+ # --- end of package_nofail (...) ---
if is_package is None:
# unfiltered variant
for dirpath, dirnames, filenames in os.walk ( self.distdir ):
- kw ['distdir'] = dirpath if dirpath != self.distdir else None
+ distdir = dirpath if dirpath != self.distdir else None
- for pkg in filenames:
- yield PackageInfo ( filename=pkg, **kw )
+ for filename in filenames:
+ pkg = package_nofail ( filename, distdir )
+ if pkg is not None:
+ yield pkg
elif hasattr ( is_package, '__call__' ):
# filtered variant (adds an if is_package... before yield)
for dirpath, dirnames, filenames in os.walk ( self.distdir ):
- kw ['distdir'] = dirpath if dirpath != self.distdir else None
+ distdir = dirpath if dirpath != self.distdir else None
- for pkg in filenames:
- if is_package ( os.path.join ( dirpath, pkg ) ):
- yield PackageInfo ( filename=pkg, **kw )
+ for filename in filenames:
+ if is_package ( os.path.join ( dirpath, filename ) ):
+ pkg = package_nofail ( filename, distdir )
+ if pkg is not None:
+ yield pkg
elif log_filtered:
- self.logger.debug ( "filtered '%s': not a package" % pkg )
+ self.logger.debug (
+ "filtered %r: not a package" % filename
+ )
else:
# faulty variant, raises Exception
- raise Exception ( "is_package should either be None or a function." )
+ raise AssertionError (
+ "is_package should either be None or a function."
+ )
#yield None
# --- end of scan_distdir (...) ---
^ permalink raw reply related [flat|nested] 35+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
@ 2012-07-03 17:48 André Erdmann
0 siblings, 0 replies; 35+ messages in thread
From: André Erdmann @ 2012-07-03 17:48 UTC (permalink / raw
To: gentoo-commits
commit: df59c48ecedca4c4bc52b5d9309559cb95fe2e2b
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jul 3 17:45:33 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jul 3 17:45:33 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=df59c48e
add sync_enabled keyword to Repo
---
roverlay/remote/repolist.py | 4 ++--
1 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/roverlay/remote/repolist.py b/roverlay/remote/repolist.py
index cee4940..a32f53d 100644
--- a/roverlay/remote/repolist.py
+++ b/roverlay/remote/repolist.py
@@ -7,10 +7,10 @@ from roverlay.remote.repoloader import read_repofile
class RepoList ( object ):
"""Controls several Repo objects."""
- def __init__ ( self ):
+ def __init__ ( self, sync_enabled=True ):
self.repos = list()
- self.sync_enabled = True
+ self.sync_enabled = sync_enabled
self.logger = logging.getLogger ( self.__class__.__name__ )
^ permalink raw reply related [flat|nested] 35+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
@ 2012-06-27 14:46 André Erdmann
0 siblings, 0 replies; 35+ messages in thread
From: André Erdmann @ 2012-06-27 14:46 UTC (permalink / raw
To: gentoo-commits
commit: 6c44579da0f7e9cbc4b889a7449ab995b8761908
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Wed Jun 27 14:39:27 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Wed Jun 27 14:39:27 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=6c44579d
remote module
* merged RsyncRepo (repo.py) and RsyncJob (rsync.py)
* added comments
* added a package filter for scan_distdir()
modified: roverlay/remote/basicrepo.py
deleted: roverlay/remote/repo.py
modified: roverlay/remote/repolist.py
modified: roverlay/remote/repoloader.py
modified: roverlay/remote/rsync.py
---
roverlay/remote/basicrepo.py | 43 ++++++++++++++++--
roverlay/remote/repo.py | 58 ------------------------
roverlay/remote/repolist.py | 76 +++++++++++++++++++++++++++-----
roverlay/remote/repoloader.py | 10 +++-
roverlay/remote/rsync.py | 98 +++++++++++++++++++++++++++++-----------
5 files changed, 182 insertions(+), 103 deletions(-)
diff --git a/roverlay/remote/basicrepo.py b/roverlay/remote/basicrepo.py
index 1574850..fa6c5da 100644
--- a/roverlay/remote/basicrepo.py
+++ b/roverlay/remote/basicrepo.py
@@ -1,4 +1,5 @@
import os.path
+import logging
from roverlay import config
from roverlay.packageinfo import PackageInfo
@@ -10,10 +11,18 @@ LOCALREPO_SRC_URI = 'http://localhost/R-Packages'
SYNC_SUCCESS = 1
SYNC_FAIL = 2
+SYNC_DONE = SYNC_SUCCESS | SYNC_FAIL
REPO_READY = 4
-
def normalize_uri ( uri, protocol, force_protocol=False ):
+ """Returns an uri that is prefixed by its protocol ('http://', ...).
+ Does nothing if protocol evaluates to False.
+
+ arguments:
+ * uri --
+ * protocol -- e.g. 'http'
+ * force_protocol -- replace an existing protocol
+ """
if not protocol:
return uri
@@ -42,7 +51,12 @@ class LocalRepo ( object ):
* directory -- distfiles dir, defaults to <DISTFILES root>/<name>
* src_uri -- SRC_URI, defaults to http://localhost/R-Packages/<name>
"""
- self.name = name
+ self.name = name
+
+ self.logger = logging.getLogger (
+ self.__class__.__name__ + ':' + self.name
+ )
+
if directory is None:
self.distdir = os.path.join (
config.get_or_fail ( [ 'DISTFILES', 'root' ] ),
@@ -63,28 +77,44 @@ class LocalRepo ( object ):
# --- end of __init__ (...) ---
def ready ( self ):
+ """Returns True if this repo is ready (for package scanning using
+ scan_distdir).
+ """
return bool ( self.sync_status & REPO_READY )
+ # --- end of ready (...) ---
def fail ( self ):
+ """Returns True if sync failed for this repo."""
return bool ( self.sync_status & SYNC_FAIL )
+ # --- end of fail (...) ---
def offline ( self ):
- return 0 == self.sync_status & SYNC_SUCCESS
+ """Returns True if this repo is offline (not synced)."""
+ return 0 == self.sync_status & SYNC_DONE
+ # --- end of offline (...) ---
def _set_ready ( self, is_synced ):
- """comment TODO"""
+ """Sets the sync status of this repo to READY.
+
+ arguments:
+ * is_synced -- whether this repo has been synced
+ """
if is_synced:
self.sync_status = SYNC_SUCCESS | REPO_READY
else:
self.sync_status = REPO_READY
+ # --- end of _set_ready (...) ---
def _set_fail ( self ):
+ """Sets the sync status of this repo to FAIL."""
self.sync_status = SYNC_FAIL
+ # --- end of _set_fail (...) ---
def __str__ ( self ):
return "repo '%s': DISTDIR '%s', SRC_URI '%s'" % (
self.name, self.distdir, self.src_uri
)
+ # --- end of __str__ (...) ---
def get_name ( self ):
"""Returns the name of this repository."""
@@ -137,7 +167,7 @@ class LocalRepo ( object ):
return status
# --- end of sync (...) ---
- def scan_distdir ( self, is_package=None ):
+ def scan_distdir ( self, is_package=None, log_filtered=True ):
"""Generator that scans the local distfiles dir of this repo and
yields PackageInfo instances.
@@ -166,6 +196,8 @@ class LocalRepo ( object ):
for pkg in filenames:
if is_package ( os.path.join ( dirpath, pkg ) ):
yield PackageInfo ( filename=pkg, **kw )
+ elif log_filtered:
+ self.logger.debug ( "filtered '%s': not a package" % pkg )
else:
@@ -282,6 +314,7 @@ class RemoteRepo ( LocalRepo ):
return "repo '%s': DISTDIR '%s', SRC_URI '%s', REMOTE_URI '%s'" % (
self.name, self.distdir, self.src_uri, self.remote_uri
)
+ # --- end of __str__ (...) ---
# --- end of RemoteRepo ---
diff --git a/roverlay/remote/repo.py b/roverlay/remote/repo.py
deleted file mode 100644
index febfaae..0000000
--- a/roverlay/remote/repo.py
+++ /dev/null
@@ -1,58 +0,0 @@
-
-import logging
-
-#from roverlay.remote.basicrepo import LocalRepo, RemoteRepo
-from roverlay.remote.basicrepo import RemoteRepo
-
-from roverlay.remote.rsync import RsyncJob
-
-class RsyncRepo ( RemoteRepo ):
-
- def __init__ (
- self, name,
- directory=None, src_uri=None, rsync_uri=None, base_uri=None,
- **rsync_kw
- ):
- # super's init: name, remote protocol, directory_kw, **uri_kw
- # using '' as remote protocol which leaves uris unchanged when
- # normalizing them for rsync usage
- super ( RsyncRepo, self ) . __init__ (
- name, '', directory=directory,
- src_uri=src_uri, remote_uri=rsync_uri, base_uri=base_uri
- )
- self.rsync_extra = rsync_kw
-
- self.sync_protocol = 'rsync'
- # --- end of __init__ (...) ---
-
-
- def _dosync ( self ):
- retcode = None
- try:
- job = RsyncJob (
- remote=self.remote_uri, distdir=self.distdir,
- run_now=True,
- **self.rsync_extra
- )
- if job.returncode == 0:
- self._set_ready ( is_synced=True )
- return True
-
- retcode = job.returncode
- except Exception as e:
- # catch exceptions, log them and return False
- ## TODO: which exceptions to catch||pass?
- logging.exception ( e )
- retcode = '<undef>'
-
- logging.error (
- 'Repo %s cannot be used for ebuild creation due to errors '
- 'while running rsync (return code was %s).' % ( self.name, retcode )
- )
- self._set_fail()
- return False
- # --- end of _dosync (...) ---
-
- def __str__ ( self ):
- return "rsync repo '%s': DISTDIR '%s', SRC_URI '%s', RSYNC_URI '%s'" \
- % ( self.name, self.distdir, self.src_uri, self.remote_uri )
diff --git a/roverlay/remote/repolist.py b/roverlay/remote/repolist.py
index ae740dd..cee4940 100644
--- a/roverlay/remote/repolist.py
+++ b/roverlay/remote/repolist.py
@@ -1,52 +1,89 @@
+import re
import logging
from roverlay import config
from roverlay.remote.repoloader import read_repofile
-LOGGER = logging.getLogger ( 'RepoList' )
-
class RepoList ( object ):
+ """Controls several Repo objects."""
def __init__ ( self ):
self.repos = list()
self.sync_enabled = True
+ self.logger = logging.getLogger ( self.__class__.__name__ )
+
# if True: use all repos when looking for packages, even those that
# could not be synced
self.use_broken_repos = False
- def sort ( self ):
- raise Exception ( "method stub." )
+
+ # <name>_<version>.<tar suffix>
+ # '^..*_[0-9.]{1,}%s$' or '^[^_]{1,}_[0-9._-]{1,}%s$'
+ self.pkg_regex = re.compile (
+ '^..*_..*%s$' % config.get_or_fail ( 'R_PACKAGE.suffix_regex' ),
+ )
+ # --- end of __init__ (...) ---
+
+ def _pkg_filter ( self, pkg_filename ):
+ """Returns True if pkg_filename is a package, else False.
+
+ arguments:
+ * pkg_filename --
+ """
+ return self.pkg_regex.match ( pkg_filename ) is not None
+ # --- end of _pkg_filter (...) ---
def load_file ( self, _file ):
+ """Loads a repo config file and adds the repos to this RepoList.
+
+ arguments:
+ * _file --
+ """
new_repos = read_repofile ( _file )
if new_repos:
self.repos.extend ( new_repos )
# --- end of load_file (...) ---
def load ( self ):
+ """Loads the default repo config files
+ (as listed in the main configuration file).
+ """
files = config.get_or_fail ( 'REPO.config_files' )
for f in files:
self.load_file ( f )
# --- end of load (...) ---
def _queue_packages_from_repo ( self, repo, add_method ):
+ """Adds all packages from a repo using add_method.
+
+ arguments:
+ * repo --
+ * add_method -- method that is called for each package,
+ has to accept exactly one arg, the package
+ """
if not repo.ready():
if self.use_broken_repos:
# warn and continue
pass
else:
# repo cannot be used
- LOGGER.warning ( "!!" )
+ self.logger.warning ( "ignoring repo %s." % repo.name )
return False
- for p in repo.scan_distdir():
- LOGGER.debug ( "adding package %s from repo %s" % ( p, repo ) )
+ for p in repo.scan_distdir ( is_package=self._pkg_filter ):
+ self.logger.debug ( "adding package %s from repo %s" % ( p, repo ) )
add_method ( p )
# --- end of _queue_packages_from_repo (...) ---
def add_packages ( self, add_method ):
+ """Adds packages from all repos using add_method.
+
+ arguments:
+ * add_method -- method that is called for each package
+ """
+
for repo in self.repos:
self._queue_packages_from_repo ( repo, add_method )
# --- end of add_packages (...) ---
@@ -56,9 +93,20 @@ class RepoList ( object ):
when_repo_success=None, when_repo_fail=None, when_repo_done=None,
when_all_done=None
):
+ """A method that syncs all repos and is able to call other methods
+ on certain events (repo done/success/fail, all done).
+
+ arguments:
+ * when_repo_success (pkg) -- called after a successful repo sync
+ * when_repo_fail (pkg) -- called after an unsuccessful repo sync
+ * when_repo_done (pkg) -- called when a repo sync has finished
+ * when_all_done (pkg) -- called when all repo sync have finished
+ """
+
+ # try_call (f,*args,**kw) calls f (*args,**kw) unless f is None
try_call = lambda f, *x, **z : None if f is None else f ( *x, **z )
- LOGGER.debug ( "Syncing repos ..." )
+ self.logger.debug ( "Syncing repos ..." )
for repo in self.repos:
if repo.sync ( sync_enabled=self.sync_enabled ):
# repo successfully synced
@@ -73,13 +121,19 @@ class RepoList ( object ):
# --- end of _sync_all_repos_and_run (...) ---
def sync ( self ):
- LOGGER.debug ( "Syncing repos ..." )
+ """Syncs all repos."""
+ self.logger.debug ( "Syncing repos ..." )
for repo in self.repos:
repo.sync ( sync_enabled=self.sync_enabled )
# --- end of sync_all (...) ---
def sync_and_add ( self, add_method ):
- """Syncs all repos and adds packages immediately to the package queue."""
+ """Syncs all repos and adds packages immediately to the package queue
+ using add_method.
+
+ arguments:
+ * add_method (pkg) --
+ """
# TODO: _nowait? raises Exception when queue is full which is
# good in non-threaded execution
# -> timeout,..
@@ -92,4 +146,4 @@ class RepoList ( object ):
def __str__ ( self ):
return '\n'.join ( ( str ( x ) for x in self.repos ) )
-
+ # --- end of __str__ (...) ---
diff --git a/roverlay/remote/repoloader.py b/roverlay/remote/repoloader.py
index cd7c9fc..4ffaf41 100644
--- a/roverlay/remote/repoloader.py
+++ b/roverlay/remote/repoloader.py
@@ -3,7 +3,7 @@ import logging
try:
import configparser
-except ImportError as running_python2:
+except ImportError:
# configparser is named ConfigParser in python2
import ConfigParser as configparser
@@ -11,12 +11,18 @@ except ImportError as running_python2:
from roverlay import config
from roverlay.remote.basicrepo import LocalRepo
-from roverlay.remote.repo import RsyncRepo
+from roverlay.remote.rsync import RsyncRepo
LOGGER = logging.getLogger ( 'repoloader' )
def read_repofile ( repo_file, lenient=False ):
+ """Reads a repo config file. Returns the list of created repos.
+ arguments:
+ * repo_file --
+ * lenient -- if True: do not fail if file is missing, allows reading
+ of more than one file at once
+ """
parser = configparser.SafeConfigParser ( allow_no_value=False )
if lenient:
diff --git a/roverlay/remote/rsync.py b/roverlay/remote/rsync.py
index 452a9f8..4744118 100644
--- a/roverlay/remote/rsync.py
+++ b/roverlay/remote/rsync.py
@@ -2,7 +2,10 @@ import os
import sys
import subprocess
-from roverlay import config, util
+from roverlay import config, util
+
+#from roverlay.remote.basicrepo import LocalRepo, RemoteRepo
+from roverlay.remote.basicrepo import RemoteRepo
RSYNC_ENV = util.keepenv (
'PATH',
@@ -18,7 +21,6 @@ RSYNC_ENV = util.keepenv (
# non-zero return code (-> 'repo cannot be used')
RERAISE_INTERRUPT = False
-
# --recursive is not in the default opts, subdirs in CRAN/contrib are
# either R releases (2.xx.x[-patches]) or the package archive
DEFAULT_RSYNC_OPTS = (
@@ -35,19 +37,36 @@ DEFAULT_RSYNC_OPTS = (
'--chmod=ugo=r,u+w,Dugo+x', # 0755 for transferred dirs, 0644 for files
)
-class RsyncJob ( object ):
+class RsyncRepo ( RemoteRepo ):
+
def __init__ (
- self, remote=None, distdir=None, run_now=True,
- extra_opts=None, recursive=False
+ self, name,
+ directory=None, src_uri=None, rsync_uri=None, base_uri=None,
+ recursive=False, extra_opts=None
):
- self.distdir = distdir
+ """Initializes an RsyncRepo.
+
+ arguments:
+ * name --
+ * directory --
+ * src_uri --
+ * rsync_uri --
+ * base_uri --
+ * recursive -- if '--recursive' should be included in the rsync opts
+ * extra_opts -- extra opts for rsync (either None or a tuple/list/..)
+ """
+ # super's init: name, remote protocol, directory_kw, **uri_kw
+ # using '' as remote protocol which leaves uris unchanged when
+ # normalizing them for rsync usage
+ super ( RsyncRepo, self ) . __init__ (
+ name, '', directory=directory,
+ src_uri=src_uri, remote_uri=rsync_uri, base_uri=base_uri
+ )
# syncing directories, not files - always appending a slash at the end
# of remote
- if remote [-1] != '/':
- self.remote = remote + '/'
- else:
- self.remote = remote
+ if self.remote_uri [-1] != '/':
+ self.remote_uri = self.remote_uri + '/'
if recursive:
self.extra_opts = [ '--recursive' ]
@@ -56,14 +75,11 @@ class RsyncJob ( object ):
else:
self.extra_opts = extra_opts
-
- if run_now: self.run()
+ self.sync_protocol = 'rsync'
# --- end of __init__ (...) ---
def _rsync_argv ( self ):
- if self.remote is None or self.distdir is None:
- raise Exception ( "None in (remote,distdir)." )
-
+ """Returns an rsync command used for syncing."""
argv = [ 'rsync' ]
argv.extend ( DEFAULT_RSYNC_OPTS )
@@ -75,22 +91,29 @@ class RsyncJob ( object ):
if self.extra_opts:
argv.extend ( self.extra_opts )
- argv.extend ( ( self.remote, self.distdir ) )
-
+ argv.extend ( ( self.remote_uri, self.distdir ) )
# removing emty args from argv
return tuple ( filter ( None, argv ) )
- # --- end of _rsync_argv (...) ---
- def run ( self ):
+ # --- end of _rsync_argv (...) ---
- rsync_cmd = self._rsync_argv()
- print ( ' '.join ( rsync_cmd ) )
+ def _dosync ( self ):
+ """Syncs this repo. Returns True if sync succeeded, else False.
+ All exceptions(?) are catched and interpreted as sync failure.
+ """
- util.dodir ( self.distdir, mkdir_p=True )
+ retcode = '<undef>'
- # TODO pipe/log/.., running this in blocking mode until implemented
try:
+
+ rsync_cmd = self._rsync_argv()
+
+ util.dodir ( self.distdir, mkdir_p=True )
+
+ self.logger.debug ( 'running rsync cmd: ' + ' '.join ( rsync_cmd ) )
+
+
proc = subprocess.Popen (
rsync_cmd,
stdin=None, stdout=None, stderr=None,
@@ -100,7 +123,11 @@ class RsyncJob ( object ):
if proc.communicate() != ( None, None ):
raise AssertionError ( "expected None,None from communicate!" )
- self.returncode = proc.returncode
+ if proc.returncode == 0:
+ self._set_ready ( is_synced=True )
+ return True
+
+ retcode = proc.returncode
except KeyboardInterrupt:
sys.stderr.write (
@@ -108,10 +135,27 @@ class RsyncJob ( object ):
)
if 'proc' in locals():
proc.communicate()
- self.returncode = proc.returncode
+ retcode = proc.returncode
else:
- self.returncode = 130
+ retcode = 130
if RERAISE_INTERRUPT:
raise
- # --- end of start (...) ---
+
+ except Exception as e:
+ # catch exceptions, log them and return False
+ ## TODO: which exceptions to catch||pass?
+ self.logger.exception ( e )
+
+ self.logger.error (
+ 'Repo %s cannot be used for ebuild creation due to errors '
+ 'while running rsync (return code was %s).' % ( self.name, retcode )
+ )
+ self._set_fail()
+ return False
+ # --- end of _dosync (...) ---
+
+ def __str__ ( self ):
+ return "rsync repo '%s': DISTDIR '%s', SRC_URI '%s', RSYNC_URI '%s'" \
+ % ( self.name, self.distdir, self.src_uri, self.remote_uri )
+ # --- end of __str__ (...) ---
^ permalink raw reply related [flat|nested] 35+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
@ 2012-06-26 15:55 André Erdmann
0 siblings, 0 replies; 35+ messages in thread
From: André Erdmann @ 2012-06-26 15:55 UTC (permalink / raw
To: gentoo-commits
commit: b866245c1503fe74352cbbae85ec5ab4108e5f7e
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jun 26 15:53:53 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jun 26 15:53:53 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=b866245c
use parser.get without fallback keyword
* using get (configparser) without the fallback keyword if
python version is < 3.2
modified: roverlay/remote/repoloader.py
---
roverlay/remote/repoloader.py | 9 +++++++--
1 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/roverlay/remote/repoloader.py b/roverlay/remote/repoloader.py
index 94dd5b7..cd7c9fc 100644
--- a/roverlay/remote/repoloader.py
+++ b/roverlay/remote/repoloader.py
@@ -1,4 +1,4 @@
-
+import sys
import logging
try:
@@ -33,7 +33,12 @@ def read_repofile ( repo_file, lenient=False ):
for name in parser.sections():
- get = lambda a, b=None : parser.get ( name, a, raw=True, fallback=b )
+ if sys.version_info < ( 3, 2 ):
+ # FIXME replace this and use more accurate version condition
+ get = lambda a, b=None: parser.get ( name, a, raw=True ) \
+ if parser.has_option ( name, a ) else b
+ else:
+ get = lambda a, b=None : parser.get ( name, a, raw=True, fallback=b )
repo_type = get ( 'type', 'rsync' ).lower()
^ permalink raw reply related [flat|nested] 35+ messages in thread
* [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
@ 2012-06-26 15:42 André Erdmann
0 siblings, 0 replies; 35+ messages in thread
From: André Erdmann @ 2012-06-26 15:42 UTC (permalink / raw
To: gentoo-commits
commit: 2383f21652384458d0fc7a05a0c70610f22719d3
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Tue Jun 26 15:37:24 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Tue Jun 26 15:37:24 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=2383f216
remote
* can now be used to feed the overlay creation package queue
modified: roverlay/remote/basicrepo.py
modified: roverlay/remote/repo.py
modified: roverlay/remote/repolist.py
modified: roverlay/remote/repoloader.py
modified: roverlay/remote/rsync.py
---
roverlay/remote/basicrepo.py | 70 +++++++++++++++++++++++++++-------
roverlay/remote/repo.py | 18 +++++---
roverlay/remote/repolist.py | 78 +++++++++++++++++++++++++++++---------
roverlay/remote/repoloader.py | 7 +++-
roverlay/remote/rsync.py | 83 ++++++++++++++++++++++++++++------------
5 files changed, 190 insertions(+), 66 deletions(-)
diff --git a/roverlay/remote/basicrepo.py b/roverlay/remote/basicrepo.py
index 9ade3a2..1574850 100644
--- a/roverlay/remote/basicrepo.py
+++ b/roverlay/remote/basicrepo.py
@@ -8,6 +8,11 @@ DEFAULT_PROTOCOL = 'http'
LOCALREPO_SRC_URI = 'http://localhost/R-Packages'
+SYNC_SUCCESS = 1
+SYNC_FAIL = 2
+REPO_READY = 4
+
+
def normalize_uri ( uri, protocol, force_protocol=False ):
if not protocol:
@@ -53,8 +58,29 @@ class LocalRepo ( object ):
else:
self.src_uri = src_uri
+ self.sync_status = 0
+
# --- end of __init__ (...) ---
+ def ready ( self ):
+ return bool ( self.sync_status & REPO_READY )
+
+ def fail ( self ):
+ return bool ( self.sync_status & SYNC_FAIL )
+
+ def offline ( self ):
+ return 0 == self.sync_status & SYNC_SUCCESS
+
+ def _set_ready ( self, is_synced ):
+ """comment TODO"""
+ if is_synced:
+ self.sync_status = SYNC_SUCCESS | REPO_READY
+ else:
+ self.sync_status = REPO_READY
+
+ def _set_fail ( self ):
+ self.sync_status = SYNC_FAIL
+
def __str__ ( self ):
return "repo '%s': DISTDIR '%s', SRC_URI '%s'" % (
self.name, self.distdir, self.src_uri
@@ -79,7 +105,7 @@ class LocalRepo ( object ):
if package_file is None:
return self.src_uri
else:
- return '/'.join ( self.src_uri, package_file )
+ return '/'.join ( ( self.src_uri, package_file ) )
# --- end of get_src_uri (...) ---
# get_src(...) -> get_src_uri(...)
@@ -90,17 +116,26 @@ class LocalRepo ( object ):
return os.path.isdir ( self.distdir )
# --- end of exists (...) ---
- def nosync ( self ):
- """Returns True if the repo is ready for overlay creation, else False.
- Useful for basic local distfiles verification without downloading
- anything.
- """
- return self.exists()
+ def sync ( self, sync_enabled=True ):
+ """Syncs this repo."""
- # --- end of nosync (...) ---
+ status = False
+ if sync_enabled and hasattr ( self, '_dosync' ):
+ status = self._dosync()
- # sync() -> nosync(), LocalRepos don't have anything to sync
- sync = nosync
+ elif hasattr ( self, '_nosync'):
+ status = self._nosync()
+
+ else:
+ status = self.exists()
+
+ if status:
+ self._set_ready ( is_synced=sync_enabled )
+ else:
+ self._set_fail()
+
+ return status
+ # --- end of sync (...) ---
def scan_distdir ( self, is_package=None ):
"""Generator that scans the local distfiles dir of this repo and
@@ -111,19 +146,26 @@ class LocalRepo ( object ):
or None which means that all files are packages.
Defaults to None.
"""
+
+ kw = { 'origin' : self }
+
if is_package is None:
# unfiltered variant
for dirpath, dirnames, filenames in os.walk ( self.distdir ):
+ kw ['distdir'] = dirpath if dirpath != self.distdir else None
+
for pkg in filenames:
- yield PackageInfo ( filename=pkg, origin=self )
+ yield PackageInfo ( filename=pkg, **kw )
elif hasattr ( is_package, '__call__' ):
# filtered variant (adds an if is_package... before yield)
for dirpath, dirnames, filenames in os.walk ( self.distdir ):
+ kw ['distdir'] = dirpath if dirpath != self.distdir else None
+
for pkg in filenames:
if is_package ( os.path.join ( dirpath, pkg ) ):
- yield PackageInfo ( filename=pkg, origin=self )
+ yield PackageInfo ( filename=pkg, **kw )
else:
@@ -227,14 +269,14 @@ class RemoteRepo ( LocalRepo ):
# get_remote(...) -> get_remote_uri(...)
get_remote = get_remote_uri
- def sync ( self ):
+ def _dosync ( self ):
"""Gets packages from remote(s) and returns True if the repo is ready
for overlay creation, else False.
Derived classes have to implement this method.
"""
raise Exception ( "RemoteRepo does not implement sync()." )
- # --- end of sync (...) ---
+ # --- end of _dosync (...) ---
def __str__ ( self ):
return "repo '%s': DISTDIR '%s', SRC_URI '%s', REMOTE_URI '%s'" % (
diff --git a/roverlay/remote/repo.py b/roverlay/remote/repo.py
index f54f448..febfaae 100644
--- a/roverlay/remote/repo.py
+++ b/roverlay/remote/repo.py
@@ -11,7 +11,7 @@ class RsyncRepo ( RemoteRepo ):
def __init__ (
self, name,
directory=None, src_uri=None, rsync_uri=None, base_uri=None,
- extra_rsync_opts=None
+ **rsync_kw
):
# super's init: name, remote protocol, directory_kw, **uri_kw
# using '' as remote protocol which leaves uris unchanged when
@@ -20,19 +20,23 @@ class RsyncRepo ( RemoteRepo ):
name, '', directory=directory,
src_uri=src_uri, remote_uri=rsync_uri, base_uri=base_uri
)
- self.extra_rsync_opts = extra_rsync_opts
+ self.rsync_extra = rsync_kw
+
+ self.sync_protocol = 'rsync'
# --- end of __init__ (...) ---
- def sync ( self ):
+ def _dosync ( self ):
retcode = None
try:
job = RsyncJob (
remote=self.remote_uri, distdir=self.distdir,
run_now=True,
- extra_opts=self.extra_rsync_opts
+ **self.rsync_extra
)
- if job.returncode == 0: return True
+ if job.returncode == 0:
+ self._set_ready ( is_synced=True )
+ return True
retcode = job.returncode
except Exception as e:
@@ -41,13 +45,13 @@ class RsyncRepo ( RemoteRepo ):
logging.exception ( e )
retcode = '<undef>'
-
logging.error (
'Repo %s cannot be used for ebuild creation due to errors '
'while running rsync (return code was %s).' % ( self.name, retcode )
)
+ self._set_fail()
return False
- # --- end of sync (...) ---
+ # --- end of _dosync (...) ---
def __str__ ( self ):
return "rsync repo '%s': DISTDIR '%s', SRC_URI '%s', RSYNC_URI '%s'" \
diff --git a/roverlay/remote/repolist.py b/roverlay/remote/repolist.py
index 4617057..ae740dd 100644
--- a/roverlay/remote/repolist.py
+++ b/roverlay/remote/repolist.py
@@ -1,13 +1,19 @@
+import logging
from roverlay import config
-
from roverlay.remote.repoloader import read_repofile
+LOGGER = logging.getLogger ( 'RepoList' )
+
class RepoList ( object ):
def __init__ ( self ):
self.repos = list()
+
self.sync_enabled = True
+
+ # if True: use all repos when looking for packages, even those that
+ # could not be synced
self.use_broken_repos = False
def sort ( self ):
@@ -25,28 +31,62 @@ class RepoList ( object ):
self.load_file ( f )
# --- end of load (...) ---
- def sync_all ( self, package_queue=None ):
- q = None
- if package_queue is None:
- q = list()
- add = q.append
- else:
- # TODO: _nowait? raises Exception when queue is full which is
- # good in non-threaded execution
- # -> timeout,..
- add = q.put
+ def _queue_packages_from_repo ( self, repo, add_method ):
+ if not repo.ready():
+ if self.use_broken_repos:
+ # warn and continue
+ pass
+ else:
+ # repo cannot be used
+ LOGGER.warning ( "!!" )
+ return False
+
+ for p in repo.scan_distdir():
+ LOGGER.debug ( "adding package %s from repo %s" % ( p, repo ) )
+ add_method ( p )
+ # --- end of _queue_packages_from_repo (...) ---
+
+ def add_packages ( self, add_method ):
+ for repo in self.repos:
+ self._queue_packages_from_repo ( repo, add_method )
+ # --- end of add_packages (...) ---
+
+ def _sync_all_repos_and_run (
+ self,
+ when_repo_success=None, when_repo_fail=None, when_repo_done=None,
+ when_all_done=None
+ ):
+ try_call = lambda f, *x, **z : None if f is None else f ( *x, **z )
+
+ LOGGER.debug ( "Syncing repos ..." )
+ for repo in self.repos:
+ if repo.sync ( sync_enabled=self.sync_enabled ):
+ # repo successfully synced
+ try_call ( when_repo_success, repo )
+ else:
+ # else log fail <>
+ try_call ( when_repo_fail, repo )
+ try_call ( when_repo_done, repo )
- # !! TODO resume here.
+ try_call ( when_all_done )
+ # --- end of _sync_all_repos_and_run (...) ---
+ def sync ( self ):
+ LOGGER.debug ( "Syncing repos ..." )
for repo in self.repos:
- if repo.sync() if self.sync_enabled else repo.nosync():
- # scan repo and create package infos
- for p in repo.scan_distdir(): add ( p )
- elif self.use_broken_repos:
- # warn and scan repo
- ## ..
- for p in repo.scan_distdir(): add ( p )
+ repo.sync ( sync_enabled=self.sync_enabled )
+ # --- end of sync_all (...) ---
+
+ def sync_and_add ( self, add_method ):
+ """Syncs all repos and adds packages immediately to the package queue."""
+ # TODO: _nowait? raises Exception when queue is full which is
+ # good in non-threaded execution
+ # -> timeout,..
+
+ qput = lambda r: self._queue_packages_from_repo ( r, add_method )
+
+ self._sync_all_repos_and_run ( when_repo_done=qput )
# --- end of sync_all (...) ---
diff --git a/roverlay/remote/repoloader.py b/roverlay/remote/repoloader.py
index eae35c5..94dd5b7 100644
--- a/roverlay/remote/repoloader.py
+++ b/roverlay/remote/repoloader.py
@@ -44,13 +44,18 @@ def read_repofile ( repo_file, lenient=False ):
src_uri = get ( 'src_uri' )
)
elif repo_type == 'rsync':
+ extra_opts = get ( 'extra_rsync_opts' )
+ if extra_opts:
+ extra_opts = extra_opts.split ( ' ' )
+
repo = RsyncRepo (
name = get ( 'name', name ),
directory = get ( 'directory' ),
src_uri = get ( 'src_uri' ),
rsync_uri = get ( 'rsync_uri' ),
base_uri = get ( 'base_uri' ),
- extra_rsync_opts = get ( 'extra_rsync_opts' )
+ extra_opts = extra_opts,
+ recursive = get ( 'recursive', False ) == 'yes',
)
else:
LOGGER.error ( "Unknown repo type %s for %s" % ( repo_type, name ) )
diff --git a/roverlay/remote/rsync.py b/roverlay/remote/rsync.py
index e46d1db..f99c8a7 100644
--- a/roverlay/remote/rsync.py
+++ b/roverlay/remote/rsync.py
@@ -1,4 +1,5 @@
import os
+import sys
import subprocess
from roverlay import config
@@ -13,16 +14,24 @@ RSYNC_ENV = keepenv (
'RSYNC_PASSWORD',
)
+# TODO:
+# either reraise an KeyboardInterrupt while running rsync (which stops script
+# execution unless the interrupt is catched elsewhere) or just set a
+# non-zero return code (-> 'repo cannot be used')
+RERAISE_INTERRUPT = False
+
# --recursive is not in the default opts, subdirs in CRAN/contrib are
-# either R release (2.xx.x[-patches] or the package archive)
+# either R releases (2.xx.x[-patches]) or the package archive
DEFAULT_RSYNC_OPTS = (
'--links', # copy symlinks as symlinks,
'--safe-links', # but ignore links outside of tree
'--times', #
'--compress', # FIXME: add lzo if necessary
- '--delete', #
+ '--dirs', #
+ '--prune-empty-dirs', #
'--force', # allow deletion of non-empty dirs
+ '--delete', #
'--human-readable', #
'--stats', #
'--chmod=ugo=r,u+w,Dugo+x', # 0755 for transferred dirs, 0644 for files
@@ -30,11 +39,25 @@ DEFAULT_RSYNC_OPTS = (
class RsyncJob ( object ):
def __init__ (
- self, remote=None, distdir=None, run_now=True, extra_opts=None
+ self, remote=None, distdir=None, run_now=True,
+ extra_opts=None, recursive=False
):
- self.remote = remote
- self.distdir = distdir
- self.extra_opts = None
+ self.distdir = distdir
+
+ # syncing directories, not files - always appending a slash at the end
+ # of remote
+ if remote [-1] != '/':
+ self.remote = remote + '/'
+ else:
+ self.remote = remote
+
+ if recursive:
+ self.extra_opts = [ '--recursive' ]
+ if extra_opts:
+ self.extra_opts.extend ( extra_opts )
+ else:
+ self.extra_opts = extra_opts
+
if run_now: self.run()
# --- end of __init__ (...) ---
@@ -51,36 +74,46 @@ class RsyncJob ( object ):
if max_bw is not None:
argv.append ( '--bwlimit=%i' % max_bw )
- if self.extra_opts is not None:
- if isinstance ( self.extra_opts, str ) or \
- not hasattr ( self.extra_opts, '__iter__' )\
- :
- argv.append ( self.extra_opts )
- else:
- argv.extend ( self.extra_opts )
+ if self.extra_opts:
+ argv.extend ( self.extra_opts )
argv.extend ( ( self.remote, self.distdir ) )
- return argv
+
+ # removing emty args from argv
+ return tuple ( filter ( None, argv ) )
# --- end of _rsync_argv (...) ---
def run ( self ):
rsync_cmd = self._rsync_argv()
+ print ( ' '.join ( rsync_cmd ) )
os.makedirs ( self.distdir, exist_ok=True )
# TODO pipe/log/.., running this in blocking mode until implemented
+ try:
+ proc = subprocess.Popen (
+ rsync_cmd,
+ stdin=None, stdout=None, stderr=None,
+ env=RSYNC_ENV
+ )
+
+ if proc.communicate() != ( None, None ):
+ raise AssertionError ( "expected None,None from communicate!" )
+
+ self.returncode = proc.returncode
+
+ except KeyboardInterrupt:
+ sys.stderr.write (
+ "\nKeyboard interrupt - waiting for rsync to exit...\n"
+ )
+ if 'proc' in locals():
+ proc.communicate()
+ self.returncode = proc.returncode
+ else:
+ self.returncode = 130
- proc = subprocess.Popen (
- rsync_cmd,
- stdin=None, stdout=None, stderr=None,
- env=RSYNC_ENV
- )
-
- if proc.communicate() != ( None, None ):
- raise AssertionError ( "expected None,None from communicate!" )
-
- self.returncode = proc.returncode
-
+ if RERAISE_INTERRUPT:
+ raise
# --- end of start (...) ---
^ permalink raw reply related [flat|nested] 35+ messages in thread* [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/
@ 2012-06-25 18:19 André Erdmann
0 siblings, 0 replies; 35+ messages in thread
From: André Erdmann @ 2012-06-25 18:19 UTC (permalink / raw
To: gentoo-commits
commit: 9a0b0c9e1740e88ee2426fefa2c97fb7e629bdb1
Author: André Erdmann <dywi <AT> mailerd <DOT> de>
AuthorDate: Mon Jun 25 18:15:38 2012 +0000
Commit: André Erdmann <dywi <AT> mailerd <DOT> de>
CommitDate: Mon Jun 25 18:15:38 2012 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=9a0b0c9e
the remote module (used for repo syncing)
* this modules handles repositories, both local (directory)
and remote (currently only rsync)
* some parts, mainly integration into roverlay, are todo
new file: roverlay/remote/__init__.py
new file: roverlay/remote/basicrepo.py
new file: roverlay/remote/repo.py
new file: roverlay/remote/repolist.py
new file: roverlay/remote/repoloader.py
new file: roverlay/remote/rsync.py
---
roverlay/remote/__init__.py | 1 +
roverlay/remote/basicrepo.py | 245 +++++++++++++++++++++++++++++++++++++++++
roverlay/remote/repo.py | 54 +++++++++
roverlay/remote/repolist.py | 55 +++++++++
roverlay/remote/repoloader.py | 66 +++++++++++
roverlay/remote/rsync.py | 86 ++++++++++++++
6 files changed, 507 insertions(+), 0 deletions(-)
diff --git a/roverlay/remote/__init__.py b/roverlay/remote/__init__.py
new file mode 100644
index 0000000..e7521be
--- /dev/null
+++ b/roverlay/remote/__init__.py
@@ -0,0 +1 @@
+from roverlay.remote.repolist import RepoList
diff --git a/roverlay/remote/basicrepo.py b/roverlay/remote/basicrepo.py
new file mode 100644
index 0000000..9ade3a2
--- /dev/null
+++ b/roverlay/remote/basicrepo.py
@@ -0,0 +1,245 @@
+import os.path
+
+from roverlay import config
+from roverlay.packageinfo import PackageInfo
+
+URI_SEPARATOR = '://'
+DEFAULT_PROTOCOL = 'http'
+
+LOCALREPO_SRC_URI = 'http://localhost/R-Packages'
+
+def normalize_uri ( uri, protocol, force_protocol=False ):
+
+ if not protocol:
+ return uri
+
+ proto, sep, base_uri = uri.partition ( URI_SEPARATOR )
+ if sep != URI_SEPARATOR:
+ return URI_SEPARATOR.join ( ( protocol, uri ) )
+ elif force_protocol:
+ return URI_SEPARATOR.join ( ( protocol, base_uri ) )
+ else:
+ return uri
+# --- end of normalize_uri (...) ---
+
+class LocalRepo ( object ):
+ """
+ This class represents a local repository - all packages are assumed
+ to exist in its distfiles dir and no remote syncing will occur.
+ It's the base class for remote repos.
+ """
+
+ def __init__ ( self, name, directory=None, src_uri=None ):
+ """Initializes a LocalRepo.
+
+ arguments:
+ * name --
+ * directory -- distfiles dir, defaults to <DISTFILES root>/<name>
+ * src_uri -- SRC_URI, defaults to http://localhost/R-Packages/<name>
+ """
+ self.name = name
+ if directory is None:
+ self.distdir = os.path.join (
+ config.get_or_fail ( [ 'DISTFILES', 'root' ] ),
+ # subdir repo names like CRAN/contrib are ok,
+ # but make sure to use the correct path separator
+ self.name.replace ( '/', os.path.sep ),
+ )
+ else:
+ self.distdir = directory
+
+ if src_uri is None:
+ self.src_uri = '/'.join ( ( LOCALREPO_SRC_URI, self.name ) )
+ else:
+ self.src_uri = src_uri
+
+ # --- end of __init__ (...) ---
+
+ def __str__ ( self ):
+ return "repo '%s': DISTDIR '%s', SRC_URI '%s'" % (
+ self.name, self.distdir, self.src_uri
+ )
+
+ def get_name ( self ):
+ """Returns the name of this repository."""
+ return self.name
+ # --- end of get_name (...) ---
+
+ def get_distdir ( self ):
+ """Returns the distfiles directory of this repository."""
+ return self.distdir
+ # --- end of get_distdir (...) ---
+
+ def get_src_uri ( self, package_file=None ):
+ """Returns the SRC_URI of this repository.
+
+ arguments:
+ * package_file -- if set and not None: returns a SRC_URI for this pkg
+ """
+ if package_file is None:
+ return self.src_uri
+ else:
+ return '/'.join ( self.src_uri, package_file )
+ # --- end of get_src_uri (...) ---
+
+ # get_src(...) -> get_src_uri(...)
+ get_src = get_src_uri
+
+ def exists ( self ):
+ """Returns True if this repo locally exists."""
+ return os.path.isdir ( self.distdir )
+ # --- end of exists (...) ---
+
+ def nosync ( self ):
+ """Returns True if the repo is ready for overlay creation, else False.
+ Useful for basic local distfiles verification without downloading
+ anything.
+ """
+ return self.exists()
+
+ # --- end of nosync (...) ---
+
+ # sync() -> nosync(), LocalRepos don't have anything to sync
+ sync = nosync
+
+ def scan_distdir ( self, is_package=None ):
+ """Generator that scans the local distfiles dir of this repo and
+ yields PackageInfo instances.
+
+ arguments:
+ * is_package -- function returning True if the given file is a package
+ or None which means that all files are packages.
+ Defaults to None.
+ """
+ if is_package is None:
+ # unfiltered variant
+
+ for dirpath, dirnames, filenames in os.walk ( self.distdir ):
+ for pkg in filenames:
+ yield PackageInfo ( filename=pkg, origin=self )
+
+ elif hasattr ( is_package, '__call__' ):
+ # filtered variant (adds an if is_package... before yield)
+ for dirpath, dirnames, filenames in os.walk ( self.distdir ):
+ for pkg in filenames:
+ if is_package ( os.path.join ( dirpath, pkg ) ):
+ yield PackageInfo ( filename=pkg, origin=self )
+
+
+ else:
+ # faulty variant, raises Exception
+ raise Exception ( "is_package should either be None or a function." )
+ #yield None
+
+ # --- end of scan_distdir (...) ---
+
+# --- end of LocalRepo ---
+
+
+class RemoteRepo ( LocalRepo ):
+ """A template for remote repositories."""
+
+ def __init__ (
+ self, name, sync_proto,
+ directory=None,
+ src_uri=None, remote_uri=None, base_uri=None
+ ):
+ """Initializes a RemoteRepo.
+ Mainly consists of URI calculation that derived classes may find useful.
+
+ arguments:
+ * name --
+ * sync_proto -- protocol used for syncing (e.g. 'rsync')
+ * directory --
+ * src_uri -- src uri, if set, else calculated using base/remote uri,
+ the leading <proto>:// can be left out in which case
+ http is assumed
+ * remote_uri -- uri used for syncing, if set, else calculated using
+ base/src uri, the leading <proto>:// can be left out
+ * base_uri -- used to calculate remote/src uri,
+ example: localhost/R-packages/something
+
+ keyword condition:
+ * | { x : x in union(src,remote,base) and x not None } | >= 1
+ ^= at least one out of src/remote/base uri is not None
+ """
+ super ( RemoteRepo, self ) . __init__ ( name, directory, src_uri='' )
+
+ self.sync_proto = sync_proto
+
+ # detemerine uris
+ if src_uri is None and remote_uri is None:
+ if base_uri is None:
+ # keyword condition not met
+ raise Exception ( "Bad initialization of RemoteRepo!" )
+
+ else:
+ # using base_uri for src,remote
+ self.src_uri = URI_SEPARATOR.join (
+ ( DEFAULT_PROTOCOL, base_uri )
+ )
+
+ self.remote_uri = URI_SEPARATOR.join (
+ ( sync_proto, base_uri )
+ )
+
+ elif src_uri is None:
+ # remote_uri is not None
+ self.remote_uri = normalize_uri ( remote_uri, self.sync_proto )
+
+ if base_uri is not None:
+ # using base_uri for src_uri
+ self.src_uri = URI_SEPARATOR.join (
+ ( DEFAULT_PROTOCOL, base_uri )
+ )
+ else:
+ # using remote_uri for src_uri
+ self.src_uri = normalize_uri (
+ self.remote_uri, DEFAULT_PROTOCOL, force_protocol=True
+ )
+
+ elif remote_uri is None:
+ # src_uri is not None
+ self.src_uri = normalize_uri ( src_uri, DEFAULT_PROTOCOL )
+
+ if base_uri is not None:
+ # using base_uri for remote_uri
+ self.remote_uri = URI_SEPARATOR.join (
+ ( self.sync_proto, base_uri )
+ )
+ else:
+ # using src_uri for remote_uri
+ self.remote_uri = normalize_uri (
+ self.src_uri, self.sync_proto, force_protocol=True
+ )
+ else:
+ # remote and src not None
+ self.remote_uri = normalize_uri ( remote_uri, self.sync_proto )
+ self.src_uri = normalize_uri ( src_uri, DEFAULT_PROTOCOL )
+
+ # --- end of __init__ (...) ---
+
+ def get_remote_uri ( self ):
+ """Returns the remote uri of this RemoteRepo which used for syncing."""
+ return self.remote_uri
+ # --- end of get_remote_uri (...) ---
+
+ # get_remote(...) -> get_remote_uri(...)
+ get_remote = get_remote_uri
+
+ def sync ( self ):
+ """Gets packages from remote(s) and returns True if the repo is ready
+ for overlay creation, else False.
+
+ Derived classes have to implement this method.
+ """
+ raise Exception ( "RemoteRepo does not implement sync()." )
+ # --- end of sync (...) ---
+
+ def __str__ ( self ):
+ return "repo '%s': DISTDIR '%s', SRC_URI '%s', REMOTE_URI '%s'" % (
+ self.name, self.distdir, self.src_uri, self.remote_uri
+ )
+
+# --- end of RemoteRepo ---
+
diff --git a/roverlay/remote/repo.py b/roverlay/remote/repo.py
new file mode 100644
index 0000000..f54f448
--- /dev/null
+++ b/roverlay/remote/repo.py
@@ -0,0 +1,54 @@
+
+import logging
+
+#from roverlay.remote.basicrepo import LocalRepo, RemoteRepo
+from roverlay.remote.basicrepo import RemoteRepo
+
+from roverlay.remote.rsync import RsyncJob
+
+class RsyncRepo ( RemoteRepo ):
+
+ def __init__ (
+ self, name,
+ directory=None, src_uri=None, rsync_uri=None, base_uri=None,
+ extra_rsync_opts=None
+ ):
+ # super's init: name, remote protocol, directory_kw, **uri_kw
+ # using '' as remote protocol which leaves uris unchanged when
+ # normalizing them for rsync usage
+ super ( RsyncRepo, self ) . __init__ (
+ name, '', directory=directory,
+ src_uri=src_uri, remote_uri=rsync_uri, base_uri=base_uri
+ )
+ self.extra_rsync_opts = extra_rsync_opts
+ # --- end of __init__ (...) ---
+
+
+ def sync ( self ):
+ retcode = None
+ try:
+ job = RsyncJob (
+ remote=self.remote_uri, distdir=self.distdir,
+ run_now=True,
+ extra_opts=self.extra_rsync_opts
+ )
+ if job.returncode == 0: return True
+
+ retcode = job.returncode
+ except Exception as e:
+ # catch exceptions, log them and return False
+ ## TODO: which exceptions to catch||pass?
+ logging.exception ( e )
+ retcode = '<undef>'
+
+
+ logging.error (
+ 'Repo %s cannot be used for ebuild creation due to errors '
+ 'while running rsync (return code was %s).' % ( self.name, retcode )
+ )
+ return False
+ # --- end of sync (...) ---
+
+ def __str__ ( self ):
+ return "rsync repo '%s': DISTDIR '%s', SRC_URI '%s', RSYNC_URI '%s'" \
+ % ( self.name, self.distdir, self.src_uri, self.remote_uri )
diff --git a/roverlay/remote/repolist.py b/roverlay/remote/repolist.py
new file mode 100644
index 0000000..4617057
--- /dev/null
+++ b/roverlay/remote/repolist.py
@@ -0,0 +1,55 @@
+
+from roverlay import config
+
+from roverlay.remote.repoloader import read_repofile
+
+class RepoList ( object ):
+
+ def __init__ ( self ):
+ self.repos = list()
+ self.sync_enabled = True
+ self.use_broken_repos = False
+
+ def sort ( self ):
+ raise Exception ( "method stub." )
+
+ def load_file ( self, _file ):
+ new_repos = read_repofile ( _file )
+ if new_repos:
+ self.repos.extend ( new_repos )
+ # --- end of load_file (...) ---
+
+ def load ( self ):
+ files = config.get_or_fail ( 'REPO.config_files' )
+ for f in files:
+ self.load_file ( f )
+ # --- end of load (...) ---
+
+ def sync_all ( self, package_queue=None ):
+ q = None
+ if package_queue is None:
+ q = list()
+ add = q.append
+ else:
+ # TODO: _nowait? raises Exception when queue is full which is
+ # good in non-threaded execution
+ # -> timeout,..
+ add = q.put
+
+
+ # !! TODO resume here.
+
+ for repo in self.repos:
+ if repo.sync() if self.sync_enabled else repo.nosync():
+ # scan repo and create package infos
+ for p in repo.scan_distdir(): add ( p )
+ elif self.use_broken_repos:
+ # warn and scan repo
+ ## ..
+ for p in repo.scan_distdir(): add ( p )
+
+ # --- end of sync_all (...) ---
+
+ def __str__ ( self ):
+ return '\n'.join ( ( str ( x ) for x in self.repos ) )
+
diff --git a/roverlay/remote/repoloader.py b/roverlay/remote/repoloader.py
new file mode 100644
index 0000000..eae35c5
--- /dev/null
+++ b/roverlay/remote/repoloader.py
@@ -0,0 +1,66 @@
+
+import logging
+
+try:
+ import configparser
+except ImportError as running_python2:
+ # configparser is named ConfigParser in python2
+ import ConfigParser as configparser
+
+
+from roverlay import config
+
+from roverlay.remote.basicrepo import LocalRepo
+from roverlay.remote.repo import RsyncRepo
+
+LOGGER = logging.getLogger ( 'repoloader' )
+
+def read_repofile ( repo_file, lenient=False ):
+
+ parser = configparser.SafeConfigParser ( allow_no_value=False )
+
+ if lenient:
+ parser.read ( repo_file )
+ else:
+ fh = None
+ try:
+ fh = open ( repo_file, 'r' )
+ parser.readfp ( fh )
+ finally:
+ if fh: fh.close()
+
+ repos = list()
+
+ for name in parser.sections():
+
+ get = lambda a, b=None : parser.get ( name, a, raw=True, fallback=b )
+
+ repo_type = get ( 'type', 'rsync' ).lower()
+
+ if repo_type == 'local':
+ repo = LocalRepo (
+ name = get ( 'name', name ),
+ directory = get ( 'directory' ),
+ src_uri = get ( 'src_uri' )
+ )
+ elif repo_type == 'rsync':
+ repo = RsyncRepo (
+ name = get ( 'name', name ),
+ directory = get ( 'directory' ),
+ src_uri = get ( 'src_uri' ),
+ rsync_uri = get ( 'rsync_uri' ),
+ base_uri = get ( 'base_uri' ),
+ extra_rsync_opts = get ( 'extra_rsync_opts' )
+ )
+ else:
+ LOGGER.error ( "Unknown repo type %s for %s" % ( repo_type, name ) )
+ continue
+
+ LOGGER.debug ( 'New entry, ' + str ( repo ) )
+
+ repos.append ( repo )
+ repo = None
+
+
+ return repos
+# --- end of read_repofile (...) ---
diff --git a/roverlay/remote/rsync.py b/roverlay/remote/rsync.py
new file mode 100644
index 0000000..e46d1db
--- /dev/null
+++ b/roverlay/remote/rsync.py
@@ -0,0 +1,86 @@
+import os
+import subprocess
+
+from roverlay import config
+from roverlay.util import keepenv
+
+
+RSYNC_ENV = keepenv (
+ 'PATH',
+ 'USER',
+ 'LOGNAME',
+ 'RSYNC_PROXY',
+ 'RSYNC_PASSWORD',
+)
+
+
+# --recursive is not in the default opts, subdirs in CRAN/contrib are
+# either R release (2.xx.x[-patches] or the package archive)
+DEFAULT_RSYNC_OPTS = (
+ '--links', # copy symlinks as symlinks,
+ '--safe-links', # but ignore links outside of tree
+ '--times', #
+ '--compress', # FIXME: add lzo if necessary
+ '--delete', #
+ '--force', # allow deletion of non-empty dirs
+ '--human-readable', #
+ '--stats', #
+ '--chmod=ugo=r,u+w,Dugo+x', # 0755 for transferred dirs, 0644 for files
+)
+
+class RsyncJob ( object ):
+ def __init__ (
+ self, remote=None, distdir=None, run_now=True, extra_opts=None
+ ):
+ self.remote = remote
+ self.distdir = distdir
+ self.extra_opts = None
+
+ if run_now: self.run()
+ # --- end of __init__ (...) ---
+
+ def _rsync_argv ( self ):
+ if self.remote is None or self.distdir is None:
+ raise Exception ( "None in (remote,distdir)." )
+
+ argv = [ 'rsync' ]
+
+ argv.extend ( DEFAULT_RSYNC_OPTS )
+
+ max_bw = config.get ( 'RSYNC_BWLIMIT', None )
+ if max_bw is not None:
+ argv.append ( '--bwlimit=%i' % max_bw )
+
+ if self.extra_opts is not None:
+ if isinstance ( self.extra_opts, str ) or \
+ not hasattr ( self.extra_opts, '__iter__' )\
+ :
+ argv.append ( self.extra_opts )
+ else:
+ argv.extend ( self.extra_opts )
+
+ argv.extend ( ( self.remote, self.distdir ) )
+
+ return argv
+ # --- end of _rsync_argv (...) ---
+
+ def run ( self ):
+
+ rsync_cmd = self._rsync_argv()
+
+ os.makedirs ( self.distdir, exist_ok=True )
+
+ # TODO pipe/log/.., running this in blocking mode until implemented
+
+ proc = subprocess.Popen (
+ rsync_cmd,
+ stdin=None, stdout=None, stderr=None,
+ env=RSYNC_ENV
+ )
+
+ if proc.communicate() != ( None, None ):
+ raise AssertionError ( "expected None,None from communicate!" )
+
+ self.returncode = proc.returncode
+
+ # --- end of start (...) ---
^ permalink raw reply related [flat|nested] 35+ messages in thread
end of thread, other threads:[~2015-01-26 17:41 UTC | newest]
Thread overview: 35+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2013-07-24 9:54 [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/ André Erdmann
-- strict thread matches above, loose matches on Subject: below --
2015-01-26 17:41 André Erdmann
2014-02-16 16:45 André Erdmann
2014-02-15 19:49 André Erdmann
2014-02-15 19:49 André Erdmann
2014-02-15 19:49 André Erdmann
2013-09-03 13:15 André Erdmann
2013-09-03 8:35 André Erdmann
2013-09-02 16:21 André Erdmann
2013-08-29 15:08 André Erdmann
2013-08-07 16:10 André Erdmann
2013-08-07 16:10 André Erdmann
2013-07-24 9:54 André Erdmann
2013-07-24 9:54 André Erdmann
2013-07-23 14:57 André Erdmann
2013-07-23 9:38 [gentoo-commits] proj/R_overlay:gsoc13/next " André Erdmann
2013-07-23 14:57 ` [gentoo-commits] proj/R_overlay:master " André Erdmann
2013-07-23 9:38 [gentoo-commits] proj/R_overlay:gsoc13/next " André Erdmann
2013-07-23 14:57 ` [gentoo-commits] proj/R_overlay:master " André Erdmann
2013-07-16 16:35 [gentoo-commits] proj/R_overlay:gsoc13/next " André Erdmann
2013-07-16 16:36 ` [gentoo-commits] proj/R_overlay:master " André Erdmann
2012-08-13 18:07 André Erdmann
2012-08-11 0:01 André Erdmann
2012-08-10 15:16 André Erdmann
2012-08-10 15:16 André Erdmann
2012-08-09 9:26 André Erdmann
2012-08-02 15:14 André Erdmann
2012-08-01 7:33 André Erdmann
2012-07-31 17:51 André Erdmann
2012-07-09 17:25 André Erdmann
2012-07-06 8:15 André Erdmann
2012-07-04 18:21 André Erdmann
2012-07-04 18:21 André Erdmann
2012-07-03 17:48 André Erdmann
2012-06-27 14:46 André Erdmann
2012-06-26 15:55 André Erdmann
2012-06-26 15:42 André Erdmann
2012-06-25 18:19 André Erdmann
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox