From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from pigeon.gentoo.org ([208.92.234.80] helo=lists.gentoo.org) by finch.gentoo.org with esmtp (Exim 4.77) (envelope-from ) id 1Sn3hn-0007n9-Om for garchives@archives.gentoo.org; Fri, 06 Jul 2012 08:16:00 +0000 Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 5E3E6E05FE; Fri, 6 Jul 2012 08:15:50 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) by pigeon.gentoo.org (Postfix) with ESMTP id 1B94AE05FE for ; Fri, 6 Jul 2012 08:15:49 +0000 (UTC) Received: from hornbill.gentoo.org (hornbill.gentoo.org [94.100.119.163]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 78B351B40BF for ; Fri, 6 Jul 2012 08:15:48 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by hornbill.gentoo.org (Postfix) with ESMTP id 3A762E543D for ; Fri, 6 Jul 2012 08:15:47 +0000 (UTC) From: "André Erdmann" To: gentoo-commits@lists.gentoo.org Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "André Erdmann" Message-ID: <1341562454.cf8a0d0dd57f04d3b9da37f091a2f401c16e5b3c.dywi@gentoo> Subject: [gentoo-commits] proj/R_overlay:master commit in: roverlay/remote/ X-VCS-Repository: proj/R_overlay X-VCS-Files: roverlay/remote/basicrepo.py roverlay/remote/rsync.py X-VCS-Directories: roverlay/remote/ X-VCS-Committer: dywi X-VCS-Committer-Name: André Erdmann X-VCS-Revision: cf8a0d0dd57f04d3b9da37f091a2f401c16e5b3c X-VCS-Branch: master Date: Fri, 6 Jul 2012 08:15:47 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: quoted-printable X-Archives-Salt: a6bc0133-329a-4a31-b0dd-336af429d7ce X-Archives-Hash: f057404cb20cf460c49f0c270e3d0335 commit: cf8a0d0dd57f04d3b9da37f091a2f401c16e5b3c Author: Andr=C3=A9 Erdmann mailerd de> AuthorDate: Fri Jul 6 08:14:14 2012 +0000 Commit: Andr=C3=A9 Erdmann mailerd de> CommitDate: Fri Jul 6 08:14:14 2012 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=3Dproj/R_overlay.git= ;a=3Dcommit;h=3Dcf8a0d0d apply distroot changes to RemoteRepo/RsyncRepo * also added the possibility to retry rsync transfers on certain error co= des modified: roverlay/remote/basicrepo.py modified: roverlay/remote/rsync.py --- roverlay/remote/basicrepo.py | 6 +++- roverlay/remote/rsync.py | 56 ++++++++++++++++++++++++++++++++----= ----- 2 files changed, 47 insertions(+), 15 deletions(-) diff --git a/roverlay/remote/basicrepo.py b/roverlay/remote/basicrepo.py index 0248413..3dd09de 100644 --- a/roverlay/remote/basicrepo.py +++ b/roverlay/remote/basicrepo.py @@ -251,7 +251,7 @@ class RemoteRepo ( LocalRepo ): """A template for remote repositories.""" =20 def __init__ ( - self, name, sync_proto, + self, name, distroot, sync_proto, directory=3DNone, src_uri=3DNone, remote_uri=3DNone, base_uri=3DNone ): @@ -274,7 +274,9 @@ class RemoteRepo ( LocalRepo ): * | { x : x in union(src,remote,base) and x not None } | >=3D 1 ^=3D at least one out of src/remote/base uri is not None """ - super ( RemoteRepo, self ) . __init__ ( name, directory, src_uri=3D'' = ) + super ( RemoteRepo, self ) . __init__ ( + name, distroot, directory, src_uri=3D'' + ) =20 self.sync_proto =3D sync_proto =20 diff --git a/roverlay/remote/rsync.py b/roverlay/remote/rsync.py index 4744118..9fcc348 100644 --- a/roverlay/remote/rsync.py +++ b/roverlay/remote/rsync.py @@ -15,6 +15,15 @@ RSYNC_ENV =3D util.keepenv ( 'RSYNC_PASSWORD', ) =20 +MAX_RSYNC_RETRY =3D 3 + +RSYNC_SIGINT =3D 20 + +RETRY_ON_RETCODE =3D frozenset (( + 23, # "Partial transfer due to error" + 24, # "Partial transfer due to vanished source files" +)) + # TODO: # either reraise an KeyboardInterrupt while running rsync (which stops s= cript # execution unless the interrupt is catched elsewhere) or just set a @@ -40,7 +49,7 @@ DEFAULT_RSYNC_OPTS =3D ( class RsyncRepo ( RemoteRepo ): =20 def __init__ ( - self, name, + self, name, distroot, directory=3DNone, src_uri=3DNone, rsync_uri=3DNone, base_uri=3DNone, recursive=3DFalse, extra_opts=3DNone ): @@ -59,7 +68,7 @@ class RsyncRepo ( RemoteRepo ): # using '' as remote protocol which leaves uris unchanged when # normalizing them for rsync usage super ( RsyncRepo, self ) . __init__ ( - name, '', directory=3Ddirectory, + name, distroot=3Ddistroot, sync_proto=3D'', directory=3Ddirectory, src_uri=3Dsrc_uri, remote_uri=3Drsync_uri, base_uri=3Dbase_uri ) =20 @@ -103,6 +112,15 @@ class RsyncRepo ( RemoteRepo ): All exceptions(?) are catched and interpreted as sync failure. """ =20 + def waitfor ( p ): + if p.communicate() !=3D ( None, None ): + raise AssertionError ( "expected None,None from communicate!" ) + if p.returncode =3D=3D RSYNC_SIGINT: + raise KeyboardInterrupt ( "propagated from rsync" ) + + return p.returncode + # --- end of waitfor (...) --- + retcode =3D '' =20 try: @@ -113,31 +131,43 @@ class RsyncRepo ( RemoteRepo ): =20 self.logger.debug ( 'running rsync cmd: ' + ' '.join ( rsync_cmd ) ) =20 + retry_count =3D 0 =20 - proc =3D subprocess.Popen ( - rsync_cmd, - stdin=3DNone, stdout=3DNone, stderr=3DNone, - env=3DRSYNC_ENV - ) + proc =3D subprocess.Popen ( rsync_cmd, env=3DRSYNC_ENV ) + retcode =3D waitfor ( proc ) + del proc =20 - if proc.communicate() !=3D ( None, None ): - raise AssertionError ( "expected None,None from communicate!" ) + while retcode in RETRY_ON_RETCODE and retry_count < MAX_RSYNC_RETRY: + # this handles retcodes like + # * 24: "Partial transfer due to vanished source files" + + # FIXME replace loop condition "retcode !=3D 0" + retry_count +=3D 1 + + self.logger.warning ( + "rsync returned {!r}, retrying ((}/{})".format ( + retcode, retry_count, MAX_RSYNC_RETRY + ) + ) + + proc =3D subprocess.Popen ( rsync_cmd, env=3DRSYNC_ENV ) + retcode =3D waitfor ( proc ) + del proc =20 - if proc.returncode =3D=3D 0: + if retcode =3D=3D 0: self._set_ready ( is_synced=3DTrue ) return True =20 - retcode =3D proc.returncode =20 except KeyboardInterrupt: sys.stderr.write ( "\nKeyboard interrupt - waiting for rsync to exit...\n" ) - if 'proc' in locals(): + if 'proc' in locals() and proc is not None: proc.communicate() retcode =3D proc.returncode else: - retcode =3D 130 + retcode =3D RSYNC_SIGINT =20 if RERAISE_INTERRUPT: raise