From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id B57C1138EF4 for ; Sat, 22 Feb 2014 14:56:03 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 43629E0B75; Sat, 22 Feb 2014 14:56:03 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 82A95E0B75 for ; Sat, 22 Feb 2014 14:56:02 +0000 (UTC) Received: from spoonbill.gentoo.org (spoonbill.gentoo.org [81.93.255.5]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 73EF033F7DC for ; Sat, 22 Feb 2014 14:56:01 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by spoonbill.gentoo.org (Postfix) with ESMTP id 40F9618873 for ; Sat, 22 Feb 2014 14:56:00 +0000 (UTC) From: "André Erdmann" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "André Erdmann" Message-ID: <1392569055.ca1020e87ff4421499df80c1d1af9e6c33f9afe2.dywi@gentoo> Subject: [gentoo-commits] proj/R_overlay:master commit in: roverlay/util/, roverlay/remote/ X-VCS-Repository: proj/R_overlay X-VCS-Files: roverlay/remote/websync.py roverlay/util/progressbar.py X-VCS-Directories: roverlay/util/ roverlay/remote/ X-VCS-Committer: dywi X-VCS-Committer-Name: André Erdmann X-VCS-Revision: ca1020e87ff4421499df80c1d1af9e6c33f9afe2 X-VCS-Branch: master Date: Sat, 22 Feb 2014 14:56:00 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Archives-Salt: 1c4ff6a3-c072-4b46-abd8-da6f9eb72226 X-Archives-Hash: ca59795f6fef21ec949d1e886a2fc5ae commit: ca1020e87ff4421499df80c1d1af9e6c33f9afe2 Author: André Erdmann mailerd de> AuthorDate: Sun Feb 16 16:44:15 2014 +0000 Commit: André Erdmann mailerd de> CommitDate: Sun Feb 16 16:44:15 2014 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=proj/R_overlay.git;a=commit;h=ca1020e8 roverlay/remote/websync: show download status --- roverlay/remote/websync.py | 39 ++++++-- roverlay/util/progressbar.py | 210 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 242 insertions(+), 7 deletions(-) diff --git a/roverlay/remote/websync.py b/roverlay/remote/websync.py index 24f5cd5..06689ad 100644 --- a/roverlay/remote/websync.py +++ b/roverlay/remote/websync.py @@ -4,6 +4,7 @@ # Distributed under the terms of the GNU General Public License; # either version 2 of the License, or (at your option) any later version. +from __future__ import division from __future__ import print_function """websync, sync packages via http""" @@ -31,6 +32,7 @@ HTTPError = _urllib_error.HTTPError from roverlay import config, digest, util from roverlay.remote.basicrepo import BasicRepo +from roverlay.util.progressbar import DownloadProgressBar, NullProgressBar # number of sync retries # changed 2014-02-15: does no longer include the first run @@ -50,6 +52,15 @@ class WebsyncBase ( BasicRepo ): HTTP_ERROR_RETRY_CODES = frozenset ({ 404, 410, 500, 503 }) URL_ERROR_RETRY_CODES = frozenset ({ errno.ETIMEDOUT, }) RETRY_ON_TIMEOUT = True + PROGRESS_BAR_CLS = None + + def __new__ ( cls, *args, **kwargs ): + if cls.PROGRESS_BAR_CLS is None: + cls.PROGRESS_BAR_CLS = ( + DownloadProgressBar if VERBOSE else NullProgressBar + ) + return super ( WebsyncBase, cls ).__new__ ( cls ) + # --- end of __new__ (...) --- def __init__ ( self, name, @@ -170,27 +181,41 @@ class WebsyncBase ( BasicRepo ): bytes_fetched = 0 assert blocksize - # FIXME: debug print (?) - if VERBOSE: - print ( - "Fetching {f} from {u} ...".format ( f=package_file, u=src_uri ) - ) - # unlink the existing file first (if it exists) # this is necessary for keeping hardlinks intact (-> package mirror) util.try_unlink ( distfile ) - with open ( distfile, mode='wb' ) as fh: + with \ + open ( distfile, mode='wb' ) as fh, \ + self.PROGRESS_BAR_CLS ( + package_file.ljust(50), expected_filesize + ) as progress_bar: + + progress_bar.update ( 0 ) block = webh.read ( blocksize ) + while block: # write block to file fh.write ( block ) # ? bytelen bytes_fetched += len ( block ) + # update progress bar on every 4th block + # blocks_fetched := math.ceil ( bytes_fetched / blocksize ) + # + # Usually, only the last block's size is <= blocksize, + # so floordiv is sufficient here + # (the progress bar gets updated for the last block anyway) + # + if 0 == ( bytes_fetched // blocksize ) % 4: + progress_bar.update ( bytes_fetched ) + # get the next block block = webh.read ( blocksize ) # -- end while + + # final progress bar update (before closing the file) + progress_bar.update ( bytes_fetched ) # -- with if bytes_fetched == expected_filesize: diff --git a/roverlay/util/progressbar.py b/roverlay/util/progressbar.py new file mode 100644 index 0000000..24c934e --- /dev/null +++ b/roverlay/util/progressbar.py @@ -0,0 +1,210 @@ +# R overlay -- util, progressbar +# -*- coding: utf-8 -*- +# Copyright (C) 2014 André Erdmann +# Distributed under the terms of the GNU General Public License; +# either version 2 of the License, or (at your option) any later version. +from __future__ import division + +import abc +import sys + +import roverlay.util.objects + + +class AbstractProgressBarBase ( roverlay.util.objects.AbstractObject ): + """Abstract base class for progress bars.""" + + @abc.abstractmethod + def setup ( self, *args, **kwargs ): + """Initialization code for __init__() and reset(). + + Returns: None + + arguments: + * *args, **kwargs -- progress bar data + """ + pass + # --- end of setup (...) --- + + def reset ( self, *args, **kwargs ): + """Finalizes the current progress bar and resets it afterwards. + + Returns: None + + arguments: + * *args, **kwargs -- passed to setup() + """ + self.print_newline() + self.setup ( *args, **kwargs ) + # --- end of reset (...) --- + + def __init__ ( self, *args, **kwargs ): + """Initializes a progress bar instance by calling its setup() method. + + arguments: + * *args, **kwargs -- passed to setup() + """ + super ( AbstractProgressBarBase, self ).__init__() + self.setup ( *args, **kwargs ) + # --- end of __init__ (...) --- + + @abc.abstractmethod + def write ( self, message ): + """(Over-)writes the progress bar, using the given message. + + Note: message should not contain newline chars. + + Returns: None + + arguments: + * message -- + """ + raise NotImplementedError() + # --- end of write (...) --- + + @abc.abstractmethod + def print_newline ( self ): + """ + Finalizes the current progress bar, usually by printing a newline char. + + Returns: None + """ + raise NotImplementedError() + # --- end of print_newline (...) --- + + @abc.abstractmethod + def update ( self, *args, **kwargs ): + """Updates the progress bar using the given data. + + Returns: None + + arguments: + * *args, **kwargs -- not specified by this class + """ + raise NotImplementedError() + # --- end of update (...) --- + + def __enter__ ( self ): + # "with"-statement, setup code + return self + + def __exit__ ( self, _type, value, traceback ): + # "with"-statement, teardown code + self.print_newline() + +# --- end of AbstractProgressBarBase --- + + +class AbstractProgressBar ( AbstractProgressBarBase ): + """ + Abstract base class for progress bars that write to a stream, e.g. stdout. + """ + + CARRIAGE_RET_CHR = chr(13) + #BACKSPACE_CHR = chr(8) + + def setup ( self, stream=None ): + self.stream = ( sys.stdout if stream is None else stream ) + # --- end of __init__ (...) --- + + def write ( self, message ): + self.stream.write ( self.CARRIAGE_RET_CHR + message ) + self.stream.flush() + # --- end of write (...) --- + + def print_newline ( self ): + self.stream.write ( "\n" ) + self.stream.flush() + # --- end of print_newline (...) --- + +# --- end of AbstractProgressBar --- + + +class AbstractPercentageProgressBar ( AbstractProgressBar ): + """Base class for displaying progress as percentage 0.00%..100.00%.""" + # not a real progress bar, just a progress indicator + + # str for formatting the percentage + # by default, reserve space for 7 chars ("ddd.dd%") + # might be set by derived classes and/or instances + PERCENTAGE_FMT = "{:>7.2%}" + + def setup ( self, message_header=None, stream=None ): + super ( AbstractPercentageProgressBar, self ).setup ( stream=stream ) + self.message_header = message_header + # --- end of setup (...) --- + + @abc.abstractmethod + def get_percentage ( self, *args, **kwargs ): + """Returns a float or int expressing a percentage. + + Any value < 0 is interpreted as "UNKNOWN". + + arguments: + * *args, **kwargs -- progress information (from update()) + """ + raise NotImplementedError() + # --- end of get_percentage (...) --- + + def _update ( self, percentage ): + if self.message_header: + message = str(self.message_header) + " " + else: + message = "" + + if percentage < 0: + message += "UNKNOWN" + else: + message += self.PERCENTAGE_FMT.format ( percentage ) + + self.write ( message ) + # --- end of _update (...) --- + + def update ( self, *args, **kwargs ): + self._update ( self.get_percentage ( *args, **kwargs ) ) + # --- end of update (...) --- + +# --- end of AbstractPercentageProgressBar --- + + +class NullProgressBar ( AbstractProgressBarBase ): + """A progress bar that discards any information.""" + + def setup ( self, *args, **kwargs ): + pass + + def write ( self, *args, **kwargs ): + pass + + def print_newline ( self, *args, **kwargs ): + pass + + def update ( self, *args, **kwargs ): + pass + +# --- end of NullProgressBar --- + + +class DownloadProgressBar ( AbstractPercentageProgressBar ): + """A progress bar for file transfers, + expressing a percentage "bytes transferred / total size". + + Note: + update() shouldn't be called too often as writing to console is rather slow + """ + + def setup ( self, filename=None, filesize=None, stream=None ): + super ( DownloadProgressBar, self ).setup ( + message_header = ( + ( "Fetching " + str(filename) ) if filename else None + ), + stream = stream + ) + self.filesize = filesize + # --- end of setup (...) --- + + def get_percentage ( self, current_filesize ): + return ( current_filesize / self.filesize ) if self.filesize else -1.0 + # --- end of get_percentage (...) --- + +# --- end of DownloadProgressBar ---