[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit:     737ccc31d83f73395a22f4a19bb13fbc1e94208a
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sat Sep 27 05:05:01 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Thu Dec  4 19:56:34 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=737ccc31

Sync: Implement native postsync.d hook code

As per bug 522032, pass the repo.name and repo.sync_uri to the hooks.
With this information, the hooks can be taylored to operate for only certain repos, or only when all repos have been synced.

---
 pym/portage/sync/controller.py | 36 +++++++++++++++++++++++++-----------
 1 file changed, 25 insertions(+), 11 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 9e483e9..c860e1a 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -87,6 +87,19 @@ class SyncManager(object):
 
 		self.module_controller = portage.sync.module_controller
 		self.module_names = self.module_controller.module_names
+		postsync_dir = os.path.join(self.settings["PORTAGE_CONFIGROOT"],
+			portage.USER_CONFIG_PATH, "postsync.d")
+		hooks = []
+		for root, dirs, names in os.walk(postsync_dir, topdown=True):
+			#print("root:", root, "dirs:", dirs, "names:", names)
+			for name in names:
+				filepath = os.path.join(root, name)
+				if os.access(filepath, os.X_OK):
+					hooks.append((filepath, name))
+				else:
+					writemsg_level(" %s postsync.d hook: '%s' is not executable\n"
+						% (warn("*"), name,), level=logging.WARN, noiselevel=2)
+		self.hooks = hooks
 
 
 	def get_module_descriptions(self, mod):
@@ -129,7 +142,7 @@ class SyncManager(object):
 		taskmaster = TaskHandler(callback=self.do_callback)
 		taskmaster.run_tasks(tasks, func, status, options=task_opts)
 
-		self.perform_post_sync_hook(repo.sync_uri)
+		self.perform_post_sync_hook(repo.name, repo.sync_uri)
 
 		return self.exitcode, None
 
@@ -143,17 +156,18 @@ class SyncManager(object):
 		return
 
 
-	def perform_post_sync_hook(self, dosyncuri):
-		postsync = os.path.join(self.settings["PORTAGE_CONFIGROOT"],
-			portage.USER_CONFIG_PATH, "bin", "post_sync")
-		if os.access(postsync, os.X_OK):
-			retval = portage.process.spawn([postsync, dosyncuri],
-				env=self.settings.environ())
+	def perform_post_sync_hook(self, reponame, dosyncuri=''):
+		succeeded = os.EX_OK
+		for filepath, hook in self.hooks:
+			writemsg_level("Spawning post_sync hook: %s\n" % (hook,),
+				level=logging.ERROR, noiselevel=4)
+			retval = portage.process.spawn([filepath,
+				reponame, dosyncuri], env=self.settings.environ())
 			if retval != os.EX_OK:
-				writemsg_level(" %s spawn failed of %s\n" % (bad("*"),
-					postsync,), level=logging.ERROR, noiselevel=-1)
-			return retval
-		return os.EX_OK
+				writemsg_level(" %s Spawn failed for: %s, %s\n" % (bad("*"),
+					hook, filepath), level=logging.ERROR, noiselevel=-1)
+				succeeded = retval
+		return succeeded
 
 
 	def pre_sync(self, repo):

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit:     7f28745b044f54862732e77185c0492667af42a5
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Sep  7 01:40:25 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Thu Dec  4 19:56:34 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=7f28745b

sync/controller.py: Use assert() on tasks, func

---
 pym/portage/sync/controller.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index e1aabb7..9e483e9 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -36,8 +36,9 @@ class TaskHandler(object):
 
 	def run_tasks(self, tasks, func, status=None, verbose=True, options=None):
 		"""Runs the module tasks"""
-		if tasks is None or func is None:
-			return
+		# Ensure we have a task and function
+		assert(tasks)
+		assert(func)
 		for task in tasks:
 			inst = task()
 			show_progress = self.show_progress_bar and self.isatty

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit:     174fbc3b24168c7b42270accb4e69707b946a484
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Fri Sep  5 20:14:57 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Thu Dec  4 19:56:34 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=174fbc3b

portage/sync/controller.py: Remove list copy, module_names list is not modified

---
 pym/portage/sync/controller.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 10ae0a8..e1aabb7 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -85,7 +85,7 @@ class SyncManager(object):
 		os.umask(0o22)
 
 		self.module_controller = portage.sync.module_controller
-		self.module_names = self.module_controller.module_names[:]
+		self.module_names = self.module_controller.module_names
 
 
 	def get_module_descriptions(self, mod):

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit:     213b1da7b74a7b392292fca077f9e187eb10a503
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Oct 20 20:44:41 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Thu Dec  4 19:56:34 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=213b1da7

_sync_callback: check for md5-cache, not cache

This fixes FEATURES=metadata-transfer to work again now that the
metadata/cache directory has been replaced with metadata/md5-cache.

---
 pym/portage/sync/controller.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index c860e1a..5c58865 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -227,7 +227,8 @@ class SyncManager(object):
 			updatecache_flg = False
 
 		if updatecache_flg and \
-			os.path.exists(os.path.join(self.repo.location, 'metadata', 'cache')):
+			os.path.exists(os.path.join(
+			self.repo.location, 'metadata', 'md5-cache')):
 
 			# Only update cache for repo.location since that's
 			# the only one that's been synced here.

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit:     54bcea5d7acf3e79653f4fe0f1e8e3ed552ae342
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue Jun 17 00:42:29 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Thu Dec  4 19:56:33 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=54bcea5d

sync/syncbase.py: Fix the self.has_bin logic

---
 pym/portage/sync/syncbase.py | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/pym/portage/sync/syncbase.py b/pym/portage/sync/syncbase.py
index 2b6b8c7..94d4aab 100644
--- a/pym/portage/sync/syncbase.py
+++ b/pym/portage/sync/syncbase.py
@@ -34,16 +34,18 @@ class SyncBase(object):
 		self.repo = None
 		self.xterm_titles = None
 		self.spawn_kwargs = None
-		self.bin_command = portage.process.find_binary(bin_command)
-
-		self.has_bin = True
-		if bin_command and self.bin_command is None:
-			msg = ["Command not found: %s" % bin_command,
-			"Type \"emerge %s\" to enable %s support." % (bin_pkg, bin_command)]
-			for l in msg:
-				writemsg_level("!!! %s\n" % l,
-					level=self.logger.ERROR, noiselevel=-1)
-			self.has_bin = False
+		self.bin_command = None
+		self.has_bin = False
+		if bin_command:
+			self.bin_command = portage.process.find_binary(bin_command)
+			if self.bin_command is None:
+				msg = ["Command not found: %s" % bin_command,
+				"Type \"emerge %s\" to enable %s support." % (bin_pkg, bin_command)]
+				for l in msg:
+					writemsg_level("!!! %s\n" % l,
+						level=self.logger.ERROR, noiselevel=-1)
+			else:
+				self.has_bin = True
 
 
 	def _kwargs(self, kwargs):

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit:     f51290f5c93eff9d4167ee01792a50172989c375
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue Jun 17 18:44:43 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Thu Dec  4 19:56:33 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=f51290f5

portage/sync: Add additional output info for unsupported sync-types

Comment out some debug print()'s

---
 pym/portage/sync/__init__.py      | 5 +++--
 pym/portage/sync/config_checks.py | 4 ++++
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/pym/portage/sync/__init__.py b/pym/portage/sync/__init__.py
index 6d2a732..b74c89e 100644
--- a/pym/portage/sync/__init__.py
+++ b/pym/portage/sync/__init__.py
@@ -11,12 +11,12 @@ sync_manager = None
 
 path = os.path.join(os.path.dirname(__file__), "modules")
 # initial development debug info
-print("module path:", path)
+#print("module path:", path)
 
 module_controller = Modules(path=path, namepath="portage.sync.modules")
 
 # initial development debug info
-print(module_controller.module_names)
+#print(module_controller.module_names)
 module_names = module_controller.module_names[:]
 
 
@@ -43,6 +43,7 @@ def get_syncer(settings=None, logger=None):
 
 def validate_config(repo, logger):
 	'''Validate the repos.conf settings for the repo'''
+	global module_names, module_controller
 	if not check_type(repo, logger, module_names):
 		return False
 

diff --git a/pym/portage/sync/config_checks.py b/pym/portage/sync/config_checks.py
index 8ef1974..db316aa 100644
--- a/pym/portage/sync/config_checks.py
+++ b/pym/portage/sync/config_checks.py
@@ -26,6 +26,10 @@ def check_type(repo, logger, module_names):
 			_("Repository '%s' has sync-type attribute set to unsupported value: '%s'")
 			% (repo.name, repo.sync_type),
 			level=logger.ERROR, noiselevel=-1)
+		writemsg_level("!!! %s\n" %
+			_("Installed sync-types are: '%s'")
+			% (str(module_names)),
+			level=logger.ERROR, noiselevel=-1)
 		return False
 	return True
 

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit:     5e37e1aebb371b0ee36a70e0e166af8398cff926
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Thu Dec  4 19:49:22 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Thu Dec  4 19:58:35 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=5e37e1ae

portage/sync/controller.py: Wrap variables with _unicode_decode for writemsg_level() calls

Fixes "BytesWarning: str() on a bytes instance" errors in py3.

---
 pym/portage/sync/controller.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index b7c668c..21aa7a7 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -20,6 +20,7 @@ bad = create_color_func("BAD")
 warn = create_color_func("WARN")
 from portage.package.ebuild.doebuild import _check_temp_dir
 from portage.metadata import action_metadata
+from portage import _unicode_decode
 
 
 class TaskHandler(object):
@@ -98,7 +99,7 @@ class SyncManager(object):
 					hooks.append((filepath, name))
 				else:
 					writemsg_level(" %s postsync.d hook: '%s' is not executable\n"
-						% (warn("*"), name,), level=logging.WARN, noiselevel=2)
+						% (warn("*"), _unicode_decode(name),), level=logging.WARN, noiselevel=2)
 		self.hooks = hooks
 
 
@@ -159,13 +160,15 @@ class SyncManager(object):
 	def perform_post_sync_hook(self, reponame, dosyncuri='', repolocation=''):
 		succeeded = os.EX_OK
 		for filepath, hook in self.hooks:
-			writemsg_level("Spawning post_sync hook: %s\n" % (hook,),
+			writemsg_level("Spawning post_sync hook: %s\n"
+				% (_unicode_decode(hook)),
 				level=logging.ERROR, noiselevel=4)
 			retval = portage.process.spawn([filepath,
 				reponame, dosyncuri, repolocation], env=self.settings.environ())
 			if retval != os.EX_OK:
 				writemsg_level(" %s Spawn failed for: %s, %s\n" % (bad("*"),
-					hook, filepath), level=logging.ERROR, noiselevel=-1)
+					_unicode_decode(hook), filepath),
+					level=logging.ERROR, noiselevel=-1)
 				succeeded = retval
 		return succeeded
 
@@ -182,7 +185,8 @@ class SyncManager(object):
 		except OSError:
 			st = None
 		if st is None:
-			writemsg_level(">>> '%s' not found, creating it." % repo.location)
+			writemsg_level(">>> '%s' not found, creating it."
+				% _unicode_decode(repo.location))
 			portage.util.ensure_dirs(repo.location, mode=0o755)
 			st = os.stat(repo.location)
 

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit:     fc65d8afbbfdcc52073d07818288c693ad04596a
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Thu Dec  4 17:03:20 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Thu Dec  4 19:56:36 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=fc65d8af

portage/sync/controller.py: Add repo location to parameters passed to post_sync hooks

---
 pym/portage/sync/controller.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 5c58865..b7c668c 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -142,7 +142,7 @@ class SyncManager(object):
 		taskmaster = TaskHandler(callback=self.do_callback)
 		taskmaster.run_tasks(tasks, func, status, options=task_opts)
 
-		self.perform_post_sync_hook(repo.name, repo.sync_uri)
+		self.perform_post_sync_hook(repo.name, repo.sync_uri, repo.location)
 
 		return self.exitcode, None
 
@@ -156,13 +156,13 @@ class SyncManager(object):
 		return
 
 
-	def perform_post_sync_hook(self, reponame, dosyncuri=''):
+	def perform_post_sync_hook(self, reponame, dosyncuri='', repolocation=''):
 		succeeded = os.EX_OK
 		for filepath, hook in self.hooks:
 			writemsg_level("Spawning post_sync hook: %s\n" % (hook,),
 				level=logging.ERROR, noiselevel=4)
 			retval = portage.process.spawn([filepath,
-				reponame, dosyncuri], env=self.settings.environ())
+				reponame, dosyncuri, repolocation], env=self.settings.environ())
 			if retval != os.EX_OK:
 				writemsg_level(" %s Spawn failed for: %s, %s\n" % (bad("*"),
 					hook, filepath), level=logging.ERROR, noiselevel=-1)

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit:     a3991f7727be0eb973aad1b120d248a0739be453
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sat Dec  6 22:54:36 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Sun Dec  7 05:22:51 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=a3991f77

portage/sync/controller.py: Make a repo.postsync.d directory

This then runs per-repo postsync hooks only on scripts in the repo.postsync.d directory.
This also maintains compatibility with existing scripts in the postsync.d dir or other
sub-directories.
Both postsync.d directories support subdirectories.
Scripts are run in sorted order.

---
 pym/portage/sync/controller.py | 36 ++++++++++++++++++++++--------------
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 21aa7a7..41f3830 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -20,7 +20,9 @@ bad = create_color_func("BAD")
 warn = create_color_func("WARN")
 from portage.package.ebuild.doebuild import _check_temp_dir
 from portage.metadata import action_metadata
+from portage import OrderedDict
 from portage import _unicode_decode
+from portage import util
 
 
 class TaskHandler(object):
@@ -88,19 +90,21 @@ class SyncManager(object):
 
 		self.module_controller = portage.sync.module_controller
 		self.module_names = self.module_controller.module_names
-		postsync_dir = os.path.join(self.settings["PORTAGE_CONFIGROOT"],
-			portage.USER_CONFIG_PATH, "postsync.d")
-		hooks = []
-		for root, dirs, names in os.walk(postsync_dir, topdown=True):
-			#print("root:", root, "dirs:", dirs, "names:", names)
-			for name in names:
-				filepath = os.path.join(root, name)
+		self.hooks = {}
+		for _dir in ["repo.postsync.d", "postsync.d"]:
+			postsync_dir = os.path.join(self.settings["PORTAGE_CONFIGROOT"],
+				portage.USER_CONFIG_PATH, _dir)
+			hooks = OrderedDict()
+			for filepath in util._recursive_file_list(postsync_dir):
+				name = filepath.split(postsync_dir)[1].lstrip(os.sep)
 				if os.access(filepath, os.X_OK):
-					hooks.append((filepath, name))
+					hooks[filepath] = name
 				else:
-					writemsg_level(" %s postsync.d hook: '%s' is not executable\n"
-						% (warn("*"), _unicode_decode(name),), level=logging.WARN, noiselevel=2)
-		self.hooks = hooks
+					writemsg_level(" %s %s hook: '%s' is not executable\n"
+						% (warn("*"), _dir, _unicode_decode(name),),
+						level=logging.WARN, noiselevel=2)
+			self.hooks[_dir] = hooks
+		print(self.hooks)
 
 
 	def get_module_descriptions(self, mod):
@@ -159,15 +163,19 @@ class SyncManager(object):
 
 	def perform_post_sync_hook(self, reponame, dosyncuri='', repolocation=''):
 		succeeded = os.EX_OK
-		for filepath, hook in self.hooks:
+		if reponame:
+			_hooks = self.hooks["repo.postsync.d"]
+		else:
+			_hooks = self.hooks["postsync.d"]
+		for filepath in _hooks:
 			writemsg_level("Spawning post_sync hook: %s\n"
-				% (_unicode_decode(hook)),
+				% (_unicode_decode(_hooks[filepath])),
 				level=logging.ERROR, noiselevel=4)
 			retval = portage.process.spawn([filepath,
 				reponame, dosyncuri, repolocation], env=self.settings.environ())
 			if retval != os.EX_OK:
 				writemsg_level(" %s Spawn failed for: %s, %s\n" % (bad("*"),
-					_unicode_decode(hook), filepath),
+					_unicode_decode(_hooks[filepath]), filepath),
 					level=logging.ERROR, noiselevel=-1)
 				succeeded = retval
 		return succeeded

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit:     8a5ee0a68764a272c554ed534fcb252e81f89118
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Dec  7 09:12:13 2014 +0000
Commit:     Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Sun Dec  7 09:12:49 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=8a5ee0a6

portage/sync/controller.py: Remove a debug print() missed

Fixes commit: a3991f7727be0eb973aad1b120d248a0739be453
Author: Brian Dolbec <dolsen <AT> gentoo.org> (Sat 06 Dec 2014 02:54:36 PM PST)
Subject: portage/sync/controller.py: Make a repo.postsync.d directory

---
 pym/portage/sync/controller.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 41f3830..6b9fb2c 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -104,7 +104,6 @@ class SyncManager(object):
 						% (warn("*"), _dir, _unicode_decode(name),),
 						level=logging.WARN, noiselevel=2)
 			self.hooks[_dir] = hooks
-		print(self.hooks)
 
 
 	def get_module_descriptions(self, mod):

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Michał Górny]

commit:     9a2b1e6096e35508e9f101fa0478101493ef0335
Author:     Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Fri Dec  5 22:54:27 2014 +0000
Commit:     Michał Górny <mgorny <AT> gentoo <DOT> org>
CommitDate: Sun Dec  7 22:57:07 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=9a2b1e60

sync: ensure sync_{umask,user} is respected when creating repo

---
 pym/portage/sync/controller.py | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index ab6eb21..9001298 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -192,11 +192,6 @@ class SyncManager(object):
 			st = os.stat(repo.location)
 		except OSError:
 			st = None
-		if st is None:
-			writemsg_level(">>> '%s' not found, creating it."
-				% _unicode_decode(repo.location))
-			portage.util.ensure_dirs(repo.location, mode=0o755)
-			st = os.stat(repo.location)
 
 		self.usersync_uid = None
 		spawn_kwargs = {}
@@ -249,7 +244,24 @@ class SyncManager(object):
 				spawn_kwargs["groups"] = [gid]
 			if home is not None:
 				spawn_kwargs["env"]["HOME"] = home
-		elif ('usersync' in self.settings.features and
+
+		if st is None:
+			perms = {'mode': 0o755}
+			# respect sync-user if set
+			if 'umask' in spawn_kwargs:
+				perms['mode'] &= ~spawn_kwargs['umask']
+			if 'uid' in spawn_kwargs:
+				perms['uid'] = spawn_kwargs['uid']
+			if 'gid' in spawn_kwargs:
+				perms['gid'] = spawn_kwargs['gid']
+
+			writemsg_level(">>> '%s' not found, creating it."
+				% _unicode_decode(repo.location))
+			portage.util.ensure_dirs(repo.location, **perms)
+			st = os.stat(repo.location)
+
+		if (repo.sync_user is None and
+			'usersync' in self.settings.features and
 			portage.data.secpass >= 2 and
 			(st.st_uid != os.getuid() and st.st_mode & 0o700 or
 			st.st_gid != os.getgid() and st.st_mode & 0o070)):

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Michał Górny]

commit:     768b9e920fe0919d87537665ae3ce88007630cc1
Author:     Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Sat Jan 17 13:03:53 2015 +0000
Commit:     Michał Górny <mgorny <AT> gentoo <DOT> org>
CommitDate: Sun Jan 18 18:04:25 2015 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=768b9e92

sync: fix module_names enumeration

Fix module_names enumeration to consider all modules. Before, the first
module on the list was omitted ('cvs' in this case).

---
 pym/portage/sync/controller.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 9001298..3d217db 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -119,7 +119,7 @@ class SyncManager(object):
 		self.callback = callback or self._sync_callback
 		self.repo = repo
 		self.exitcode = 1
-		if repo.sync_type in self.module_names[1:]:
+		if repo.sync_type in self.module_names:
 			tasks = [self.module_controller.get_class(repo.sync_type)]
 		else:
 			msg = "\n%s: Sync module '%s' is not an installed/known type'\n" \

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit:     e4f9c7251a34e780315271e6fb97dfddcb86d85a
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon Feb  9 19:43:47 2015 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Mon Feb  9 20:11:44 2015 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=e4f9c725

portage/sync/controller.py: Fix commit 7909ed55ba, failure to import writemsg

Author: Michał Górny <mgorny <AT> gentoo.org> (Fri 05 Dec 2014 02:40:13 PM PST)
Subjetc: sync: allow overriding sync-user for the repository
SHA: 7909ed55ba45b8a94f4a29148e43953eb58aa184

---
 pym/portage/sync/controller.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index d2c606d..1583767 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -14,7 +14,7 @@ from portage import os
 from portage.progress import ProgressBar
 #from portage.emaint.defaults import DEFAULT_OPTIONS
 #from portage.util._argparse import ArgumentParser
-from portage.util import writemsg_level
+from portage.util import writemsg, writemsg_level
 from portage.output import create_color_func
 good = create_color_func("GOOD")
 bad = create_color_func("BAD")

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit:     a1091d4b0c538cfabc1f67e7758d7c9bab71a040
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon Feb  9 19:51:01 2015 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Mon Feb  9 20:11:44 2015 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=a1091d4b

portage/sync/controller.py: Fix missed self.portdb assignment bug 539478

In _sync_callback() action_metadata() call, portdb was needed as one of it's parameters.
But it was not defined when it was moved into task_opts dictionary to be passed to the sync module.

---
 pym/portage/sync/controller.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 1583767..6dec3c8 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -135,7 +135,7 @@ class SyncManager(object):
 		task_opts = {
 			'emerge_config': emerge_config,
 			'logger': self.logger,
-			'portdb': self.trees[self.settings['EROOT']]['porttree'].dbapi,
+			'portdb': self.portdb,
 			'repo': repo,
 			'settings': self.settings,
 			'spawn_kwargs': self.spawn_kwargs,
@@ -192,6 +192,7 @@ class SyncManager(object):
 			% (repo.name, repo.location)
 		self.logger(self.xterm_titles, msg)
 		writemsg_level(msg + "\n")
+		self.portdb = self.trees[self.settings['EROOT']]['porttree'].dbapi
 		try:
 			st = os.stat(repo.location)
 		except OSError:

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit:     7a16988f4765963896170b92253ebbd4344216ea
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Wed Apr 22 16:58:38 2015 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Wed Apr 22 16:58:38 2015 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=7a16988f

portage/sync/controller.py: Fix postsync hook regression (bug 547414)

It seems the migration from the portage-utils postsync hook script to the native portage one
causes problems.  This is due to the fact the portage-utils supplied script did not pass on any 
arguments, even though emerge did pass on the uri to it.

X-Gentoo-Bug: 547414
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=547414

 pym/portage/sync/controller.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 6dec3c8..307487f 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -175,8 +175,13 @@ class SyncManager(object):
 			writemsg_level("Spawning post_sync hook: %s\n"
 				% (_unicode_decode(_hooks[filepath])),
 				level=logging.ERROR, noiselevel=4)
-			retval = portage.process.spawn([filepath,
-				reponame, dosyncuri, repolocation], env=self.settings.environ())
+			if reponame:
+				retval = portage.process.spawn(
+					[filepath, reponame, dosyncuri, repolocation],
+					env=self.settings.environ())
+			else:
+				retval = portage.process.spawn([filepath],
+					env=self.settings.environ())
 			if retval != os.EX_OK:
 				writemsg_level(" %s Spawn failed for: %s, %s\n" % (bad("*"),
 					_unicode_decode(_hooks[filepath]), filepath),

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Brian Dolbec]

commit:     c0e1d514c1c8159aa732d23a527640432a9c076d
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Tue Jul 14 20:40:23 2015 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Tue Jul 14 21:28:32 2015 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=c0e1d514

portage/sync.__init__.py: Trap KeyError in module_specific_options()

Return an empty frozenset if the sync module is older and does not contain a
module_specific_option definition in it's module_spec.
Makes the module_specific_options addition backwards compatible with existing sync modules.

 pym/portage/sync/__init__.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/pym/portage/sync/__init__.py b/pym/portage/sync/__init__.py
index b0e0ebe..32b2c45 100644
--- a/pym/portage/sync/__init__.py
+++ b/pym/portage/sync/__init__.py
@@ -31,9 +31,11 @@ def module_specific_options(repo):
 	global module_controller
 
 	if repo.sync_type:
-		opts = frozenset([opt for opt in
-			module_controller.modules[repo.sync_type]['module_specific_options']])
-		return opts
+		try:
+			return frozenset(
+				module_controller.modules[repo.sync_type]['module_specific_options'])
+		except KeyError:
+			pass
 	return frozenset()
 
 

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit:     4a3f6ce8e5c64b7447bb32851ee91e19faf18be3
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Aug 29 19:56:50 2015 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Aug 30 23:50:58 2015 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=4a3f6ce8

sync: include metadata/layout.conf with profile submodule (bug 559122)

Since metadata/layout.conf settings such as profile-formats affect the
interpretation of profiles, make the profiles submodule include this
file (which is all that's needed for a minimal binhost client).

X-Gentoo-Bug: 559122
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=559122
Acked-by: Brian Dolbec <dolsen <AT> gentoo.org>

 pym/portage/sync/__init__.py | 6 +++---
 pym/portage/sync/syncbase.py | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/__init__.py b/pym/portage/sync/__init__.py
index 32b2c45..805b1f2 100644
--- a/pym/portage/sync/__init__.py
+++ b/pym/portage/sync/__init__.py
@@ -9,9 +9,9 @@ from portage.sync.controller import SyncManager
 from portage.sync.config_checks import check_type
 
 _SUBMODULE_PATH_MAP = OrderedDict([
-	('glsa', 'metadata/glsa'),
-	('news', 'metadata/news'),
-	('profiles', 'profiles'),
+	('glsa', ('metadata/glsa',)),
+	('news', ('metadata/news',)),
+	('profiles', ('metadata/layout.conf', 'profiles')),
 ])
 
 path = os.path.join(os.path.dirname(__file__), "modules")

diff --git a/pym/portage/sync/syncbase.py b/pym/portage/sync/syncbase.py
index d30d69d..6aaa9c4 100644
--- a/pym/portage/sync/syncbase.py
+++ b/pym/portage/sync/syncbase.py
@@ -99,7 +99,7 @@ class SyncBase(object):
 		emerge_config = self.options.get('emerge_config')
 		if emerge_config is not None:
 			for name in emerge_config.opts.get('--sync-submodule', []):
-				paths.append(_SUBMODULE_PATH_MAP[name])
+				paths.extend(_SUBMODULE_PATH_MAP[name])
 		return tuple(paths)
 
 

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit:     200d876b01dd11521cdcd9bfa07abdca165d24e8
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Sep 23 16:16:45 2015 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Sep 23 22:31:36 2015 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=200d876b

SyncManager.async: initialize attributes before fork (bug 561234)

Since commit 496ff326dc18890889d1ea5d2aec590394635960, the pre_sync
method executes in a forked child process. Therefore, initialize
attributes before the fork (in the async method).

Fixes: 496ff326dc18 ("sync repositories in parallel (bug 557426)")
X-Gentoo-Bug: 561234
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=561234
Acked-by: Brian Dolbec <dolsen <AT> gentoo.org>

 pym/portage/sync/controller.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index e992cc4..28dbc57 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -115,13 +115,16 @@ class SyncManager(object):
 		return []
 
 	def async(self, emerge_config=None, repo=None):
+		self.emerge_config = emerge_config
+		self.settings, self.trees, self.mtimedb = emerge_config
+		self.xterm_titles = "notitles" not in self.settings.features
+		self.portdb = self.trees[self.settings['EROOT']]['porttree'].dbapi
 		proc = AsyncFunction(target=self.sync,
 			kwargs=dict(emerge_config=emerge_config, repo=repo))
 		proc.addExitListener(self._sync_callback)
 		return proc
 
 	def sync(self, emerge_config=None, repo=None):
-		self.emerge_config = emerge_config
 		self.callback = None
 		self.repo = repo
 		self.exitcode = 1
@@ -199,13 +202,10 @@ class SyncManager(object):
 
 
 	def pre_sync(self, repo):
-		self.settings, self.trees, self.mtimedb = self.emerge_config
-		self.xterm_titles = "notitles" not in self.settings.features
 		msg = ">>> Syncing repository '%s' into '%s'..." \
 			% (repo.name, repo.location)
 		self.logger(self.xterm_titles, msg)
 		writemsg_level(msg + "\n")
-		self.portdb = self.trees[self.settings['EROOT']]['porttree'].dbapi
 		try:
 			st = os.stat(repo.location)
 		except OSError:

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit:     4205ec912aebf2e3b0bd673fcacb576b1f344329
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Oct  5 23:01:49 2015 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Oct  6 16:25:37 2015 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=4205ec91

SyncRepos.async: group sync and callback as composite task (bug 562264)

Group together the sync operation and the callback as a single task,
so that the callback will not execute concurrently with another sync
operation for --jobs=1.

X-Gentoo-Bug: 562264
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=562264
Acked-by: Brian Dolbec <dolsen <AT> gentoo.org>

 pym/portage/sync/controller.py | 39 +++++++++++++++++++++++++++++++++++----
 1 file changed, 35 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 28dbc57..159b9c0 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -25,6 +25,7 @@ from portage.util._async.AsyncFunction import AsyncFunction
 from portage import OrderedDict
 from portage import _unicode_decode
 from portage import util
+from _emerge.CompositeTask import CompositeTask
 
 
 class TaskHandler(object):
@@ -119,10 +120,9 @@ class SyncManager(object):
 		self.settings, self.trees, self.mtimedb = emerge_config
 		self.xterm_titles = "notitles" not in self.settings.features
 		self.portdb = self.trees[self.settings['EROOT']]['porttree'].dbapi
-		proc = AsyncFunction(target=self.sync,
-			kwargs=dict(emerge_config=emerge_config, repo=repo))
-		proc.addExitListener(self._sync_callback)
-		return proc
+		return SyncRepo(sync_task=AsyncFunction(target=self.sync,
+			kwargs=dict(emerge_config=emerge_config, repo=repo)),
+			sync_callback=self._sync_callback)
 
 	def sync(self, emerge_config=None, repo=None):
 		self.callback = None
@@ -343,3 +343,34 @@ class SyncManager(object):
 			action_metadata(self.settings, self.portdb, self.emerge_config.opts,
 				porttrees=[repo.location])
 
+
+class SyncRepo(CompositeTask):
+	"""
+	Encapsulates a sync operation and the callback which executes afterwards,
+	so both can be considered as a single composite task. This is useful
+	since we don't want to consider a particular repo's sync operation as
+	complete until after the callback has executed (bug 562264).
+
+	The kwargs and result properties expose attributes that are accessed
+	by SyncScheduler.
+	"""
+
+	__slots__ = ('sync_task', 'sync_callback')
+
+	@property
+	def kwargs(self):
+		return self.sync_task.kwargs
+
+	@property
+	def result(self):
+		return self.sync_task.result
+
+	def _start(self):
+		self._start_task(self.sync_task, self._sync_task_exit)
+
+	def _sync_task_exit(self, sync_task):
+		self._current_task = None
+		self.returncode = sync_task.returncode
+		self.sync_callback(self.sync_task)
+		self._async_wait()
+

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit:     baeff1e7a7306081123d9a31b24c62d59ae73abb
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun Nov 22 20:05:37 2015 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Nov 24 16:15:04 2015 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=baeff1e7

SyncManager: redirect command stderr to stdout (bug 566132)

X-Gentoo-Bug: 566132
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=566132
Acked-by: Alexander Berntsen <bernalex <AT> gentoo.org>

 pym/portage/sync/controller.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index 4595293..e71ba67 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -218,6 +218,13 @@ class SyncManager(object):
 
 		self.usersync_uid = None
 		spawn_kwargs = {}
+		# Redirect command stderr to stdout, in order to prevent
+		# spurious cron job emails (bug 566132).
+		spawn_kwargs["fd_pipes"] = {
+			0: sys.__stdin__.fileno(),
+			1: sys.__stdout__.fileno(),
+			2: sys.__stdout__.fileno()
+		}
 		spawn_kwargs["env"] = self.settings.environ()
 		if repo.sync_user is not None:
 			def get_sync_user_data(sync_user):

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit:     32b372bf79633bbfe6c7b1f5bca2f290a32695d4
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Dec 12 22:24:04 2015 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Dec 12 23:40:41 2015 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=32b372bf

SyncManager.sync: always return 4-tuple (bug 567932)

Since commit 496ff326dc18890889d1ea5d2aec590394635960, invalid repos.conf
settings or failure of the pre_sync method caused SyncManager.sync to
return an incorrect number of values.

Fixes: 496ff326dc18 ("sync repositories in parallel (bug 557426)")
X-Gentoo-Bug: 567932
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=567932
Acked-by: Brian Dolbec <dolsen <AT> gentoo.org>

 pym/portage/sync/controller.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index e71ba67..92be3cb 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -129,16 +129,17 @@ class SyncManager(object):
 		self.repo = repo
 		self.exitcode = 1
 		self.updatecache_flg = False
+		hooks_enabled = master_hooks or not repo.sync_hooks_only_on_change
 		if repo.sync_type in self.module_names:
 			tasks = [self.module_controller.get_class(repo.sync_type)]
 		else:
 			msg = "\n%s: Sync module '%s' is not an installed/known type'\n" \
 				% (bad("ERROR"), repo.sync_type)
-			return self.exitcode, msg
+			return self.exitcode, msg, self.updatecache_flg, hooks_enabled
 
 		rval = self.pre_sync(repo)
 		if rval != os.EX_OK:
-			return rval, None
+			return rval, None, self.updatecache_flg, hooks_enabled
 
 		# need to pass the kwargs dict to the modules
 		# so they are available if needed.
@@ -157,7 +158,6 @@ class SyncManager(object):
 		taskmaster = TaskHandler(callback=self.do_callback)
 		taskmaster.run_tasks(tasks, func, status, options=task_opts)
 
-		hooks_enabled = False
 		if (master_hooks or self.updatecache_flg or
 			not repo.sync_hooks_only_on_change):
 			hooks_enabled = True

[gentoo-commits] proj/portage:master commit in: pym/portage/sync/

[Zac Medico]

commit:     0d1d2b8dccdd5ce9f99358fe842fb968a0423a25
Author:     Nicolas Porcel <nicolasporcel06 <AT> gmail <DOT> com>
AuthorDate: Sun Sep 17 23:29:27 2017 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Sep 17 23:59:11 2017 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=0d1d2b8d

Fix emerge --info when using webrsync (bug 630538)

When calling retrieve_head from a SyncBase object, it is expected to
raise a NotImplementedError. However, all classes that do not inherit
from NewBase will raise an AttributeError which is not caught.

Fixes: 0e1699ad6b3f ("emerge: Add head commit per repo to --info")

 pym/portage/sync/syncbase.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pym/portage/sync/syncbase.py b/pym/portage/sync/syncbase.py
index 05e4d69d4..43b667fb0 100644
--- a/pym/portage/sync/syncbase.py
+++ b/pym/portage/sync/syncbase.py
@@ -102,6 +102,10 @@ class SyncBase(object):
 				paths.extend(_SUBMODULE_PATH_MAP[name])
 		return tuple(paths)
 
+	def retrieve_head(self, **kwargs):
+		'''Get information about the head commit'''
+		raise NotImplementedError
+
 
 class NewBase(SyncBase):
 	'''Subclasses Syncbase adding a new() and runs it
@@ -133,7 +137,3 @@ class NewBase(SyncBase):
 		'''Update existing repository
 		'''
 		raise NotImplementedError
-
-	def retrieve_head(self, **kwargs):
-		'''Get information about the head commit'''
-		raise NotImplementedError