[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     966f66269f41f46f29895f7c9b6240f7babd5094
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Dec 20 22:13:17 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Dec 20 22:13:17 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=966f6626

fix a NoResultFound in get_job_status_waiting_on_guest

 pym/tbc/sqlquerys.py | 5 +++--
 pym/tbc/sync.py      | 4 +++-
 pym/tbc/updatedb.py  | 4 +++-
 3 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index 0276131..0ec5fc9 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -45,8 +45,9 @@ def get_jobs(session, config_id):
 	return JobsInfo
 
 def get_job_status_waiting_on_guest(session):
-	JobsInfo = session.query(Jobs).filter_by(Status = 'Waiting_on_guest').one()
-	if JobsInfo == []:
+	try:
+		JobsInfo = session.query(Jobs).filter_by(Status = 'Waiting_on_guest').one()
+	except NoResultFound as e:
 		return None
 	return JobInfo.JobId
 

diff --git a/pym/tbc/sync.py b/pym/tbc/sync.py
index dee13ff..a93b84f 100644
--- a/pym/tbc/sync.py
+++ b/pym/tbc/sync.py
@@ -66,7 +66,9 @@ def git_sync_main(session):
 			time.sleep(60)
 
 	job_id = get_job_status_waiting_on_guest(session)
-	update_job_list(session, 'Runing', job_id)
+	if not job_id is None:
+		update_job_list(session, 'Runing', job_id)
+
 	# check git diffs witch get updated and pass that to a dict
 	# fetch and merge the repo
 	repo_cp_dict = {}

diff --git a/pym/tbc/updatedb.py b/pym/tbc/updatedb.py
index b6539e4..7fce867 100644
--- a/pym/tbc/updatedb.py
+++ b/pym/tbc/updatedb.py
@@ -94,7 +94,9 @@ def update_cpv_db(session, repo_cp_dict, config_id, tbc_settings):
 		time.sleep(30)
 
 	job_id = get_job_status_waiting_on_guest(session)
-	update_job_list(session, 'Runing', job_id)
+	if not job_id is None:
+		update_job_list(session, 'Runing', job_id)
+
 	log_msg = "Checking categories, package, ebuilds"
 	write_log(session, log_msg, "info", config_id, 'updatedb.update_cpv_db')
 	new_build_jobs_list = []

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     236dc594e15792d1b53501b7595edd571993e03b
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Feb  5 02:52:40 2018 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Feb  5 02:52:40 2018 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=236dc594

use perform_checksum for checksum on files

 pym/tbc/build_job.py   | 3 ++-
 pym/tbc/build_log.py   | 4 ++--
 pym/tbc/check_setup.py | 5 +++--
 pym/tbc/package.py     | 3 ++-
 pym/tbc/updatedb.py    | 3 ++-
 5 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/pym/tbc/build_job.py b/pym/tbc/build_job.py
index d0d72c3..8d10a68 100644
--- a/pym/tbc/build_job.py
+++ b/pym/tbc/build_job.py
@@ -12,6 +12,7 @@ from portage import _encodings
 from portage import _unicode_decode
 from portage.versions import cpv_getkey
 from portage.dep import check_required_use
+from portage.checksum import perform_checksum
 from tbc.depclean import do_depclean
 from tbc.flags import tbc_use_flags
 from tbc.qachecks import check_file_in_manifest
@@ -34,7 +35,7 @@ class build_job_action(object):
 		pkgdir = portdb.getRepositoryPath(repo) + "/" + cp
 		build_use_flags_list = []
 		try:
-			ebuild_version_checksum_tree = portage.checksum.sha256hash(pkgdir + "/" + package + "-" + build_dict['ebuild_version'] + ".ebuild")[0]
+			ebuild_version_checksum_tree = perform_checksum(pkgdir + "/" + package + "-" + build_dict['ebuild_version'] + ".ebuild", "SHA256")[0]
 		except:
 			ebuild_version_checksum_tree = None
 		if ebuild_version_checksum_tree == build_dict['checksum']:

diff --git a/pym/tbc/build_log.py b/pym/tbc/build_log.py
index 762ebd4..e9e5dd0 100644
--- a/pym/tbc/build_log.py
+++ b/pym/tbc/build_log.py
@@ -14,7 +14,7 @@ from portage.util import writemsg, \
 	writemsg_level, writemsg_stdout
 from portage import _encodings
 from portage import _unicode_encode
-
+from portage.checksum import perform_checksum
 from _emerge.main import parse_opts
 
 portage.proxy.lazyimport.lazyimport(globals(),
@@ -106,7 +106,7 @@ def get_build_dict_db(session, config_id, settings, tbc_settings_dict, pkg):
 	else:
 		build_dict['build_useflags'] = use_flagsDict
 	pkgdir = myportdb.getRepositoryPath(repo) + "/" + categories + "/" + package
-	ebuild_version_checksum_tree = portage.checksum.sha256hash(pkgdir+ "/" + package + "-" + ebuild_version + ".ebuild")[0]
+	ebuild_version_checksum_tree = perform_checksum(pkgdir+ "/" + package + "-" + ebuild_version + ".ebuild", "SHA256")[0]
 	build_dict['checksum'] = ebuild_version_checksum_tree
 	ebuild_id_list, status = get_ebuild_id_db(session, build_dict['checksum'], build_dict['package_id'], build_dict['ebuild_version'])
 	if status:

diff --git a/pym/tbc/check_setup.py b/pym/tbc/check_setup.py
index a5f53e7..dcd7a0c 100644
--- a/pym/tbc/check_setup.py
+++ b/pym/tbc/check_setup.py
@@ -7,6 +7,7 @@ import os
 import errno
 
 from portage.exception import DigestException, FileNotFound, ParseError, PermissionDenied
+from portage.checksum import perform_checksum
 from tbc.text import get_file_text
 from tbc.sqlquerys import get_config_all_info, add_logs, get_configmetadata_info, get_setup_info
 from tbc.sync import git_pull
@@ -29,7 +30,7 @@ def check_make_conf(session, config_id):
 		# Check if we have some error in the file. (portage.util.getconfig)
 		# Check if we envorment error with the config. (settings.validate)
 		try:
-			make_conf_checksum_tree = portage.checksum.sha256hash(make_conf_file)[0]
+			make_conf_checksum_tree = perform_checksum(make_conf_file, "SHA256")[0]
 			portage.util.getconfig(make_conf_file, tolerant=0, allow_sourcing=True, expand=True)
 			mysettings = portage.config(config_root = check_config_dir)
 			mysettings.validate()
@@ -61,7 +62,7 @@ def check_configure_guest(session, config_id):
 	# Check if we have some error in the file (portage.util.getconfig)
 	# Check if we envorment error with the config (settings.validate)
 	try:
-		make_conf_checksum_tree = portage.checksum.sha256hash(make_conf_file)[0]
+		make_conf_checksum_tree = perform_checksum(make_conf_file, "SHA256")[0]
 		portage.util.getconfig(make_conf_file, tolerant=0, allow_sourcing=True, expand=True)
 		mysettings = portage.config(config_root = "/")
 		mysettings.validate()

diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index 721f9b1..84e3907 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -9,6 +9,7 @@ import git
 import portage
 import datetime
 from portage.xml.metadata import MetaDataXML
+from portage.checksum import perform_checksum
 from tbc.flags import tbc_use_flags
 from tbc.text import get_ebuild_cvs_revision, get_log_text_dict
 from tbc.flags import tbc_use_flags
@@ -111,7 +112,7 @@ class tbc_package(object):
 		ebuild_file = pkgdir + "/" + package + "-" + ebuild_version_tree + ".ebuild"
 		# Make a checksum of the ebuild
 		try:
-			ebuild_version_checksum_tree = portage.checksum.sha256hash(ebuild_file)[0]
+			ebuild_version_checksum_tree = perform_checksum(ebuild_file, "SHA256")[0]
 		except:
 			ebuild_version_checksum_tree = "0"
 			log_msg = "QA: Can't checksum the ebuild file. %s on repo %s" % (cpv, repo,)

diff --git a/pym/tbc/updatedb.py b/pym/tbc/updatedb.py
index 437cfd5..d32f0ac 100644
--- a/pym/tbc/updatedb.py
+++ b/pym/tbc/updatedb.py
@@ -9,6 +9,7 @@ import time
 import re
 import portage
 from portage.xml.metadata import MetaDataXML
+from portage.checksum import perform_checksum
 from sqlalchemy.orm import scoped_session, sessionmaker
 from tbc.ConnectionManager import NewConnection
 from tbc.sqlquerys import get_package_info, update_repo_db, \
@@ -46,7 +47,7 @@ def get_categories_metadataDict(pkgdir):
 		pkg_md = MetaDataXML(pkgdir + "/metadata.xml", None)
 		metadata_xml_descriptions_tree = re.sub('\t', '', pkg_md.descriptions()[0])
 		categories_metadataDict['metadata_xml_descriptions'] = re.sub('\n', '', metadata_xml_descriptions_tree)
-		categories_metadataDict['metadata_xml_checksum'] =  portage.checksum.sha256hash(pkgdir + "/metadata.xml")[0]
+		categories_metadataDict['metadata_xml_checksum'] =  perform_checksum(pkgdir + "/metadata.xml", "SHA256")[0]
 		return categories_metadataDict
 
 def update_cpv_db_pool(mysettings, myportdb, cp, repo, tbc_settings, config_id):

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     700a263237acc4beb17442ff66b4e70dd0947bce
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Jan 15 19:57:10 2017 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Jan 15 19:57:10 2017 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=700a2632

update web irker adress

 pym/tbc/build_log.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/tbc/build_log.py b/pym/tbc/build_log.py
index 2a00b95..762ebd4 100644
--- a/pym/tbc/build_log.py
+++ b/pym/tbc/build_log.py
@@ -328,7 +328,7 @@ def add_buildlog_main(settings, pkg, trees):
 				else:
 					build_msg = "BUILD: FAILD"
 		tbc_config =  get_tbc_config(session)
-		msg = "Package: %s Repo: %s %s %s %s Weblink http://%s/new/logs/build/%s\n" % (pkg.cpv, pkg.repo, build_msg, repoman_msg, qa_msg, tbc_config.WebIrker, log_id,)
+		msg = "Package: %s Repo: %s %s %s %s Weblink http://%s/%s\n" % (pkg.cpv, pkg.repo, build_msg, repoman_msg, qa_msg, tbc_config.WebIrker, log_id,)
 		write_log(session, msg, "info", config_id, 'build_log.add_buildlog_main')
 		send_irk(msg, tbc_config.HostIrker)
 	session.close

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     7d547c4e12884b4f2914eb9e151d3f043978ba42
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Jan  8 15:31:57 2017 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Jan  8 15:31:57 2017 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=7d547c4e

add slot and homepage

 pym/tbc/db_mapping.py | 3 +++
 pym/tbc/package.py    | 2 +-
 pym/tbc/sqlquerys.py  | 6 ++++--
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/pym/tbc/db_mapping.py b/pym/tbc/db_mapping.py
index 96ae75b..f48ef56 100644
--- a/pym/tbc/db_mapping.py
+++ b/pym/tbc/db_mapping.py
@@ -293,6 +293,9 @@ class EbuildsMetadata(Base):
 	New = Column('new', Boolean, default=False)
 	Updated = Column('updated', Boolean, default=False)
 	Descriptions = Column('descriptions', Text)
+	Slot = Column('slot', String(10))
+	Homepage = Column('homepage', String(200))
+	License = Column('license', String(200))
 	__tablename__ = 'ebuilds_metadata'
 
 class TbcConfig(Base):

diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index 4bbb63e..721f9b1 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -299,7 +299,7 @@ class tbc_package(object):
 			# take package descriptions from the ebuilds
 			if package_metadataDict[package_id]['metadata_xml_descriptions'] != packageDict[cpv]['ebuild_version_descriptions_tree']:
 				package_metadataDict[package_id]['metadata_xml_descriptions'] = packageDict[cpv]['ebuild_version_descriptions_tree']
-		package_metadataDict[package_id][new] = True
+		package_metadataDict[package_id]['new'] = True
 		self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list)
 		log_msg = "C %s:%s ... Done." % (cp, repo)
 		write_log(self._session, log_msg, "info", self._config_id, 'packages.add_new_package_db')

diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index de60af7..9c962f4 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -432,7 +432,9 @@ def add_new_ebuild_sql(session, packageDict):
 			except (MultipleResultsFound) as e:
 				# FIXME
 				sys.exit()
-		session.add(EbuildsMetadata(EbuildId = EbuildInfo.EbuildId, New = v['new'], Updated = v['updated'], Commit = v['git_commit'], CommitMsg = v['git_commit_msg'], Descriptions = v['ebuild_version_descriptions_tree']))
+		session.add(EbuildsMetadata(EbuildId = EbuildInfo.EbuildId, New = v['new'], Updated = v['updated'], Commit = v['git_commit'], \
+			CommitMsg = v['git_commit_msg'], Descriptions = v['ebuild_version_descriptions_tree'], Slot = v['ebuild_version_metadata_tree'][2], \
+			Homepage = v['ebuild_version_metadata_tree'][5], License = v['ebuild_version_metadata_tree'][6]))
 		session.commit()
 		ebuild_id_list.append(EbuildInfo.EbuildId)
 		restrictions = []
@@ -636,7 +638,7 @@ def reset_new_updated(session):
 		for x in EMInfo:
 			x.New = False
 			session.flush()
-try:
+	try:
 		BLInfo = session.query(BuildLogs).filter(BuildLogs.New == True).all()
 	except NoResultFound as e:
 		pass

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     9ea28d6fc2bc00eaf64739cf6739ff52bbfe1056
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Jan  8 15:32:49 2017 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Jan  8 15:32:49 2017 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=9ea28d6f

add header.txt to ignore files

 pym/tbc/sync.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/tbc/sync.py b/pym/tbc/sync.py
index a93b84f..8488c50 100644
--- a/pym/tbc/sync.py
+++ b/pym/tbc/sync.py
@@ -72,7 +72,7 @@ def git_sync_main(session):
 	# check git diffs witch get updated and pass that to a dict
 	# fetch and merge the repo
 	repo_cp_dict = {}
-	search_list = [ '^metadata', '^eclass', '^licenses', '^profiles', '^scripts', '^skel.', ]
+	search_list = [ '^metadata', '^eclass', '^licenses', '^profiles', '^scripts', '^skel.', '^header.txt']
 	for repo_dir in git_repos_list(myportdb):
 		reponame = myportdb.getRepositoryName(repo_dir)
 		repo = git.Repo(repo_dir)

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     35875a290ff4f094f6a9f2299f2c9e9c301482d9
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Dec 22 18:31:02 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Dec 22 18:31:02 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=35875a29

add new on BuildLogs and BuildJobs

 pym/tbc/db_mapping.py |  2 ++
 pym/tbc/sqlquerys.py  | 22 +++++++++++++++++++---
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/pym/tbc/db_mapping.py b/pym/tbc/db_mapping.py
index 91ae5dd..96ae75b 100644
--- a/pym/tbc/db_mapping.py
+++ b/pym/tbc/db_mapping.py
@@ -136,6 +136,7 @@ class BuildJobs(Base):
 	Status = Column('status', Enum('Waiting','Building','Looked',))
 	BuildNow = Column('build_now', Boolean, default=False)
 	RemoveBin = Column('removebin', Boolean ,default=False)
+	New = Column('new', Boolean, default=False)
 	TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow)
 	__tablename__ = 'build_jobs'
 
@@ -188,6 +189,7 @@ class BuildLogs(Base):
 	SummeryText = Column('summery_text', Text)
 	LogHash = Column('log_hash', String(100))
 	BugId = Column('bug_id', Integer, default=0)
+	New = Column('new', Boolean, default=False)
 	TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow)
 	__tablename__ = 'build_logs'
 

diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index 6542c0a..de60af7 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -54,7 +54,7 @@ def get_job_status_waiting_on_guest(session):
 def update_job_list(session, status, job_id):
 	JobInfo = session.query(Jobs).filter_by(JobId = job_id).one()
 	JobInfo.Status = status
-	if status = 'Done':
+	if status == 'Done':
 		JobInfo.TimeStamp = datetime.datetime.utcnow()
 	session.commit()
 
@@ -244,7 +244,7 @@ def add_new_buildlog(session, build_dict, build_log_dict):
 		return None, False
 
 	def build_log_id_no_match(build_dict, build_log_dict):
-		NewBuildLog = BuildLogs(EbuildId = build_dict['ebuild_id'], Fail = build_log_dict['fail'], SummeryText = build_log_dict['build_error'], LogHash = build_log_dict['log_hash'])
+		NewBuildLog = BuildLogs(EbuildId = build_dict['ebuild_id'], Fail = build_log_dict['fail'], SummeryText = build_log_dict['build_error'], LogHash = build_log_dict['log_hash'], New = True)
 		session.add(NewBuildLog)
 		session.flush()
 		build_log_id = NewBuildLog.BuildLogId
@@ -532,7 +532,7 @@ def get_package_info_from_package_id(session, package_id):
 	return PackageInfo, CategoryInfo, RepoInfo
 
 def add_new_build_job(session, ebuild_id, setup_id, use_flagsDict, config_id):
-	NewBuildJobs =BuildJobs(EbuildId = ebuild_id, SetupId = setup_id, ConfigId = config_id, Status = 'Waiting', BuildNow = False, RemoveBin = True)
+	NewBuildJobs = BuildJobs(EbuildId = ebuild_id, SetupId = setup_id, ConfigId = config_id, Status = 'Waiting', BuildNow = False, RemoveBin = True, New = True)
 	session.add(NewBuildJobs)
 	session.flush()
 	build_job_id = NewBuildJobs.BuildJobId
@@ -636,6 +636,22 @@ def reset_new_updated(session):
 		for x in EMInfo:
 			x.New = False
 			session.flush()
+try:
+		BLInfo = session.query(BuildLogs).filter(BuildLogs.New == True).all()
+	except NoResultFound as e:
+		pass
+	else:
+		for x in BLInfo:
+			x.New = False
+			session.flush()
+	try:
+		BJInfo = session.query(BuildJobs).filter(BuildJobs.New == True).all()
+	except NoResultFound as e:
+		pass
+	else:
+		for x in BJInfo:
+			x.New = False
+			session.flush()
 	try:
 		EMInfo = session.query(EbuildsMetadata).filter(EbuildsMetadata.Updated == True).all()
 	except NoResultFound as e:

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     e8fbf308450b9b257f1cd16532591caba51cc192
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Dec 22 11:09:24 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Dec 22 11:09:24 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=e8fbf308

reset new and updated on ebuild and package metadata

 pym/tbc/sqlquerys.py | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index 1d69151..a64c5ef 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -616,3 +616,30 @@ def add_old_category(session, category_id):
 	CategorysInfo = session.query(Categories).filter_by(CategoryId = category_id).one()
 	CategorysInfo.Active = False
 	session.commit()
+
+def reset_new_updated(session):
+	try:
+		PMInfo = session.query(PackagesMetadata).filter(PackagesMetadata.New == True).all()
+	except NoResultFound as e:
+		pass
+	else:
+		for x in PMInfo:
+			x.New = False
+			session.flush()
+	try:
+		EMInfo = session.query(EbuildsMetadata).filter(EbuildsMetadata.New == True).all()
+	except NoResultFound as e:
+		pass
+	else:
+		for x in EMInfo:
+			x.New = False
+			session.flush()
+	try:
+		EMInfo = session.query(EbuildsMetadata).filter(EbuildsMetadata.Updated == True).all()
+	except NoResultFound as e:
+		pass
+	else:
+		for x in EMInfo:
+			x.Updated = False
+			session.flush()
+	session.commit()

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     3ac0465df0049240f20ccb0efee65463203f7542
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Dec 22 11:09:50 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Dec 22 11:09:50 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=3ac0465d

reset new and updated on ebuild and package metadata

 pym/tbc/updatedb.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/pym/tbc/updatedb.py b/pym/tbc/updatedb.py
index 7fce867..437cfd5 100644
--- a/pym/tbc/updatedb.py
+++ b/pym/tbc/updatedb.py
@@ -13,7 +13,8 @@ from sqlalchemy.orm import scoped_session, sessionmaker
 from tbc.ConnectionManager import NewConnection
 from tbc.sqlquerys import get_package_info, update_repo_db, \
 	update_categories_db, get_configmetadata_info, get_config_all_info, add_new_build_job, \
-	get_config_info, get_setup_info, get_job_status_waiting_on_guest, update_job_list
+	get_config_info, get_setup_info, get_job_status_waiting_on_guest, update_job_list, \
+	reset_new_updated
 from tbc.check_setup import check_make_conf
 from tbc.package import tbc_package
 # Get the options from the config file tbc.conf
@@ -156,6 +157,7 @@ def update_cpv_db(session, repo_cp_dict, config_id, tbc_settings):
 
 def update_db_main(session, repo_cp_dict, config_id):
 	# Main
+	reset_new_updated(session)
 	if repo_cp_dict == {}:
 		return True
 	# Logging

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     1513ff38cfc184098d7a96e22a5ecaec72f863d1
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Dec 22 11:23:44 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Dec 22 11:23:44 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=1513ff38

update timestamp when jobs is done

 pym/tbc/sqlquerys.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index a64c5ef..6542c0a 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -54,6 +54,8 @@ def get_job_status_waiting_on_guest(session):
 def update_job_list(session, status, job_id):
 	JobInfo = session.query(Jobs).filter_by(JobId = job_id).one()
 	JobInfo.Status = status
+	if status = 'Done':
+		JobInfo.TimeStamp = datetime.datetime.utcnow()
 	session.commit()
 
 def get_config_all_info(session):

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     f8bbed36aa7ed37a73b7d06347a9b8558c8dc728
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Dec 22 10:14:34 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Dec 22 10:14:34 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=f8bbed36

fix commit to git_commit

 pym/tbc/db_mapping.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/tbc/db_mapping.py b/pym/tbc/db_mapping.py
index 964df54..91ae5dd 100644
--- a/pym/tbc/db_mapping.py
+++ b/pym/tbc/db_mapping.py
@@ -286,7 +286,7 @@ class EbuildsKeywords(Base):
 class EbuildsMetadata(Base):
 	Id =  Column('id', Integer, primary_key=True)
 	EbuildId = Column('ebuild_id', ForeignKey('ebuilds.ebuild_id'))
-	Commit = Column('commit', String(100))
+	Commit = Column('git_commit', String(100))
 	CommitMsg = Column('git_commit_msg', String(200))
 	New = Column('new', Boolean, default=False)
 	Updated = Column('updated', Boolean, default=False)

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     4834c9751340622feb34f4c3d747517826499788
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Dec 22 10:10:27 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Dec 22 10:10:27 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=4834c975

add git commit msg

 pym/tbc/db_mapping.py |  1 +
 pym/tbc/package.py    | 14 ++++++++++----
 pym/tbc/sqlquerys.py  |  2 +-
 3 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/pym/tbc/db_mapping.py b/pym/tbc/db_mapping.py
index aba6ee1..964df54 100644
--- a/pym/tbc/db_mapping.py
+++ b/pym/tbc/db_mapping.py
@@ -287,6 +287,7 @@ class EbuildsMetadata(Base):
 	Id =  Column('id', Integer, primary_key=True)
 	EbuildId = Column('ebuild_id', ForeignKey('ebuilds.ebuild_id'))
 	Commit = Column('commit', String(100))
+	CommitMsg = Column('git_commit_msg', String(200))
 	New = Column('new', Boolean, default=False)
 	Updated = Column('updated', Boolean, default=False)
 	Descriptions = Column('descriptions', Text)

diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index 3c5664c..4bbb63e 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -92,10 +92,14 @@ class tbc_package(object):
 	def get_git_log_ebuild(self, repodir, ebuild_file):
 		git_log_ebuild = ''
 		g = git.Git(repodir)
+		index = 1
+		git_log_dict = {}
 		for line in g.log('-n 1', ebuild_file).splitlines():
-			if re.search('^commit', line):
-				git_log_ebuild = re.sub('commit', '', line)
-		return git_log_ebuild
+			git_log_dict[index] = line
+			index = index + 1
+		git_ebuild_commit = re.sub('commit ', '', git_log_dict[1])
+		git_ebuild_commit_msg = re.sub('    ', '', git_log_dict[5])
+		return git_ebuild_commit, git_ebuild_commit_msg
 
 	def get_packageDict(self, pkgdir, cpv, repo):
 
@@ -115,9 +119,10 @@ class tbc_package(object):
 			log_msg = "C %s:%s ... Fail." % (cpv, repo)
 			write_log(self._session, log_msg, "info", self._config_id, 'packages.get_packageDict')
 			git_commit = '0'
+			git_commit_msg = '0'
 		else:
 			repodir =self._myportdb.getRepositoryPath(repo)
-			git_commit = self.get_git_log_ebuild(repodir, ebuild_file)
+			git_commit, git_commit_msg = self.get_git_log_ebuild(repodir, ebuild_file)
 
 		# Get the ebuild metadata
 		ebuild_version_metadata_tree = self.get_ebuild_metadata(cpv, repo)
@@ -140,6 +145,7 @@ class tbc_package(object):
 		attDict['ebuild_version_metadata_tree'] = ebuild_version_metadata_tree
 		#attDict['ebuild_version_text_tree'] = ebuild_version_text_tree[0]
 		attDict['git_commit'] = git_commit
+		attDict['git_commit_msg'] = git_commit_msg
 		attDict['new'] = False
 		attDict['updated'] = False
 		attDict['ebuild_version_descriptions_tree'] = ebuild_version_metadata_tree[7]

diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index c9141ef..1d69151 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -430,7 +430,7 @@ def add_new_ebuild_sql(session, packageDict):
 			except (MultipleResultsFound) as e:
 				# FIXME
 				sys.exit()
-		session.add(EbuildsMetadata(EbuildId = EbuildInfo.EbuildId, New = v['new'], Updated = v['updated'], Commit = v['git_commit'], Descriptions = v['ebuild_version_descriptions_tree']))
+		session.add(EbuildsMetadata(EbuildId = EbuildInfo.EbuildId, New = v['new'], Updated = v['updated'], Commit = v['git_commit'], CommitMsg = v['git_commit_msg'], Descriptions = v['ebuild_version_descriptions_tree']))
 		session.commit()
 		ebuild_id_list.append(EbuildInfo.EbuildId)
 		restrictions = []

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     682b192ef7f23ddebb56206e263a540e557ba515
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Dec 22 09:58:30 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Dec 22 09:58:30 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=682b192e

change new to updated in the update run and some typos

 pym/tbc/package.py   | 5 ++---
 pym/tbc/sqlquerys.py | 2 +-
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index 0a5749d..3c5664c 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -141,7 +141,7 @@ class tbc_package(object):
 		#attDict['ebuild_version_text_tree'] = ebuild_version_text_tree[0]
 		attDict['git_commit'] = git_commit
 		attDict['new'] = False
-		attDict['update'] = False
+		attDict['updated'] = False
 		attDict['ebuild_version_descriptions_tree'] = ebuild_version_metadata_tree[7]
 		return attDict
 
@@ -264,7 +264,6 @@ class tbc_package(object):
 		mytree = []
 		mytree.append(repodir)
 		pkgdir = repodir + "/" + cp # Get RepoDIR + cp
-		
 		package_id = add_new_package_sql(self._session, cp, repo)
 
 		# Check cp with repoman full
@@ -377,7 +376,7 @@ class tbc_package(object):
 				# N = New ebuild
 				log_msg = "N %s:%s" % (cpv, repo,)
 				write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')
-				packageDict[cpv]['new'] = True
+				packageDict[cpv]['updated'] = True
 				package_updated = True
 			elif  ebuild_version_checksum_tree != ebuild_version_manifest_checksum_db:
 				# U = Updated ebuild

diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index 0ec5fc9..c9141ef 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -49,7 +49,7 @@ def get_job_status_waiting_on_guest(session):
 		JobsInfo = session.query(Jobs).filter_by(Status = 'Waiting_on_guest').one()
 	except NoResultFound as e:
 		return None
-	return JobInfo.JobId
+	return JobsInfo.JobId
 
 def update_job_list(session, status, job_id):
 	JobInfo = session.query(Jobs).filter_by(JobId = job_id).one()

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     7c8cea8c8af2b8393b59ac1bf23917c130f9b369
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Dec 20 21:20:30 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Dec 20 21:20:30 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=7c8cea8c

fix a typo in update_package_metadata

 pym/tbc/sqlquerys.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index aa9c2a7..0276131 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -511,7 +511,7 @@ def update_package_metadata(session, package_metadataDict):
 		try:
 			PackagesMetadataInfo = session.query(PackagesMetadata).filter_by(PackageId = k).one()
 		except NoResultFound as e:
-			session.add(PackagesMetadata(PackageId = k, Gitlog = v['git_changlog'], Descriptions = v['metadata_xml_descriptions'], v['new']))
+			session.add(PackagesMetadata(PackageId = k, Gitlog = v['git_changlog'], Descriptions = v['metadata_xml_descriptions'], New = v['new']))
 			session.commit()
 		else:
 			PackagesMetadataInfo.Gitlog = v['git_changlog']

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     603f968772a11e3406f774967f76bbbcb5ece729
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Dec 20 21:15:24 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Dec 20 21:15:24 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=603f9687

fix a typo in add_new_ebuild_sql

 pym/tbc/sqlquerys.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index 901db08..aa9c2a7 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -429,7 +429,7 @@ def add_new_ebuild_sql(session, packageDict):
 			except (MultipleResultsFound) as e:
 				# FIXME
 				sys.exit()
-		session.add(EbuildsMetadata(EbuildId = EbuildInfo.EbuildId, New = v['new'], Updated = v['updated'], = v['git_commit'], Descriptions = v['ebuild_version_descriptions_tree']))
+		session.add(EbuildsMetadata(EbuildId = EbuildInfo.EbuildId, New = v['new'], Updated = v['updated'], Commit = v['git_commit'], Descriptions = v['ebuild_version_descriptions_tree']))
 		session.commit()
 		ebuild_id_list.append(EbuildInfo.EbuildId)
 		restrictions = []

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     f028abe51998fb054cf463861c920babed43a4cf
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Dec 20 19:57:16 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Dec 20 19:57:16 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=f028abe5

fix a typo in irk code

 pym/tbc/irk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/tbc/irk.py b/pym/tbc/irk.py
index b00f816..3b3589b 100644
--- a/pym/tbc/irk.py
+++ b/pym/tbc/irk.py
@@ -23,6 +23,6 @@ def irk(target, message, server = DEFAULT_SERVER):
 def send_irk(msg, host):
 	target = "tinderbox-cluster"
 	try:
-		irk(target, msg, server = (host, 6659)
+		irk(target, msg, server = (host, 6659))
 	except socket.error as e:
 		sys.stderr.write("irk: write to server failed: %r\n" % e)

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     7df70c0bf6d0a8cd4a2773091778a44ada419294
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Dec 20 19:59:12 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Dec 20 19:59:12 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=7df70c0b

add new,update on ebuilds and package

 pym/tbc/db_mapping.py | 2 ++
 pym/tbc/package.py    | 8 +++++---
 pym/tbc/sqlquerys.py  | 4 ++--
 3 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/pym/tbc/db_mapping.py b/pym/tbc/db_mapping.py
index d3fead1..304dcd8 100644
--- a/pym/tbc/db_mapping.py
+++ b/pym/tbc/db_mapping.py
@@ -106,6 +106,7 @@ class PackagesMetadata(Base):
 	PackageId = Column('package_id', Integer, ForeignKey('packages.package_id'))
 	Gitlog = Column('gitlog', Text)
 	Descriptions = Column('descriptions', Text)
+	New = Column('new', Boolean, default=False)
 	__tablename__ = 'packages_metadata'
 
 class Ebuilds(Base):
@@ -287,6 +288,7 @@ class EbuildsMetadata(Base):
 	EbuildId = Column('ebuild_id', ForeignKey('ebuilds.ebuild_id'))
 	Commit = Column('commit', String(100))
 	New = Column('new', Boolean, default=False)
+	Updated = Column('updated', Boolean, default=False)
 	Descriptions = Column('descriptions', Text)
 	__tablename__ = 'ebuilds_metadata'
 

diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index 6e42af3..0a5749d 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -141,6 +141,7 @@ class tbc_package(object):
 		#attDict['ebuild_version_text_tree'] = ebuild_version_text_tree[0]
 		attDict['git_commit'] = git_commit
 		attDict['new'] = False
+		attDict['update'] = False
 		attDict['ebuild_version_descriptions_tree'] = ebuild_version_metadata_tree[7]
 		return attDict
 
@@ -195,7 +196,6 @@ class tbc_package(object):
 		package_metadataDict = {}
 		md_email_list = []
 		herd = None
-		attDict['metadata_xml_email'] = False
 		try:
 			pkg_md = MetaDataXML(pkgdir + "/metadata.xml", herd)
 		except:
@@ -217,6 +217,7 @@ class tbc_package(object):
 				write_log(self._session, log_msg, "warning", self._config_id, 'packages.get_package_metadataDict')
 		attDict['git_changlog'] = self.get_git_changelog_text(repodir, cp)
 		attDict['metadata_xml_descriptions'] = ''
+		attDict['new'] = False
 		package_metadataDict[package_id] = attDict
 		return package_metadataDict
 
@@ -288,12 +289,12 @@ class tbc_package(object):
 		old_ebuild_id_list = []
 		for cpv in sorted(ebuild_list_tree):
 			packageDict[cpv] = self.get_packageDict(pkgdir, cpv, repo)
+			packageDict[cpv]['new'] = True
 
 			# take package descriptions from the ebuilds
 			if package_metadataDict[package_id]['metadata_xml_descriptions'] != packageDict[cpv]['ebuild_version_descriptions_tree']:
 				package_metadataDict[package_id]['metadata_xml_descriptions'] = packageDict[cpv]['ebuild_version_descriptions_tree']
-
-		packageDict[cpv]['new'] = True
+		package_metadataDict[package_id][new] = True
 		self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list)
 		log_msg = "C %s:%s ... Done." % (cp, repo)
 		write_log(self._session, log_msg, "info", self._config_id, 'packages.add_new_package_db')
@@ -382,6 +383,7 @@ class tbc_package(object):
 				# U = Updated ebuild
 				log_msg = "U %s:%s" % (cpv, repo,)
 				write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')
+				packageDict[cpv]['updated'] = True
 				package_updated = True
 			else:
 				# Remove cpv from packageDict and add ebuild to new ebuils list

diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index 60f4ba8..770e84e 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -423,7 +423,7 @@ def add_new_ebuild_sql(session, packageDict):
 			except (MultipleResultsFound) as e:
 				# FIXME
 				sys.exit()
-		session.add(EbuildsMetadata(EbuildId = EbuildInfo.EbuildId, New = v['new'], Commit = v['git_commit'], Descriptions = v['ebuild_version_descriptions_tree']))
+		session.add(EbuildsMetadata(EbuildId = EbuildInfo.EbuildId, New = v['new'], Updated = v['updated'], = v['git_commit'], Descriptions = v['ebuild_version_descriptions_tree']))
 		session.commit()
 		ebuild_id_list.append(EbuildInfo.EbuildId)
 		restrictions = []
@@ -505,7 +505,7 @@ def update_package_metadata(session, package_metadataDict):
 		try:
 			PackagesMetadataInfo = session.query(PackagesMetadata).filter_by(PackageId = k).one()
 		except NoResultFound as e:
-			session.add(PackagesMetadata(PackageId = k, Gitlog = v['git_changlog'], Descriptions = v['metadata_xml_descriptions']))
+			session.add(PackagesMetadata(PackageId = k, Gitlog = v['git_changlog'], Descriptions = v['metadata_xml_descriptions'], v['new']))
 			session.commit()
 		else:
 			PackagesMetadataInfo.Gitlog = v['git_changlog']

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     ccd8dad636b0f58585c0c3c5eac7f80d138600ee
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Dec 20 20:25:03 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Dec 20 20:25:03 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=ccd8dad6

add Waiting_on_guest

 pym/tbc/db_mapping.py | 2 +-
 pym/tbc/jobs.py       | 4 ++--
 pym/tbc/sqlquerys.py  | 6 ++++++
 pym/tbc/sync.py       | 5 ++++-
 pym/tbc/updatedb.py   | 4 +++-
 5 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/pym/tbc/db_mapping.py b/pym/tbc/db_mapping.py
index 304dcd8..aba6ee1 100644
--- a/pym/tbc/db_mapping.py
+++ b/pym/tbc/db_mapping.py
@@ -39,7 +39,7 @@ class Logs(Base):
 class Jobs(Base):
 	JobId = Column('job_id', Integer, primary_key=True)
 	JobType = Column('job_type', Enum('updatedb', 'esync', 'removeold_cpv'))
-	Status = Column('status', Enum('Runing', 'Done', 'Waiting'))
+	Status = Column('status', Enum('Runing', 'Done', 'Waiting', 'Waiting_on_guest'))
 	User = Column('user', String(50))
 	ConfigId = Column('config_id', Integer, ForeignKey('configs.config_id'))
 	RunConfigId = Column('run_config_id', Integer, ForeignKey('configs.config_id'))

diff --git a/pym/tbc/jobs.py b/pym/tbc/jobs.py
index efe8623..800fe87 100644
--- a/pym/tbc/jobs.py
+++ b/pym/tbc/jobs.py
@@ -46,7 +46,7 @@ def jobs_main(session, config_id):
 			#	log_msg = "Job %s did fail." % (job_id,)
 			#	write_log(session, log_msg, "info", config_id, 'jobs_main')
 		elif job == "esync":
-			update_job_list(session, "Runing", job_id)
+			update_job_list(session, "Waiting_on_guest", job_id)
 			log_msg = "Job %s is runing." % (job_id,)
 			write_log(session, log_msg, "info", config_id, 'jobs_main')
 			if update_db_main(session, git_sync_main(session), config_id):
@@ -58,7 +58,7 @@ def jobs_main(session, config_id):
 				log_msg = "Job %s did fail." % (job_id,)
 				write_log(session, log_msg, "info", config_id, 'jobs_main')
 		elif job == "updatedb":
-			update_job_list(session, "Runing", job_id)
+			update_job_list(session, "Waiting_on_guest", job_id)
 			log_msg = "Job %s is runing." % (job_id,)
 			write_log(session, log_msg, "info", config_id, 'jobs_main')
 			if update_db_main(session, None, config_id):

diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index 770e84e..901db08 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -44,6 +44,12 @@ def get_jobs(session, config_id):
 		return None
 	return JobsInfo
 
+def get_job_status_waiting_on_guest(session):
+	JobsInfo = session.query(Jobs).filter_by(Status = 'Waiting_on_guest').one()
+	if JobsInfo == []:
+		return None
+	return JobInfo.JobId
+
 def update_job_list(session, status, job_id):
 	JobInfo = session.query(Jobs).filter_by(JobId = job_id).one()
 	JobInfo.Status = status

diff --git a/pym/tbc/sync.py b/pym/tbc/sync.py
index 7063bfb..dee13ff 100644
--- a/pym/tbc/sync.py
+++ b/pym/tbc/sync.py
@@ -11,7 +11,8 @@ import re
 import git
 
 from tbc.sqlquerys import get_config_id_fqdn, add_logs, get_config_all_info, \
-	get_configmetadata_info, get_config_info, get_setup_info
+	get_configmetadata_info, get_config_info, get_setup_info, get_job_status_waiting_on_guest, \
+	update_job_list
 from tbc.readconf import read_config_settings
 from tbc.log import write_log
 
@@ -64,6 +65,8 @@ def git_sync_main(session):
 		else:
 			time.sleep(60)
 
+	job_id = get_job_status_waiting_on_guest(session)
+	update_job_list(session, 'Runing', job_id)
 	# check git diffs witch get updated and pass that to a dict
 	# fetch and merge the repo
 	repo_cp_dict = {}

diff --git a/pym/tbc/updatedb.py b/pym/tbc/updatedb.py
index 0753d92..b6539e4 100644
--- a/pym/tbc/updatedb.py
+++ b/pym/tbc/updatedb.py
@@ -13,7 +13,7 @@ from sqlalchemy.orm import scoped_session, sessionmaker
 from tbc.ConnectionManager import NewConnection
 from tbc.sqlquerys import get_package_info, update_repo_db, \
 	update_categories_db, get_configmetadata_info, get_config_all_info, add_new_build_job, \
-	get_config_info, get_setup_info
+	get_config_info, get_setup_info, get_job_status_waiting_on_guest, update_job_list
 from tbc.check_setup import check_make_conf
 from tbc.package import tbc_package
 # Get the options from the config file tbc.conf
@@ -93,6 +93,8 @@ def update_cpv_db(session, repo_cp_dict, config_id, tbc_settings):
 			break
 		time.sleep(30)
 
+	job_id = get_job_status_waiting_on_guest(session)
+	update_job_list(session, 'Runing', job_id)
 	log_msg = "Checking categories, package, ebuilds"
 	write_log(session, log_msg, "info", config_id, 'updatedb.update_cpv_db')
 	new_build_jobs_list = []

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     a18f23061cd8ca410e6453212bf802feb8d0c602
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Dec 10 16:16:56 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Dec 10 16:16:56 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=a18f2306

add port for irker

 pym/tbc/irk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/tbc/irk.py b/pym/tbc/irk.py
index 3186cb5..b00f816 100644
--- a/pym/tbc/irk.py
+++ b/pym/tbc/irk.py
@@ -23,6 +23,6 @@ def irk(target, message, server = DEFAULT_SERVER):
 def send_irk(msg, host):
 	target = "tinderbox-cluster"
 	try:
-		irk(target, msg, server = host)
+		irk(target, msg, server = (host, 6659)
 	except socket.error as e:
 		sys.stderr.write("irk: write to server failed: %r\n" % e)

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     d88b81146838c007345ca0c87cd6d51b6f624f27
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Dec  5 21:11:08 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Dec  5 21:11:08 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=d88b8114

Set email to maintainer-needed if email not found

 pym/tbc/package.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index 255ec67..6e42af3 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -211,7 +211,9 @@ class tbc_package(object):
 			if md_email_list != []:
 				attDict['metadata_xml_email'] = md_email_list
 			else:
-				log_msg = "Metadata file %s missing Email" % (pkgdir + "/metadata.xml")
+				md_email_list.append('maintainer-needed@gentoo.org')
+				attDict['metadata_xml_email'] = md_email_list
+				log_msg = "Metadata file %s missing Email. Setting it to maintainer-needed" % (pkgdir + "/metadata.xml")
 				write_log(self._session, log_msg, "warning", self._config_id, 'packages.get_package_metadataDict')
 		attDict['git_changlog'] = self.get_git_changelog_text(repodir, cp)
 		attDict['metadata_xml_descriptions'] = ''

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     cc61c64c0c2bfb3bdfe32eb0030db2ab69483346
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Nov 20 15:37:42 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Nov 20 15:37:42 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=cc61c64c

ImportError: No module named 'repoman.checks.ebuilds'

 pym/tbc/qachecks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/tbc/qachecks.py b/pym/tbc/qachecks.py
index 892642f..976cb2a 100644
--- a/pym/tbc/qachecks.py
+++ b/pym/tbc/qachecks.py
@@ -14,7 +14,7 @@ from portage import os, _encodings, _unicode_decode, _unicode_encode
 from portage.exception import DigestException, FileNotFound, ParseError, PermissionDenied
 from _emerge.Package import Package
 from _emerge.RootConfig import RootConfig
-from repoman.checks.ebuilds.checks import run_checks
+from repoman.modules.scan.ebuild.checks import run_checks
 from tbc.repoman import repoman_main
 from tbc.sqlquerys import get_configmetadata_info, get_config_info, get_setup_info
 import portage

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     7af6ebcf89684464488684ee3dc349827ddd7f46
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Apr  5 21:35:47 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Apr  5 21:35:47 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=7af6ebcf

remove log file from summery text

 pym/tbc/build_log.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/pym/tbc/build_log.py b/pym/tbc/build_log.py
index 5476592..2a00b95 100644
--- a/pym/tbc/build_log.py
+++ b/pym/tbc/build_log.py
@@ -292,16 +292,17 @@ def add_buildlog_main(settings, pkg, trees):
 	build_log_dict = {}
 	build_log_dict = get_buildlog_info(session, settings, pkg, build_dict, config_id)
 	error_log_list = build_log_dict['error_log_list']
+	build_log_dict['logfilename'] = settings.get("PORTAGE_LOG_FILE").split(host_config)[1]
 	build_error = ""
 	log_hash = hashlib.sha256()
 	build_error = ""
 	if error_log_list != []:
 		for log_line in error_log_list:
-			build_error = build_error + log_line
+			if not re.search(build_log_dict['logfilename'], log_line):
+				build_error = build_error + log_line
 		log_hash.update(build_error.encode('utf-8'))
 	build_log_dict['build_error'] = build_error
 	build_log_dict['log_hash'] = log_hash.hexdigest()
-	build_log_dict['logfilename'] = settings.get("PORTAGE_LOG_FILE").split(host_config)[1]
 	log_msg = "Logfile name: %s" % (settings.get("PORTAGE_LOG_FILE"),)
 	write_log(session, log_msg, "info", config_id, 'build_log.add_buildlog_main')
 	build_log_dict['emerge_info'] = get_emerge_info_id(settings, trees, session, config_id)

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     2fe8d223790b5f40b42e41f3ed80b59d408aaa05
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Mar  6 17:47:58 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Mar  6 17:47:58 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=2fe8d223

fix a typo for package env files

 pym/tbc/build_job.py | 23 ++++++++---------------
 1 file changed, 8 insertions(+), 15 deletions(-)

diff --git a/pym/tbc/build_job.py b/pym/tbc/build_job.py
index e8ab829..d0d72c3 100644
--- a/pym/tbc/build_job.py
+++ b/pym/tbc/build_job.py
@@ -103,21 +103,14 @@ class build_job_action(object):
 					f.write(filetext)
 					f.write('\n')
 					f.close
-		if enable_test_features:
-			filetext = k + ' ' + 'test.conf'
-			log_msg = "filetext: %s" % filetext
-			add_logs(self._session, log_msg, "info", self._config_id)
-			with open("/etc/portage/package.env/99_env", "a") as f:
-				f.write(filetext)
-				f.write('\n')
-				f.close
-			filetext = '=' + k + ' ' + 'test.conf'
-			log_msg = "filetext: %s" % filetext
-			add_logs(self._session, log_msg, "info", self._config_id)
-			with open("/etc/portage/package.env/99_env", "a") as f:
-				f.write(filetext)
-				f.write('\n')
-				f.close
+			if enable_test_features:
+				filetext = '=' + k + ' ' + 'test.conf'
+				log_msg = "filetext: %s" % filetext
+				add_logs(self._session, log_msg, "info", self._config_id)
+				with open("/etc/portage/package.env/99_env", "a") as f:
+					f.write(filetext)
+					f.write('\n')
+					f.close
 
 		log_msg = "build_cpv_list: %s" % (build_cpv_list,)
 		add_logs(self._session, log_msg, "info", self._config_id)

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     75fd66c5a04ba87ae92005b335fd5f854e00b5e5
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Mar  3 15:27:15 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Mar  3 15:27:15 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=75fd66c5

fix test too many values to unpack

 pym/tbc/flags.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/tbc/flags.py b/pym/tbc/flags.py
index 10c9390..eb3f782 100644
--- a/pym/tbc/flags.py
+++ b/pym/tbc/flags.py
@@ -195,7 +195,7 @@ class tbc_use_flags(object):
 		return iuse_flags, final_flags
 
 	def comper_useflags(self, build_dict):
-		iuse_flags, use_enable = self.get_flags()
+		iuse_flags, use_enable, usemasked = self.get_flags()
 		iuse = []
 		build_use_flags_dict = build_dict['build_useflags']
 		build_use_flags_list = []

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     e57dd6203e4c04ac18c4b5333bf6c78b1a1eff11
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Wed Mar  2 20:50:59 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Wed Mar  2 20:50:59 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=e57dd620

add more support for use test

 pym/tbc/build_job.py  | 49 ++++++++++++++++++++++++++++++++++++-------------
 pym/tbc/db_mapping.py |  6 ++++--
 pym/tbc/flags.py      | 10 ++--------
 pym/tbc/package.py    | 18 +++++++++++++-----
 pym/tbc/sqlquerys.py  | 15 ++++++---------
 5 files changed, 61 insertions(+), 37 deletions(-)

diff --git a/pym/tbc/build_job.py b/pym/tbc/build_job.py
index dcec3b9..e8ab829 100644
--- a/pym/tbc/build_job.py
+++ b/pym/tbc/build_job.py
@@ -67,6 +67,14 @@ class build_job_action(object):
 		build_cpv_list = []
 		depclean_fail = True
 		disable_test_features = False
+		enable_test_features = False
+		restrictions_test = False
+		restrictions_list= get_ebuild_restrictions(self._session, build_dict['ebuild_id'])
+		if restrictions_list:
+			if "test" in restrictions_list:
+				restrictions_test = True
+		if restrictions_test and "test" in settings.features:
+			disable_test_features = True
 		for k, build_use_flags_list in buildqueru_cpv_dict.items():
 			build_cpv_list.append("=" + k)
 			if not build_use_flags_list == None:
@@ -80,22 +88,37 @@ class build_job_action(object):
 					f.write(filetext)
 					f.write('\n')
 					f.close
-		if not build_dict['build_useflags'] is None:
-			if "test" in build_dict['build_useflags'] and "test" in settings.features:
-				if build_dict['build_useflags']['test'] is False:
-					disable_test_features = True
-		restrictions_dict = get_ebuild_restrictions(self._session, build_dict['ebuild_id'])
-		if restrictions_dict:
-			if "test" in restrictions_dict:
-				disable_test_features = True
-		if disable_test_features:
-			filetext = k + ' ' + 'notest.conf'
+
+			if not build_dict['build_useflags'] is None:
+				if "test" in build_dict['build_useflags']:
+					if build_dict['build_useflags']['test'] is False and "test" in settings.features:
+						disable_test_features = True
+					if build_dict['build_useflags']['test'] is True and not disable_test_features and "test" not in settings.features:
+						enable_test_features = True
+			if disable_test_features:
+				filetext = '=' + k + ' ' + 'notest.conf'
+				log_msg = "filetext: %s" % filetext
+				add_logs(self._session, log_msg, "info", self._config_id)
+				with open("/etc/portage/package.env/99_env", "a") as f:
+					f.write(filetext)
+					f.write('\n')
+					f.close
+		if enable_test_features:
+			filetext = k + ' ' + 'test.conf'
 			log_msg = "filetext: %s" % filetext
 			add_logs(self._session, log_msg, "info", self._config_id)
-			with open("/etc/portage/package.env", "a") as f:
+			with open("/etc/portage/package.env/99_env", "a") as f:
 				f.write(filetext)
 				f.write('\n')
 				f.close
+			filetext = '=' + k + ' ' + 'test.conf'
+			log_msg = "filetext: %s" % filetext
+			add_logs(self._session, log_msg, "info", self._config_id)
+			with open("/etc/portage/package.env/99_env", "a") as f:
+				f.write(filetext)
+				f.write('\n')
+				f.close
+
 		log_msg = "build_cpv_list: %s" % (build_cpv_list,)
 		add_logs(self._session, log_msg, "info", self._config_id)
 
@@ -138,8 +161,8 @@ class build_job_action(object):
 			os.remove("/etc/portage/package.use/99_autounmask")
 			with open("/etc/portage/package.use/99_autounmask", "a") as f:
 				f.close
-			os.remove("/etc/portage/package.env")
-			with open("/etc/portage/package.env", "a") as f:
+			os.remove("/etc/portage/package.env/99_env")
+			with open("/etc/portage/package.env/99_env/", "a") as f:
 				f.close
 		except:
 			pass

diff --git a/pym/tbc/db_mapping.py b/pym/tbc/db_mapping.py
index 86884bf..34ac324 100644
--- a/pym/tbc/db_mapping.py
+++ b/pym/tbc/db_mapping.py
@@ -12,11 +12,13 @@ class Keywords(Base):
 	KeywordId = Column('keyword_id', Integer, primary_key=True)
 	Keyword = Column('keyword', String)
 	__tablename__ = 'keywords'
-	
+
 class Setups(Base):
 	SetupId = Column('setup_id', Integer, primary_key=True)
 	Setup = Column('setup', String(100))
 	Profile = Column('profile', String(150))
+	Test = Column('test', Boolean, default=False)
+	Repoman = Column('repoman', Boolean, default=False)
 	__tablename__ = 'setups'
 
 class Configs(Base):
@@ -25,7 +27,7 @@ class Configs(Base):
 	SetupId = Column('setup_id', Integer, ForeignKey('setups.setup_id'))
 	Host = Column('default_config', Boolean, default=False)
 	__tablename__ = 'configs'
-	
+
 class Logs(Base):
 	LogId = Column('log_id', Integer, primary_key=True)
 	ConfigId = Column('config_id', Integer, ForeignKey('configs.config_id'))

diff --git a/pym/tbc/flags.py b/pym/tbc/flags.py
index 82b7d42..10c9390 100644
--- a/pym/tbc/flags.py
+++ b/pym/tbc/flags.py
@@ -1,14 +1,8 @@
-#!/usr/bin/python
-#
-# Copyright 1998-2015 Gentoo Foundation
+# Copyright 1998-2016 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 # Origin flags.py from portage public api repo
 from __future__ import print_function
-from _emerge.main import parse_opts
-from _emerge.depgraph import backtrack_depgraph, depgraph, resume_depgraph
-from _emerge.create_depgraph_params import create_depgraph_params
-from _emerge.actions import load_emerge_config
 import portage
 import os
 
@@ -163,7 +157,7 @@ class tbc_use_flags(object):
 		iuse_flags = self.filter_flags(self.get_iuse(), use_expand_hidden, usemasked, useforced)
 		#flags = filter_flags(use_flags, use_expand_hidden, usemasked, useforced)
 		final_flags = self.filter_flags(final_use, use_expand_hidden, usemasked, useforced)
-		return iuse_flags, final_flags
+		return iuse_flags, final_flags, usemasked
 
 	def get_flags_looked(self):
 		"""Retrieves all information needed to filter out hidden, masked, etc.

diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index c069c7e..255ec67 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -56,16 +56,20 @@ class tbc_package(object):
 
 					# Get the iuse and use flags for that config/setup and cpv
 					init_useflags = tbc_use_flags(mysettings_setup, myportdb_setup, build_cpv)
-					iuse_flags_list, final_use_list = init_useflags.get_flags()
+					iuse_flags_list, final_use_list, usemasked = init_useflags.get_flags()
 					iuse_flags_list2 = []
 					for iuse_line in iuse_flags_list:
 						iuse_flags_list2.append( init_useflags.reduce_flag(iuse_line))
-
+					enable_test = False
+					if SetupInfo.Test:
+						if not "test" in usemasked:
+							enable_test = True
 					# Dict the needed info
 					attDict = {}
 					attDict['cpv'] = build_cpv
 					attDict['useflags'] = final_use_list
 					attDict['iuse'] = iuse_flags_list2
+					attDict['test'] = enable_test
 					config_cpv_dict[ConfigInfo.SetupId] = attDict
 
 				# Clean some cache
@@ -156,6 +160,7 @@ class tbc_package(object):
 					use_flagsDict[x] = True
 				for x in use_disable:
 					use_flagsDict[x] = False
+				enable_test = v['test']
 				# Unpack packageDict
 				i = 0
 				for k, v in packageDict.items():
@@ -163,9 +168,12 @@ class tbc_package(object):
 
 					# Comper and add the cpv to buildqueue
 					if build_cpv == k:
-						restrictions_dict = get_ebuild_restrictions(self._session, ebuild_id)
-						if restrictions_dict:
-							if "test" in restrictions_dict and "test" in use_flagsDict:
+						# check if we need to enable or disable test
+						if "test" in use_flagsDict and enable_test:
+							use_flagsDict['test'] = True
+						restrictions_list = get_ebuild_restrictions(self._session, ebuild_id)
+						if restrictions_list:
+							if "test" in restrictions_list and "test" in use_flagsDict:
 								use_flagsDict['test'] = False
 						add_new_build_job(self._session, ebuild_id, setup_id, use_flagsDict, self._config_id)
 						# B = Build cpv use-flags config

diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index 85621fe..60f4ba8 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -561,16 +561,13 @@ def get_ebuild_id_db(session, checksum, package_id, ebuild_version):
 
 def get_ebuild_restrictions(session, ebuild_id):
 	restrictions = []
-	try:
-		EbuildsRestrictionsInfo = session.query(EbuildsRestrictions).filter_by(EbuildId = ebuild_id).one()
-	except NoResultFound as e:
+	EbuildsRestrictionsInfos = session.query(EbuildsRestrictions).filter_by(EbuildId = ebuild_id).all()
+	if EbuildsRestrictionsInfos == []:
 		return False
-	except MultipleResultsFound as e:
-		EbuildsRestrictionsInfos = session.query(EbuildsRestrictions).filter_by(EbuildId = ebuild_id).all()
-		for EbuildsRestrictionsInfo in EbuildsRestrictionsInfos:
-			restrictions.append(session.query(Restrictions).filter_by(RestrictionId = EbuildsRestrictionsInfo.RestrictionId).one())
-		return restrictions
-	return restrictions.append(session.query(Restrictions).filter_by(RestrictionId = EbuildsRestrictionsInfo.RestrictionId).one())
+	for EbuildsRestrictionsInfo in EbuildsRestrictionsInfos:
+		RestrictionsInfo = session.query(Restrictions).filter_by(RestrictionId = EbuildsRestrictionsInfo.RestrictionId).one()
+		restrictions.append(RestrictionsInfo.Restriction)
+	return restrictions
 
 def add_repoman_log(session, package_id, repoman_log, repoman_hash):
 	try:

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     495e43e67bbf4a013b088e0add0b36edae644d21
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Feb 29 23:01:59 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Feb 29 23:01:59 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=495e43e6

fix a bug in repoman logging multiple repeat

 pym/tbc/build_log.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/pym/tbc/build_log.py b/pym/tbc/build_log.py
index 8b6ef0b..d053d57 100644
--- a/pym/tbc/build_log.py
+++ b/pym/tbc/build_log.py
@@ -41,16 +41,16 @@ def check_repoman_full(session, pkgdir, package_id, config_id, cpv=False):
 	status = repoman_full(session, pkgdir, config_id)
 	repoman_hash = hashlib.sha256()
 	if cpv:
-		element = cpv.split('/')
-		pv = element[1]
+		ebuild_version_tree = portage.versions.cpv_getversion(cpv)
 	if status:
 		repoman_dict = {}
 		for k, v in status.items():
 			repoman_log2 = []
 			for line in v:
-				if not cpv:
-					repoman_log2.append(line)
-				elif cpv and re.search(pv + '.ebuild', line):
+				if cpv:
+					if re.search(ebuild_version_tree, line):
+						repoman_log2.append(line)
+				else: 
 					repoman_log2.append(line)
 			if not repoman_log2 == []:
 				repoman_dict[k] = repoman_log2

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     c5bc414fb226ee5ab38c4d0062e2a534d9b92a8a
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Feb 22 18:34:24 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Feb 22 18:34:24 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=c5bc414f

remove multi cpu support in remove_old_cpv job

 pym/tbc/old_cpv.py   | 47 ++++++++++++++---------------------------------
 pym/tbc/sqlquerys.py | 18 +++---------------
 2 files changed, 17 insertions(+), 48 deletions(-)

diff --git a/pym/tbc/old_cpv.py b/pym/tbc/old_cpv.py
index 3c9ffd4..e97b121 100644
--- a/pym/tbc/old_cpv.py
+++ b/pym/tbc/old_cpv.py
@@ -2,61 +2,42 @@
 # Distributed under the terms of the GNU General Public License v2
 
 from __future__ import print_function
-import multiprocessing
 from datetime import datetime
 from tbc.log import write_log
-from sqlalchemy.orm import scoped_session, sessionmaker
-from tbc.ConnectionManager import NewConnection
-from tbc.readconf import  read_config_settings
 from tbc.sqlquerys import get_category_list_info, get_package_list_info, get_ebuild_list_info, \
 	get_build_job_all, del_old_build_jobs, del_old_ebuild, del_old_package, add_old_category
 
-def remove_old_ebuilds(package_id, config_id, tbc_settings, cp):
-	today = datetime.utcnow()
-	session_factory = sessionmaker(bind=NewConnection(tbc_settings))
-	Session = scoped_session(session_factory)
-	session2 = Session()
-	EbuildsInfo = get_ebuild_list_info(session2, package_id)
+def remove_old_ebuilds(session, package_id, config_id, cp, today):
+	EbuildsInfo = get_ebuild_list_info(session, package_id)
 	for EbuildInfo in EbuildsInfo:
 		cpv = cp + '-' + EbuildInfo.Version
 		log_msg = "Checking: %s" % (cpv,)
-		write_log(session2, log_msg, "info", config_id, 'old_cpv.remove_old_ebuilds')
+		write_log(session, log_msg, "info", config_id, 'old_cpv.remove_old_ebuilds')
 		if not EbuildInfo.Active:
 			duration = today - EbuildInfo.TimeStamp
 			if duration.days > 30:
 				log_msg = "Removing: %s" % (cpv,)
-				write_log(session2, log_msg, "info", config_id, 'old_cpv.remove_old_ebuilds')
-				build_job_id_list = get_build_job_all(session2, EbuildInfo.EbuildId)
+				write_log(session, log_msg, "info", config_id, 'old_cpv.remove_old_ebuilds')
+				build_job_id_list = get_build_job_all(session, EbuildInfo.EbuildId)
 				if build_job_id_list != []:
 					for build_job in build_job_id_list:
-						del_old_build_jobs(session2, build_job.BuildJobId)
-				del_old_ebuild(session2, EbuildInfo.EbuildId)
-	if not get_ebuild_list_info(session2, package_id):
+						del_old_build_jobs(session, build_job.BuildJobId)
+				del_old_ebuild(session, EbuildInfo.EbuildId)
+	if get_ebuild_list_info(session, package_id) == []:
 		log_msg = "Removing: %s" % (cp,)
-		write_log(session2, log_msg, "info", config_id, 'old_cpv.remove_old_cpv_ebuilds')
-		del_old_package(session2, package_id)
-	session2.close
-	Session.remove()
+		write_log(session, log_msg, "info", config_id, 'old_cpv.remove_old_cpv_ebuilds')
+		del_old_package(session, package_id)
 
 def remove_old_cpv_main(session, config_id):
-	tbc_settings = read_config_settings()
-	# Use all cores when multiprocessing
-	#pool_cores = multiprocessing.cpu_count()
-	#pool = multiprocessing.Pool(processes = pool_cores)
-
+	today = datetime.utcnow()
 	CategorysInfo = get_category_list_info(session)
 	for CategoryInfo in CategorysInfo:
 		log_msg = "Checking: %s" % (CategoryInfo.Category,)
-		write_log(session2, log_msg, "info", config_id, 'old_cpv.remove_old_cpv_main')
+		write_log(session, log_msg, "info", config_id, 'old_cpv.remove_old_cpv_main')
 		PackagesInfo = get_package_list_info(session, CategoryInfo.CategoryId)
 		for PackageInfo in PackagesInfo:
 			cp = CategoryInfo.Category + '/' + PackageInfo.Package
-			# pool.apply_async( remove_old_ebuilds, (Package.PackageId, config_id, tbc_settings, cp,))
-			# use this when debuging
-			remove_old_ebuilds(PackageInfo.PackageId, config_id, tbc_settings, cp,)
+			remove_old_ebuilds(session, PackageInfo.PackageId, config_id, cp, today)
 
-		#close and join the multiprocessing pools
-		# pool.close()
-		# pool.join()
-		if not get_package_list_info(session, CategoryInfo.CategoryId):
+		if get_package_list_info(session, CategoryInfo.CategoryId) == []:
 			 add_old_category(session, CategoryInfo.CategoryId)

diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index 75e3d30..85621fe 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -585,25 +585,13 @@ def add_repoman_log(session, package_id, repoman_log, repoman_hash):
 			session.commit()
 
 def get_category_list_info(session):
-	try:
-		CategorysInfo = session.query(Categories).all()
-	except NoResultFound as e:
-		return False
-	return CategorysInfo
+	return session.query(Categories).all()
 
 def get_package_list_info(session, category_id):
-	try:
-		PackagesInfo = session.query(Packages).filter_by(CategoryId = category_id).all()
-	except NoResultFound as e:
-		return False
-	return PackagesInfo
+	return session.query(Packages).filter_by(CategoryId = category_id).all()
 
 def get_ebuild_list_info(session, package_id):
-	try:
-		EbuildsInfo = session.query(Ebuilds).filter_by(PackageId = package_id).all()
-	except NoResultFound as e:
-		return False
-	return EbuildsInfo
+	return session.query(Ebuilds).filter_by(PackageId = package_id).all()
 
 def del_old_ebuild(session, ebuild_id):
 	session.query(EbuildsRestrictions).filter(EbuildsRestrictions.EbuildId == ebuild_id).delete()

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     7ee36aabda72a4f6006d0fb0810ba8be5e80d9c4
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Feb 22 18:11:52 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Feb 22 18:11:52 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=7ee36aab

redone remove_old_cpv job

 pym/tbc/old_cpv.py | 21 +++++++--------------
 1 file changed, 7 insertions(+), 14 deletions(-)

diff --git a/pym/tbc/old_cpv.py b/pym/tbc/old_cpv.py
index cc80428..3c9ffd4 100644
--- a/pym/tbc/old_cpv.py
+++ b/pym/tbc/old_cpv.py
@@ -9,8 +9,7 @@ from sqlalchemy.orm import scoped_session, sessionmaker
 from tbc.ConnectionManager import NewConnection
 from tbc.readconf import  read_config_settings
 from tbc.sqlquerys import get_category_list_info, get_package_list_info, get_ebuild_list_info, \
-	get_build_job_all, del_old_build_jobs, del_old_ebuild, add_old_package,  del_old_package, \
-	add_old_category
+	get_build_job_all, del_old_build_jobs, del_old_ebuild, del_old_package, add_old_category
 
 def remove_old_ebuilds(package_id, config_id, tbc_settings, cp):
 	today = datetime.utcnow()
@@ -33,12 +32,13 @@ def remove_old_ebuilds(package_id, config_id, tbc_settings, cp):
 						del_old_build_jobs(session2, build_job.BuildJobId)
 				del_old_ebuild(session2, EbuildInfo.EbuildId)
 	if not get_ebuild_list_info(session2, package_id):
-		add_old_package(session2, package_id)
+		log_msg = "Removing: %s" % (cp,)
+		write_log(session2, log_msg, "info", config_id, 'old_cpv.remove_old_cpv_ebuilds')
+		del_old_package(session2, package_id)
 	session2.close
 	Session.remove()
 
 def remove_old_cpv_main(session, config_id):
-	today = datetime.datetime.utcnow()
 	tbc_settings = read_config_settings()
 	# Use all cores when multiprocessing
 	#pool_cores = multiprocessing.cpu_count()
@@ -51,16 +51,9 @@ def remove_old_cpv_main(session, config_id):
 		PackagesInfo = get_package_list_info(session, CategoryInfo.CategoryId)
 		for PackageInfo in PackagesInfo:
 			cp = CategoryInfo.Category + '/' + PackageInfo.Package
-			if PackageInfo.Active:
-				# pool.apply_async( remove_old_ebuilds, (Package.PackageId, config_id, tbc_settings, cp,))
-				# use this when debuging
-			 	remove_old_ebuilds(PackageInfo.PackageId, config_id, tbc_settings, cp,)
-			else:
-				duration = today - PackageInfo.TimeStamp
-				if duration.days > 30:
-					log_msg = "Removing: %s" % (cp,)
-					write_log(session2, log_msg, "info", config_id, 'old_cpv.remove_old_cpv_main')
-					del_old_package(session, PackageInfo.PackageId)
+			# pool.apply_async( remove_old_ebuilds, (Package.PackageId, config_id, tbc_settings, cp,))
+			# use this when debuging
+			remove_old_ebuilds(PackageInfo.PackageId, config_id, tbc_settings, cp,)
 
 		#close and join the multiprocessing pools
 		# pool.close()

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     5e5291855b1206e942125bc8fd3bca360c7fdcb5
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Feb 22 17:09:41 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Feb 22 17:09:41 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=5e529185

fix typo in remove_old_cpv_main()

 pym/tbc/old_cpv.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pym/tbc/old_cpv.py b/pym/tbc/old_cpv.py
index 027f171..cc80428 100644
--- a/pym/tbc/old_cpv.py
+++ b/pym/tbc/old_cpv.py
@@ -38,7 +38,7 @@ def remove_old_ebuilds(package_id, config_id, tbc_settings, cp):
 	Session.remove()
 
 def remove_old_cpv_main(session, config_id):
-	today = datetime.datetime.utcnow
+	today = datetime.datetime.utcnow()
 	tbc_settings = read_config_settings()
 	# Use all cores when multiprocessing
 	#pool_cores = multiprocessing.cpu_count()
@@ -65,5 +65,5 @@ def remove_old_cpv_main(session, config_id):
 		#close and join the multiprocessing pools
 		# pool.close()
 		# pool.join()
-		if not get_package_list_info(session, CategoryId = CategoryInfo.CategoryId):
-			 add_old_category(session, category_id)
+		if not get_package_list_info(session, CategoryInfo.CategoryId):
+			 add_old_category(session, CategoryInfo.CategoryId)

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     426c02c2521b862ed7876c3fdb3018b34f843224
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Feb 20 12:12:50 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Feb 20 12:12:50 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=426c02c2

Remove the profile/config and some prints

 pym/tbc/flags.py |  3 ---
 pym/tbc/sync.py  | 14 --------------
 2 files changed, 17 deletions(-)

diff --git a/pym/tbc/flags.py b/pym/tbc/flags.py
index eea48f3..82b7d42 100644
--- a/pym/tbc/flags.py
+++ b/pym/tbc/flags.py
@@ -117,7 +117,6 @@ class tbc_use_flags(object):
 		# reset cpv filter
 		self._mysettings.reset()
 		self._mysettings.lock()
-		print(usemask)
 		return use, use_expand_hidden, usemask, useforce
 
 	def get_all_cpv_use_looked(self):
@@ -218,7 +217,6 @@ class tbc_use_flags(object):
 			use_flagsDict[x] = True
 		for x in use_disable:
 			use_flagsDict[x] = False
-		print("use_flagsDict", use_flagsDict)
 		for k, v in use_flagsDict.items():
 			if build_use_flags_dict[k] != v:
 				if build_use_flags_dict[k]:
@@ -227,5 +225,4 @@ class tbc_use_flags(object):
 					build_use_flags_list.append("-" + k)
 		if build_use_flags_list == []:
 			build_use_flags_list = None
-		print(build_use_flags_list)
 		return build_use_flags_list

diff --git a/pym/tbc/sync.py b/pym/tbc/sync.py
index 59560f0..7063bfb 100644
--- a/pym/tbc/sync.py
+++ b/pym/tbc/sync.py
@@ -63,12 +63,6 @@ def git_sync_main(session):
 			GuestBusy = False
 		else:
 			time.sleep(60)
-	#remove the needed base profile clone
-	try:
-		os.remove(mysettings['PORTDIR'] + "/profiles/config/parent")
-		os.rmdir(mysettings['PORTDIR'] + "/profiles/config")
-	except:
-		pass
 
 	# check git diffs witch get updated and pass that to a dict
 	# fetch and merge the repo
@@ -105,14 +99,6 @@ def git_sync_main(session):
 			write_log(session, log_msg, "info", config_id, 'sync.git_sync_main')
 		log_msg = "Checking repo %s Done" % (reponame)
 		write_log(session, log_msg, "info", config_id, 'sync.git_sync_main')
-	# Need to add a clone of profiles/base for reading the tree
-	try:
-		os.mkdir(mysettings['PORTDIR'] + "/profiles/config", 0o777)
-		with open(mysettings['PORTDIR'] + "/profiles/config/parent", "w") as f:
-			f.write("../base\n")
-			f.close()
-	except:
-		pass
 
 	log_msg = "Repo sync ... Done."
 	write_log(session, log_msg, "info", config_id, 'sync.git_sync_main')

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     14b0918ae227f4135477946c8e6c9d9919b56fbf
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Wed Feb 10 20:25:25 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Wed Feb 10 20:25:25 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=14b0918a

fix so we mark any error as fail and some small fixes

 pym/tbc/build_log.py  | 11 +++++++----
 pym/tbc/db_mapping.py |  6 ++++++
 pym/tbc/package.py    |  2 +-
 pym/tbc/sqlquerys.py  |  8 ++++++--
 4 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/pym/tbc/build_log.py b/pym/tbc/build_log.py
index c70dc76..8b6ef0b 100644
--- a/pym/tbc/build_log.py
+++ b/pym/tbc/build_log.py
@@ -30,7 +30,7 @@ from tbc.sqlquerys import get_config_id, get_ebuild_id_db, add_new_buildlog, \
 	get_package_info, get_build_job_id, get_use_id, get_config_info, get_hilight_info, get_error_info_list, \
 	add_e_info, get_fail_times, add_fail_times, update_fail_times, del_old_build_jobs, add_old_ebuild, \
 	update_buildjobs_status, add_repoman_qa, get_config_id_fqdn, get_setup_info, \
-	add_repoman_log
+	add_repoman_log,  get_tbc_config
 from tbc.log import write_log
 
 from sqlalchemy.orm import sessionmaker
@@ -224,13 +224,13 @@ def get_buildlog_info(session, settings, pkg, build_dict, config_id):
 	for k, v in sorted(hilight_dict.items()):
 		if v['startline'] == v['endline']:
 			error_log_list.append(logfile_text_dict[k])
-			if v['hilight_css_id'] == 3: # qa = 3 and 4
+			if v['hilight_css_id'] == 3: # qa = 3
 				qa_error_list.append(logfile_text_dict[k])
 		else:
 			i = k
 			while i != (v['endline'] + 1):
 				error_log_list.append(logfile_text_dict[i])
-				if v['hilight_css_id'] == 3: # qa = 3 and 4
+				if v['hilight_css_id'] == 3: # qa = 3
 					qa_error_list.append(logfile_text_dict[i])
 				i = i +1
 
@@ -245,8 +245,10 @@ def get_buildlog_info(session, settings, pkg, build_dict, config_id):
 	build_log_dict['fail'] = False
 	if repoman_error_list:
 		sum_build_log_list.append(1) # repoman = 1
+		build_log_dict['fail'] = True
 	if qa_error_list != []:
 		sum_build_log_list.append(2) # qa = 2
+		build_log_dict['fail'] = True
 	else:
 		qa_error_list = False
 	for error_log_line in error_log_list:
@@ -324,7 +326,8 @@ def add_buildlog_main(settings, pkg, trees):
 					qa_msg = "QA: FAILD"
 				else:
 					build_msg = "BUILD: FAILD"
-		msg = "Package: %s Repo: %s %s %s %s Weblink http://77.110.8.67/new/logs/build/%s\n" % (pkg.cpv, pkg.repo, build_msg, repoman_msg, qa_msg, log_id,)
+		tbc_config =  get_tbc_config(session)
+		msg = "Package: %s Repo: %s %s %s %s Weblink http://%s/new/logs/build/%s\n" % (pkg.cpv, pkg.repo, build_msg, repoman_msg, qa_msg, tbc_config.WebIker, log_id,)
 		write_log(session, msg, "info", config_id, 'build_log.add_buildlog_main')
 		send_irk(msg)
 	session.close

diff --git a/pym/tbc/db_mapping.py b/pym/tbc/db_mapping.py
index b0ab23f..9be0d0a 100644
--- a/pym/tbc/db_mapping.py
+++ b/pym/tbc/db_mapping.py
@@ -287,3 +287,9 @@ class EbuildsMetadata(Base):
 	New = Column('new', Boolean, default=False)
 	Descriptions = Column('descriptions', Text)
 	__tablename__ = 'ebuilds_metadata'
+
+class TbcConfig(Base):
+	Id =  Column('id', Integer, primary_key=True)
+	WebIker = Column('webinker', String)
+	WebBug = Column('webbug', String)
+	__tablename__ = 'tbc_config'

diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index 510a5d2..c069c7e 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -136,7 +136,7 @@ class tbc_package(object):
 		attDict['ebuild_version_metadata_tree'] = ebuild_version_metadata_tree
 		#attDict['ebuild_version_text_tree'] = ebuild_version_text_tree[0]
 		attDict['git_commit'] = git_commit
-		attDict['New'] = False
+		attDict['new'] = False
 		attDict['ebuild_version_descriptions_tree'] = ebuild_version_metadata_tree[7]
 		return attDict
 

diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index 5a8a183..f9f14a2 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -9,12 +9,16 @@ from tbc.db_mapping import Configs, Logs, ConfigsMetaData, Jobs, BuildJobs, Pack
 	HiLightCss, BuildLogsHiLight, BuildLogsEmergeOptions, BuildLogsErrors, ErrorsInfo, EmergeInfo, BuildLogsUse, \
 	BuildJobsEmergeOptions, EbuildsMetadata, EbuildsIUse, Restrictions, EbuildsRestrictions, EbuildsKeywords, \
 	Keywords, PackagesMetadata, Emails, PackagesEmails, Setups, BuildLogsRepoman, CategoriesMetadata, \
-	PackagesRepoman, BuildLogsQa
+	PackagesRepoman, BuildLogsQa, TbcConfig
 from tbc.log import write_log
 from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
 from sqlalchemy import and_, or_
 
 # Guest Functions
+def get_tbc_config(session):
+	TbcConfigInfo = session.query(TbcConfig).one()
+	return TbcConfigInfo
+
 def get_config_id(session, setup, host):
 	SetupInfo = session.query(Setups).filter_by(Setup = setup).one()
 	ConfigInfo = session.query(Configs).filter_by(SetupId = SetupInfo.SetupId).filter_by(Hostname = host).one()
@@ -419,7 +423,7 @@ def add_new_ebuild_sql(session, packageDict):
 			except (MultipleResultsFound) as e:
 				# FIXME
 				sys.exit()
-		session.add(EbuildsMetadata(EbuildId = EbuildInfo.EbuildId, Commit = v['git_commit'], Descriptions = v['ebuild_version_descriptions_tree']))
+		session.add(EbuildsMetadata(EbuildId = EbuildInfo.EbuildId, New = v['new'], Commit = v['git_commit'], Descriptions = v['ebuild_version_descriptions_tree']))
 		session.commit()
 		ebuild_id_list.append(EbuildInfo.EbuildId)
 		restrictions = []

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     d1cf1a085c322865694c738c23e655421b9364ce
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Feb  2 23:14:54 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Feb  2 23:14:54 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=d1cf1a08

split build error from repoman/qa

 pym/tbc/build_log.py | 4 +---
 pym/tbc/sync.py      | 2 +-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/pym/tbc/build_log.py b/pym/tbc/build_log.py
index b194561..c70dc76 100644
--- a/pym/tbc/build_log.py
+++ b/pym/tbc/build_log.py
@@ -241,16 +241,14 @@ def get_buildlog_info(session, settings, pkg, build_dict, config_id):
 	pkgdir = myportdb.getRepositoryPath(build_dict['repo']) + "/" + categories + "/" + package
 	repoman_error_list = check_repoman_full(session, pkgdir, build_dict['package_id'], config_id, build_dict['cpv'])
 	build_log_dict = {}
+	error_search_line = "^ \\* ERROR: "
 	build_log_dict['fail'] = False
 	if repoman_error_list:
 		sum_build_log_list.append(1) # repoman = 1
-		build_log_dict['fail'] = True
 	if qa_error_list != []:
 		sum_build_log_list.append(2) # qa = 2
-		build_log_dict['fail'] = True
 	else:
 		qa_error_list = False
-	error_search_line = "^ \\* ERROR: "
 	for error_log_line in error_log_list:
 		if re.search(error_search_line, error_log_line):
 			build_log_dict['fail'] = True

diff --git a/pym/tbc/sync.py b/pym/tbc/sync.py
index 52d84ab..59560f0 100644
--- a/pym/tbc/sync.py
+++ b/pym/tbc/sync.py
@@ -73,7 +73,7 @@ def git_sync_main(session):
 	# check git diffs witch get updated and pass that to a dict
 	# fetch and merge the repo
 	repo_cp_dict = {}
-	search_list = [ '^metadata', '^eclass', '^licenses', '^profiles', '^scripts', '^skel.ebuild',]
+	search_list = [ '^metadata', '^eclass', '^licenses', '^profiles', '^scripts', '^skel.', ]
 	for repo_dir in git_repos_list(myportdb):
 		reponame = myportdb.getRepositoryName(repo_dir)
 		repo = git.Repo(repo_dir)

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     28dc9209b9e9787a1c9dc968f68ad89051a699f6
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Jan 31 13:40:04 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Jan 31 13:40:04 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=28dc9209

small fixes

 pym/tbc/package.py   |  2 +-
 pym/tbc/sqlquerys.py |  4 ++--
 pym/tbc/sync.py      | 20 ++++++++++----------
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index cf711ff..510a5d2 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -376,7 +376,7 @@ class tbc_package(object):
 			else:
 				# Remove cpv from packageDict and add ebuild to new ebuils list
 				del packageDict[cpv]
-				ebuild_id , status = get_ebuild_id_db(self._session, ebuild_version_checksum_tree, package_id)
+				ebuild_id , status = get_ebuild_id_db(self._session, ebuild_version_checksum_tree, package_id, ebuild_version_tree)
 				new_ebuild_id_list.append(ebuild_id)
 		self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list)
 

diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index 2fdb7f1..9ada5dd 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -504,10 +504,10 @@ def update_package_metadata(session, package_metadataDict):
 		try:
 			PackagesMetadataInfo = session.query(PackagesMetadata).filter_by(PackageId = k).one()
 		except NoResultFound as e:
-			session.add(PackagesMetadata(PackageId = k, Changlog = v['changelog_text'], Descriptions = v['metadata_xml_descriptions']))
+			session.add(PackagesMetadata(PackageId = k, Gitlog = v['git_changlog'], Descriptions = v['metadata_xml_descriptions']))
 			session.commit()
 		else:
-			PackagesMetadataInfo.Changlog = v['changelog_text']
+			PackagesMetadataInfo.Gitlog = v['git_changlog']
 			PackagesMetadataInfo.Descriptions = v['metadata_xml_descriptions']
 			session.commit()
 		if v['metadata_xml_email']:

diff --git a/pym/tbc/sync.py b/pym/tbc/sync.py
index 2d22e74..52d84ab 100644
--- a/pym/tbc/sync.py
+++ b/pym/tbc/sync.py
@@ -73,7 +73,7 @@ def git_sync_main(session):
 	# check git diffs witch get updated and pass that to a dict
 	# fetch and merge the repo
 	repo_cp_dict = {}
-	search_list = [ '^metadata', '^eclass', '^licenses', '^profiles', '^scripts',]
+	search_list = [ '^metadata', '^eclass', '^licenses', '^profiles', '^scripts', '^skel.ebuild',]
 	for repo_dir in git_repos_list(myportdb):
 		reponame = myportdb.getRepositoryName(repo_dir)
 		repo = git.Repo(repo_dir)
@@ -87,15 +87,15 @@ def git_sync_main(session):
 			repo_diff = repo.git.diff('origin', '--name-only')
 			write_log(session, 'Git dir diff:\n%s' % (repo_diff,), "debug", config_id, 'sync.git_sync_main')
 			for diff_line in repo_diff.splitlines():
-                                find_search = True
-                                for search_line in search_list:
-                                        if re.search(search_line, diff_line):
-                                                find_search = False
-                                if find_search:
-                                        splited_diff_line = re.split('/', diff_line)
-                                        cp = splited_diff_line[0] + '/' + splited_diff_line[1]
-                                        if not cp in cp_list:
-                                                cp_list.append(cp)
+				find_search = True
+				for search_line in search_list:
+					if re.search(search_line, diff_line):
+						find_search = False
+				if find_search:
+					splited_diff_line = re.split('/', diff_line)
+					cp = splited_diff_line[0] + '/' + splited_diff_line[1]
+					if not cp in cp_list:
+						cp_list.append(cp)
 			attr['cp_list'] = cp_list
 			write_log(session, 'Git CP Diff: %s' % (cp_list,), "debug", config_id, 'sync.git_sync_main')
 			repo_cp_dict[reponame] = attr

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     cc16d58199d60e5f65b86c3f8a59063e52c8cdae
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Wed Jan 27 05:22:11 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Wed Jan 27 05:22:11 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=cc16d581

check all ebuilds in package instead of only check Manifest small fix

 pym/tbc/package.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index 60e0de5..cf711ff 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -284,7 +284,7 @@ class tbc_package(object):
 				package_metadataDict[package_id]['metadata_xml_descriptions'] = packageDict[cpv]['ebuild_version_descriptions_tree']
 
 		packageDict[cpv]['new'] = True
-		self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_mtime_tree)
+		self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list)
 		log_msg = "C %s:%s ... Done." % (cp, repo)
 		write_log(self._session, log_msg, "info", self._config_id, 'packages.add_new_package_db')
 
@@ -378,7 +378,7 @@ class tbc_package(object):
 				del packageDict[cpv]
 				ebuild_id , status = get_ebuild_id_db(self._session, ebuild_version_checksum_tree, package_id)
 				new_ebuild_id_list.append(ebuild_id)
-		self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_mtime_tree)
+		self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list)
 
 		if package_updated:
 			# Check cp with repoman full

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     4b895c761e79db78b1c3640d5d053c93fdd98277
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Jan 26 23:17:20 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Jan 26 23:17:20 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=4b895c76

fix for Dups of checksums

 pym/tbc/build_log.py | 6 +++---
 pym/tbc/package.py   | 2 +-
 pym/tbc/sqlquerys.py | 6 +++---
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/pym/tbc/build_log.py b/pym/tbc/build_log.py
index d4fa0ef..b194561 100644
--- a/pym/tbc/build_log.py
+++ b/pym/tbc/build_log.py
@@ -108,16 +108,16 @@ def get_build_dict_db(session, config_id, settings, tbc_settings_dict, pkg):
 	pkgdir = myportdb.getRepositoryPath(repo) + "/" + categories + "/" + package
 	ebuild_version_checksum_tree = portage.checksum.sha256hash(pkgdir+ "/" + package + "-" + ebuild_version + ".ebuild")[0]
 	build_dict['checksum'] = ebuild_version_checksum_tree
-	ebuild_id_list, status = get_ebuild_id_db(session, build_dict['checksum'], build_dict['package_id'])
+	ebuild_id_list, status = get_ebuild_id_db(session, build_dict['checksum'], build_dict['package_id'], build_dict['ebuild_version'])
 	if status:
 		if ebuild_id_list is None:
 			log_msg = "%s:%s Don't have any ebuild_id!" % (pkg.cpv, repo,)
-			add_logs(session, log_msg, "error", config_id)
+			write_log(session, log_msg, "error", config_id, 'build_log.get_build_dict_db')
 		else:
 			old_ebuild_id_list = []
 			for ebuild_id in ebuild_id_list:
 				log_msg = "%s:%s:%s Dups of checksums" % (pkg.cpv, repo, ebuild_id,)
-				add_logs(session, log_msg, "error", config_id)
+				write_log(session, log_msg, "error", config_id, 'build_log.get_build_dict_db')
 				old_ebuild_id_list.append(ebuild_id)
 			add_old_ebuild(session, old_ebuild_id_list)
 		return

diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index bb797de..60e0de5 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -351,7 +351,7 @@ class tbc_package(object):
 			elif fail:
 				dupe_ebuild_id_list = []
 				for checksum in checksums_db:
-					ebuilds_id , status = get_ebuild_id_db(self._session, checksum, package_id)
+					ebuilds_id , status = get_ebuild_id_db(self._session, checksum, package_id, ebuild_version_tree)
 					for ebuild_id in ebuilds_id:
 						log_msg = "U %s:%s:%s Dups of checksums" % (cpv, repo, ebuild_id,)
 						write_log(self._session, log_msg, "warning", self._config_id, 'packages.update_package_db')

diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index 4161461..2fdb7f1 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -545,13 +545,13 @@ def get_ebuild_checksums(session, package_id, ebuild_version):
 		return ebuild_checksum_list, True
 	return EbuildInfo.Checksum, False
 
-def get_ebuild_id_db(session, checksum, package_id):
+def get_ebuild_id_db(session, checksum, package_id, ebuild_version):
 	try:
-		EbuildInfos = session.query(Ebuilds).filter_by(PackageId = package_id).filter_by(Checksum = checksum).one()
+		EbuildInfos = session.query(Ebuilds).filter_by(PackageId = package_id).filter_by(Checksum = checksum).filter_by(Version = ebuild_version).filter_by(Active = True).one()
 	except NoResultFound as e:
 		return None, True
 	except MultipleResultsFound as e:
-		EbuildInfos = session.query(Ebuilds).filter_by(PackageId = package_id).filter_by(Checksum = checksum).all()
+		EbuildInfos = session.query(Ebuilds).filter_by(PackageId = package_id).filter_by(Checksum = checksum).filter_by(Version = ebuild_version).filter_by(Active = True).all()
 		ebuilds_id = []
 		for EbuildInfo in EbuildInfos:
 			ebuilds_id.append(EbuildInfo.EbuildId)

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     85d70df173a81e8257172d39be1ae971be54422b
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Jan 26 20:30:38 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Jan 26 20:30:38 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=85d70df1

check all ebuilds in package instead of only check Manifest

 pym/tbc/build_log.py  |   5 +-
 pym/tbc/db_mapping.py |   1 -
 pym/tbc/package.py    | 151 +++++++++++++++++++++-----------------------------
 pym/tbc/sqlquerys.py  |   5 --
 pym/tbc/sync.py       |  20 +++++--
 5 files changed, 80 insertions(+), 102 deletions(-)

diff --git a/pym/tbc/build_log.py b/pym/tbc/build_log.py
index a16cdc6..ec4aaa7 100644
--- a/pym/tbc/build_log.py
+++ b/pym/tbc/build_log.py
@@ -29,7 +29,7 @@ from tbc.ConnectionManager import NewConnection
 from tbc.sqlquerys import get_config_id, get_ebuild_id_db, add_new_buildlog, \
 	get_package_info, get_build_job_id, get_use_id, get_config_info, get_hilight_info, get_error_info_list, \
 	add_e_info, get_fail_times, add_fail_times, update_fail_times, del_old_build_jobs, add_old_ebuild, \
-	update_buildjobs_status, update_mtime_sql, add_repoman_qa, get_config_id_fqdn, get_setup_info, \
+	update_buildjobs_status, add_repoman_qa, get_config_id_fqdn, get_setup_info, \
 	add_repoman_log
 from tbc.log import write_log
 
@@ -112,8 +112,7 @@ def get_build_dict_db(session, config_id, settings, tbc_settings_dict, pkg):
 	if status:
 		if ebuild_id_list is None:
 			log_msg = "%s:%s Don't have any ebuild_id!" % (pkg.cpv, repo,)
-			add_logs(session, log_msg, "info", config_id)
-			update_mtime_sql(session, build_dict['package_id'], 0)
+			add_logs(session, log_msg, "error", config_id)
 		else:
 			old_ebuild_id_list = []
 			for ebuild_id in ebuild_id_list:

diff --git a/pym/tbc/db_mapping.py b/pym/tbc/db_mapping.py
index 2636bb6..d9fcf25 100644
--- a/pym/tbc/db_mapping.py
+++ b/pym/tbc/db_mapping.py
@@ -84,7 +84,6 @@ class Packages(Base):
 	CategoryId = Column('category_id', Integer, ForeignKey('categories.category_id'))
 	Package = Column('package',String(150))
 	RepoId = Column('repo_id', Integer, ForeignKey('repos.repo_id'))
-	Mtime = Column('mtime', DateTime, default=0)
 	Active = Column('active', Boolean, default=False)
 	TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow)
 	__tablename__ = 'packages'

diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index 65a0f40..68574cf 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -17,7 +17,7 @@ from tbc.build_log import check_repoman_full
 from tbc.log import write_log
 from tbc.sqlquerys import get_package_info, get_config_info, \
 	add_new_build_job, add_new_ebuild_sql, get_ebuild_id_list, add_old_ebuild, \
-	get_package_metadata_sql, update_package_metadata, update_mtime_sql, \
+	get_package_metadata_sql, update_package_metadata, \
 	get_package_info_from_package_id, get_config_all_info, add_new_package_sql, \
 	get_ebuild_checksums, get_ebuild_id_db, get_configmetadata_info, get_setup_info, \
 	get_ebuild_info_ebuild_id, get_ebuild_restrictions, add_old_package
@@ -210,7 +210,7 @@ class tbc_package(object):
 		package_metadataDict[package_id] = attDict
 		return package_metadataDict
 
-	def add_package(self, packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_mtime_tree):
+	def add_package(self, packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list):
 		# Use packageDict to update the db
 		ebuild_id_list = add_new_ebuild_sql(self._session, packageDict)
 
@@ -227,9 +227,6 @@ class tbc_package(object):
 		# update package metadata
 		update_package_metadata(self._session, package_metadataDict)
 
-		# update the cp manifest checksum
-		update_mtime_sql(self._session, package_id, manifest_mtime_tree)
-
 		# Get the best cpv for the configs and add it to config_cpv_listDict
 		PackageInfo, CategoryInfo, RepoInfo = get_package_info_from_package_id(self._session, package_id)
 		cp = CategoryInfo.Category + '/' + PackageInfo.Package
@@ -244,18 +241,6 @@ class tbc_package(object):
 		# Add the ebuild to the build jobs table if needed
 		self.add_new_build_job_db(ebuild_id_list, packageDict, config_cpv_listDict)
 
-	def get_manifest_mtime_tree(self, pkgdir, cp, repo, mytree):
-		# Get the cp manifest file mtime.
-		try:
-			mtime = os.path.getmtime(pkgdir + "/Manifest")
-		except:
-			log_msg = "QA: Can't checksum the Manifest file. :%s:%s" % (cp, repo,)
-			write_log(self._session, log_msg, "warning", self._config_id, 'packages.get_manifest_mtime_tree')
-			log_msg = "C %s:%s ... Fail." % (cp, repo)
-			write_log(self._session, log_msg, "warning", self._config_id, 'packages.get_manifest_mtime_tree')
-			return False
-		return datetime.datetime.fromtimestamp(mtime).replace(microsecond=0)
-
 	def add_new_package_db(self, cp, repo):
 		# Add new categories package ebuild to tables package and ebuilds
 		# C = Checking
@@ -268,9 +253,6 @@ class tbc_package(object):
 		mytree = []
 		mytree.append(repodir)
 		pkgdir = repodir + "/" + cp # Get RepoDIR + cp
-		manifest_mtime_tree = self.get_manifest_mtime_tree(pkgdir, cp, repo, mytree)
-		if not manifest_mtime_tree:
-			return None
 		
 		package_id = add_new_package_sql(self._session, cp, repo)
 
@@ -332,83 +314,78 @@ class tbc_package(object):
 		
 		mytree = []
 		mytree.append(repodir)
-		manifest_mtime_tree = self.get_manifest_mtime_tree(pkgdir, cp, repo, mytree)
-		if not manifest_mtime_tree:
+
+		# Get the ebuild list for cp
+		old_ebuild_id_list = []
+		ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree)
+		if ebuild_list_tree == []:
+			log_msg = "QA: Can't get the ebuilds list. %s:%s" % (cp, repo,)
+			write_log(self._session, log_msg, "error", self._config_id, 'packages.update_package_db')
+			log_msg = "C %s:%s ... Fail." % (cp, repo)
+			write_log(self._session, log_msg, "warning", self._config_id, 'packages.update_package_db')
 			return None
 
-		# if we NOT have the same mtime in the db update the package
-		if PackageInfo.Mtime is None or manifest_mtime_tree > PackageInfo.Mtime:
+		package_metadataDict = self.get_package_metadataDict(pkgdir, repodir, package_id, cp)
+		packageDict ={}
+		new_ebuild_id_list = []
+		package_updated = False
+		for cpv in sorted(ebuild_list_tree):
 
-			# U = Update
-			log_msg = "U %s:%s" % (cp, repo)
-			write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')
+			# split out ebuild version
+			ebuild_version_tree = portage.versions.cpv_getversion(cpv)
 
-			# Get the ebuild list for cp
-			old_ebuild_id_list = []
-			ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree)
-			if ebuild_list_tree == []:
-				log_msg = "QA: Can't get the ebuilds list. %s:%s" % (cp, repo,)
-				write_log(self._session, log_msg, "error", self._config_id, 'packages.update_package_db')
-				log_msg = "C %s:%s ... Fail." % (cp, repo)
-				write_log(self._session, log_msg, "warning", self._config_id, 'packages.update_package_db')
-				return None
+			# Get packageDict for cpv
+			packageDict[cpv] = self.get_packageDict(pkgdir, cpv, repo)
+
+			# take package descriptions from the ebuilds
+			if package_metadataDict[package_id]['metadata_xml_descriptions'] != packageDict[cpv]['ebuild_version_descriptions_tree']:
+				package_metadataDict[package_id]['metadata_xml_descriptions'] = packageDict[cpv]['ebuild_version_descriptions_tree']
+
+			# Get the checksum of the ebuild in tree and db
+			ebuild_version_checksum_tree = packageDict[cpv]['checksum']
+			checksums_db, fail = get_ebuild_checksums(self._session, package_id, ebuild_version_tree)
+
+			# check if we have dupes of the checksum from db
+			if checksums_db is None:
+				ebuild_version_manifest_checksum_db = None
+			elif fail:
+				dupe_ebuild_id_list = []
+				for checksum in checksums_db:
+					ebuilds_id , status = get_ebuild_id_db(self._session, checksum, package_id)
+					for ebuild_id in ebuilds_id:
+						log_msg = "U %s:%s:%s Dups of checksums" % (cpv, repo, ebuild_id,)
+						write_log(self._session, log_msg, "warning", self._config_id, 'packages.update_package_db')
+						dupe_ebuild_id_list.append(ebuild_id)
+				add_old_ebuild(self._session, dupe_ebuild_id_list)
+				ebuild_version_manifest_checksum_db = None
+			else:
+				ebuild_version_manifest_checksum_db = checksums_db
+
+			# Check if the checksum have change
+			if ebuild_version_manifest_checksum_db is None:
+				# N = New ebuild
+				log_msg = "N %s:%s" % (cpv, repo,)
+				write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')
+				packageDict[cpv]['new'] = True
+				package_updated = True
+			elif  ebuild_version_checksum_tree != ebuild_version_manifest_checksum_db:
+				# U = Updated ebuild
+				log_msg = "U %s:%s" % (cpv, repo,)
+				write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')
+				package_updated = True
+			else:
+				# Remove cpv from packageDict and add ebuild to new ebuils list
+				del packageDict[cpv]
+				ebuild_id , status = get_ebuild_id_db(self._session, ebuild_version_checksum_tree, package_id)
+				new_ebuild_id_list.append(ebuild_id)
+		self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_mtime_tree)
 
+		if package_updated:
 			# Check cp with repoman full
 			status = check_repoman_full(self._session, pkgdir, package_id, self._config_id)
 			if status:
 				log_msg = "Repoman %s::%s ... Fail." % (cp, repo)
 				write_log(self._session, log_msg, "warning", self._config_id, 'packages.update_package_db')
-			package_metadataDict = self.get_package_metadataDict(pkgdir, repodir, package_id, cp)
-			packageDict ={}
-			new_ebuild_id_list = []
-			for cpv in sorted(ebuild_list_tree):
-
-				# split out ebuild version
-				ebuild_version_tree = portage.versions.cpv_getversion(cpv)
-
-				# Get packageDict for cpv
-				packageDict[cpv] = self.get_packageDict(pkgdir, cpv, repo)
-
-				# take package descriptions from the ebuilds
-				if package_metadataDict[package_id]['metadata_xml_descriptions'] != packageDict[cpv]['ebuild_version_descriptions_tree']:
-					package_metadataDict[package_id]['metadata_xml_descriptions'] = packageDict[cpv]['ebuild_version_descriptions_tree']
-
-				# Get the checksum of the ebuild in tree and db
-				ebuild_version_checksum_tree = packageDict[cpv]['checksum']
-				checksums_db, fail = get_ebuild_checksums(self._session, package_id, ebuild_version_tree)
-
-				# check if we have dupes of the checksum from db
-				if checksums_db is None:
-					ebuild_version_manifest_checksum_db = None
-				elif fail:
-					dupe_ebuild_id_list = []
-					for checksum in checksums_db:
-						ebuilds_id , status = get_ebuild_id_db(self._session, checksum, package_id)
-						for ebuild_id in ebuilds_id:
-							log_msg = "U %s:%s:%s Dups of checksums" % (cpv, repo, ebuild_id,)
-							write_log(self._session, log_msg, "warning", self._config_id, 'packages.update_package_db')
-							dupe_ebuild_id_list.append(ebuild_id)
-					add_old_ebuild(self._session, dupe_ebuild_id_list)
-					ebuild_version_manifest_checksum_db = None
-				else:
-					ebuild_version_manifest_checksum_db = checksums_db
-
-				# Check if the checksum have change
-				if ebuild_version_manifest_checksum_db is None:
-					# N = New ebuild
-					log_msg = "N %s:%s" % (cpv, repo,)
-					write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')
-					packageDict[cpv]['new'] = True
-				elif  ebuild_version_checksum_tree != ebuild_version_manifest_checksum_db:
-					# U = Updated ebuild
-					log_msg = "U %s:%s" % (cpv, repo,)
-					write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')
-				else:
-					# Remove cpv from packageDict and add ebuild to new ebuils list
-					del packageDict[cpv]
-					ebuild_id , status = get_ebuild_id_db(self._session, ebuild_version_checksum_tree, package_id)
-					new_ebuild_id_list.append(ebuild_id)
-			self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_mtime_tree)
 
 		log_msg = "C %s:%s ... Done." % (cp, repo)
 		write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')

diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index e43b8fc..4161461 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -515,11 +515,6 @@ def update_package_metadata(session, package_metadataDict):
 				EmailInfo = update_email_info(session, email)
 				PackagesEmailInfo = update_package_email_info(session, EmailInfo.EmailId, k)
 
-def update_mtime_sql(session, package_id, manifest_mtime_tree):
-	PackagesInfo = session.query(Packages).filter_by(PackageId = package_id).one()
-	PackagesInfo.Mtime = manifest_mtime_tree
-	session.commit()
-
 def get_package_info_from_package_id(session, package_id):
 	PackageInfo = session.query(Packages).filter_by(PackageId = package_id).one()
 	CategoryInfo = session.query(Categories).filter_by(CategoryId = PackageInfo.CategoryId).one()

diff --git a/pym/tbc/sync.py b/pym/tbc/sync.py
index 1732f1f..2d22e74 100644
--- a/pym/tbc/sync.py
+++ b/pym/tbc/sync.py
@@ -70,9 +70,10 @@ def git_sync_main(session):
 	except:
 		pass
 
-	# check git diffs witch Manifests get updated and pass that to a dict
+	# check git diffs witch get updated and pass that to a dict
 	# fetch and merge the repo
 	repo_cp_dict = {}
+	search_list = [ '^metadata', '^eclass', '^licenses', '^profiles', '^scripts',]
 	for repo_dir in git_repos_list(myportdb):
 		reponame = myportdb.getRepositoryName(repo_dir)
 		repo = git.Repo(repo_dir)
@@ -82,14 +83,21 @@ def git_sync_main(session):
 		if not repouptodate:
 			cp_list = []
 			attr = {}
-			# We check for Manifest changes and add the package to a list
+			# We check for dir changes and add the package to a list
 			repo_diff = repo.git.diff('origin', '--name-only')
-			write_log(session, 'Git diff: %s' % (repo_diff,), "debug", config_id, 'sync.git_sync_main')
+			write_log(session, 'Git dir diff:\n%s' % (repo_diff,), "debug", config_id, 'sync.git_sync_main')
 			for diff_line in repo_diff.splitlines():
-				if re.search("Manifest$", diff_line):
-						cp = re.sub('/Manifest', '', diff_line)
-						cp_list.append(cp)
+                                find_search = True
+                                for search_line in search_list:
+                                        if re.search(search_line, diff_line):
+                                                find_search = False
+                                if find_search:
+                                        splited_diff_line = re.split('/', diff_line)
+                                        cp = splited_diff_line[0] + '/' + splited_diff_line[1]
+                                        if not cp in cp_list:
+                                                cp_list.append(cp)
 			attr['cp_list'] = cp_list
+			write_log(session, 'Git CP Diff: %s' % (cp_list,), "debug", config_id, 'sync.git_sync_main')
 			repo_cp_dict[reponame] = attr
 			git_merge(repo, info_list[0])
 		else:

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     71d06a89176b48e1dbdbb971b703422334efc540
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Jan 25 22:48:27 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Jan 25 22:48:27 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=71d06a89

add support for logging part2

 pym/tbc/build_log.py |  2 +-
 pym/tbc/package.py   | 65 +++++++++++++++++++++++++++-------------------------
 pym/tbc/sqlquerys.py |  2 +-
 3 files changed, 36 insertions(+), 33 deletions(-)

diff --git a/pym/tbc/build_log.py b/pym/tbc/build_log.py
index 70cd091..a16cdc6 100644
--- a/pym/tbc/build_log.py
+++ b/pym/tbc/build_log.py
@@ -304,7 +304,7 @@ def add_buildlog_main(settings, pkg, trees):
 	build_log_dict['log_hash'] = log_hash.hexdigest()
 	build_log_dict['logfilename'] = settings.get("PORTAGE_LOG_FILE").split(host_config)[1]
 	log_msg = "Logfile name: %s" % (settings.get("PORTAGE_LOG_FILE"),)
-        write_log(session, log_msg, "info", config_id, 'build_log.add_buildlog_main')
+	write_log(session, log_msg, "info", config_id, 'build_log.add_buildlog_main')
 	build_log_dict['emerge_info'] = get_emerge_info_id(settings, trees, session, config_id)
 	log_id = add_new_buildlog(session, build_dict, build_log_dict)
 

diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index 72108a2..65a0f40 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -7,6 +7,7 @@ import hashlib
 import os
 import git
 import portage
+import datetime
 from portage.xml.metadata import MetaDataXML
 from tbc.flags import tbc_use_flags
 from tbc.text import get_ebuild_cvs_revision, get_log_text_dict
@@ -83,6 +84,7 @@ class tbc_package(object):
 				if ebuild_auxdb_list[i] == '':
 					ebuild_auxdb_list[i] = ''
 			return ebuild_auxdb_list
+
 	def get_git_log_ebuild(self, repodir, ebuild_file):
 		git_log_ebuild = ''
 		g = git.Git(repodir)
@@ -105,11 +107,12 @@ class tbc_package(object):
 		except:
 			ebuild_version_checksum_tree = "0"
 			log_msg = "QA: Can't checksum the ebuild file. %s on repo %s" % (cpv, repo,)
-			write_log(session, log_msg, "warning", config_id, 'packages.get_packageDict')
+			write_log(self._session, log_msg, "warning", self._config_id, 'packages.get_packageDict')
 			log_msg = "C %s:%s ... Fail." % (cpv, repo)
-			write_log(session, log_msg, "info", config_id, 'packages.get_packageDict')
+			write_log(self._session, log_msg, "info", self._config_id, 'packages.get_packageDict')
 			git_commit = '0'
 		else:
+			repodir =self._ myportdb.getRepositoryPath(repo)
 			git_commit = self.get_git_log_ebuild(repodir, ebuild_file)
 
 		# Get the ebuild metadata
@@ -119,7 +122,7 @@ class tbc_package(object):
 		# so it can be updated next time we update the db
 		if not ebuild_version_metadata_tree:
 			log_msg = " QA: %s have broken metadata on repo %s" % (cpv, repo)
-			write_log(session, log_msg, "warning", config_id, 'packages.get_packageDict')
+			write_log(self._session, log_msg, "warning", self._config_id, 'packages.get_packageDict')
 			ebuild_version_metadata_tree = ['','','','','','','','','','','','','','','','','','','','','','','','','']
 			ebuild_version_checksum_tree = '0'
 
@@ -168,17 +171,17 @@ class tbc_package(object):
 						# B = Build cpv use-flags config
 						# FIXME log_msg need a fix to log the use flags corect.
 						log_msg = "B %s:%s USE: %s Setup: %s" % (k, v['repo'], use_flagsDict, setup_id,)
-						write_log(session, log_msg, "info", config_id, 'packages.add_new_build_job_db')
+						write_log(self._session, log_msg, "info", self._config_id, 'packages.add_new_build_job_db')
 					i = i +1
 
-	def get_git_log_pkg_text(self, repodir, cp):
+	def get_git_changelog_text(self, repodir, cp):
 		n = '5'
 		git_log_pkg = ''
 		g = git.Git(repodir)
 		git_log_pkg = g.log('-n ' + n, '--grep=' + cp)
 		return git_log_pkg
 
-	def get_package_metadataDict(self, pkgdir, package_id):
+	def get_package_metadataDict(self, pkgdir, repodir, package_id, cp):
 		# Make package_metadataDict
 		attDict = {}
 		package_metadataDict = {}
@@ -189,7 +192,7 @@ class tbc_package(object):
 			pkg_md = MetaDataXML(pkgdir + "/metadata.xml", herd)
 		except:
 			log_msg = "Metadata file %s is missing or has errors" % (pkgdir + "/metadata.xml")
-			write_log(session, log_msg, "warning", config_id, 'packages.get_package_metadataDict')
+			write_log(self._session, log_msg, "warning", self._config_id, 'packages.get_package_metadataDict')
 		else:
 			tmp_herds = pkg_md.herds()
 			if tmp_herds != ():
@@ -201,8 +204,8 @@ class tbc_package(object):
 				attDict['metadata_xml_email'] = md_email_list
 			else:
 				log_msg = "Metadata file %s missing Email" % (pkgdir + "/metadata.xml")
-				write_log(session, log_msg, "warning", config_id, 'packages.get_package_metadataDict')
-		attDict['git_log_pkg_text'] = self.get_git_log_pkg_text(repodir, cp)
+				write_log(self._session, log_msg, "warning", self._config_id, 'packages.get_package_metadataDict')
+		attDict['git_changlog'] = self.get_git_changelog_text(repodir, cp)
 		attDict['metadata_xml_descriptions'] = ''
 		package_metadataDict[package_id] = attDict
 		return package_metadataDict
@@ -247,9 +250,9 @@ class tbc_package(object):
 			mtime = os.path.getmtime(pkgdir + "/Manifest")
 		except:
 			log_msg = "QA: Can't checksum the Manifest file. :%s:%s" % (cp, repo,)
-			write_log(session, log_msg, "warning", config_id, 'packages.get_manifest_mtime_tree')
+			write_log(self._session, log_msg, "warning", self._config_id, 'packages.get_manifest_mtime_tree')
 			log_msg = "C %s:%s ... Fail." % (cp, repo)
-			write_log(session, log_msg, "warning", config_id, 'packages.get_manifest_mtime_tree')
+			write_log(self._session, log_msg, "warning", self._config_id, 'packages.get_manifest_mtime_tree')
 			return False
 		return datetime.datetime.fromtimestamp(mtime).replace(microsecond=0)
 
@@ -258,9 +261,9 @@ class tbc_package(object):
 		# C = Checking
 		# N = New Package
 		log_msg = "C %s:%s" % (cp, repo)
-		write_log(session, log_msg, "info", config_id, 'packages.add_new_package_db')
+		write_log(self._session, log_msg, "info", self._config_id, 'packages.add_new_package_db')
 		log_msg = "N %s:%s" % (cp, repo)
-		write_log(session, log_msg, "info", config_id, 'packages.add_new_package_db')
+		write_log(self._session, log_msg, "info", self._config_id, 'packages.add_new_package_db')
 		repodir = self._myportdb.getRepositoryPath(repo)
 		mytree = []
 		mytree.append(repodir)
@@ -275,16 +278,16 @@ class tbc_package(object):
 		status = check_repoman_full(self._session, pkgdir, package_id, self._config_id)
 		if status:
 			log_msg = "Repoman %s::%s ... Fail." % (cp, repo)
-			write_log(session, log_msg, "warning", config_id, 'packages.add_new_package_db')
+			write_log(self._session, log_msg, "warning", self._config_id, 'packages.add_new_package_db')
 
-		package_metadataDict = self.get_package_metadataDict(pkgdir, package_id)
+		package_metadataDict = self.get_package_metadataDict(pkgdir, repodir, package_id, cp)
 		# Get the ebuild list for cp
 		ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree)
 		if ebuild_list_tree == []:
 			log_msg = "QA: Can't get the ebuilds list. %s:%s" % (cp, repo,)
-			write_log(session, log_msg, "error", config_id, 'packages.add_new_package_db')
+			write_log(self._session, log_msg, "error", self._config_id, 'packages.add_new_package_db')
 			log_msg = "C %s:%s ... Fail." % (cp, repo)
-			write_log(session, log_msg, "warning", config_id, 'packages.add_new_package_db')
+			write_log(self._session, log_msg, "warning", self._config_id, 'packages.add_new_package_db')
 			return None
 
 		# Make the needed packageDict with ebuild infos so we can add it later to the db.
@@ -301,7 +304,7 @@ class tbc_package(object):
 		packageDict[cpv]['new'] = True
 		self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_mtime_tree)
 		log_msg = "C %s:%s ... Done." % (cp, repo)
-		write_log(session, log_msg, "info", config_id, 'packages.add_new_package_db')
+		write_log(self._session, log_msg, "info", self._config_id, 'packages.add_new_package_db')
 
 	def update_package_db(self, package_id):
 		# Update the categories and package with new info
@@ -310,7 +313,7 @@ class tbc_package(object):
 		cp = CategoryInfo.Category + '/' + PackageInfo.Package
 		repo = RepoInfo.Repo
 		log_msg = "C %s:%s" % (cp, repo)
-		write_log(session, log_msg, "info", config_id, 'packages.update_package_db')
+		write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')
 		repodir = self._myportdb.getRepositoryPath(repo)
 		pkgdir = repodir + "/" + cp # Get RepoDIR + cp
 		if not os.path.isdir(pkgdir):
@@ -320,16 +323,16 @@ class tbc_package(object):
 				cpv = cp + "-" + EbuildInfo.Version
 				# R =  remove ebuild
 				log_msg = "R %s:%s" % (cpv, repo,)
-				write_log(session, log_msg, "info", config_id, 'packages.update_package_db')
+				write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')
 			add_old_ebuild(self._session, old_ebuild_id_list)
 			add_old_package(self._session, package_id)
 			log_msg = "C %s:%s ... Done." % (cp, repo)
-			write_log(session, log_msg, "info", config_id, 'packages.update_package_db')
+			write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')
 			return None
 		
 		mytree = []
 		mytree.append(repodir)
-		manifest_mtime_tree = self.get_manifest_checksum_tree(pkgdir, cp, repo, mytree)
+		manifest_mtime_tree = self.get_manifest_mtime_tree(pkgdir, cp, repo, mytree)
 		if not manifest_mtime_tree:
 			return None
 
@@ -338,24 +341,24 @@ class tbc_package(object):
 
 			# U = Update
 			log_msg = "U %s:%s" % (cp, repo)
-			write_log(session, log_msg, "info", config_id, 'packages.update_package_db')
+			write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')
 
 			# Get the ebuild list for cp
 			old_ebuild_id_list = []
 			ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree)
 			if ebuild_list_tree == []:
 				log_msg = "QA: Can't get the ebuilds list. %s:%s" % (cp, repo,)
-				write_log(session, log_msg, "error", config_id, 'packages.update_package_db')
+				write_log(self._session, log_msg, "error", self._config_id, 'packages.update_package_db')
 				log_msg = "C %s:%s ... Fail." % (cp, repo)
-				write_log(session, log_msg, "warning", config_id, 'packages.update_package_db')
+				write_log(self._session, log_msg, "warning", self._config_id, 'packages.update_package_db')
 				return None
 
 			# Check cp with repoman full
 			status = check_repoman_full(self._session, pkgdir, package_id, self._config_id)
 			if status:
 				log_msg = "Repoman %s::%s ... Fail." % (cp, repo)
-				write_log(session, log_msg, "warning", config_id, 'packages.update_package_db')
-			package_metadataDict = self.get_package_metadataDict(pkgdir, package_id)
+				write_log(self._session, log_msg, "warning", self._config_id, 'packages.update_package_db')
+			package_metadataDict = self.get_package_metadataDict(pkgdir, repodir, package_id, cp)
 			packageDict ={}
 			new_ebuild_id_list = []
 			for cpv in sorted(ebuild_list_tree):
@@ -383,7 +386,7 @@ class tbc_package(object):
 						ebuilds_id , status = get_ebuild_id_db(self._session, checksum, package_id)
 						for ebuild_id in ebuilds_id:
 							log_msg = "U %s:%s:%s Dups of checksums" % (cpv, repo, ebuild_id,)
-							write_log(session, log_msg, "warning", config_id, 'packages.update_package_db')
+							write_log(self._session, log_msg, "warning", self._config_id, 'packages.update_package_db')
 							dupe_ebuild_id_list.append(ebuild_id)
 					add_old_ebuild(self._session, dupe_ebuild_id_list)
 					ebuild_version_manifest_checksum_db = None
@@ -394,12 +397,12 @@ class tbc_package(object):
 				if ebuild_version_manifest_checksum_db is None:
 					# N = New ebuild
 					log_msg = "N %s:%s" % (cpv, repo,)
-					write_log(session, log_msg, "info", config_id, 'packages.update_package_db')
+					write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')
 					packageDict[cpv]['new'] = True
 				elif  ebuild_version_checksum_tree != ebuild_version_manifest_checksum_db:
 					# U = Updated ebuild
 					log_msg = "U %s:%s" % (cpv, repo,)
-					write_log(session, log_msg, "info", config_id, 'packages.update_package_db')
+					write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')
 				else:
 					# Remove cpv from packageDict and add ebuild to new ebuils list
 					del packageDict[cpv]
@@ -408,4 +411,4 @@ class tbc_package(object):
 			self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_mtime_tree)
 
 		log_msg = "C %s:%s ... Done." % (cp, repo)
-		write_log(session, log_msg, "info", config_id, 'packages.update_package_db')
+		write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')

diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index 5100bec..e43b8fc 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -422,7 +422,7 @@ def add_new_ebuild_sql(session, packageDict):
 			except (MultipleResultsFound) as e:
 				# FIXME
 				sys.exit()
-		session.add(EbuildsMetadata(EbuildId = EbuildInfo.EbuildId, Revision = v['ebuild_version_revision_tree'], Descriptions = v['ebuild_version_descriptions_tree']))
+		session.add(EbuildsMetadata(EbuildId = EbuildInfo.EbuildId, Commit = v['git_commit'], Descriptions = v['ebuild_version_descriptions_tree']))
 		session.commit()
 		ebuild_id_list.append(EbuildInfo.EbuildId)
 		restrictions = []

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     250f837e65c6e28381ca57ecbe97085974902bdf
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Jan 18 20:28:04 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Jan 18 20:28:04 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=250f837e

move the hash for the build log

 pym/tbc/build_log.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pym/tbc/build_log.py b/pym/tbc/build_log.py
index 79dbe2a..7b3784a 100644
--- a/pym/tbc/build_log.py
+++ b/pym/tbc/build_log.py
@@ -28,7 +28,7 @@ from tbc.ConnectionManager import NewConnection
 from tbc.sqlquerys import add_logs, get_config_id, get_ebuild_id_db, add_new_buildlog, \
 	get_package_info, get_build_job_id, get_use_id, get_config_info, get_hilight_info, get_error_info_list, \
 	add_e_info, get_fail_times, add_fail_times, update_fail_times, del_old_build_jobs, add_old_ebuild, \
-	update_buildjobs_status, update_manifest_sql, add_repoman_qa, get_config_id_fqdn, get_setup_info, \
+	update_buildjobs_status, update_mtime_sql, add_repoman_qa, get_config_id_fqdn, get_setup_info, \
 	add_repoman_log
 from sqlalchemy.orm import sessionmaker
 
@@ -107,7 +107,7 @@ def get_build_dict_db(session, config_id, settings, tbc_settings_dict, pkg):
 		if ebuild_id_list is None:
 			log_msg = "%s:%s Don't have any ebuild_id!" % (pkg.cpv, repo,)
 			add_logs(session, log_msg, "info", config_id)
-			update_manifest_sql(session, build_dict['package_id'], "0")
+			update_mtime_sql(session, build_dict['package_id'], 0)
 		else:
 			old_ebuild_id_list = []
 			for ebuild_id in ebuild_id_list:
@@ -293,7 +293,7 @@ def add_buildlog_main(settings, pkg, trees):
 	if error_log_list != []:
 		for log_line in error_log_list:
 			build_error = build_error + log_line
-			log_hash.update(log_line.encode('utf-8'))
+		log_hash.update(build_error.encode('utf-8'))
 	build_log_dict['build_error'] = build_error
 	build_log_dict['log_hash'] = log_hash.hexdigest()
 	build_log_dict['logfilename'] = settings.get("PORTAGE_LOG_FILE").split(host_config)[1]
@@ -322,7 +322,7 @@ def add_buildlog_main(settings, pkg, trees):
 					qa_msg = "QA: FAILD"
 				else:
 					build_msg = "BUILD: FAILD"
-		msg = "Package: %s Repo: %s %s %s %s Weblink http://foo.gg.oo/buildpackage/%s\n" % (pkg.cpv, pkg.repo, build_msg, repoman_msg, qa_msg, log_id,)
+		msg = "Package: %s Repo: %s %s %s %s Weblink http://foo.gg.oo/new/logs/build/%s\n" % (pkg.cpv, pkg.repo, build_msg, repoman_msg, qa_msg, log_id,)
 		send_irk(msg)
 	session.close
 

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     80b205f448c096b57772d42706e5180f2c4849e0
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Jan 10 12:54:58 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Jan 10 12:54:58 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=80b205f4

change from checksum to mtime on Manifest

 pym/tbc/db_mapping.py |  2 +-
 pym/tbc/package.py    | 83 ++++++++++++++++++++++++++-------------------------
 pym/tbc/sqlquerys.py  |  6 ++--
 3 files changed, 46 insertions(+), 45 deletions(-)

diff --git a/pym/tbc/db_mapping.py b/pym/tbc/db_mapping.py
index 82318a1..2636bb6 100644
--- a/pym/tbc/db_mapping.py
+++ b/pym/tbc/db_mapping.py
@@ -84,7 +84,7 @@ class Packages(Base):
 	CategoryId = Column('category_id', Integer, ForeignKey('categories.category_id'))
 	Package = Column('package',String(150))
 	RepoId = Column('repo_id', Integer, ForeignKey('repos.repo_id'))
-	Checksum = Column('checksum', String(100))
+	Mtime = Column('mtime', DateTime, default=0)
 	Active = Column('active', Boolean, default=False)
 	TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow)
 	__tablename__ = 'packages'

diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index 091f75f..40de495 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -15,7 +15,7 @@ from tbc.qachecks import check_repoman
 from tbc.build_log import check_repoman_full
 from tbc.sqlquerys import add_logs, get_package_info, get_config_info, \
 	add_new_build_job, add_new_ebuild_sql, get_ebuild_id_list, add_old_ebuild, \
-	get_package_metadata_sql, update_package_metadata, update_manifest_sql, \
+	get_package_metadata_sql, update_package_metadata, update_mtime_sql, \
 	get_package_info_from_package_id, get_config_all_info, add_new_package_sql, \
 	get_ebuild_checksums, get_ebuild_id_db, get_configmetadata_info, get_setup_info, \
 	get_ebuild_info_ebuild_id, get_ebuild_restrictions, add_old_package
@@ -206,7 +206,7 @@ class tbc_package(object):
 		package_metadataDict[package_id] = attDict
 		return package_metadataDict
 
-	def add_package(self, packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_checksum_tree):
+	def add_package(self, packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_mtime_tree):
 		# Use packageDict to update the db
 		ebuild_id_list = add_new_ebuild_sql(self._session, packageDict)
 
@@ -224,7 +224,7 @@ class tbc_package(object):
 		update_package_metadata(self._session, package_metadataDict)
 
 		# update the cp manifest checksum
-		update_manifest_sql(self._session, package_id, manifest_checksum_tree)
+		update_mtime_sql(self._session, package_id, manifest_mtime_tree)
 
 		# Get the best cpv for the configs and add it to config_cpv_listDict
 		PackageInfo, CategoryInfo, RepoInfo = get_package_info_from_package_id(self._session, package_id)
@@ -240,20 +240,17 @@ class tbc_package(object):
 		# Add the ebuild to the build jobs table if needed
 		self.add_new_build_job_db(ebuild_id_list, packageDict, config_cpv_listDict)
 
-	def get_manifest_checksum_tree(self, pkgdir, cp, repo, mytree):
-		# Get the cp manifest file checksum.
+	def get_manifest_mtime_tree(self, pkgdir, cp, repo, mytree):
+		# Get the cp manifest file mtime.
 		try:
-			manifest_checksum_tree = portage.checksum.sha256hash(pkgdir + "/Manifest")[0]
+			mtime = os.path.getmtime(pkgdir + "/Manifest")
 		except:
-			# Get the package list from the repo
-			package_list_tree =self. _myportdb.cp_all(trees=mytree)
-			if cp in package_list_tree:
-				log_msg = "QA: Can't checksum the Manifest file. :%s:%s" % (cp, repo,)
-				add_logs(self._session, log_msg, "error", self._config_id)
-				log_msg = "C %s:%s ... Fail." % (cp, repo)
-				add_logs(self._session, log_msg, "error", self._config_id)
-			return "0"
-		return manifest_checksum_tree
+			log_msg = "QA: Can't checksum the Manifest file. :%s:%s" % (cp, repo,)
+			add_logs(self._session, log_msg, "error", self._config_id)
+			log_msg = "C %s:%s ... Fail." % (cp, repo)
+			add_logs(self._session, log_msg, "error", self._config_id)
+			return False
+		return datetime.datetime.fromtimestamp(mtime).replace(microsecond=0)
 
 	def add_new_package_db(self, cp, repo):
 		# Add new categories package ebuild to tables package and ebuilds
@@ -267,8 +264,10 @@ class tbc_package(object):
 		mytree = []
 		mytree.append(repodir)
 		pkgdir = repodir + "/" + cp # Get RepoDIR + cp
-		manifest_checksum_tree = self.get_manifest_checksum_tree(pkgdir, cp, repo, mytree)
-
+		manifest_mtime_tree = self.get_manifest_mtime_tree(pkgdir, cp, repo, mytree)
+		if not manifest_mtime_tree:
+			return None
+		
 		package_id = add_new_package_sql(self._session, cp, repo)
 
 		# Check cp with repoman full
@@ -299,7 +298,7 @@ class tbc_package(object):
 				package_metadataDict[package_id]['metadata_xml_descriptions'] = packageDict[cpv]['ebuild_version_descriptions_tree']
 
 		packageDict[cpv]['new'] = True
-		self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_checksum_tree)
+		self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_mtime_tree)
 		log_msg = "C %s:%s ... Done." % (cp, repo)
 		add_logs(self._session, log_msg, "info", self._config_id)
 
@@ -313,12 +312,28 @@ class tbc_package(object):
 		add_logs(self._session, log_msg, "info", self._config_id)
 		repodir = self._myportdb.getRepositoryPath(repo)
 		pkgdir = repodir + "/" + cp # Get RepoDIR + cp
+		if not os.path.isdir(pkgdir):
+			old_ebuild_id_list = get_ebuild_id_list(self._session, package_id)
+			for ebuild_id in old_ebuild_id_list:
+				EbuildInfo = get_ebuild_info_ebuild_id(self._session, ebuild_id)
+				cpv = cp + "-" + EbuildInfo.Version
+				# R =  remove ebuild
+				log_msg = "R %s:%s" % (cpv, repo,)
+				add_logs(self._session, log_msg, "info", self._config_id)
+			add_old_ebuild(self._session, old_ebuild_id_list)
+			add_old_package(self._session, package_id)
+			log_msg = "C %s:%s ... Done." % (cp, repo)
+			add_logs(self._session, log_msg, "info", self._config_id)
+			return None
+		
 		mytree = []
 		mytree.append(repodir)
-		manifest_checksum_tree = self.get_manifest_checksum_tree(pkgdir, cp, repo, mytree)
+		manifest_mtime_tree = self.get_manifest_checksum_tree(pkgdir, cp, repo, mytree)
+		if not manifest_mtime_tree:
+			return None
 
-		# if we NOT have the same checksum in the db update the package
-		if manifest_checksum_tree != PackageInfo.Checksum:
+		# if we NOT have the same mtime in the db update the package
+		if PackageInfo.Mtime is None or manifest_mtime_tree > PackageInfo.Mtime:
 
 			# U = Update
 			log_msg = "U %s:%s" % (cp, repo)
@@ -328,25 +343,11 @@ class tbc_package(object):
 			old_ebuild_id_list = []
 			ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree)
 			if ebuild_list_tree == []:
-				if manifest_checksum_tree == "0":
-					old_ebuild_id_list = get_ebuild_id_list(self._session, package_id)
-					for ebuild_id in old_ebuild_id_list:
-						EbuildInfo = get_ebuild_info_ebuild_id(self._session, ebuild_id)
-						cpv = cp + "-" + EbuildInfo.Version
-						# R =  remove ebuild
-						log_msg = "R %s:%s" % (cpv, repo,)
-						add_logs(self._session, log_msg, "info", self._config_id)
-						if not os.path.isdir(pkgdir):
-							add_old_package(self._session, package_id)
-					add_old_ebuild(self._session, old_ebuild_id_list)
-					log_msg = "C %s:%s ... Done." % (cp, repo)
-					add_logs(self._session, log_msg, "info", self._config_id)
-				else:
-					log_msg = "QA: Can't get the ebuilds list. %s:%s" % (cp, repo,)
-					add_logs(self._session, log_msg, "info", self._config_id)
-					log_msg = "C %s:%s ... Fail." % (cp, repo)
-					add_logs(self._session, log_msg, "info", self._config_id)
-				return
+				log_msg = "QA: Can't get the ebuilds list. %s:%s" % (cp, repo,)
+				add_logs(self._session, log_msg, "info", self._config_id)
+				log_msg = "C %s:%s ... Fail." % (cp, repo)
+				add_logs(self._session, log_msg, "info", self._config_id)
+				return None
 
 			# Check cp with repoman full
 			status = check_repoman_full(self._session, pkgdir, package_id, self._config_id)
@@ -403,7 +404,7 @@ class tbc_package(object):
 					del packageDict[cpv]
 					ebuild_id , status = get_ebuild_id_db(self._session, ebuild_version_checksum_tree, package_id)
 					new_ebuild_id_list.append(ebuild_id)
-			self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_checksum_tree)
+			self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_mtime_tree)
 
 		log_msg = "C %s:%s ... Done." % (cp, repo)
 		add_logs(self._session, log_msg, "info", self._config_id)

diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index 7791e6e..e93c96d 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -466,7 +466,7 @@ def add_new_package_sql(session, cp, repo):
 	repo_id = RepoInfo.RepoId
 	CategoriesInfo = get_category_info(session, categories)
 	category_id = CategoriesInfo.CategoryId
-	session.add(Packages(Package = package, CategoryId = category_id, RepoId = repo_id, Checksum = '0', Active = True))
+	session.add(Packages(Package = package, CategoryId = category_id, RepoId = repo_id, Active = True))
 	session.commit()
 	PackageInfo = get_package_info(session, categories, package, repo)
 	return PackageInfo.PackageId
@@ -512,9 +512,9 @@ def update_package_metadata(session, package_metadataDict):
 				EmailInfo = update_email_info(session, email)
 				PackagesEmailInfo = update_package_email_info(session, EmailInfo.EmailId, k)
 
-def update_manifest_sql(session, package_id, manifest_checksum_tree):
+def update_mtime_sql(session, package_id, manifest_mtime_tree):
 	PackagesInfo = session.query(Packages).filter_by(PackageId = package_id).one()
-	PackagesInfo.Checksum = manifest_checksum_tree
+	PackagesInfo.Mtime = manifest_mtime_tree
 	session.commit()
 
 def get_package_info_from_package_id(session, package_id):

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     2e76eff16ecad10b5106080359e49c896bea950f
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Jan  7 07:35:20 2016 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Jan  7 07:35:20 2016 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=2e76eff1

add support for repoman cpv

 pym/tbc/build_log.py | 52 +++++++++++++++++++++++++++++++++++-----------------
 1 file changed, 35 insertions(+), 17 deletions(-)

diff --git a/pym/tbc/build_log.py b/pym/tbc/build_log.py
index 52db083..79dbe2a 100644
--- a/pym/tbc/build_log.py
+++ b/pym/tbc/build_log.py
@@ -32,25 +32,35 @@ from tbc.sqlquerys import add_logs, get_config_id, get_ebuild_id_db, add_new_bui
 	add_repoman_log
 from sqlalchemy.orm import sessionmaker
 
-def check_repoman_full(session, pkgdir, package_id, config_id):
+def check_repoman_full(session, pkgdir, package_id, config_id, cpv=False):
 	# Check cp with repoman repoman full
 	status = repoman_full(session, pkgdir, config_id)
 	repoman_hash = hashlib.sha256()
-	repoman_log = ""
+	if cpv:
+		element = cpv.split('/')
+		pv = element[1]
 	if status:
+		repoman_dict = {}
 		for k, v in status.items():
-			repoman_line = k + '/n'
-			repoman_hash.update(repoman_line.encode('utf-8'))
-			repoman_log = repoman_log + repoman_line
+			repoman_log2 = []
 			for line in v:
-				repoman_line = line + '/n'
-				repoman_hash.update(repoman_line.encode('utf-8'))
-				repoman_log = repoman_log + repoman_line
-		add_repoman_log(session, package_id, repoman_log, repoman_hash.hexdigest())
-		return repoman_log
-	else:
-		return status
-
+				if not cpv:
+					repoman_log2.append(line)
+				elif cpv and re.search(pv + '.ebuild', line):
+					repoman_log2.append(line)
+			if not repoman_log2 == []:
+				repoman_dict[k] = repoman_log2
+		if not repoman_dict == {}:
+			repoman_log = ""
+			for k, v in repoman_dict.items():
+				repoman_log = repoman_log + k + "\n"
+				repoman_hash.update(k.encode('utf-8'))
+				for line in v:
+					repoman_log = repoman_log + line + "\n"
+					repoman_hash.update(line.encode('utf-8'))
+			add_repoman_log(session, package_id, repoman_log, repoman_hash.hexdigest())
+			return repoman_log
+	return False
 
 def get_build_dict_db(session, config_id, settings, tbc_settings_dict, pkg):
 	myportdb = portage.portdbapi(mysettings=settings)
@@ -224,7 +234,7 @@ def get_buildlog_info(session, settings, pkg, build_dict, config_id):
 	categories = element[0]
 	package = element[1]
 	pkgdir = myportdb.getRepositoryPath(build_dict['repo']) + "/" + categories + "/" + package
-	repoman_error_list = check_repoman_full(session, pkgdir, build_dict['package_id'], config_id)
+	repoman_error_list = check_repoman_full(session, pkgdir, build_dict['package_id'], config_id, build_dict['cpv'])
 	build_log_dict = {}
 	build_log_dict['fail'] = False
 	if repoman_error_list:
@@ -301,10 +311,18 @@ def add_buildlog_main(settings, pkg, trees):
 		log_msg = "Package: %s:%s is logged." % (pkg.cpv, pkg.repo,)
 		add_logs(session, log_msg, "info", config_id)
 		print("\n>>> Logging %s:%s\n" % (pkg.cpv, pkg.repo,))
+		build_msg = "BUILD: PASS"
+		qa_msg = "QA: PASS"
+		repoman_msg = "REPOMAN: PASS"
 		if build_log_dict['fail']:
-			msg = "Package: %s Repo: %s FAILD Weblink http://foo.gg.oo/buildpackage/%s\n" % (pkg.cpv, pkg.repo, log_id,)
-		else:
-			msg = "Package: %s Repo: %s PASS Weblink http://foo.gg.oo/buildpackage/%s\n" % (pkg.cpv, pkg.repo, log_id,)
+			for error_id in build_log_dict['summary_error_list']:
+				if error_id == 1:
+					repoman_msg = "REPOMAN: FAILD"
+				elif error_id ==2:
+					qa_msg = "QA: FAILD"
+				else:
+					build_msg = "BUILD: FAILD"
+		msg = "Package: %s Repo: %s %s %s %s Weblink http://foo.gg.oo/buildpackage/%s\n" % (pkg.cpv, pkg.repo, build_msg, repoman_msg, qa_msg, log_id,)
 		send_irk(msg)
 	session.close
 

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     8f4a556c399d1efd129a5a339c1d65e0801d1471
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Dec 21 22:53:31 2015 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Dec 21 22:53:31 2015 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=8f4a556c

add support for irker

 pym/tbc/build_log.py |  7 ++++++-
 pym/tbc/irk.py       | 28 ++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/pym/tbc/build_log.py b/pym/tbc/build_log.py
index bbe35d4..52db083 100644
--- a/pym/tbc/build_log.py
+++ b/pym/tbc/build_log.py
@@ -19,7 +19,7 @@ from _emerge.main import parse_opts
 portage.proxy.lazyimport.lazyimport(globals(),
 	'tbc.actions:action_info,load_emerge_config',
 )
-
+from tbc.irk import send_irk
 from tbc.qachecks import check_repoman, repoman_full
 from tbc.text import get_log_text_dict
 from tbc.readconf import read_config_settings
@@ -301,6 +301,11 @@ def add_buildlog_main(settings, pkg, trees):
 		log_msg = "Package: %s:%s is logged." % (pkg.cpv, pkg.repo,)
 		add_logs(session, log_msg, "info", config_id)
 		print("\n>>> Logging %s:%s\n" % (pkg.cpv, pkg.repo,))
+		if build_log_dict['fail']:
+			msg = "Package: %s Repo: %s FAILD Weblink http://foo.gg.oo/buildpackage/%s\n" % (pkg.cpv, pkg.repo, log_id,)
+		else:
+			msg = "Package: %s Repo: %s PASS Weblink http://foo.gg.oo/buildpackage/%s\n" % (pkg.cpv, pkg.repo, log_id,)
+		send_irk(msg)
 	session.close
 
 def log_fail_queru(session, build_dict, settings):

diff --git a/pym/tbc/irk.py b/pym/tbc/irk.py
new file mode 100644
index 0000000..01765c7
--- /dev/null
+++ b/pym/tbc/irk.py
@@ -0,0 +1,28 @@
+from __future__ import unicode_literals
+
+import json
+import socket
+import sys
+
+DEFAULT_SERVER = ("192.168.0.5", 6659)
+
+def connect(server = DEFAULT_SERVER):
+	return socket.create_connection(server)
+
+def send(s, target, message):
+	data = {"to": target, "privmsg" : message}
+	s.sendall(json.dumps(data).encode('ascii'))
+
+def irk(target, message, server = DEFAULT_SERVER):
+	s = connect(server)
+	if "irc:" not in target and "ircs:" not in target:
+		target = "irc://chat.freenode.net/{0}".format(target)
+		send(s, target, message)
+	s.close()
+
+def send_irk(msg):
+	target = "tinderbox-cluster"
+	try:
+		irk(target, msg)
+	except socket.error as e:
+		sys.stderr.write("irk: write to server failed: %r\n" % e)
\ No newline at end of file

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     9d888380b877c7e310fc7799962776d17a242821
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Dec 20 00:12:08 2015 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Dec 20 00:12:08 2015 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=9d888380

fix for thin manifest

 pym/tbc/qachecks.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/pym/tbc/qachecks.py b/pym/tbc/qachecks.py
index acb080e..892642f 100644
--- a/pym/tbc/qachecks.py
+++ b/pym/tbc/qachecks.py
@@ -22,10 +22,6 @@ import portage
 def check_file_in_manifest(pkgdir, mysettings, portdb, cpv, build_use_flags_list, repo):
 	myfetchlistdict = portage.FetchlistDict(pkgdir, mysettings, portdb)
 	my_manifest = portage.Manifest(pkgdir, mysettings['DISTDIR'], fetchlist_dict=myfetchlistdict, manifest1_compat=False, from_scratch=False)
-	ebuild_version = portage.versions.cpv_getversion(cpv)
-	package = portage.versions.cpv_getkey(cpv).split("/")[1]
-	if my_manifest.findFile(package + "-" + ebuild_version + ".ebuild") is None:
-		return "Ebuild file not found.\n"
 	tree = portdb.getRepositoryPath(repo)
 	cpv_fetchmap = portdb.getFetchMap(cpv, useflags=build_use_flags_list, mytree=tree)
 	mysettings.unlock()
@@ -37,12 +33,12 @@ def check_file_in_manifest(pkgdir, mysettings, portdb, cpv, build_use_flags_list
 	finally:
 		mysettings.lock()
 	try:
-		my_manifest.checkCpvHashes(cpv, checkDistfiles=True, onlyDistfiles=False, checkMiscfiles=True)
+		my_manifest.checkCpvHashes(cpv, checkDistfiles=True, onlyDistfiles=True, checkMiscfiles=False)
 	except:
 		return "Can't fetch the file or the hash failed.\n"
 	try:
 		portdb.fetch_check(cpv, useflags=build_use_flags_list, mysettings=mysettings, all=False)
-	except:	
+	except:
 		return "Fetch check failed.\n"
 	return
 

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     6698f841f9f77bad00a963ae61bf0625d5ce35c3
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Oct  6 19:31:35 2015 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Oct  6 19:31:35 2015 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=6698f841

add more repoman fixes

 pym/tbc/package.py  | 2 +-
 pym/tbc/qachecks.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index ef9b453..091f75f 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -109,7 +109,7 @@ class tbc_package(object):
 			add_logs(self._session, log_msg, "info", self._config_id)
 			git_commit = '0'
 		else:
-			git_commit = self.get_git_log_ebuild(repodir, ebuild_file):
+			git_commit = self.get_git_log_ebuild(repodir, ebuild_file)
 
 		# Get the ebuild metadata
 		ebuild_version_metadata_tree = self.get_ebuild_metadata(cpv, repo)

diff --git a/pym/tbc/qachecks.py b/pym/tbc/qachecks.py
index 67b1448..451f315 100644
--- a/pym/tbc/qachecks.py
+++ b/pym/tbc/qachecks.py
@@ -14,7 +14,7 @@ from portage import os, _encodings, _unicode_decode, _unicode_encode
 from portage.exception import DigestException, FileNotFound, ParseError, PermissionDenied
 from _emerge.Package import Package
 from _emerge.RootConfig import RootConfig
-from tbc.repoman.checks.ebuilds.checks import run_checks
+from repoman.checks.ebuilds.checks import run_checks
 from tbc.repoman.main import repoman_main
 from tbc.sqlquerys import get_configmetadata_info, get_config_info, get_setup_info
 import portage

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     988511359bd3f58aa91f8cd94179b8f44779719a
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Sep  6 20:28:38 2015 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Sep  6 20:28:38 2015 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=98851135

add git log file commit ID

 pym/tbc/package.py | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index 6224846..ef9b453 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -82,6 +82,13 @@ class tbc_package(object):
 				if ebuild_auxdb_list[i] == '':
 					ebuild_auxdb_list[i] = ''
 			return ebuild_auxdb_list
+	def get_git_log_ebuild(self, repodir, ebuild_file):
+		git_log_ebuild = ''
+		g = git.Git(repodir)
+		for line in g.log('-n 1', ebuild_file).splitlines():
+			if re.search('^commit', line):
+				git_log_ebuild = re.sub('commit', '', line)
+		return git_log_ebuild
 
 	def get_packageDict(self, pkgdir, cpv, repo):
 
@@ -90,10 +97,10 @@ class tbc_package(object):
 		element = portage.versions.cpv_getkey(cpv).split('/')
 		categories = element[0]
 		package = element[1]
-
+		ebuild_file = pkgdir + "/" + package + "-" + ebuild_version_tree + ".ebuild"
 		# Make a checksum of the ebuild
 		try:
-			ebuild_version_checksum_tree = portage.checksum.sha256hash(pkgdir + "/" + package + "-" + ebuild_version_tree + ".ebuild")[0]
+			ebuild_version_checksum_tree = portage.checksum.sha256hash(ebuild_file)[0]
 		except:
 			ebuild_version_checksum_tree = "0"
 			log_msg = "QA: Can't checksum the ebuild file. %s on repo %s" % (cpv, repo,)
@@ -102,8 +109,7 @@ class tbc_package(object):
 			add_logs(self._session, log_msg, "info", self._config_id)
 			git_commit = '0'
 		else:
-			git_commit = '0'
-		#FIXME The git commit need to be fixed or when $ID get fixed
+			git_commit = self.get_git_log_ebuild(repodir, ebuild_file):
 
 		# Get the ebuild metadata
 		ebuild_version_metadata_tree = self.get_ebuild_metadata(cpv, repo)
@@ -177,14 +183,13 @@ class tbc_package(object):
 		package_metadataDict = {}
 		md_email_list = []
 		herd = None
+		attDict['metadata_xml_email'] = False
 		try:
 			pkg_md = MetaDataXML(pkgdir + "/metadata.xml", herd)
 		except:
 			log_msg = "Metadata file %s is missing or has errors" % (pkgdir + "/metadata.xml")
 			add_logs(self._session, log_msg, "qa", self._config_id)
-			attDict['metadata_xml_email'] = False
 		else:
-			attDict['git_log_pkg_text'] = self.get_git_log_pkg_text(repodir, cp)
 			tmp_herds = pkg_md.herds()
 			if tmp_herds != ():
 				attDict['metadata_xml_herds'] = tmp_herds[0]
@@ -196,7 +201,7 @@ class tbc_package(object):
 			else:
 				log_msg = "Metadata file %s missing Email" % (pkgdir + "/metadata.xml")
 				add_logs(self._session, log_msg, "qa", self._config_id)
-				attDict['metadata_xml_email'] = False
+		attDict['git_log_pkg_text'] = self.get_git_log_pkg_text(repodir, cp)
 		attDict['metadata_xml_descriptions'] = ''
 		package_metadataDict[package_id] = attDict
 		return package_metadataDict

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     aac3ed04e3bc5d555b2432ea40dd262335dfd776
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Sep  3 21:02:29 2015 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Sep  3 21:02:29 2015 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=aac3ed04

Add support move from cvs to git

 pym/tbc/db_mapping.py |  7 ++--
 pym/tbc/package.py    | 91 ++++++++++++++++++-----------------------------
 pym/tbc/qachecks.py   | 97 ---------------------------------------------------
 pym/tbc/sync.py       | 17 ++++-----
 4 files changed, 44 insertions(+), 168 deletions(-)

diff --git a/pym/tbc/db_mapping.py b/pym/tbc/db_mapping.py
index dbae5a8..82318a1 100644
--- a/pym/tbc/db_mapping.py
+++ b/pym/tbc/db_mapping.py
@@ -103,7 +103,7 @@ class PackagesEmails(Base):
 class PackagesMetadata(Base):
 	Id = Column('id', Integer, primary_key=True)
 	PackageId = Column('package_id', Integer, ForeignKey('packages.package_id'))
-	Changlog = Column('changlog', Text)
+	Gitlog = Column('gitlog', Text)
 	Descriptions = Column('descriptions', Text)
 	__tablename__ = 'packages_metadata'
 
@@ -278,6 +278,7 @@ class EbuildsKeywords(Base):
 class EbuildsMetadata(Base):
 	Id =  Column('id', Integer, primary_key=True)
 	EbuildId = Column('ebuild_id', ForeignKey('ebuilds.ebuild_id'))
-	Revision = Column('revision', String(30))
-	Descriptions = Column('descriptions', String(200))
+	Commit = Column('commit', String(100))
+	New = Column('new', Boolean, default=False)
+	Descriptions = Column('descriptions', Text)
 	__tablename__ = 'ebuilds_metadata'

diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index 980b089..6224846 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -5,12 +5,13 @@ from __future__ import print_function
 import re
 import hashlib
 import os
+import git
 import portage
 from portage.xml.metadata import MetaDataXML
 from tbc.flags import tbc_use_flags
 from tbc.text import get_ebuild_cvs_revision, get_log_text_dict
 from tbc.flags import tbc_use_flags
-from tbc.qachecks import digestcheck, check_repoman
+from tbc.qachecks import check_repoman
 from tbc.build_log import check_repoman_full
 from tbc.sqlquerys import add_logs, get_package_info, get_config_info, \
 	add_new_build_job, add_new_ebuild_sql, get_ebuild_id_list, add_old_ebuild, \
@@ -99,9 +100,10 @@ class tbc_package(object):
 			add_logs(self._session, log_msg, "info", self._config_id)
 			log_msg = "C %s:%s ... Fail." % (cpv, repo)
 			add_logs(self._session, log_msg, "info", self._config_id)
-			ebuild_version_cvs_revision_tree = '0'
+			git_commit = '0'
 		else:
-			ebuild_version_cvs_revision_tree = get_ebuild_cvs_revision(pkgdir + "/" + package + "-" + ebuild_version_tree + ".ebuild")
+			git_commit = '0'
+		#FIXME The git commit need to be fixed or when $ID get fixed
 
 		# Get the ebuild metadata
 		ebuild_version_metadata_tree = self.get_ebuild_metadata(cpv, repo)
@@ -123,7 +125,8 @@ class tbc_package(object):
 		attDict['checksum']= ebuild_version_checksum_tree
 		attDict['ebuild_version_metadata_tree'] = ebuild_version_metadata_tree
 		#attDict['ebuild_version_text_tree'] = ebuild_version_text_tree[0]
-		attDict['ebuild_version_revision_tree'] = ebuild_version_cvs_revision_tree
+		attDict['git_commit'] = git_commit
+		attDict['New'] = False
 		attDict['ebuild_version_descriptions_tree'] = ebuild_version_metadata_tree[7]
 		return attDict
 
@@ -161,33 +164,12 @@ class tbc_package(object):
 						add_logs(self._session, log_msg, "info", self._config_id)
 					i = i +1
 
-	def get_changelog_text(self, pkgdir):
-		changelog_text_dict, max_text_lines = get_log_text_dict(pkgdir + "/ChangeLog")
-		spec = 3
-		spec_tmp = 1
-		changelog_text_tree = ''
-		for index, text_line in changelog_text_dict.items():
-			if index == max_text_lines:
-				if not re.search('^\n', text_line):
-					changelog_text_tree = changelog_text_tree + text_line
-				break
-			elif re.search('^#', text_line):
-				pass
-			elif re.search('^\n', text_line) and re.search('^#', changelog_text_dict[index - 1]):
-				pass
-			elif re.search('^\n', text_line) and re.search('^\*', changelog_text_dict[index + 1]):
-				changelog_text_tree = changelog_text_tree + text_line
-				spec_tmp = spec_tmp + 1
-				spec = spec + 1
-			elif re.search('^\n', text_line) and not re.search('^\*', changelog_text_dict[index + 1]):
-				if spec_tmp == spec:
-					break
-				else:
-					spec_tmp = spec_tmp + 1
-					changelog_text_tree = changelog_text_tree + text_line
-			else:
-				changelog_text_tree = changelog_text_tree + text_line
-		return changelog_text_tree
+	def get_git_log_pkg_text(self, repodir, cp):
+		n = '5'
+		git_log_pkg = ''
+		g = git.Git(repodir)
+		git_log_pkg = g.log('-n ' + n, '--grep=' + cp)
+		return git_log_pkg
 
 	def get_package_metadataDict(self, pkgdir, package_id):
 		# Make package_metadataDict
@@ -195,20 +177,26 @@ class tbc_package(object):
 		package_metadataDict = {}
 		md_email_list = []
 		herd = None
-		pkg_md = MetaDataXML(pkgdir + "/metadata.xml", herd)
-		attDict['changelog_text'] =  self.get_changelog_text(pkgdir)
-		tmp_herds = pkg_md.herds()
-		if tmp_herds != ():
-			attDict['metadata_xml_herds'] = tmp_herds[0]
-			md_email_list.append(attDict['metadata_xml_herds'] + '@gentoo.org')
-		for maint in pkg_md.maintainers():
-			md_email_list.append(maint.email)
-		if md_email_list != []:
-			attDict['metadata_xml_email'] = md_email_list
-		else:
-			log_msg = "Metadata file %s missing Email" % (pkgdir + "/metadata.xml")
+		try:
+			pkg_md = MetaDataXML(pkgdir + "/metadata.xml", herd)
+		except:
+			log_msg = "Metadata file %s is missing or has errors" % (pkgdir + "/metadata.xml")
 			add_logs(self._session, log_msg, "qa", self._config_id)
 			attDict['metadata_xml_email'] = False
+		else:
+			attDict['git_log_pkg_text'] = self.get_git_log_pkg_text(repodir, cp)
+			tmp_herds = pkg_md.herds()
+			if tmp_herds != ():
+				attDict['metadata_xml_herds'] = tmp_herds[0]
+				md_email_list.append(attDict['metadata_xml_herds'] + '@gentoo.org')
+			for maint in pkg_md.maintainers():
+				md_email_list.append(maint.email)
+			if md_email_list != []:
+				attDict['metadata_xml_email'] = md_email_list
+			else:
+				log_msg = "Metadata file %s missing Email" % (pkgdir + "/metadata.xml")
+				add_logs(self._session, log_msg, "qa", self._config_id)
+				attDict['metadata_xml_email'] = False
 		attDict['metadata_xml_descriptions'] = ''
 		package_metadataDict[package_id] = attDict
 		return package_metadataDict
@@ -248,7 +236,6 @@ class tbc_package(object):
 		self.add_new_build_job_db(ebuild_id_list, packageDict, config_cpv_listDict)
 
 	def get_manifest_checksum_tree(self, pkgdir, cp, repo, mytree):
-
 		# Get the cp manifest file checksum.
 		try:
 			manifest_checksum_tree = portage.checksum.sha256hash(pkgdir + "/Manifest")[0]
@@ -261,13 +248,6 @@ class tbc_package(object):
 				log_msg = "C %s:%s ... Fail." % (cp, repo)
 				add_logs(self._session, log_msg, "error", self._config_id)
 			return "0"
-		fail_msg = digestcheck(self._mysettings, pkgdir)
-		if fail_msg:
-			log_msg = "QA: Manifest file has errors. :%s:%s" % (cp, repo,)
-			add_logs(self._session, log_msg, "error", self._config_id)
-			log_msg = "C %s:%s ... Fail." % (cp, repo)
-			add_logs(self._session, log_msg, "error", self._config_id)
-			return None
 		return manifest_checksum_tree
 
 	def add_new_package_db(self, cp, repo):
@@ -282,10 +262,7 @@ class tbc_package(object):
 		mytree = []
 		mytree.append(repodir)
 		pkgdir = repodir + "/" + cp # Get RepoDIR + cp
-
 		manifest_checksum_tree = self.get_manifest_checksum_tree(pkgdir, cp, repo, mytree)
-		if manifest_checksum_tree is None:
-			return
 
 		package_id = add_new_package_sql(self._session, cp, repo)
 
@@ -316,6 +293,7 @@ class tbc_package(object):
 			if package_metadataDict[package_id]['metadata_xml_descriptions'] != packageDict[cpv]['ebuild_version_descriptions_tree']:
 				package_metadataDict[package_id]['metadata_xml_descriptions'] = packageDict[cpv]['ebuild_version_descriptions_tree']
 
+		packageDict[cpv]['new'] = True
 		self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_checksum_tree)
 		log_msg = "C %s:%s ... Done." % (cp, repo)
 		add_logs(self._session, log_msg, "info", self._config_id)
@@ -332,10 +310,7 @@ class tbc_package(object):
 		pkgdir = repodir + "/" + cp # Get RepoDIR + cp
 		mytree = []
 		mytree.append(repodir)
-
 		manifest_checksum_tree = self.get_manifest_checksum_tree(pkgdir, cp, repo, mytree)
-		if manifest_checksum_tree is None:
-			return
 
 		# if we NOT have the same checksum in the db update the package
 		if manifest_checksum_tree != PackageInfo.Checksum:
@@ -361,7 +336,6 @@ class tbc_package(object):
 					add_old_ebuild(self._session, old_ebuild_id_list)
 					log_msg = "C %s:%s ... Done." % (cp, repo)
 					add_logs(self._session, log_msg, "info", self._config_id)
-
 				else:
 					log_msg = "QA: Can't get the ebuilds list. %s:%s" % (cp, repo,)
 					add_logs(self._session, log_msg, "info", self._config_id)
@@ -414,6 +388,7 @@ class tbc_package(object):
 					# N = New ebuild
 					log_msg = "N %s:%s" % (cpv, repo,)
 					add_logs(self._session, log_msg, "info", self._config_id)
+					packageDict[cpv]['new'] = True
 				elif  ebuild_version_checksum_tree != ebuild_version_manifest_checksum_db:
 					# U = Updated ebuild
 					log_msg = "U %s:%s" % (cpv, repo,)

diff --git a/pym/tbc/qachecks.py b/pym/tbc/qachecks.py
index 766b572..67b1448 100644
--- a/pym/tbc/qachecks.py
+++ b/pym/tbc/qachecks.py
@@ -19,103 +19,6 @@ from tbc.repoman.main import repoman_main
 from tbc.sqlquerys import get_configmetadata_info, get_config_info, get_setup_info
 import portage
 
-# Copy of portage.digestcheck() but without the writemsg() stuff
-def digestcheck(mysettings, pkgdir):
-	"""
-	Verifies checksums. Assumes all files have been downloaded.
-	@rtype: int
-	@return: 1 on success and 0 on failure
-	"""
-	strict=False
-	myfiles = []
-	justmanifest = None
-	mf = None
-
-	if mysettings.get("EBUILD_SKIP_MANIFEST") == "1":
-		return False
-	hash_filter = _hash_filter(mysettings.get("PORTAGE_CHECKSUM_FILTER", ""))
-	if hash_filter.transparent:
-		hash_filter = None
-	if mf is None:
-		mf = mysettings.repositories.get_repo_for_location(
-			os.path.dirname(os.path.dirname(pkgdir)))
-		mf = mf.load_manifest(pkgdir, mysettings["DISTDIR"])
-	try:
-		if not mf.thin and strict and "PORTAGE_PARALLEL_FETCHONLY" not in mysettings:
-			if mf.fhashdict.get("EBUILD"):
-				mf.checkTypeHashes("EBUILD", hash_filter=hash_filter)
-			if mf.fhashdict.get("AUX"):
-				mf.checkTypeHashes("AUX", hash_filter=hash_filter)
-			if mf.fhashdict.get("MISC"):
-				mf.checkTypeHashes("MISC", ignoreMissingFiles=True,
-					hash_filter=hash_filter)
-		for f in myfiles:
-			ftype = mf.findFile(f)
-			if ftype is None:
-				if mf.allow_missing:
-					continue
-				return ("\n!!! Missing digest for '%s'\n") % f
-			mf.checkFileHashes(ftype, f, hash_filter=hash_filter)
-	except FileNotFound as e:
-		return ("\n!!! A file listed in the Manifest could not be found: %s\n") % str(e)
-	except DigestException as e:
-		return ("!!! Digest verification failed: %s\nReason: %s\nGot: %s\nExpected: %s\n") \
-				 % (e.value[0], e.value[1], e.value[2], e.value[3])
-	if mf.thin or mf.allow_missing:
-		# In this case we ignore any missing digests that
-		# would otherwise be detected below.
-		return False
-	# Make sure that all of the ebuilds are actually listed in the Manifest.
-	for f in os.listdir(pkgdir):
-		pf = None
-		if f[-7:] == '.ebuild':
-			pf = f[:-7]
-		if pf is not None and not mf.hasFile("EBUILD", f):
-			return ("!!! A file is not listed in the Manifest: '%s'\n") % os.path.join(pkgdir, f)
-
-	# epatch will just grab all the patches out of a directory, so we have to
-	# make sure there aren't any foreign files that it might grab.
-	filesdir = os.path.join(pkgdir, "files")
-
-	for parent, dirs, files in os.walk(filesdir):
-		try:
-			parent = _unicode_decode(parent,
-				encoding=_encodings['fs'], errors='strict')
-		except UnicodeDecodeError:
-			parent = _unicode_decode(parent,
-				encoding=_encodings['fs'], errors='replace')
-			return ("!!! Path contains invalid character(s) for encoding '%s': '%s'\n") % (_encodings['fs'], parent)
-		for d in dirs:
-			d_bytes = d
-			try:
-				d = _unicode_decode(d,
-					encoding=_encodings['fs'], errors='strict')
-			except UnicodeDecodeError:
-				d = _unicode_decode(d,
-					encoding=_encodings['fs'], errors='replace')
-				return ("!!! Path contains invalid character(s) for encoding '%s': '%s'\n") % (_encodings['fs'], os.path.join(parent, d))
-			if d.startswith(".") or d == "CVS":
-				dirs.remove(d_bytes)
-		for f in files:
-			try:
-				f = _unicode_decode(f,
-					encoding=_encodings['fs'], errors='strict')
-			except UnicodeDecodeError:
-				f = _unicode_decode(f,
-					encoding=_encodings['fs'], errors='replace')
-				if f.startswith("."):
-					continue
-				f = os.path.join(parent, f)[len(filesdir) + 1:]
-				return ("!!! File name contains invalid character(s) for encoding '%s': '%s'\n") % (_encodings['fs'], f)
-			if f.startswith("."):
-				continue
-			f = os.path.join(parent, f)[len(filesdir) + 1:]
-			file_type = mf.findFile(f)
-			if file_type != "AUX" and not f.startswith("digest-"):
-				return ("!!! A file is not listed in the Manifest: '%s'\n") % os.path.join(filesdir, f)
-	return False
-
-
 def check_file_in_manifest(pkgdir, mysettings, portdb, cpv, build_use_flags_list, repo):
 	myfetchlistdict = portage.FetchlistDict(pkgdir, mysettings, portdb)
 	my_manifest = portage.Manifest(pkgdir, mysettings['DISTDIR'], fetchlist_dict=myfetchlistdict, manifest1_compat=False, from_scratch=False)

diff --git a/pym/tbc/sync.py b/pym/tbc/sync.py
index 12c4c9f..610f357 100644
--- a/pym/tbc/sync.py
+++ b/pym/tbc/sync.py
@@ -74,20 +74,16 @@ def git_sync_main(session):
 	for repo_dir in git_repos_list(myportdb):
 		reponame = myportdb.getRepositoryName(repo_dir)
 		repo = git.Repo(repo_dir)
+		log_msg = "Checking repo %s" % (reponame)
+		add_logs(session, log_msg, "info", config_id)
 		info_list, repouptodate = git_fetch(repo)
 		if not repouptodate:
 			cp_list = []
 			attr = {}
 			# We check for Manifest changes and add the package to a list
-			for diff_line in repo.git.diff('origin').splitlines():
-				if re.search("^diff --git.*/Manifest", diff_line):
-					diff_line2 = re.split(' b/', re.sub('diff --git', '', diff_line))
-					diff_line3 = re.sub(' a/', '', diff_line2[0])
-					if diff_line3 == diff_line2[1] or "Manifest" in diff_line3:
-						cp = re.sub('/Manifest', '', diff_line3)
-						cp_list.append(cp)
-					else:
-						cp = re.sub('/Manifest', '', diff_line2[1])
+			for diff_line in repo.git.diff('origin', '--name-only').splitlines():
+				if re.search("Manifest$", diff_line):
+						cp = re.sub('/Manifest', '', diff_line)
 						cp_list.append(cp)
 			attr['cp_list'] = cp_list
 			repo_cp_dict[reponame] = attr
@@ -95,7 +91,8 @@ def git_sync_main(session):
 		else:
 			log_msg = "Repo %s is up to date" % (reponame)
 			add_logs(session, log_msg, "info", config_id)
-	
+		log_msg = "Checking repo %s Done" % (reponame)
+		add_logs(session, log_msg, "info", config_id)
 	# Need to add a clone of profiles/base for reading the tree
 	try:
 		os.mkdir(mysettings['PORTDIR'] + "/profiles/config", 0o777)

[gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/

[Magnus Granberg]

commit:     1cbba36a9ea54c96b751728c98e30e0c23fac006
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Wed Aug  5 01:09:51 2015 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Wed Aug  5 01:09:51 2015 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=1cbba36a

add remove package and fix some repoman stuff

 pym/tbc/build_log.py |  4 ++--
 pym/tbc/package.py   | 38 ++++++++++++++++++++++----------------
 pym/tbc/sqlquerys.py |  5 +++++
 3 files changed, 29 insertions(+), 18 deletions(-)

diff --git a/pym/tbc/build_log.py b/pym/tbc/build_log.py
index 71c14b5..d72ef91 100644
--- a/pym/tbc/build_log.py
+++ b/pym/tbc/build_log.py
@@ -32,7 +32,7 @@ from tbc.sqlquerys import add_logs, get_config_id, get_ebuild_id_db, add_new_bui
 	add_repoman_log
 from sqlalchemy.orm import sessionmaker
 
-def repoman_check_full(session, pkgdir, package_id, config_id):
+def check_repoman_full(session, pkgdir, package_id, config_id):
 	# Check cp with repoman repoman full
 	status = repoman_full(session, pkgdir, config_id)
 	repoman_hash = hashlib.sha256()
@@ -224,7 +224,7 @@ def get_buildlog_info(session, settings, pkg, build_dict):
 	categories = element[0]
 	package = element[1]
 	pkgdir = myportdb.getRepositoryPath(build_dict['repo']) + "/" + categories + "/" + package
-	repoman_error_list = repoman_check_full(session, pkgdir, build_dict['package_id'], config_id)
+	repoman_error_list = check_repoman_full(session, pkgdir, build_dict['package_id'], config_id)
 	build_log_dict = {}
 	build_log_dict['fail'] = False
 	if repoman_error_list:

diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index 657c90d..0bb41eb 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -16,7 +16,7 @@ from tbc.sqlquerys import add_logs, get_package_info, get_config_info, \
 	get_package_metadata_sql, update_package_metadata, update_manifest_sql, \
 	get_package_info_from_package_id, get_config_all_info, add_new_package_sql, \
 	get_ebuild_checksums, get_ebuild_id_db, get_configmetadata_info, get_setup_info, \
-	get_ebuild_info_ebuild_id, get_ebuild_restrictions
+	get_ebuild_info_ebuild_id, get_ebuild_restrictions, add_old_package
 
 class tbc_package(object):
 
@@ -159,6 +159,7 @@ class tbc_package(object):
 						log_msg = "B %s:%s USE: %s Setup: %s" % (k, v['repo'], use_flagsDict, setup_id,)
 						add_logs(self._session, log_msg, "info", self._config_id)
 					i = i +1
+
 	def get_changelog_text(self, pkgdir):
 		changelog_text_dict, max_text_lines = get_log_text_dict(pkgdir + "/ChangeLog")
 		spec = 3
@@ -214,7 +215,7 @@ class tbc_package(object):
 	def add_package(self, packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_checksum_tree):
 		# Use packageDict to update the db
 		ebuild_id_list = add_new_ebuild_sql(self._session, packageDict)
-		
+
 		# Make old ebuilds unactive
 		for ebuild_id in ebuild_id_list:
 			new_ebuild_id_list.append(ebuild_id)
@@ -286,12 +287,13 @@ class tbc_package(object):
 			return
 
 		package_id = add_new_package_sql(self._session, cp, repo)
-		
+
 		# Check cp with repoman full
 		status = check_repoman_full(self._session, pkgdir, package_id, self._config_id)
 		if status:
-					log_msg = "Repoman %s:%s ... Fail." % (cpv, repo)
-					add_logs(self._session, log_msg, "error", self._config_id)
+			log_msg = "Repoman %s::%s ... Fail." % (cp, repo)
+			add_logs(self._session, log_msg, "error", self._config_id)
+
 		package_metadataDict = self.get_package_metadataDict(pkgdir, package_id)
 		# Get the ebuild list for cp
 		ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree)
@@ -308,7 +310,7 @@ class tbc_package(object):
 		old_ebuild_id_list = []
 		for cpv in sorted(ebuild_list_tree):
 			packageDict[cpv] = self.get_packageDict(pkgdir, cpv, repo)
-			
+
 			# take package descriptions from the ebuilds
 			if package_metadataDict[package_id]['metadata_xml_descriptions'] != packageDict[cpv]['ebuild_version_descriptions_tree']:
 				package_metadataDict[package_id]['metadata_xml_descriptions'] = packageDict[cpv]['ebuild_version_descriptions_tree']
@@ -329,6 +331,7 @@ class tbc_package(object):
 		pkgdir = repodir + "/" + cp # Get RepoDIR + cp
 		mytree = []
 		mytree.append(repodir)
+
 		manifest_checksum_tree = self.get_manifest_checksum_tree(pkgdir, cp, repo, mytree)
 		if manifest_checksum_tree is None:
 			return
@@ -340,17 +343,11 @@ class tbc_package(object):
 			log_msg = "U %s:%s" % (cp, repo)
 			add_logs(self._session, log_msg, "info", self._config_id)
 
-			# Check cp with repoman full
-			status = check_repoman_full(self._session, pkgdir, package_id, self._config_id)
-			if status:
-					log_msg = "Repoman %s:%s ... Fail." % (cpv, repo)
-					add_logs(self._session, log_msg, "error", self._config_id)
-
 			# Get the ebuild list for cp
 			old_ebuild_id_list = []
 			ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree)
 			if ebuild_list_tree == []:
-				if manifest_checksum_tree != "0":
+				if manifest_checksum_tree == "0":
 					old_ebuild_id_list = get_ebuild_id_list(self._session, package_id)
 					for ebuild_id in old_ebuild_id_list:
 						EbuildInfo = get_ebuild_info_ebuild_id(self._session, ebuild_id)
@@ -358,15 +355,24 @@ class tbc_package(object):
 						# R =  remove ebuild
 						log_msg = "R %s:%s" % (cpv, repo,)
 						add_logs(self._session, log_msg, "info", self._config_id)
+						if not os.path.isdir(pkgdir):
+							add_old_package(self._session, package_id)
 					add_old_ebuild(self._session, old_ebuild_id_list)
 					log_msg = "C %s:%s ... Done." % (cp, repo)
 					add_logs(self._session, log_msg, "info", self._config_id)
+
 				else:
 					log_msg = "QA: Can't get the ebuilds list. %s:%s" % (cp, repo,)
 					add_logs(self._session, log_msg, "info", self._config_id)
 					log_msg = "C %s:%s ... Fail." % (cp, repo)
 					add_logs(self._session, log_msg, "info", self._config_id)
 				return
+
+			# Check cp with repoman full
+			status = check_repoman_full(self._session, pkgdir, package_id, self._config_id)
+			if status:
+				log_msg = "Repoman %s::%s ... Fail." % (cp, repo)
+				add_logs(self._session, log_msg, "error", self._config_id)
 			package_metadataDict = self.get_package_metadataDict(pkgdir, package_id)
 			packageDict ={}
 			new_ebuild_id_list = []
@@ -374,14 +380,14 @@ class tbc_package(object):
 
 				# split out ebuild version
 				ebuild_version_tree = portage.versions.cpv_getversion(cpv)
-				
+
 				# Get packageDict for cpv
 				packageDict[cpv] = self.get_packageDict(pkgdir, cpv, repo)
-				
+
 				# take package descriptions from the ebuilds
 				if package_metadataDict[package_id]['metadata_xml_descriptions'] != packageDict[cpv]['ebuild_version_descriptions_tree']:
 					package_metadataDict[package_id]['metadata_xml_descriptions'] = packageDict[cpv]['ebuild_version_descriptions_tree']
-					
+
 				# Get the checksum of the ebuild in tree and db
 				ebuild_version_checksum_tree = packageDict[cpv]['checksum']
 				checksums_db, fail = get_ebuild_checksums(self._session, package_id, ebuild_version_tree)

diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index 4d2a5e5..7791e6e 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -453,6 +453,11 @@ def add_old_ebuild(session, old_ebuild_list):
 			for build_job in build_job_id_list:
 				del_old_build_jobs(session, build_job.BuildJobId)
 
+def add_old_package(session, package_id):
+	PackagesInfo = session.query(Packages).filter_by(PackageId = package_id).one()
+	PackagesInfo.Active = False
+	session.commit()
+
 def add_new_package_sql(session, cp, repo):
 	element = cp.split('/')
 	categories = element[0]