* [gentoo-commits] proj/tinderbox-cluster:master commit in: tbc/pym/
@ 2015-04-18 20:27 Magnus Granberg
0 siblings, 0 replies; 20+ messages in thread
From: Magnus Granberg @ 2015-04-18 20:27 UTC (permalink / raw
To: gentoo-commits
commit: c78602c4fc3043be4c0c722f0b5886a372ad9382
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Apr 18 20:26:35 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Apr 18 20:26:35 2015 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=c78602c4
add code/files in tbc/pym for the backend python code
tbc/pym/ConnectionManager.py | 22 ++
tbc/pym/__init__.py | 0
tbc/pym/build_depgraph.py | 65 ++++++
tbc/pym/build_job.py | 166 ++++++++++++++
tbc/pym/build_log.py | 336 ++++++++++++++++++++++++++++
tbc/pym/buildquerydb.py | 102 +++++++++
tbc/pym/check_setup.py | 74 +++++++
tbc/pym/db_mapping.py | 260 ++++++++++++++++++++++
tbc/pym/depclean.py | 53 +++++
tbc/pym/flags.py | 231 +++++++++++++++++++
tbc/pym/jobs.py | 86 ++++++++
tbc/pym/old_cpv.py | 97 ++++++++
tbc/pym/package.py | 355 ++++++++++++++++++++++++++++++
tbc/pym/readconf.py | 58 +++++
tbc/pym/sqlquerys.py | 512 +++++++++++++++++++++++++++++++++++++++++++
tbc/pym/sync.py | 73 ++++++
tbc/pym/text.py | 49 +++++
tbc/pym/updatedb.py | 135 ++++++++++++
18 files changed, 2674 insertions(+)
diff --git a/tbc/pym/ConnectionManager.py b/tbc/pym/ConnectionManager.py
new file mode 100644
index 0000000..40abfd5
--- /dev/null
+++ b/tbc/pym/ConnectionManager.py
@@ -0,0 +1,22 @@
+# Copyright 1998-2015 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from __future__ import print_function
+import logging
+
+def NewConnection(tbc_settings_dict):
+ backend=tbc_settings_dict['sql_backend']
+ host=tbc_settings_dict['sql_host']
+ user=tbc_settings_dict['sql_user']
+ password=tbc_settings_dict['sql_passwd']
+ database=tbc_settings_dict['sql_db']
+ if backend == 'mysql':
+ try:
+ from sqlalchemy import create_engine
+ except ImportError:
+ print("Please install a recent version of dev-python/sqlalchemy for Python")
+ sys.exit(1)
+ #logging.basicConfig()
+ #logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
+ mysqldriver = 'mysql+mysqlconnector'
+ return create_engine(mysqldriver + '://' + user + ':' + password + '@' + host + '/' + database, pool_recycle=120)
diff --git a/tbc/pym/__init__.py b/tbc/pym/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tbc/pym/build_depgraph.py b/tbc/pym/build_depgraph.py
new file mode 100644
index 0000000..812e576
--- /dev/null
+++ b/tbc/pym/build_depgraph.py
@@ -0,0 +1,65 @@
+# Copyright 1998-2015 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from __future__ import print_function
+from _emerge.create_depgraph_params import create_depgraph_params
+from _emerge.depgraph import backtrack_depgraph
+import portage
+portage.proxy.lazyimport.lazyimport(globals(),
+ 'tbc.actions:load_emerge_config',
+)
+from portage.exception import PackageSetNotFound
+
+from tbc.build_log import log_fail_queru
+
+def build_mydepgraph(settings, trees, mtimedb, myopts, myparams, myaction, myfiles, spinner, build_dict, session):
+ try:
+ success, mydepgraph, favorites = backtrack_depgraph(
+ settings, trees, myopts, myparams, myaction, myfiles, spinner)
+ except portage.exception.PackageSetNotFound as e:
+ root_config = trees[settings["ROOT"]]["root_config"]
+ display_missing_pkg_set(root_config, e.value)
+ build_dict['type_fail'] = "depgraph fail"
+ build_dict['check_fail'] = True
+ else:
+ if not success:
+ repeat = True
+ repeat_times = 0
+ while repeat:
+ if mydepgraph._dynamic_config._needed_p_mask_changes:
+ build_dict['type_fail'] = "Mask package or dep"
+ build_dict['check_fail'] = True
+ elif mydepgraph._dynamic_config._needed_use_config_changes:
+ mydepgraph._display_autounmask()
+ build_dict['type_fail'] = "Need use change"
+ build_dict['check_fail'] = True
+ elif mydepgraph._dynamic_config._slot_conflict_handler:
+ build_dict['type_fail'] = "Slot blocking"
+ build_dict['check_fail'] = True
+ elif mydepgraph._dynamic_config._circular_deps_for_display:
+ build_dict['type_fail'] = "Circular Deps"
+ build_dict['check_fail'] = True
+ elif mydepgraph._dynamic_config._unsolvable_blockers:
+ build_dict['type_fail'] = "Blocking packages"
+ build_dict['check_fail'] = True
+ else:
+ build_dict['type_fail'] = "Dep calc fail"
+ build_dict['check_fail'] = True
+ mydepgraph.display_problems()
+ if repeat_times is 2:
+ repeat = False
+ log_fail_queru(session, build_dict, settings)
+ else:
+ repeat_times = repeat_times + 1
+ settings, trees, mtimedb = load_emerge_config()
+ myparams = create_depgraph_params(myopts, myaction)
+ try:
+ success, mydepgraph, favorites = backtrack_depgraph(
+ settings, trees, myopts, myparams, myaction, myfiles, spinner)
+ except portage.exception.PackageSetNotFound as e:
+ root_config = trees[settings["ROOT"]]["root_config"]
+ display_missing_pkg_set(root_config, e.value)
+ if success:
+ repeat = False
+
+ return success, settings, trees, mtimedb, mydepgraph
diff --git a/tbc/pym/build_job.py b/tbc/pym/build_job.py
new file mode 100644
index 0000000..0167882
--- /dev/null
+++ b/tbc/pym/build_job.py
@@ -0,0 +1,166 @@
+# Copyright 1998-2015 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from __future__ import print_function
+import portage
+import os
+import re
+import sys
+import signal
+
+from portage import _encodings
+from portage import _unicode_decode
+from portage.versions import cpv_getkey
+from portage.dep import check_required_use
+from tbc.manifest import tbc_manifest
+from tbc.depclean import do_depclean
+from tbc.flags import tbc_use_flags
+from tbc.main import emerge_main
+from tbc.build_log import log_fail_queru
+from tbc.actions import load_emerge_config
+from tbc.sqlquerys import add_tbc_logs, get_packages_to_build, update_buildjobs_status, is_build_job_done
+
+class build_job_action(object):
+
+ def __init__(self, config_id, session):
+ self._config_id = config_id
+ self._session = session
+
+ def make_build_list(self, build_dict, settings, portdb):
+ cp = build_dict['cp']
+ repo = build_dict['repo']
+ package = build_dict['package']
+ cpv = build_dict['cpv']
+ pkgdir = portdb.getRepositoryPath(repo) + "/" + cp
+ init_manifest = tbc_manifest(settings, pkgdir)
+ build_use_flags_list = []
+ try:
+ ebuild_version_checksum_tree = portage.checksum.sha256hash(pkgdir + "/" + package + "-" + build_dict['ebuild_version'] + ".ebuild")[0]
+ except:
+ ebuild_version_checksum_tree = None
+ if ebuild_version_checksum_tree == build_dict['checksum']:
+ manifest_error = init_manifest.check_file_in_manifest(portdb, cpv, build_use_flags_list, repo)
+ if manifest_error is None:
+ init_flags = tbc_use_flags(settings, portdb, cpv)
+ build_use_flags_list = init_flags.comper_useflags(build_dict)
+ log_msg = "build_use_flags_list %s" % (build_use_flags_list,)
+ add_tbc_logs(self._session, log_msg, "info", self._config_id)
+ manifest_error = init_manifest.check_file_in_manifest(portdb, cpv, build_use_flags_list, repo)
+ if manifest_error is None:
+ build_dict['check_fail'] = False
+ build_cpv_dict = {}
+ build_cpv_dict[cpv] = build_use_flags_list
+ log_msg = "build_cpv_dict: %s" % (build_cpv_dict,)
+ add_tbc_logs(self._session, log_msg, "info", self._config_id)
+ return build_cpv_dict
+ build_dict['type_fail'] = "Manifest error"
+ build_dict['check_fail'] = True
+ log_msg = "Manifest error: %s:%s" % (cpv, manifest_error)
+ add_tbc_logs(self._session, log_msg, "info", self._config_id)
+ else:
+ build_dict['type_fail'] = "Wrong ebuild checksum"
+ build_dict['check_fail'] = True
+ if build_dict['check_fail'] is True:
+ log_fail_queru(self._session, build_dict, settings)
+ return None
+
+ def build_procces(self, buildqueru_cpv_dict, build_dict, settings, portdb):
+ build_cpv_list = []
+ depclean_fail = True
+ for k, build_use_flags_list in buildqueru_cpv_dict.items():
+ build_cpv_list.append("=" + k)
+ if not build_use_flags_list == None:
+ build_use_flags = ""
+ for flags in build_use_flags_list:
+ build_use_flags = build_use_flags + flags + " "
+ filetext = '=' + k + ' ' + build_use_flags
+ log_msg = "filetext: %s" % filetext
+ add_tbc_logs(self._session, log_msg, "info", self._config_id)
+ with open("/etc/portage/package.use/99_autounmask", "a") as f:
+ f.write(filetext)
+ f.write('\n')
+ f.close
+ log_msg = "build_cpv_list: %s" % (build_cpv_list,)
+ add_tbc_logs(self._session, log_msg, "info", self._config_id)
+
+ # We remove the binary package if removebin is true
+ if build_dict['removebin']:
+ package = build_dict['package']
+ pv = package + "-" + build_dict['ebuild_version']
+ binfile = settings['PKGDIR'] + "/" + build_dict['category'] + "/" + pv + ".tbz2"
+ try:
+ os.remove(binfile)
+ except:
+ log_msg = "Binary file was not removed or found: %s" % (binfile,)
+ add_tbc_logs(self._session, log_msg, "info", self._config_id)
+
+ argscmd = []
+ for emerge_option in build_dict['emerge_options']:
+ if emerge_option == '--depclean':
+ pass
+ elif emerge_option == '--nodepclean':
+ pass
+ elif emerge_option == '--nooneshot':
+ pass
+ else:
+ if not emerge_option in argscmd:
+ argscmd.append(emerge_option)
+ for build_cpv in build_cpv_list:
+ argscmd.append(build_cpv)
+ print("Emerge options: %s" % argscmd)
+ log_msg = "argscmd: %s" % (argscmd,)
+ add_tbc_logs(self._session, log_msg, "info", self._config_id)
+
+ # Call main_emerge to build the package in build_cpv_list
+ print("Build: %s" % build_dict)
+ update_buildjobs_status(self._session, build_dict['build_job_id'], 'Building', self._config_id)
+ build_fail = emerge_main(argscmd, build_dict, self._session)
+ # Run depclean
+ if '--depclean' in build_dict['emerge_options'] and not '--nodepclean' in build_dict['emerge_options']:
+ depclean_fail = do_depclean()
+ try:
+ os.remove("/etc/portage/package.use/99_autounmask")
+ with open("/etc/portage/package.use/99_autounmask", "a") as f:
+ f.close
+ except:
+ pass
+
+ if is_build_job_done(self._session, build_dict['build_job_id']):
+ update_buildjobs_status(self._session, build_dict['build_job_id'], 'Looked', self._config_id)
+ log_msg = "build_job %s was not removed" % (build_dict['build_job_id'],)
+ add_tbc_logs(self._session, log_msg, "info", self._config_id)
+ print("qurery was not removed")
+ build_dict['type_fail'] = "Querey was not removed"
+ build_dict['check_fail'] = True
+ log_fail_queru(self._session, build_dict, settings)
+ if build_fail is True:
+ build_dict['type_fail'] = "Emerge faild"
+ build_dict['check_fail'] = True
+ log_msg = "Emerge faild!"
+ add_tbc_logs(self._session, log_msg, "info", self._config_id)
+ return True
+ return False
+
+ def procces_build_jobs(self):
+ build_dict = {}
+ build_dict = get_packages_to_build(self._session, self._config_id)
+ if build_dict is None:
+ return
+ print("build_dict: %s" % (build_dict,))
+ log_msg = "build_dict: %s" % (build_dict,)
+ add_tbc_logs(self._session, log_msg, "info", self._config_id)
+ if not build_dict['ebuild_id'] is None and build_dict['checksum'] is not None:
+ settings, trees, mtimedb = load_emerge_config()
+ portdb = trees[settings["ROOT"]]["porttree"].dbapi
+ buildqueru_cpv_dict = self.make_build_list(build_dict, settings, portdb)
+ log_msg = "buildqueru_cpv_dict: %s" % (buildqueru_cpv_dict,)
+ add_tbc_logs(self._session, log_msg, "info", self._config_id)
+ if buildqueru_cpv_dict is None:
+ return
+ fail_build_procces = self.build_procces(buildqueru_cpv_dict, build_dict, settings, portdb)
+ return
+ if not build_dict['emerge_options'] is [] and build_dict['ebuild_id'] is None:
+ return
+ if not build_dict['ebuild_id'] is None and build_dict['emerge_options'] is None:
+ pass
+ # del_old_queue(self._session, build_dict['queue_id'])
diff --git a/tbc/pym/build_log.py b/tbc/pym/build_log.py
new file mode 100644
index 0000000..a7d5314
--- /dev/null
+++ b/tbc/pym/build_log.py
@@ -0,0 +1,336 @@
+# Copyright 1998-2015 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from __future__ import print_function
+import re
+import os
+import platform
+import hashlib
+
+from portage.versions import catpkgsplit, cpv_getversion
+import portage
+from portage.util import writemsg, \
+ writemsg_level, writemsg_stdout
+from portage import _encodings
+from portage import _unicode_encode
+
+from _emerge.main import parse_opts
+
+portage.proxy.lazyimport.lazyimport(globals(),
+ 'tbc.actions:action_info,load_emerge_config',
+)
+
+from tbc.repoman_tbc import tbc_repoman
+from tbc.text import get_log_text_dict
+from tbc.package import tbc_package
+from tbc.readconf import get_conf_settings
+from tbc.flags import tbc_use_flags
+from tbc.ConnectionManager import NewConnection
+from tbc.sqlquerys import add_tbc_logs, get_config_id, get_ebuild_id_db, add_new_buildlog, \
+ get_package_info, get_build_job_id, get_use_id, get_config_info, get_hilight_info, get_error_info_list, \
+ add_e_info, get_fail_times, add_fail_times, update_fail_times, del_old_build_jobs, add_old_ebuild, \
+ update_buildjobs_status
+from sqlalchemy.orm import sessionmaker
+
+def get_build_dict_db(session, config_id, settings, pkg):
+ myportdb = portage.portdbapi(mysettings=settings)
+ cpvr_list = catpkgsplit(pkg.cpv, silent=1)
+ categories = cpvr_list[0]
+ package = cpvr_list[1]
+ repo = pkg.repo
+ ebuild_version = cpv_getversion(pkg.cpv)
+ log_msg = "Logging %s:%s" % (pkg.cpv, repo,)
+ add_tbc_logs(session, log_msg, "info", config_id)
+ PackageInfo = get_package_info(session, categories, package, repo)
+ build_dict = {}
+ build_dict['ebuild_version'] = ebuild_version
+ build_dict['package_id'] = PackageInfo.PackageId
+ build_dict['cpv'] = pkg.cpv
+ build_dict['categories'] = categories
+ build_dict['package'] = package
+ build_dict['config_id'] = config_id
+ init_useflags = tbc_use_flags(settings, myportdb, pkg.cpv)
+ iuse_flags_list, final_use_list = init_useflags.get_flags_pkg(pkg, settings)
+ iuse = []
+ for iuse_line in iuse_flags_list:
+ iuse.append(init_useflags.reduce_flag(iuse_line))
+ iuse_flags_list2 = list(set(iuse))
+ use_enable = final_use_list
+ use_disable = list(set(iuse_flags_list2).difference(set(use_enable)))
+ use_flagsDict = {}
+ for x in use_enable:
+ use_id = get_use_id(session, x)
+ use_flagsDict[use_id] = True
+ for x in use_disable:
+ use_id = get_use_id(session, x)
+ use_flagsDict[use_id] = False
+ if use_enable == [] and use_disable == []:
+ build_dict['build_useflags'] = None
+ else:
+ build_dict['build_useflags'] = use_flagsDict
+ pkgdir = myportdb.getRepositoryPath(repo) + "/" + categories + "/" + package
+ ebuild_version_checksum_tree = portage.checksum.sha256hash(pkgdir+ "/" + package + "-" + ebuild_version + ".ebuild")[0]
+ build_dict['checksum'] = ebuild_version_checksum_tree
+ ebuild_id_list, status = get_ebuild_id_db(session, build_dict['checksum'], build_dict['package_id'])
+ if status:
+ if ebuild_id_list is None:
+ log_msg = "%s:%s Don't have any ebuild_id!" % (pkg.cpv, repo,)
+ add_tbc_logs(session, log_msg, "info", config_id)
+ update_manifest_sql(session, build_dict['package_id'], "0")
+ init_package = tbc_package(session, settings, myportdb)
+ init_package.update_package_db(build_dict['package_id'])
+ ebuild_id_list, status = get_ebuild_id_db(session, build_dict['checksum'], build_dict['package_id'])
+ if status and ebuild_id is None:
+ log_msg = "%s:%s Don't have any ebuild_id!" % (pkg.cpv, repo,)
+ add_tbc_logs(session, log_msg, "error", config_id)
+ else:
+ old_ebuild_id_list = []
+ for ebuild_id in ebuild_id_list:
+ log_msg = "%s:%s:%s Dups of checksums" % (pkg.cpv, repo, ebuild_id,)
+ add_tbc_logs(session, log_msg, "error", config_id)
+ old_ebuild_id_list.append(ebuild_id)
+ add_old_ebuild(session, old_ebuild_id_list)
+ return
+ build_dict['ebuild_id'] = ebuild_id_list
+
+ build_job_id = get_build_job_id(session, build_dict)
+ if build_job_id is None:
+ build_dict['build_job_id'] = None
+ else:
+ build_dict['build_job_id'] = build_job_id
+ return build_dict
+
+def search_buildlog(session, logfile_text_dict, max_text_lines):
+ log_search_list = get_hilight_info(session)
+ hilight_list = []
+ for index, text_line in logfile_text_dict.items():
+ for search_pattern in log_search_list:
+ if re.search(search_pattern.HiLightSearch, text_line):
+ hilight_tmp = {}
+ hilight_tmp['startline'] = index - search_pattern.HiLightStart
+ hilight_tmp['hilight'] = search_pattern.HiLightCssId
+ if search_pattern.HiLightSearchEnd == "":
+ hilight_tmp['endline'] = index + search_pattern.HiLightEnd
+ if hilight_tmp['endline'] > max_text_lines:
+ hilight_tmp['endline'] = max_text_lines
+ elif not search_pattern.HiLightSearchEnd == "" and (index + 1) >= max_text_lines:
+ hilight_tmp['endline'] = max_text_lines
+ else:
+ i = index + 1
+ match = True
+ while match:
+ if i >= max_text_lines:
+ match = False
+ break
+ if re.search(search_pattern.HiLightSearchPattern, logfile_text_dict[i]) and re.search(search_pattern.HiLightSearchPattern, logfile_text_dict[i + 1]):
+ for search_pattern2 in log_search_list:
+ if re.search(search_pattern2.HiLightSearch, logfile_text_dict[i]):
+ match = False
+ if match:
+ i = i + 1
+ elif re.search(search_pattern.HiLightSearchPattern, logfile_text_dict[i]) and re.search(search_pattern.HiLightSearchEnd, logfile_text_dict[i + 1]):
+ i = i + 1
+ match = False
+ else:
+ match = False
+ if i >= max_text_lines:
+ hilight_tmp['endline'] = max_text_lines
+ if re.search(search_pattern.HiLightSearchEnd, logfile_text_dict[i]):
+ hilight_tmp['endline'] = i
+ else:
+ hilight_tmp['endline'] = i - 1
+ hilight_list.append(hilight_tmp)
+
+ new_hilight_dict = {}
+ for hilight_tmp in hilight_list:
+ add_new_hilight = True
+ add_new_hilight_middel = None
+ for k, v in sorted(new_hilight_dict.items()):
+ if hilight_tmp['startline'] == hilight_tmp['endline']:
+ if v['endline'] == hilight_tmp['startline'] or v['startline'] == hilight_tmp['startline']:
+ add_new_hilight = False
+ if hilight_tmp['startline'] > v['startline'] and hilight_tmp['startline'] < v['endline']:
+ add_new_hilight = False
+ add_new_hilight_middel = k
+ else:
+ if v['endline'] == hilight_tmp['startline'] or v['startline'] == hilight_tmp['startline']:
+ add_new_hilight = False
+ if hilight_tmp['startline'] > v['startline'] and hilight_tmp['startline'] < v['endline']:
+ add_new_hilight = False
+ if add_new_hilight is True:
+ adict = {}
+ adict['startline'] = hilight_tmp['startline']
+ adict['hilight_css_id'] = hilight_tmp['hilight']
+ adict['endline'] = hilight_tmp['endline']
+ new_hilight_dict[hilight_tmp['startline']] = adict
+ if not add_new_hilight_middel is None:
+ adict1 = {}
+ adict2 = {}
+ adict3 = {}
+ adict1['startline'] = new_hilight_dict[add_new_hilight_middel]['startline']
+ adict1['endline'] = hilight_tmp['startline'] -1
+ adict1['hilight_css_id'] = new_hilight_dict[add_new_hilight_middel]['hilight']
+ adict2['startline'] = hilight_tmp['startline']
+ adict2['hilight_css_id'] = hilight_tmp['hilight']
+ adict2['endline'] = hilight_tmp['endline']
+ adict3['startline'] = hilight_tmp['endline'] + 1
+ adict3['hilight_css_id'] = new_hilight_dict[add_new_hilight_middel]['hilight']
+ adict3['endline'] = new_hilight_dict[add_new_hilight_middel]['endline']
+ del new_hilight_dict[add_new_hilight_middel]
+ new_hilight_dict[adict1['startline']] = adict1
+ new_hilight_dict[adict2['startline']] = adict2
+ new_hilight_dict[adict3['startline']] = adict3
+ return new_hilight_dict
+
+def get_buildlog_info(session, settings, pkg, build_dict):
+ myportdb = portage.portdbapi(mysettings=settings)
+ init_repoman = tbc_repoman(settings, myportdb)
+ logfile_text_dict, max_text_lines = get_log_text_dict(settings.get("PORTAGE_LOG_FILE"))
+ hilight_dict = search_buildlog(session, logfile_text_dict, max_text_lines)
+ error_log_list = []
+ qa_error_list = []
+ repoman_error_list = []
+ sum_build_log_list = []
+ error_info_list = get_error_info_list(session)
+ for k, v in sorted(hilight_dict.items()):
+ if v['startline'] == v['endline']:
+ error_log_list.append(logfile_text_dict[k ])
+ if v['hilight_css_id'] == "3" or v['hilight_css_id'] == "4": # qa = 3 and 4
+ qa_error_list.append(logfile_text_dict[k])
+ else:
+ i = k
+ while i != (v['endline'] + 1):
+ error_log_list.append(logfile_text_dict[i])
+ if v['hilight_css_id'] == "3" or v['hilight_css_id'] == "4": # qa = 3 and 4
+ qa_error_list.append(logfile_text_dict[i])
+ i = i +1
+
+ # Run repoman check_repoman()
+ repoman_error_list = init_repoman.check_repoman(build_dict['cpv'], pkg.repo)
+ if repoman_error_list != []:
+ sum_build_log_list.append("1") # repoman = 1
+ if qa_error_list != []:
+ sum_build_log_list.append("2") # qa = 2
+ error_search_line = "^ \\* ERROR: "
+ for error_log_line in error_log_list:
+ if re.search(error_search_line, error_log_line):
+ for error_info in error_info_list:
+ if re.search(error_info.ErrorSearch, error_log_line):
+ sum_build_log_list.append(error_info.ErrorId)
+ build_log_dict = {}
+ build_log_dict['repoman_error_list'] = repoman_error_list
+ build_log_dict['qa_error_list'] = qa_error_list
+ build_log_dict['error_log_list'] = error_log_list
+ build_log_dict['summary_error_list'] = sum_build_log_list
+ build_log_dict['hilight_dict'] = hilight_dict
+ return build_log_dict
+
+def get_emerge_info_id(settings, trees, session, config_id):
+ args = []
+ args.append("--info")
+ myaction, myopts, myfiles = parse_opts(args, silent=True)
+ status, emerge_info_list = action_info(settings, trees, myopts, myfiles)
+ emerge_info = ""
+ return "\n".join(emerge_info_list)
+
+def add_buildlog_main(settings, pkg, trees):
+ reader=get_conf_settings()
+ tbc_settings_dict=reader.read_tbc_settings_all()
+ config = tbc_settings_dict['tbc_config']
+ hostname =tbc_settings_dict['hostname']
+ host_config = hostname + "/" + config
+ Session = sessionmaker(bind=NewConnection(tbc_settings_dict))
+ session = Session()
+ config_id = get_config_id(session, config, hostname)
+ if pkg.type_name == "binary":
+ build_dict = None
+ else:
+ build_dict = get_build_dict_db(session, config_id, settings, pkg)
+ if build_dict is None:
+ log_msg = "Package %s:%s is NOT logged." % (pkg.cpv, pkg.repo,)
+ add_tbc_logs(session, log_msg, "info", config_id)
+ session.close
+ return
+ build_log_dict = {}
+ build_log_dict = get_buildlog_info(session, settings, pkg, build_dict)
+ error_log_list = build_log_dict['error_log_list']
+ build_error = ""
+ log_hash = hashlib.sha256()
+ build_error = ""
+ if error_log_list != []:
+ for log_line in error_log_list:
+ build_error = build_error + log_line
+ log_hash.update(log_line.encode('utf-8'))
+ build_log_dict['build_error'] = build_error
+ build_log_dict['log_hash'] = log_hash.hexdigest()
+ build_log_dict['logfilename'] = settings.get("PORTAGE_LOG_FILE").split(host_config)[1]
+ log_msg = "Logfile name: %s" % (settings.get("PORTAGE_LOG_FILE"),)
+ add_tbc_logs(session, log_msg, "info", config_id)
+ build_log_dict['emerge_info'] = get_emerge_info_id(settings, trees, session, config_id)
+ log_id = add_new_buildlog(session, build_dict, build_log_dict)
+
+ if log_id is None:
+ log_msg = "Package %s:%s is NOT logged." % (pkg.cpv, pkg.repo,)
+ add_tbc_logs(session, log_msg, "info", config_id)
+ else:
+ os.chmod(settings.get("PORTAGE_LOG_FILE"), 0o664)
+ log_msg = "Package: %s:%s is logged." % (pkg.cpv, pkg.repo,)
+ add_tbc_logs(session, log_msg, "info", config_id)
+ print("\n>>> Logging %s:%s\n" % (pkg.cpv, pkg.repo,))
+ session.close
+
+def log_fail_queru(session, build_dict, settings):
+ config_id = build_dict['config_id']
+ if get_fail_times(session, build_dict):
+ fail_querue_dict = {}
+ fail_querue_dict['build_job_id'] = build_dict['build_job_id']
+ fail_querue_dict['fail_type'] = build_dict['type_fail']
+ fail_querue_dict['fail_times'] = 1
+ add_fail_times(session, fail_querue_dict)
+ update_buildjobs_status(session, build_dict['build_job_id'], 'Waiting', config_id)
+ else:
+ build_log_dict = {}
+ error_log_list = []
+ qa_error_list = []
+ repoman_error_list = []
+ sum_build_log_list = []
+ sum_build_log_list.append("2")
+ error_log_list.append(build_dict['type_fail'])
+ build_log_dict['repoman_error_list'] = repoman_error_list
+ build_log_dict['qa_error_list'] = qa_error_list
+ build_log_dict['summary_error_list'] = sum_build_log_list
+ if build_dict['type_fail'] == 'merge fail':
+ error_log_list = []
+ for k, v in build_dict['failed_merge'].items():
+ error_log_list.append(v['fail_msg'])
+ build_log_dict['error_log_list'] = error_log_list
+ build_error = ""
+ if error_log_list != []:
+ for log_line in error_log_list:
+ build_error = build_error + log_line
+ build_log_dict['build_error'] = build_error
+ summary_error = ""
+ if sum_build_log_list != []:
+ for sum_log_line in sum_build_log_list:
+ summary_error = summary_error + " " + sum_log_line
+ build_log_dict['log_hash'] = '0'
+ useflagsdict = {}
+ if build_dict['build_useflags'] == {}:
+ for k, v in build_dict['build_useflags'].items():
+ use_id = get_use_id(session, k)
+ useflagsdict[use_id] = v
+ build_dict['build_useflags'] = useflagsdict
+ else:
+ build_dict['build_useflags'] = None
+ if settings.get("PORTAGE_LOG_FILE") is not None:
+ ConfigInfo= get_config_info(session, config_id)
+ host_config = ConfigInfo.Hostname +"/" + ConfigInfo.Config
+ build_log_dict['logfilename'] = settings.get("PORTAGE_LOG_FILE").split(host_config)[1]
+ os.chmod(settings.get("PORTAGE_LOG_FILE"), 0o664)
+ else:
+ build_log_dict['logfilename'] = ""
+ build_log_dict['hilight_dict'] = {}
+ settings2, trees, tmp = load_emerge_config()
+ build_log_dict['emerge_info'] = get_emerge_info_id(settings2, trees, session, config_id)
+ log_id = add_new_buildlog(session, build_dict, build_log_dict)
+ del_old_build_jobs(session, build_dict['build_job_id'])
diff --git a/tbc/pym/buildquerydb.py b/tbc/pym/buildquerydb.py
new file mode 100644
index 0000000..7fe7f82
--- /dev/null
+++ b/tbc/pym/buildquerydb.py
@@ -0,0 +1,102 @@
+# Copyright 1998-2015 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import sys
+import os
+
+# Get the options from the config file set in tbc.readconf
+from tbc.readconf import get_conf_settings
+reader=get_conf_settings()
+tbc_settings_dict=reader.read_tbc_settings_all()
+config_profile = tbc_settings_dict['tbc_config']
+
+from tbc.check_setup import check_make_conf
+from tbc.sync import git_pull
+from tbc.package import tbc_package
+import portage
+import multiprocessing
+
+def add_cpv_query_pool(mysettings, myportdb, config_id, cp, repo):
+ conn =0
+ init_package = tbc_package(mysettings, myportdb)
+ # FIXME: remove the check for tbc when in tree
+ if cp != "dev-python/tbc":
+ build_dict = {}
+ packageDict = {}
+ ebuild_id_list = []
+ # split the cp to categories and package
+ element = cp.split('/')
+ categories = element[0]
+ package = element[1]
+ log_msg = "C %s:%s" % (cp, repo,)
+ add_tbc_logs(conn, log_msg, "info", config_id)
+ pkgdir = self._myportdb.getRepositoryPath(repo) + "/" + cp
+ config_id_list = []
+ config_id_list.append(config_id)
+ config_cpv_listDict = init_package.config_match_ebuild(cp, config_id_list)
+ if config_cpv_listDict != {}:
+ cpv = config_cpv_listDict[config_id]['cpv']
+ packageDict[cpv] = init_package.get_packageDict(pkgdir, cpv, repo)
+ build_dict['checksum'] = packageDict[cpv]['ebuild_version_checksum_tree']
+ build_dict['package_id'] = get_package_id(conn, categories, package, repo)
+ build_dict['ebuild_version'] = packageDict[cpv]['ebuild_version_tree']
+ ebuild_id = get_ebuild_id_db_checksum(conn, build_dict)
+ if ebuild_id is not None:
+ ebuild_id_list.append(ebuild_id)
+ init_package.add_new_ebuild_buildquery_db(ebuild_id_list, packageDict, config_cpv_listDict)
+ log_msg = "C %s:%s ... Done." % (cp, repo,)
+ add_tbc_logs(conn, log_msg, "info", config_id)
+ return
+
+def add_buildquery_main(config_id):
+ conn = 0
+ config_setup = get_config(conn, config_id)
+ log_msg = "Adding build jobs for: %s" % (config_setup,)
+ add_tbc_logs(conn, log_msg, "info", config_id)
+ check_make_conf()
+ log_msg = "Check configs done"
+ add_tbc_logs(conn, log_msg, "info", config_profile)
+ # Get default config from the configs table and default_config=1
+ default_config_root = "/var/cache/tbc/" + tbc_settings_dict['tbc_gitreponame'] + "/" + config_setup + "/"
+ # Set config_root (PORTAGE_CONFIGROOT) to default_config_root
+ mysettings = portage.config(config_root = default_config_root)
+ myportdb = portage.portdbapi(mysettings=mysettings)
+ init_package = tbc_package(mysettings, myportdb)
+ log_msg = "Setting default config to: %s" % (config_setup)
+ add_tbc_logs(conn, log_msg, "info", config_is)
+ # Use all exept 2 cores when multiprocessing
+ pool_cores= multiprocessing.cpu_count()
+ if pool_cores >= 3:
+ use_pool_cores = pool_cores - 2
+ else:
+ use_pool_cores = 1
+ pool = multiprocessing.Pool(processes=use_pool_cores)
+
+ repo_trees_list = myportdb.porttrees
+ for repo_dir in repo_trees_list:
+ repo = myportdb.getRepositoryName(repo_dir)
+ repo_dir_list = []
+ repo_dir_list.append(repo_dir)
+
+ # Get the package list from the repo
+ package_list_tree = myportdb.cp_all(trees=repo_dir_list)
+ for cp in sorted(package_list_tree):
+ pool.apply_async(add_cpv_query_pool, (mysettings, myportdb, config_id, cp, repo,))
+ pool.close()
+ pool.join()
+ log_msg = "Adding build jobs for: %s ... Done." % (config_setup,)
+ add_tbc_logs(conn, log_msg, "info", config_profile)
+ return True
+
+def del_buildquery_main(config_id):
+ conn=0
+ config_setup = get_config(conn, config_id)
+ log_msg = "Removeing build jobs for: %s" % (config_setup,)
+ add_tbc_logs(conn, log_msg, "info", config_id)
+ build_job_id_list = get_build_jobs_id_list_config(conn, config_id)
+ if build_job_id_list is not None:
+ for build_job_id in build_job_id_list:
+ del_old_build_jobs(conn, build_job_id)
+ log_msg = "Removeing build jobs for: %s ... Done." % (config_setup,)
+ add_tbc_logs(conn, log_msg, "info", config_id)
+ return True
diff --git a/tbc/pym/check_setup.py b/tbc/pym/check_setup.py
new file mode 100644
index 0000000..46d4391
--- /dev/null
+++ b/tbc/pym/check_setup.py
@@ -0,0 +1,74 @@
+# Copyright 1998-2015 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from __future__ import print_function
+import portage
+import os
+import errno
+
+from portage.exception import DigestException, FileNotFound, ParseError, PermissionDenied
+from tbc.text import get_file_text
+from tbc.sqlquerys import get_config_all_info, add_tbc_logs, get_configmetadata_info, get_setup_info
+
+def check_make_conf(session, config_id, tbc_settings_dict):
+ log_msg = "Checking configs for changes and errors"
+ add_tbc_logs(session, log_msg, "info", config_id)
+ configsDict = {}
+ for ConfigInfo in get_config_all_info(session):
+ attDict={}
+ # Set the config dir
+ SetupInfo = get_setup_info(session, ConfigInfo.ConfigId)
+ check_config_dir = "/var/cache/tbc/" + tbc_settings_dict['tbc_gitreponame'] + "/" + ConfigInfo.Hostname +"/" + SetupInfo.Setup + "/"
+ make_conf_file = check_config_dir + "etc/portage/make.conf"
+ ConfigsMetaDataInfo = get_configmetadata_info(session, ConfigInfo.ConfigId)
+ # Check if we can take a checksum on it.
+ # Check if we have some error in the file. (portage.util.getconfig)
+ # Check if we envorment error with the config. (settings.validate)
+ try:
+ make_conf_checksum_tree = portage.checksum.sha256hash(make_conf_file)[0]
+ portage.util.getconfig(make_conf_file, tolerant=0, allow_sourcing=True, expand=True)
+ mysettings = portage.config(config_root = check_config_dir)
+ mysettings.validate()
+ # With errors we update the db on the config and disable the config
+ except ParseError as e:
+ ConfigsMetaDataInfo.ConfigErrorText = str(e)
+ ConfigsMetaDataInfo.Active = False
+ log_msg = "%s FAIL!" % (ConfigInfo.Hostname,)
+ add_tbc_logs(session, log_msg, "info", config_id)
+ session.commit()
+ else:
+ ConfigsMetaDataInfo.Active = True
+ log_msg = "%s PASS" % (ConfigInfo.Hostname,)
+ add_tbc_logs(session, log_msg, "info", config_id)
+ session.commit()
+ if make_conf_checksum_tree != ConfigsMetaDataInfo.Checksum:
+ ConfigsMetaDataInfo.MakeConfText = get_file_text(make_conf_file)
+ ConfigsMetaDataInfo.Checksum = make_conf_checksum_tree
+ session.commit()
+ log_msg = "Checking configs for changes and errors ... Done"
+ add_tbc_logs(session, log_msg, "info", config_id)
+
+def check_make_conf_guest(session, config_id):
+ make_conf_file = "/etc/portage/make.conf"
+ # Check if we can open the file and close it
+ # Check if we have some error in the file (portage.util.getconfig)
+ # Check if we envorment error with the config (settings.validate)
+ try:
+ make_conf_checksum_tree = portage.checksum.sha256hash(make_conf_file)[0]
+ portage.util.getconfig(make_conf_file, tolerant=0, allow_sourcing=True, expand=True)
+ mysettings = portage.config(config_root = "/")
+ mysettings.validate()
+ # With errors we return false
+ except Exception as e:
+ return False
+ ConfigsMetaDataInfo = get_configmetadata_info(session, config_id)
+ print('make_conf_checksum_tree', make_conf_checksum_tree)
+ print('make_conf_checksum_db', ConfigsMetaDataInfo.Checksum)
+ if make_conf_checksum_tree != ConfigsMetaDataInfo.Checksum:
+ return False
+ return True
+
+def check_configure_guest(session, config_id):
+ pass_make_conf = check_make_conf_guest(session, config_id)
+ print(pass_make_conf)
+ return pass_make_conf
\ No newline at end of file
diff --git a/tbc/pym/db_mapping.py b/tbc/pym/db_mapping.py
new file mode 100644
index 0000000..c215456
--- /dev/null
+++ b/tbc/pym/db_mapping.py
@@ -0,0 +1,260 @@
+# Copyright 1998-2015 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import datetime
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy import Column, Integer, String, Boolean, DateTime, Enum, Text, ForeignKey
+from sqlalchemy.orm import relationship, backref
+
+Base = declarative_base()
+
+class Keywords(Base):
+ KeywordId = Column('keyword_id', Integer, primary_key=True)
+ Keyword = Column('keyword', String)
+ __tablename__ = 'keywords'
+
+class Setups(Base):
+ SetupId = Column('setup_id', Integer, primary_key=True)
+ Setup = Column('setup', String(100))
+ __tablename__ = 'setups'
+
+class Configs(Base):
+ ConfigId = Column('config_id', Integer, primary_key=True)
+ Hostname = Column('hostname', String(150))
+ SetupId = Column('setup_id', Integer, ForeignKey('setups.setup_id'))
+ Host = Column('default_config', Boolean, default=False)
+ __tablename__ = 'configs'
+
+class Logs(Base):
+ LogId = Column('log_id', Integer, primary_key=True)
+ ConfigId = Column('config_id', Integer, ForeignKey('configs.config_id'))
+ LogType = Column('log_type', Enum('info','error','debug','qa','repoman'))
+ Msg = Column('msg', Text)
+ TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow)
+ __tablename__ = 'logs'
+
+class Jobs(Base):
+ JobId = Column('job_id', Integer, primary_key=True)
+ JobType = Column('job_type', Enum('updatedb', 'esync', 'gsync'))
+ Status = Column('status', Enum('Runing', 'Done', 'Waiting'))
+ User = Column('user', String(50))
+ ConfigId = Column('config_id', Integer, ForeignKey('configs.config_id'))
+ RunConfigId = Column('run_config_id', Integer, ForeignKey('configs.config_id'))
+ TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow)
+ __tablename__ = 'jobs'
+
+class ConfigsMetaData(Base):
+ Id = Column('id', Integer, primary_key=True)
+ ConfigId = Column('config_id', Integer, ForeignKey('configs.config_id'))
+ Profile = Column('profile', String(150))
+ KeywordId = Column('keyword_id', Integer, ForeignKey('keywords.keyword_id'))
+ MakeConfText = Column('make_conf_text', Text)
+ Checksum = Column('checksum', String(100))
+ ConfigSync = Column('configsync', Boolean, default=False)
+ Active = Column('active', Boolean, default=False)
+ ConfigErrorText = Column('config_error_text', Text)
+ Updateing = Column('updateing', Boolean, default=False)
+ Status = Column('status', Enum('Stopped', 'Runing', 'Waiting'))
+ Auto = Column('auto', Boolean, default=False)
+ GitWww = Column('git_www', String(200))
+ TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow)
+ __tablename__ = 'configs_metadata'
+
+class Categories(Base):
+ CategoryId = Column('category_id', Integer, primary_key=True)
+ Category = Column('category', String(150))
+ Active = Column('active', Boolean, default=True)
+ TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow)
+ __tablename__ = 'categories'
+
+class Repos(Base):
+ RepoId = Column('repo_id', Integer, primary_key=True)
+ Repo = Column('repo', String(100))
+ __tablename__ = 'repos'
+
+class Packages(Base):
+ PackageId = Column('package_id', Integer, primary_key=True)
+ CategoryId = Column('category_id', Integer, ForeignKey('categories.category_id'))
+ Package = Column('package',String(150))
+ RepoId = Column('repo_id', Integer, ForeignKey('repos.repo_id'))
+ Checksum = Column('checksum', String(100))
+ Active = Column('active', Boolean, default=False)
+ TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow)
+ __tablename__ = 'packages'
+
+class Emails(Base):
+ EmailId = Column('email_id', Integer, primary_key=True)
+ Email = Column('email', String(150))
+ __tablename__ = 'emails'
+
+class PackagesEmails(Base):
+ Id = Column('id', Integer, primary_key=True)
+ PackageId = Column('package_id', Integer, ForeignKey('packages.package_id'))
+ EmailId = Column('email_id', Integer, ForeignKey('emails.email_id'))
+ __tablename__ = 'packages_emails'
+
+class PackagesMetadata(Base):
+ Id = Column('id', Integer, primary_key=True)
+ PackageId = Column('package_id', Integer, ForeignKey('packages.package_id'))
+ Checksum = Column('checksum', String(100))
+ __tablename__ = 'packages_metadata'
+
+class Ebuilds(Base):
+ EbuildId = Column('ebuild_id', Integer, primary_key=True)
+ PackageId = Column('package_id', Integer, ForeignKey('packages.package_id'))
+ Version = Column('version', String(150))
+ Checksum = Column('checksum', String(100))
+ Active = Column('active', Boolean, default=False)
+ TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow)
+ __tablename__ = 'ebuilds'
+
+class EmergeOptions(Base):
+ EmergeOptionId = Column('eoption_id', Integer, primary_key=True)
+ EOption = Column('eoption', String(45))
+ __tablename__ = 'emerge_options'
+
+class ConfigsEmergeOptions(Base):
+ ConfigId = Column('config_id', Integer, ForeignKey('configs.config_id'), primary_key=True)
+ EOptionId = Column('eoption_id', Integer, ForeignKey('emerge_options.eoption_id'))
+ __tablename__ = 'configs_emerge_options'
+
+class BuildJobs(Base):
+ BuildJobId = Column('build_job_id', Integer, primary_key=True)
+ EbuildId = Column('ebuild_id', Integer, ForeignKey('ebuilds.ebuild_id'))
+ SetupId = Column('setup_id', Integer, ForeignKey('setups.setup_id'))
+ ConfigId = Column('config_id', Integer, ForeignKey('configs.config_id'))
+ Status = Column('status', Enum('Waiting','Building','Looked',))
+ BuildNow = Column('build_now', Boolean, default=False)
+ RemoveBin = Column('removebin', Boolean ,default=False)
+ TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow)
+ __tablename__ = 'build_jobs'
+
+class BuildJobsEmergeOptions(Base):
+ Id = Column('id', Integer, primary_key=True)
+ BuildJobId = Column('build_job_id', Integer, ForeignKey('build_jobs.build_job_id'))
+ EOption = Column('eoption_id', Integer, ForeignKey('emerge_options.eoption_id'))
+ __tablename__ = 'build_jobs_emerge_options'
+
+class BuildJobsRedo(Base):
+ Id = Column('id', Integer, primary_key=True)
+ BuildJobId = Column('build_job_id', Integer, ForeignKey('build_jobs.build_job_id'))
+ FailTimes = Column('fail_times', Integer)
+ FailType = Column('fail_type', String(50))
+ TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow)
+ __tablename__ = 'build_jobs_redo'
+
+class Uses(Base):
+ UseId = Column('use_id', Integer, primary_key=True)
+ Flag = Column('flag', String(150))
+ __tablename__ = 'uses'
+
+class BuildJobsUse(Base):
+ Id = Column('id', Integer, primary_key=True)
+ BuildJobId = Column('build_job_id', Integer, ForeignKey('build_jobs.build_job_id'))
+ UseId = Column('use_id', Integer, ForeignKey('uses.use_id'))
+ Status = Column('status', Boolean, default=False)
+ __tablename__ = 'build_jobs_use'
+
+class HiLightCss(Base):
+ HiLightCssId = Column('hilight_css_id', Integer, primary_key=True)
+ HiLightCssName = Column('hilight_css_name', String(30))
+ HiLightCssCollor = Column('hilight_css_collor', String(30))
+ __tablename__ = 'hilight_css'
+
+class HiLight(Base):
+ HiLightId = Column('hilight_id', Integer, primary_key=True)
+ HiLightSearch = Column('hilight_search', String(50))
+ HiLightSearchEnd = Column('hilight_search_end', String(50))
+ HiLightSearchPattern = Column('hilight_search_pattern', String(50))
+ HiLightCssId = Column('hilight_css_id', Integer, ForeignKey('hilight_css.hilight_css_id'))
+ HiLightStart = Column('hilight_start', Integer)
+ HiLightEnd = Column('hilight_end', Integer)
+ __tablename__ = 'hilight'
+
+class BuildLogs(Base):
+ BuildLogId = Column('build_log_id', Integer, primary_key=True)
+ EbuildId = Column('ebuild_id', Integer, ForeignKey('ebuilds.ebuild_id'))
+ Fail = Column('fail', Boolean, default=False)
+ SummeryText = Column('summery_text', Text)
+ LogHash = Column('log_hash', String(100))
+ BugId = Column('bug_id', Integer, default=0)
+ TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow)
+ __tablename__ = 'build_logs'
+
+class EmergeInfo(Base):
+ EInfoId = Column('einfo_id', Integer, primary_key=True)
+ EmergeInfoText = Column('emerge_info_text', Text)
+ __tablename__ = 'emerge_info'
+
+class BuildLogsConfig(Base):
+ LogId = Column('log_id', Integer, primary_key=True)
+ BuildLogId = Column('build_log_id', Integer, ForeignKey('build_logs.build_log_id'))
+ ConfigId = Column('config_id', Integer, ForeignKey('configs.config_id'))
+ EInfoId = Column('einfo_id', Integer, ForeignKey('emerge_info.einfo_id'))
+ LogName = Column('logname', String(450))
+ TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow)
+ __tablename__ = 'build_logs_config'
+
+class BuildLogsHiLight(Base):
+ BuildLogHiLightId = Column('id', Integer, primary_key=True)
+ LogId = Column('log_id', Integer, ForeignKey('build_logs_config.log_id'))
+ StartLine = Column('start_line', Integer)
+ EndLine = Column('end_line', Integer)
+ HiLightCssId = Column('hilight_css_id', Integer, ForeignKey('hilight_css.hilight_css_id'))
+ __tablename__ = 'build_logs_hilight'
+
+class BuildLogsEmergeOptions(Base):
+ Id = Column('id', Integer, primary_key=True)
+ BuildLogId = Column('build_log_id', Integer, ForeignKey('Build_logs.Build_log_id'))
+ EmergeOptionId = Column('eoption_id', Integer, ForeignKey('emerge_options.eoption_id'))
+ __tablename__ = 'build_logs_emerge_options'
+
+class BuildLogsUse(Base):
+ Id = Column('id', Integer, primary_key=True)
+ BuildLogId = Column('build_log_id', Integer, ForeignKey('build_logs.build_log_id'))
+ UseId = Column('use_id', Integer, ForeignKey('uses.use_id'))
+ Status = Column('status', Boolean, default=False)
+ __tablename__ = 'build_logs_use'
+
+class ErrorsInfo(Base):
+ ErrorId = Column('error_id', Integer, primary_key=True)
+ ErrorName = Column('error_name', String)
+ ErrorSearch = Column('error_search', String)
+ __tablename__ = 'errors_info'
+
+class BuildLogsErrors(Base):
+ BuildLogErrorId = Column('id', Integer, primary_key=True)
+ BuildLogId = Column('build_log_id', Integer, ForeignKey('build_logs.build_log_id'))
+ ErrorId = Column('error_id', Integer, ForeignKey('errors_info.error_id'))
+ __tablename__ = 'build_logs_errors'
+
+class Restrictions(Base):
+ RestrictionId = Column('restriction_id', Integer, primary_key=True)
+ Restriction = Column('restriction', String(150))
+ __tablename__ = 'restrictions'
+
+class EbuildsRestrictions(Base):
+ Id = Column('id', Integer, primary_key=True)
+ EbuildId = Column('ebuild_id', ForeignKey('ebuilds.ebuild_id'))
+ RestrictionId = Column('restriction_id', ForeignKey('restrictions.restriction_id'))
+ __tablename__ = 'ebuilds_restrictions'
+
+class EbuildsIUse(Base):
+ Id = Column('id', Integer, primary_key=True)
+ EbuildId = Column('ebuild_id', ForeignKey('ebuilds.ebuild_id'))
+ UseId = Column('use_id', ForeignKey('uses.use_id'))
+ Status = Column('status', Boolean, default=False)
+ __tablename__= 'ebuilds_iuse'
+
+class EbuildsKeywords(Base):
+ Id = Column('id', Integer, primary_key=True)
+ EbuildId = Column('ebuild_id', ForeignKey('ebuilds.ebuild_id'))
+ KeywordId = Column('keyword_id', ForeignKey('keywords.keyword_id'))
+ Status = Column('status', Enum('Stable','Unstable','Negative'))
+ __tablename__ = 'ebuilds_keywords'
+
+class EbuildsMetadata(Base):
+ Id = Column('id', Integer, primary_key=True)
+ EbuildId = Column('ebuild_id', ForeignKey('ebuilds.ebuild_id'))
+ Revision = Column('revision', String(30))
+ __tablename__ = 'ebuilds_metadata'
diff --git a/tbc/pym/depclean.py b/tbc/pym/depclean.py
new file mode 100644
index 0000000..3154ac5
--- /dev/null
+++ b/tbc/pym/depclean.py
@@ -0,0 +1,53 @@
+# Copyright 1998-2015 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from __future__ import print_function
+import portage
+from portage._sets.base import InternalPackageSet
+from _emerge.main import parse_opts
+from tbc.actions import load_emerge_config, action_depclean, calc_depclean
+
+def do_depclean():
+ mysettings, mytrees, mtimedb = load_emerge_config()
+ myroot = mysettings["ROOT"]
+ root_config = mytrees[myroot]["root_config"]
+ psets = root_config.setconfig.psets
+ args_set = InternalPackageSet(allow_repo=True)
+ spinner=None
+ scheduler=None
+ tmpcmdline = []
+ tmpcmdline.append("--depclean")
+ tmpcmdline.append("--pretend")
+ print("depclean",tmpcmdline)
+ myaction, myopts, myfiles = parse_opts(tmpcmdline, silent=False)
+ if myfiles:
+ args_set.update(myfiles)
+ matched_packages = False
+ for x in args_set:
+ if vardb.match(x):
+ matched_packages = True
+ if not matched_packages:
+ return 0
+
+ rval, cleanlist, ordered, req_pkg_count, unresolvable = calc_depclean(mysettings, mytrees, mtimedb["ldpath"], myopts, myaction, args_set, spinner)
+ print('rval, cleanlist, ordered, req_pkg_count, unresolvable', rval, cleanlist, ordered, req_pkg_count, unresolvable)
+ if unresolvable != []:
+ return True
+ if cleanlist != []:
+ conflict_package_list = []
+ for depclean_cpv in cleanlist:
+ if portage.versions.cpv_getkey(depclean_cpv) in list(psets["system"]):
+ conflict_package_list.append(depclean_cpv)
+ if portage.versions.cpv_getkey(depclean_cpv) in list(psets['selected']):
+ conflict_package_list.append(depclean_cpv)
+ print('conflict_package_list', conflict_package_list)
+ if conflict_package_list == []:
+ tmpcmdline = []
+ tmpcmdline.append("--depclean")
+ myaction, myopts, myfiles = parse_opts(tmpcmdline, silent=False)
+ rval = action_depclean(mysettings, mytrees, mtimedb["ldpath"], myopts, myaction, myfiles, spinner, scheduler=None)
+ return True
+ else:
+ print("conflicting packages: %s", conflict_package_list)
+ return True
+ return True
diff --git a/tbc/pym/flags.py b/tbc/pym/flags.py
new file mode 100644
index 0000000..eea48f3
--- /dev/null
+++ b/tbc/pym/flags.py
@@ -0,0 +1,231 @@
+#!/usr/bin/python
+#
+# Copyright 1998-2015 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+# Origin flags.py from portage public api repo
+from __future__ import print_function
+from _emerge.main import parse_opts
+from _emerge.depgraph import backtrack_depgraph, depgraph, resume_depgraph
+from _emerge.create_depgraph_params import create_depgraph_params
+from _emerge.actions import load_emerge_config
+import portage
+import os
+
+class tbc_use_flags(object):
+
+ def __init__(self, mysettings, myportdb, cpv):
+ self._mysettings = mysettings
+ self._myportdb = myportdb
+ self._cpv = cpv
+
+ def get_iuse(self):
+ """Gets the current IUSE flags from the tree
+ To be used when a gentoolkit package object is not needed
+ @type: cpv: string
+ @param cpv: cat/pkg-ver
+ @rtype list
+ @returns [] or the list of IUSE flags
+ """
+ return self._myportdb.aux_get(self._cpv, ["IUSE"])[0].split()
+
+ def reduce_flag(self, flag):
+ """Absolute value function for a USE flag
+ @type flag: string
+ @param flag: the use flag to absolute.
+ @rtype: string
+ @return absolute USE flag
+ """
+ if flag[0] in ["+","-"]:
+ return flag[1:]
+ else:
+ return flag
+
+ def reduce_flags(self, the_list):
+ """Absolute value function for a USE flag list
+ @type the_list: list
+ @param the_list: the use flags to absolute.
+ @rtype: list
+ @return absolute USE flags
+ """
+ r=[]
+ for member in the_list:
+ r.append(self.reduce_flag(member))
+ return r
+
+ def filter_flags(self, use, use_expand_hidden, usemasked, useforced):
+ """Filter function to remove hidden or otherwise not normally
+ visible USE flags from a list.
+ @type use: list
+ @param use: the USE flag list to be filtered.
+ @type use_expand_hidden: list
+ @param use_expand_hidden: list of flags hidden.
+ @type usemasked: list
+ @param usemasked: list of masked USE flags.
+ @type useforced: list
+ @param useforced: the forced USE flags.
+ @rtype: list
+ @return the filtered USE flags.
+ """
+ # clean out some environment flags, since they will most probably
+ # be confusing for the user
+ for f in use_expand_hidden:
+ f=f.lower() + "_"
+ for x in use:
+ if f in x:
+ use.remove(x)
+ # clean out any arch's
+ archlist = self._mysettings["PORTAGE_ARCHLIST"].split()
+ for a in use[:]:
+ if a in archlist:
+ use.remove(a)
+ # clean out any abi_ flag
+ for a in use[:]:
+ if a.startswith("abi_"):
+ use.remove(a)
+ # clean out any python_ flag
+ for a in use[:]:
+ if a.startswith("python_"):
+ use.remove(a)
+
+ # dbl check if any from usemasked or useforced are still there
+ masked = usemasked + useforced
+ for a in use[:]:
+ if a in masked:
+ use.remove(a)
+ return use
+
+ def get_all_cpv_use(self):
+ """Uses portage to determine final USE flags and settings for an emerge
+ @type cpv: string
+ @param cpv: eg cat/pkg-ver
+ @rtype: lists
+ @return use, use_expand_hidden, usemask, useforce
+ """
+ use = None
+ self._mysettings.unlock()
+ try:
+ self._mysettings.setcpv(self._cpv, use_cache=None, mydb=self._myportdb)
+ use = self._mysettings['PORTAGE_USE'].split()
+ use_expand_hidden = self._mysettings["USE_EXPAND_HIDDEN"].split()
+ usemask = list(self._mysettings.usemask)
+ useforce = list(self._mysettings.useforce)
+ except KeyError:
+ self._mysettings.reset()
+ self._mysettings.lock()
+ return [], [], [], []
+ # reset cpv filter
+ self._mysettings.reset()
+ self._mysettings.lock()
+ print(usemask)
+ return use, use_expand_hidden, usemask, useforce
+
+ def get_all_cpv_use_looked(self):
+ """Uses portage to determine final USE flags and settings for an emerge
+ @type cpv: string
+ @param cpv: eg cat/pkg-ver
+ @rtype: lists
+ @return use, use_expand_hidden, usemask, useforce
+ """
+ # use = self._mysettings['PORTAGE_USE'].split()
+ use = os.environ['USE'].split()
+ use_expand_hidden = self._mysettings["USE_EXPAND_HIDDEN"].split()
+ usemask = list(self._mysettings.usemask)
+ useforce = list(self._mysettings.useforce)
+ return use, use_expand_hidden, usemask, useforce
+
+ def get_all_cpv_use_pkg(self, pkg, settings):
+ """Uses portage to determine final USE flags and settings for an emerge
+ @type cpv: string
+ @param cpv: eg cat/pkg-ver
+ @rtype: lists
+ @return use, use_expand_hidden, usemask, useforce
+ """
+ # use = self._mysettings['PORTAGE_USE'].split()
+ use_list = list(pkg.use.enabled)
+ use_expand_hidden = settings["USE_EXPAND_HIDDEN"].split()
+ usemask = list(settings.usemask)
+ useforced = list(settings.useforce)
+ return use_list, use_expand_hidden, usemask, useforced
+
+ def get_flags(self):
+ """Retrieves all information needed to filter out hidden, masked, etc.
+ USE flags for a given package.
+
+ @type cpv: string
+ @param cpv: eg. cat/pkg-ver
+ @type final_setting: boolean
+ @param final_setting: used to also determine the final
+ enviroment USE flag settings and return them as well.
+ @rtype: list or list, list
+ @return IUSE or IUSE, final_flags
+ """
+ final_use, use_expand_hidden, usemasked, useforced = self.get_all_cpv_use()
+ iuse_flags = self.filter_flags(self.get_iuse(), use_expand_hidden, usemasked, useforced)
+ #flags = filter_flags(use_flags, use_expand_hidden, usemasked, useforced)
+ final_flags = self.filter_flags(final_use, use_expand_hidden, usemasked, useforced)
+ return iuse_flags, final_flags
+
+ def get_flags_looked(self):
+ """Retrieves all information needed to filter out hidden, masked, etc.
+ USE flags for a given package.
+
+ @type cpv: string
+ @param cpv: eg. cat/pkg-ver
+ @type final_setting: boolean
+ @param final_setting: used to also determine the final
+ enviroment USE flag settings and return them as well.
+ @rtype: list or list, list
+ @return IUSE or IUSE, final_flags
+ """
+ final_use, use_expand_hidden, usemasked, useforced = self.get_all_cpv_use_looked()
+ iuse_flags = self.filter_flags(self.get_iuse(), use_expand_hidden, usemasked, useforced)
+ #flags = filter_flags(use_flags, use_expand_hidden, usemasked, useforced)
+ final_flags = self.filter_flags(final_use, use_expand_hidden, usemasked, useforced)
+ return iuse_flags, final_flags
+
+ def get_flags_pkg(self, pkg, settings):
+ """Retrieves all information needed to filter out hidden, masked, etc.
+ USE flags for a given package.
+ @type cpv: string
+ @param cpv: eg. cat/pkg-ver
+ @type final_setting: boolean
+ @param final_setting: used to also determine the final
+ enviroment USE flag settings and return them as well.
+ @rtype: list or list, list
+ @return IUSE or IUSE, final_flags
+ """
+ final_use, use_expand_hidden, usemasked, useforced = self.get_all_cpv_use_pkg(pkg, settings)
+ iuse_flags = self.filter_flags(list(pkg.iuse.all), use_expand_hidden, usemasked, useforced)
+ #flags = filter_flags(use_flags, use_expand_hidden, usemasked, useforced)
+ final_flags = self.filter_flags(final_use, use_expand_hidden, usemasked, useforced)
+ return iuse_flags, final_flags
+
+ def comper_useflags(self, build_dict):
+ iuse_flags, use_enable = self.get_flags()
+ iuse = []
+ build_use_flags_dict = build_dict['build_useflags']
+ build_use_flags_list = []
+ if use_enable == []:
+ if build_use_flags_dict is None:
+ return None
+ for iuse_line in iuse_flags:
+ iuse.append(self.reduce_flag(iuse_line))
+ iuse_flags_list = list(set(iuse))
+ use_disable = list(set(iuse_flags_list).difference(set(use_enable)))
+ use_flagsDict = {}
+ for x in use_enable:
+ use_flagsDict[x] = True
+ for x in use_disable:
+ use_flagsDict[x] = False
+ print("use_flagsDict", use_flagsDict)
+ for k, v in use_flagsDict.items():
+ if build_use_flags_dict[k] != v:
+ if build_use_flags_dict[k]:
+ build_use_flags_list.append(k)
+ else:
+ build_use_flags_list.append("-" + k)
+ if build_use_flags_list == []:
+ build_use_flags_list = None
+ print(build_use_flags_list)
+ return build_use_flags_list
diff --git a/tbc/pym/jobs.py b/tbc/pym/jobs.py
new file mode 100644
index 0000000..840932e
--- /dev/null
+++ b/tbc/pym/jobs.py
@@ -0,0 +1,86 @@
+# Copyright 1998-2015 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from __future__ import print_function
+from tbc.sync import sync_tree
+#from tbc.buildquerydb import add_buildquery_main, del_buildquery_main
+from tbc.updatedb import update_db_main
+from tbc.sqlquerys import get_config_id, add_tbc_logs, get_jobs, update_job_list
+
+def jobs_main(session, config_id):
+ JobsInfo = get_jobs(session, config_id)
+ if JobsInfo is None:
+ return
+ for JobInfo in JobsInfo:
+ job = JobInfo.JobType
+ run_config_id = JobInfo.RunConfigId
+ job_id = JobInfo.JobId
+ log_msg = "Job: %s Type: %s" % (job_id, job,)
+ add_tbc_logs(session, log_msg, "info", config_id)
+ if job == "addbuildquery":
+ update_job_list(session, "Runing", job_id)
+ log_msg = "Job %s is runing." % (job_id,)
+ add_tbc_logs(session, log_msg, "info", config_id)
+ #result = add_buildquery_main(run_config_id)
+ #if result is True:
+ # update_job_list(session, "Done", job_id)
+ # log_msg = "Job %s is done.." % (job_id,)
+ # add_tbc_logs(session, log_msg, "info", config_id)
+ #else:
+ # update_job_list(session, "Fail", job_id)
+ # log_msg = "Job %s did fail." % (job_id,)
+ # add_tbc_logs(session, log_msg, "info", config_id)
+ elif job == "delbuildquery":
+ update_job_list(session, "Runing", job_id)
+ log_msg = "Job %s is runing." % (job_id,)
+ add_tbc_logs(session, log_msg, "info", config_id)
+ #result = del_buildquery_main(config_id)
+ #if result is True:
+ # update_job_list(session, "Done", job_id)
+ # log_msg = "Job %s is done.." % (job_id,)
+ # add_tbc_logs(session, log_msg, "info", config_id)
+ #else:
+ # update_job_list(session, "Fail", job_id)
+ # log_msg = "Job %s did fail." % (job_id,)
+ # add_tbc_logs(session, log_msg, "info", config_id)
+ elif job == "gsync":
+ update_job_list(session, "Runing", job_id)
+ log_msg = "Job %s is runing." % (job_id,)
+ add_tbc_logs(session, log_msg, "info", config_id)
+ result = True
+ # result = git_pull(session)
+ if result:
+ update_job_list(session, "Done", job_id)
+ log_msg = "Job %s is done.." % (job_id,)
+ add_tbc_logs(session, log_msg, "info", config_id)
+ else:
+ update_job_list(session, "Fail", job_id)
+ log_msg = "Job %s did fail." % (job_id,)
+ add_tbc_logs(session, log_msg, "info", config_id)
+ elif job == "esync":
+ update_job_list(session, "Runing", job_id)
+ log_msg = "Job %s is runing." % (job_id,)
+ add_tbc_logs(session, log_msg, "info", config_id)
+ result = sync_tree(session)
+ if result:
+ update_job_list(session, "Done", job_id)
+ log_msg = "Job %s is done.." % (job_id,)
+ add_tbc_logs(session, log_msg, "info", config_id)
+ else:
+ update_job_list(session, "Fail", job_id)
+ log_msg = "Job %s did fail." % (job_id,)
+ add_tbc_logs(session, log_msg, "info", config_id)
+ elif job == "updatedb":
+ update_job_list(session, "Runing", job_id)
+ log_msg = "Job %s is runing." % (job_id,)
+ add_tbc_logs(session, log_msg, "info", config_id)
+ result = update_db_main(session, config_id)
+ if result:
+ update_job_list(session, "Done", job_id)
+ log_msg = "Job %s is done.." % (job_id,)
+ add_tbc_logs(session, log_msg, "info", config_id)
+ else:
+ update_job_list(session, "Fail", job_id)
+ log_msg = "Job %s did fail." % (job_id,)
+ add_tbc_logs(session, log_msg, "info", config_id)
+ return
diff --git a/tbc/pym/old_cpv.py b/tbc/pym/old_cpv.py
new file mode 100644
index 0000000..dbeba56
--- /dev/null
+++ b/tbc/pym/old_cpv.py
@@ -0,0 +1,97 @@
+# Copyright 1998-2015 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from __future__ import print_function
+import logging
+from tbc.readconf import get_conf_settings
+reader=get_conf_settings()
+tbc_settings_dict=reader.read_tbc_settings_all()
+# make a CM
+from tbc.ConnectionManager import connectionManager
+CM=connectionManager(tbc_settings_dict)
+#selectively import the pgsql/mysql querys
+if CM.getName()=='pgsql':
+ from tbc.pgsql_querys import *
+
+class tbc_old_cpv(object):
+
+ def __init__(self, myportdb, mysettings):
+ self._mysettings = mysettings
+ self._myportdb = myportdb
+
+ def mark_old_ebuild_db(self, package_id):
+ conn=CM.getConnection()
+ # Get the ebuild list for cp
+ cp, repo = get_cp_repo_from_package_id(conn, package_id)
+ mytree = []
+ mytree.append(self._myportdb.getRepositoryPath(repo))
+ ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree)
+ # Get ebuild list on categories, package in the db
+ ebuild_list_db = cp_list_db(conn, package_id)
+ # Check if don't have the ebuild in the tree
+ # Add it to the no active list
+ old_ebuild_list = []
+ for ebuild_line in ebuild_list_db:
+ cpv_db = cp + "-" + ebuild_line[0]
+ if not cpv_db in ebuild_list_tree:
+ old_ebuild_list.append(ebuild_line)
+ # Set no active on ebuilds in the db that no longer in tree
+ if old_ebuild_list != []:
+ for old_ebuild in old_ebuild_list:
+ logging.info("O %s/%s-%s", categories, package, old_ebuild[0])
+ add_old_ebuild(conn,package_id, old_ebuild_list)
+ # Check if we have older no activ ebuilds then 60 days
+ ebuild_old_list_db = cp_list_old_db(conn,package_id)
+ # Delete older ebuilds in the db
+ if ebuild_old_list_db != []:
+ for del_ebuild_old in ebuild_old_list_db:
+ logging.info("D %s/%s-%s", categories, package, del_ebuild_old[1])
+ del_old_ebuild(conn,ebuild_old_list_db)
+ CM.putConnection(conn)
+
+ def mark_old_package_db(self, package_id_list_tree):
+ conn=CM.getConnection()
+ # Get categories/package list from db
+ package_list_db = cp_all_db(conn)
+ old_package_id_list = []
+ # Check if don't have the categories/package in the tree
+ # Add it to the no active list
+ for package_line in package_list_db:
+ if not package_line in package_id_list_tree:
+ old_package_id_list.append(package_line)
+ # Set no active on categories/package and ebuilds in the db that no longer in tree
+ if old_package_id_list != []:
+ mark_old_list = add_old_package(conn,old_package_id_list)
+ if mark_old_list != []:
+ for x in mark_old_list:
+ element = get_cp_from_package_id(conn,x)
+ logging.info("O %s", element[0])
+ # Check if we have older no activ categories/package then 60 days
+ del_package_id_old_list = cp_all_old_db(conn,old_package_id_list)
+ # Delete older categories/package and ebuilds in the db
+ if del_package_id_old_list != []:
+ for i in del_package_id_old_list:
+ element = get_cp_from_package_id(conn,i)
+ logging.info("D %s", element)
+ del_old_package(conn,del_package_id_old_list)
+ CM.putConnection(conn)
+
+ def mark_old_categories_db(self):
+ conn=CM.getConnection()
+ # Get categories list from the tree and db
+ categories_list_tree = self._mysettings.categories
+ categories_list_db =get_categories_db(conn)
+ categories_old_list = []
+ # Check if don't have the categories in the tree
+ # Add it to the no active list
+ for categories_line in categories_list_db:
+ if not categories_line[0] in categories_list_tree:
+ old_c = get_old_categories(conn,categories_line[0])
+ if old_c is not None:
+ categories_old_list.append(categories_line)
+ # Delete older categories in the db
+ if categories_old_list != []:
+ for real_old_categories in categories_old_list:
+ del_old_categories(conn,real_old_categoriess)
+ logging.info("D %s", real_old_categories)
+ CM.putConnection(conn)
\ No newline at end of file
diff --git a/tbc/pym/package.py b/tbc/pym/package.py
new file mode 100644
index 0000000..af1bff5
--- /dev/null
+++ b/tbc/pym/package.py
@@ -0,0 +1,355 @@
+# Copyright 1998-2015 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from __future__ import print_function
+import portage
+from portage.xml.metadata import MetaDataXML
+from tbc.flags import tbc_use_flags
+from tbc.manifest import tbc_manifest
+from tbc.text import get_ebuild_cvs_revision
+from tbc.flags import tbc_use_flags
+from tbc.sqlquerys import add_tbc_logs, get_package_info, get_config_info, \
+ add_new_build_job, add_new_ebuild_sql, get_ebuild_id_list, add_old_ebuild, \
+ get_package_metadata_sql, update_package_metadata, update_manifest_sql, \
+ get_package_info_from_package_id, get_config_all_info, add_new_package_sql, \
+ get_ebuild_checksums, get_ebuild_id_db, get_configmetadata_info, get_setup_info
+from tbc.readconf import get_conf_settings
+
+class tbc_package(object):
+
+ def __init__(self, session, mysettings, myportdb, config_id, tbc_settings_dict):
+ self._session = session
+ self._mysettings = mysettings
+ self._myportdb = myportdb
+ self._config_id = config_id
+ self._tbc_settings_dict = tbc_settings_dict
+
+ def change_config(self, host_config):
+ # Change config_root config_setup = table config
+ my_new_setup = "/var/cache/tbc/" + self._tbc_settings_dict['tbc_gitreponame'] + "/" + host_config + "/"
+ mysettings_setup = portage.config(config_root = my_new_setup)
+ return mysettings_setup
+
+ def config_match_ebuild(self, cp, config_list):
+ config_cpv_listDict ={}
+ if config_list == []:
+ return config_cpv_listDict
+ for config_id in config_list:
+ ConfigInfo = get_config_info(self._session, config_id)
+ ConfigsMetaData = get_configmetadata_info(self._session, config_id)
+ if ConfigsMetaData.Auto and ConfigsMetaData.Active and ConfigsMetaData.Status != 'Stopped':
+ SetupInfo = get_setup_info(self._session, config_id)
+ mysettings_setup = self.change_config(ConfigInfo.Hostname + "/" + SetupInfo.Setup)
+ myportdb_setup = portage.portdbapi(mysettings=mysettings_setup)
+
+ # Get the latest cpv from portage with the config that we can build
+ build_cpv = myportdb_setup.xmatch('bestmatch-visible', cp)
+
+ # Check if could get cpv from portage and add it to the config_cpv_listDict.
+ if build_cpv != "" and not ConfigInfo.SetupId in config_cpv_listDict:
+
+ # Get the iuse and use flags for that config/setup and cpv
+ init_useflags = tbc_use_flags(mysettings_setup, myportdb_setup, build_cpv)
+ iuse_flags_list, final_use_list = init_useflags.get_flags()
+ iuse_flags_list2 = []
+ for iuse_line in iuse_flags_list:
+ iuse_flags_list2.append( init_useflags.reduce_flag(iuse_line))
+
+ # Dict the needed info
+ attDict = {}
+ attDict['cpv'] = build_cpv
+ attDict['useflags'] = final_use_list
+ attDict['iuse'] = iuse_flags_list2
+ config_cpv_listDict[ConfigInfo.SetupId] = attDict
+
+ # Clean some cache
+ myportdb_setup.close_caches()
+ portage.portdbapi.portdbapi_instances.remove(myportdb_setup)
+ return config_cpv_listDict
+
+ def get_ebuild_metadata(self, cpv, repo):
+ # Get the auxdbkeys infos for the ebuild
+ try:
+ ebuild_auxdb_list = self._myportdb.aux_get(cpv, portage.auxdbkeys, myrepo=repo)
+ except:
+ ebuild_auxdb_list = None
+ else:
+ for i in range(len(ebuild_auxdb_list)):
+ if ebuild_auxdb_list[i] == '':
+ ebuild_auxdb_list[i] = ''
+ return ebuild_auxdb_list
+
+ def get_packageDict(self, pkgdir, cpv, repo):
+
+ #Get categories, package and version from cpv
+ ebuild_version_tree = portage.versions.cpv_getversion(cpv)
+ element = portage.versions.cpv_getkey(cpv).split('/')
+ categories = element[0]
+ package = element[1]
+
+ # Make a checksum of the ebuild
+ try:
+ ebuild_version_checksum_tree = portage.checksum.sha256hash(pkgdir + "/" + package + "-" + ebuild_version_tree + ".ebuild")[0]
+ except:
+ ebuild_version_checksum_tree = "0"
+ log_msg = "QA: Can't checksum the ebuild file. %s on repo %s" % (cpv, repo,)
+ add_tbc_logs(self._session, log_msg, "info", self._config_id)
+ log_msg = "C %s:%s ... Fail." % (cpv, repo)
+ add_tbc_logs(self._session, log_msg, "info", self._config_id)
+ ebuild_version_cvs_revision_tree = '0'
+ else:
+ ebuild_version_cvs_revision_tree = get_ebuild_cvs_revision(pkgdir + "/" + package + "-" + ebuild_version_tree + ".ebuild")
+
+ # Get the ebuild metadata
+ ebuild_version_metadata_tree = self.get_ebuild_metadata(cpv, repo)
+ # if there some error to get the metadata we add rubish to the
+ # ebuild_version_metadata_tree and set ebuild_version_checksum_tree to 0
+ # so it can be updated next time we update the db
+ if ebuild_version_metadata_tree is None:
+ log_msg = " QA: %s have broken metadata on repo %s" % (cpv, repo)
+ add_tbc_logs(self._session, log_msg, "info", self._config_id)
+ ebuild_version_metadata_tree = ['','','','','','','','','','','','','','','','','','','','','','','','','']
+ ebuild_version_checksum_tree = '0'
+
+ # add the ebuild info to the dict packages
+ PackageInfo = get_package_info(self._session, categories, package, repo)
+ attDict = {}
+ attDict['package_id'] = PackageInfo.PackageId
+ attDict['repo'] = repo
+ attDict['ebuild_version'] = ebuild_version_tree
+ attDict['checksum']= ebuild_version_checksum_tree
+ attDict['ebuild_version_metadata_tree'] = ebuild_version_metadata_tree
+ #attDict['ebuild_version_text_tree'] = ebuild_version_text_tree[0]
+ attDict['ebuild_version_revision_tree'] = ebuild_version_cvs_revision_tree
+ return attDict
+
+ def add_new_build_job_db(self, ebuild_id_list, packageDict, config_cpv_listDict):
+ # Get the needed info from packageDict and config_cpv_listDict and put that in buildqueue
+ # Only add it if ebuild_version in packageDict and config_cpv_listDict match
+ if config_cpv_listDict is not None:
+ # Unpack config_cpv_listDict
+ for setup_id, v in config_cpv_listDict.items():
+ build_cpv = v['cpv']
+ iuse_flags_list = list(set(v['iuse']))
+ use_enable= v['useflags']
+ use_disable = list(set(iuse_flags_list).difference(set(use_enable)))
+ # Make a dict with enable and disable use flags for ebuildqueuedwithuses
+ use_flagsDict = {}
+ for x in use_enable:
+ use_flagsDict[x] = True
+ for x in use_disable:
+ use_flagsDict[x] = False
+ # Unpack packageDict
+ i = 0
+ for k, v in packageDict.items():
+ ebuild_id = ebuild_id_list[i]
+
+ # Comper and add the cpv to buildqueue
+ if build_cpv == k:
+ add_new_build_job(self._session, ebuild_id, setup_id, use_flagsDict, self._config_id)
+ # B = Build cpv use-flags config
+ # FIXME log_msg need a fix to log the use flags corect.
+ log_msg = "B %s:%s USE: %s Setup: %s" % (k, v['repo'], use_flagsDict, setup_id,)
+ add_tbc_logs(self._session, log_msg, "info", self._config_id)
+ i = i +1
+
+ def get_package_metadataDict(self, pkgdir, package_id):
+ # Make package_metadataDict
+ attDict = {}
+ package_metadataDict = {}
+ md_email_list = []
+ # changelog_checksum_tree = portage.checksum.sha256hash(pkgdir + "/ChangeLog")
+ # changelog_text_tree = get_file_text(pkgdir + "/ChangeLog")
+ herd = None
+ pkg_md = MetaDataXML(pkgdir + "/metadata.xml", herd)
+ #metadata_xml_text_tree = get_file_text(pkgdir + "/metadata.xml")
+ # attDict['changelog_checksum'] = changelog_checksum_tree[0]
+ # attDict['changelog_text'] = changelog_text_tree
+ tmp_herds = pkg_md.herds()
+ if tmp_herds != ():
+ attDict['metadata_xml_herds'] = tmp_herds[0]
+ md_email_list.append(attDict['metadata_xml_herds'] + '@gentoo.org')
+ for maint in pkg_md.maintainers():
+ md_email_list.append(maint.email)
+ if md_email_list != []:
+ attDict['metadata_xml_email'] = md_email_list
+ else:
+ log_msg = "Metadata file %s missing Email" % (pkgdir + "/metadata.xml")
+ add_tbc_logs(self._session, log_msg, "qa", self._config_id)
+ attDict['metadata_xml_email'] = False
+ attDict['metadata_xml_checksum'] = portage.checksum.sha256hash(pkgdir + "/metadata.xml")[0]
+ #attDict['metadata_xml_text'] = metadata_xml_text_tree
+ package_metadataDict[package_id] = attDict
+ return package_metadataDict
+
+ def add_package(self, packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_checksum_tree):
+ # Use packageDict to update the db
+ ebuild_id_list = add_new_ebuild_sql(self._session, packageDict)
+
+ # Make old ebuilds unactive
+ for ebuild_id in ebuild_id_list:
+ new_ebuild_id_list.append(ebuild_id)
+ for ebuild_id in get_ebuild_id_list(self._session, package_id):
+ if not ebuild_id in new_ebuild_id_list:
+ if not ebuild_id in old_ebuild_id_list:
+ old_ebuild_id_list.append(ebuild_id)
+ if not old_ebuild_id_list == []:
+ add_old_ebuild(self._session, old_ebuild_id_list)
+ PackagesMetadataInfo = get_package_metadata_sql(self._session, package_id)
+ if PackagesMetadataInfo:
+ package_metadata_checksum_sql = PackagesMetadataInfo.Checksum
+ else:
+ package_metadata_checksum_sql = None
+ if package_metadata_checksum_sql is None or package_metadata_checksum_sql != package_metadataDict[package_id]['metadata_xml_checksum']:
+ update_package_metadata(self._session, package_metadataDict)
+
+ # update the cp manifest checksum
+ update_manifest_sql(self._session, package_id, manifest_checksum_tree)
+
+ # Get the best cpv for the configs and add it to config_cpv_listDict
+ PackageInfo, CategoryInfo, RepoInfo = get_package_info_from_package_id(self._session, package_id)
+ cp = CategoryInfo.Category + '/' + PackageInfo.Package
+ config_all_info = get_config_all_info(self._session)
+ config_list = []
+ for config in get_config_all_info(self._session):
+ if config.Host is False:
+ config_list.append(config.ConfigId)
+ config_cpv_listDict = self.config_match_ebuild(cp, config_list)
+
+ # Add the ebuild to the build jobs table if needed
+ self.add_new_build_job_db(ebuild_id_list, packageDict, config_cpv_listDict)
+
+ def add_new_package_db(self, cp, repo):
+ # Add new categories package ebuild to tables package and ebuilds
+ # C = Checking
+ # N = New Package
+ log_msg = "C %s:%s" % (cp, repo)
+ add_tbc_logs(self._session, log_msg, "info", self._config_id)
+ log_msg = "N %s:%s" % (cp, repo)
+ add_tbc_logs(self._session, log_msg, "info", self._config_id)
+ repodir = self._myportdb.getRepositoryPath(repo)
+ pkgdir = repodir + "/" + cp # Get RepoDIR + cp
+
+ # Get the cp manifest file checksum.
+ try:
+ manifest_checksum_tree = portage.checksum.sha256hash(pkgdir + "/Manifest")[0]
+ except:
+ manifest_checksum_tree = "0"
+ log_msg = "QA: Can't checksum the Manifest file. :%s:%s" % (cp, repo,)
+ add_tbc_logs(self._session, log_msg, "info", self._config_id)
+ log_msg = "C %s:%s ... Fail." % (cp, repo)
+ add_tbc_logs(self._session, log_msg, "info", self._config_id)
+ return None
+ package_id = add_new_package_sql(self._session, cp, repo)
+
+ package_metadataDict = self.get_package_metadataDict(pkgdir, package_id)
+ # Get the ebuild list for cp
+ mytree = []
+ mytree.append(repodir)
+ ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree)
+ if ebuild_list_tree == []:
+ log_msg = "QA: Can't get the ebuilds list. %s:%s" % (cp, repo,)
+ add_tbc_logs(self._session, log_msg, "info", self._config_id)
+ log_msg = "C %s:%s ... Fail." % (cp, repo)
+ add_tbc_logs(self._session, log_msg, "info", self._config_id)
+ return None
+
+ # Make the needed packageDict with ebuild infos so we can add it later to the db.
+ packageDict ={}
+ new_ebuild_id_list = []
+ old_ebuild_id_list = []
+ for cpv in sorted(ebuild_list_tree):
+ packageDict[cpv] = self.get_packageDict(pkgdir, cpv, repo)
+
+ self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_checksum_tree)
+ log_msg = "C %s:%s ... Done." % (cp, repo)
+ add_tbc_logs(self._session, log_msg, "info", self._config_id)
+
+ def update_package_db(self, package_id):
+ # Update the categories and package with new info
+ # C = Checking
+ PackageInfo, CategoryInfo, RepoInfo = get_package_info_from_package_id(self._session, package_id)
+ cp = CategoryInfo.Category + '/' + PackageInfo.Package
+ repo = RepoInfo.Repo
+ log_msg = "C %s:%s" % (cp, repo)
+ add_tbc_logs(self._session, log_msg, "info", self._config_id)
+ repodir = self._myportdb.getRepositoryPath(repo)
+ pkgdir = repodir + "/" + cp # Get RepoDIR + cp
+
+ # Get the cp mainfest file checksum
+ try:
+ manifest_checksum_tree = portage.checksum.sha256hash(pkgdir + "/Manifest")[0]
+ except:
+ manifest_checksum_tree = "0"
+ log_msg = "QA: Can't checksum the Manifest file. %s:%s" % (cp, repo,)
+ add_tbc_logs(self._session, log_msg, "info", self._config_id)
+ log_msg = "C %s:%s ... Fail." % (cp, repo)
+ add_tbc_logs(self._session, log_msg, "info", self._config_id)
+ return None
+
+ # if we NOT have the same checksum in the db update the package
+ if manifest_checksum_tree != PackageInfo.Checksum:
+
+ # U = Update
+ log_msg = "U %s:%s" % (cp, repo)
+ add_tbc_logs(self._session, log_msg, "info", self._config_id)
+
+ # Get the ebuild list for cp
+ mytree = []
+ mytree.append(repodir)
+ ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree)
+ if ebuild_list_tree == []:
+ log_msg = "QA: Can't get the ebuilds list. %s:%s" % (cp, repo,)
+ add_tbc_logs(self._session, log_msg, "info", self._config_id)
+ log_msg = "C %s:%s ... Fail." % (cp, repo)
+ add_tbc_logs(self._session, log_msg, "info", self._config_id)
+ return None
+ packageDict ={}
+ new_ebuild_id_list = []
+ old_ebuild_id_list = []
+ for cpv in sorted(ebuild_list_tree):
+
+ # split out ebuild version
+ ebuild_version_tree = portage.versions.cpv_getversion(cpv)
+
+ # Get packageDict for cpv
+ packageDict[cpv] = self.get_packageDict(pkgdir, cpv, repo)
+
+ # Get the checksum of the ebuild in tree and db
+ ebuild_version_checksum_tree = packageDict[cpv]['checksum']
+ checksums_db, fail= get_ebuild_checksums(self._session, package_id, ebuild_version_tree)
+ # check if we have dupes of the checksum from db
+ if checksums_db is None:
+ ebuild_version_manifest_checksum_db = None
+ elif fail:
+ dupe_ebuild_id_list = []
+ for checksum in checksums_db:
+ ebuilds_id , status = get_ebuild_id_db(self._session, checksum, package_id)
+ for ebuild_id in ebuilds_id:
+ log_msg = "U %s:%s:%s Dups of checksums" % (cpv, repo, ebuild_id,)
+ add_tbc_logs(self._session, log_msg, "error", self._config_id)
+ dupe_ebuild_id_list.append(ebuild_id)
+ add_old_ebuild(self._session, dupe_ebuild_id_list)
+ ebuild_version_manifest_checksum_db = None
+ else:
+ ebuild_version_manifest_checksum_db = checksums_db
+
+ # Check if the checksum have change
+ if ebuild_version_manifest_checksum_db is None:
+ # N = New ebuild
+ log_msg = "N %s:%s" % (cpv, repo,)
+ add_tbc_logs(self._session, log_msg, "info", self._config_id)
+ elif ebuild_version_checksum_tree != ebuild_version_manifest_checksum_db:
+ # U = Updated ebuild
+ log_msg = "U %s:%s" % (cpv, repo,)
+ add_tbc_logs(self._session, log_msg, "info", self._config_id)
+ else:
+ # Remove cpv from packageDict and add ebuild to new ebuils list
+ del packageDict[cpv]
+ ebuild_id , status = get_ebuild_id_db(self._session, ebuild_version_checksum_tree, package_id)
+ new_ebuild_id_list.append(ebuild_id)
+ package_metadataDict = self.get_package_metadataDict(pkgdir, package_id)
+ self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_checksum_tree)
+
+ log_msg = "C %s:%s ... Done." % (cp, repo)
+ add_tbc_logs(self._session, log_msg, "info", self._config_id)
diff --git a/tbc/pym/readconf.py b/tbc/pym/readconf.py
new file mode 100644
index 0000000..db1056e
--- /dev/null
+++ b/tbc/pym/readconf.py
@@ -0,0 +1,58 @@
+# Copyright 1998-2015 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import os
+import sys
+import re
+from socket import getfqdn
+
+class get_conf_settings(object):
+# open the /etc/tbc/tbc.conf file and get the needed
+# settings for tbc
+ def __init__(self):
+ self.configfile = "/etc/tbc/tbc.conf"
+
+ def read_tbc_settings_all(self):
+ # It will return a dict with options from the configfile
+ try:
+ open_conffile = open(self.configfile, 'r')
+ except:
+ sys.exit("Fail to open config file:" + self.configfile)
+ textlines = open_conffile.readlines()
+ for line in textlines:
+ element = line.split('=')
+ if element[0] == 'SQLBACKEND': # Databas backend
+ get_sql_backend = element[1]
+ if element[0] == 'SQLDB': # Database
+ get_sql_db = element[1]
+ if element[0] == 'SQLHOST': # Host
+ get_sql_host = element[1]
+ if element[0] == 'SQLUSER': # User
+ get_sql_user = element[1]
+ if element[0] == 'SQLPASSWD': # Password
+ get_sql_passwd = element[1]
+ # Buildhost root (dir for host/setup on host)
+ if element[0] == 'ZOBCSGITREPONAME':
+ get_tbc_gitreponame = element[1]
+ # Buildhost setup (host/setup on guest)
+ if element[0] == 'ZOBCSCONFIG':
+ get_tbc_config = element[1]
+ # if element[0] == 'LOGFILE':
+ # get_tbc_logfile = element[1]
+ open_conffile.close()
+
+ tbc_settings_dict = {}
+ tbc_settings_dict['sql_backend'] = get_sql_backend.rstrip('\n')
+ tbc_settings_dict['sql_db'] = get_sql_db.rstrip('\n')
+ tbc_settings_dict['sql_host'] = get_sql_host.rstrip('\n')
+ tbc_settings_dict['sql_user'] = get_sql_user.rstrip('\n')
+ tbc_settings_dict['sql_passwd'] = get_sql_passwd.rstrip('\n')
+ tbc_settings_dict['tbc_gitreponame'] = get_tbc_gitreponame.rstrip('\n')
+ tbc_settings_dict['tbc_config'] = get_tbc_config.rstrip('\n')
+ tbc_settings_dict['hostname'] = getfqdn()
+ # tbc_settings_dict['tbc_logfile'] = get_tbc_logfile.rstrip('\n')
+ return tbc_settings_dict
+
+def read_config_settings():
+ reader = get_conf_settings()
+ return reader.read_tbc_settings_all()
diff --git a/tbc/pym/sqlquerys.py b/tbc/pym/sqlquerys.py
new file mode 100644
index 0000000..623f0b0
--- /dev/null
+++ b/tbc/pym/sqlquerys.py
@@ -0,0 +1,512 @@
+# Copyright 1998-2015 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from __future__ import print_function
+import datetime
+from tbc.db_mapping import Configs, Logs, ConfigsMetaData, Jobs, BuildJobs, Packages, Ebuilds, Repos, Categories, \
+ Uses, ConfigsEmergeOptions, EmergeOptions, HiLight, BuildLogs, BuildLogsConfig, BuildJobsUse, BuildJobsRedo, \
+ HiLightCss, BuildLogsHiLight, BuildLogsEmergeOptions, BuildLogsErrors, ErrorsInfo, EmergeInfo, BuildLogsUse, \
+ BuildJobsEmergeOptions, EbuildsMetadata, EbuildsIUse, Restrictions, EbuildsRestrictions, EbuildsKeywords, \
+ Keywords, PackagesMetadata, Emails, PackagesEmails, Setups
+from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
+from sqlalchemy import and_, or_
+
+# Guest Functions
+def get_config_id(session, setup, host):
+ SetupInfo = session.query(Setups).filter_by(Setup = setup).one()
+ ConfigInfo = session.query(Configs).filter_by(SetupId = SetupInfo.SetupId).filter_by(Hostname = host).one()
+ return ConfigInfo.ConfigId
+
+def add_tbc_logs(session, log_msg, log_type, config_id):
+ Add_Log = Logs(ConfigId = config_id, LogType = log_type, Msg = log_msg)
+ session.add(Add_Log)
+ session.commit()
+
+def update_deamon_status(session, status, config_id):
+ ConfigInfo = session.query(ConfigsMetaData).filter_by(ConfigId = config_id).one()
+ ConfigInfo.Status = status
+ session.commit()
+
+def get_jobs(session, config_id):
+ JobsInfo = session.query(Jobs).filter_by(Status = 'Waiting').filter_by(ConfigId = config_id).order_by(Jobs.JobId).all()
+ if JobsInfo == []:
+ return None
+ return JobsInfo
+
+def update_job_list(session, status, job_id):
+ JobInfo = session.query(Jobs).filter_by(JobId = job_id).one()
+ JobInfo.Status = status
+ session.commit()
+
+def get_config_all_info(session):
+ return session.query(Configs).all()
+
+def get_config_info(session, config_id):
+ ConfigInfo = session.query(Configs).filter_by(ConfigId = config_id).one()
+ return ConfigInfo
+
+def get_setup_info(session, config_id):
+ ConfigInfo = get_config_info(session, config_id)
+ SetupInfo = session.query(Setups).filter_by(SetupId = ConfigInfo.SetupId).one()
+ return SetupInfo
+
+def update_buildjobs_status(session, build_job_id, status, config_id):
+ BuildJobsInfo = session.query(BuildJobs).filter_by(BuildJobId = build_job_id).one()
+ BuildJobsInfo.Status = status
+ BuildJobsInfo.ConfigId = config_id
+ session.commit()
+
+def get_configmetadata_info(session, config_id):
+ return session.query(ConfigsMetaData).filter_by(ConfigId = config_id).one()
+
+def is_build_job_done(session, build_job_id):
+ try:
+ BuildJobsInfo = session.query(BuildJobs).filter_by(BuildJobId = build_job_id).one()
+ except NoResultFound as e:
+ return False
+ return True
+
+def get_packages_to_build(session, config_id):
+ SetupInfo = get_setup_info(session, config_id)
+ BuildJobsTmp = session.query(BuildJobs).filter(BuildJobs.SetupId==SetupInfo.SetupId). \
+ order_by(BuildJobs.BuildJobId)
+ if session.query(BuildJobs).filter_by(SetupId = SetupInfo.SetupId).filter_by(BuildNow = True).filter_by(Status = 'Waiting').all() == [] and session.query(BuildJobs).filter_by(SetupId = SetupInfo.SetupId).filter_by(Status = 'Waiting').all() == []:
+ return None
+ if not BuildJobsTmp.filter_by(BuildNow = True).first() is None:
+ BuildJobsInfo = session.query(BuildJobs).filter_by(SetupId = SetupInfo.SetupId).filter_by(BuildNow = True). \
+ filter_by(Status = 'Waiting').order_by(BuildJobs.BuildJobId).first()
+ else:
+ BuildJobsInfo = session.query(BuildJobs).filter_by(SetupId = SetupInfo.SetupId).filter_by(Status = 'Waiting').\
+ order_by(BuildJobs.BuildJobId).first()
+ update_buildjobs_status(session, BuildJobsInfo.BuildJobId, 'Looked', config_id)
+ EbuildsInfo = session.query(Ebuilds).filter_by(EbuildId = BuildJobsInfo.EbuildId).one()
+ PackagesInfo, CategoriesInfo = session.query(Packages, Categories).filter(Packages.PackageId==EbuildsInfo.PackageId).filter(Packages.CategoryId==Categories.CategoryId).one()
+ ReposInfo = session.query(Repos).filter_by(RepoId = PackagesInfo.RepoId).one()
+ uses={}
+ for BuildJobsUseInfo, UsesInfo in session.query(BuildJobsUse, Uses).filter(BuildJobsUse.BuildJobId==BuildJobsInfo.BuildJobId).filter(BuildJobsUse.UseId==Uses.UseId).all():
+ uses[UsesInfo.Flag] = BuildJobsUseInfo.Status
+ if uses == {}:
+ uses = None
+ emerge_options_list = []
+ for ConfigsEmergeOptionsInfo, EmergeOptionsInfo in session.query(ConfigsEmergeOptions, EmergeOptions). \
+ filter(ConfigsEmergeOptions.ConfigId==config_id). \
+ filter(ConfigsEmergeOptions.EOptionId==EmergeOptions.EmergeOptionId).all():
+ emerge_options_list.append(EmergeOptionsInfo.EOption)
+ build_dict={}
+ build_dict['config_id'] = config_id
+ build_dict['setup_id'] = BuildJobsInfo.SetupId
+ build_dict['build_job_id'] = BuildJobsInfo.BuildJobId
+ build_dict['ebuild_id']= EbuildsInfo.EbuildId
+ build_dict['package_id'] = EbuildsInfo.PackageId
+ build_dict['package'] = PackagesInfo.Package
+ build_dict['category'] = CategoriesInfo.Category
+ build_dict['repo'] = ReposInfo.Repo
+ build_dict['removebin'] = BuildJobsInfo.RemoveBin
+ build_dict['ebuild_version'] = EbuildsInfo.Version
+ build_dict['checksum'] = EbuildsInfo.Checksum
+ build_dict['cp'] = CategoriesInfo.Category + '/' + PackagesInfo.Package
+ build_dict['cpv'] = build_dict['cp'] + '-' + EbuildsInfo.Version
+ build_dict['build_useflags'] = uses
+ build_dict['emerge_options'] = emerge_options_list
+ return build_dict
+
+def get_category_info(session, category):
+ try:
+ CategoryInfo = session.query(Categories).filter_by(Category = category).filter_by(Active = True).one()
+ except NoResultFound as e:
+ return False
+ return CategoryInfo
+
+def get_repo_info(session, repo):
+ try:
+ RepoInfo = session.query(Repos).filter_by(Repo = repo).one()
+ except NoResultFound as e:
+ return False
+ return RepoInfo
+
+def get_package_info(session, category, package, repo):
+ CategoryInfo = get_category_info(session, category)
+ RepoInfo = get_repo_info(session, repo)
+ try:
+ PackageInfo = session.query(Packages).filter_by(CategoryId = CategoryInfo.CategoryId). \
+ filter_by(Package = package).filter_by(RepoId = RepoInfo.RepoId).filter_by(Active = True).one()
+ except NoResultFound as e:
+ return False
+ return PackageInfo
+
+def get_ebuild_info(session, build_dict):
+ EbuildInfo = session.query(Ebuilds).filter_by(Version = build_dict['ebuild_version']).filter_by(Checksum = build_dict['checksum']).\
+ filter_by(PackageId = build_dict['package_id']).filter_by(Active = True)
+ if EbuildInfo.all() == []:
+ return None, True
+ try:
+ EbuildInfo2 = EbuildInfo.one()
+ except (MultipleResultsFound) as e:
+ return EbuildInfo.all(), True
+ return EbuildInfo2, False
+
+def get_build_job_id(session, build_dict):
+ BuildJobsIdInfo = session.query(BuildJobs.BuildJobId).filter_by(EbuildId = build_dict['ebuild_id']).filter_by(ConfigId = build_dict['config_id']).all()
+ if BuildJobsIdInfo == []:
+ return None
+ for build_job_id in BuildJobsIdInfo:
+ BuildJobsUseInfo = session.query(BuildJobsUse).filter_by(BuildJobId = build_job_id.BuildJobId).all()
+ useflagsdict = {}
+ if BuildJobsUseInfo == []:
+ useflagsdict = None
+ else:
+ for x in BuildJobsUseInfo:
+ useflagsdict[x.UseId] = x.Status
+ if useflagsdict == build_dict['build_useflags']:
+ return build_job_id.BuildJobId
+ return None
+
+def get_use_id(session, use_flag):
+ try:
+ UseIdInfo = session.query(Uses).filter_by(Flag = use_flag).one()
+ except NoResultFound as e:
+ return None
+ return UseIdInfo.UseId
+
+def get_hilight_info(session):
+ return session.query(HiLight).all()
+
+def get_error_info_list(session):
+ return session.query(ErrorsInfo).all()
+
+def add_e_info(session, emerge_info):
+ AddEmergeInfo = EmergeInfo(EmergeInfoText = emerge_info)
+ session.add(AddEmergeInfo)
+ session.flush()
+ EmergeInfoId = AddEmergeInfo.EInfoId
+ session.commit()
+ return EmergeInfoId
+
+def del_old_build_jobs(session, build_job_id):
+ session.query(BuildJobsUse).filter(BuildJobsUse.BuildJobId == build_job_id).delete()
+ session.query(BuildJobsRedo).filter(BuildJobsRedo.BuildJobId == build_job_id).delete()
+ session.query(BuildJobsEmergeOptions).filter(BuildJobsEmergeOptions.BuildJobId == build_job_id).delete()
+ session.query(BuildJobs).filter(BuildJobs.BuildJobId == build_job_id).delete()
+ session.commit()
+
+def add_new_buildlog(session, build_dict, build_log_dict):
+ build_log_id_list = session.query(BuildLogs.BuildLogId).filter_by(EbuildId = build_dict['ebuild_id']).all()
+
+ def add_new_hilight(log_id, build_log_dict):
+ for k, hilight_tmp in sorted(build_log_dict['hilight_dict'].items()):
+ NewHiLight = BuildLogsHiLight(LogId = log_id, StartLine = hilight_tmp['startline'], EndLine = hilight_tmp['endline'], HiLightCssId = hilight_tmp['hilight_css_id'])
+ session.add(NewHiLight)
+ session.commit()
+
+ def build_log_id_match(build_log_id_list, build_dict, build_log_dict):
+ for build_log_id in build_log_id_list:
+ log_hash = session.query(BuildLogs.LogHash).filter_by(BuildLogId = build_log_id[0]).one()
+ use_list = session.query(BuildLogsUse).filter_by(BuildLogId = build_log_id[0]).all()
+ useflagsdict = {}
+ if use_list == []:
+ useflagsdict = None
+ else:
+ for use in use_list:
+ useflagsdict[use.UseId] = use.Status
+ if log_hash[0] == build_log_dict['log_hash'] and build_dict['build_useflags'] == useflagsdict:
+ if session.query(BuildLogsConfig).filter(BuildLogsConfig.ConfigId.in_([build_dict['config_id']])).filter_by(BuildLogId = build_log_id[0]):
+ return None, True
+ e_info_id = add_e_info(session, build_log_dict['emerge_info'])
+ NewBuildLogConfig = BuildLogsConfig(BuildLogId = build_log_id[0], ConfigId = build_dict['config_id'], LogName = build_log_dict['logfilename'], EInfoId = e_info_id)
+ session.add(NewBuildLogConfig)
+ session.commit()
+ return build_log_id[0], True
+ return None, False
+
+ def build_log_id_no_match(build_dict, build_log_dict):
+ if build_log_dict['summary_error_list'] == []:
+ NewBuildLog = BuildLogs(EbuildId = build_dict['ebuild_id'], Fail = False, SummeryText = build_log_dict['build_error'], LogHash = build_log_dict['log_hash'])
+ else:
+ NewBuildLog = BuildLogs(EbuildId = build_dict['ebuild_id'], Fail = True, SummeryText = build_log_dict['build_error'], LogHash = build_log_dict['log_hash'])
+ session.add(NewBuildLog)
+ session.flush()
+ build_log_id = NewBuildLog.BuildLogId
+ session.commit()
+ if build_log_dict['summary_error_list'] != []:
+ for error in build_log_dict['summary_error_list']:
+ NewError = BuildLogsErrors(BuildLogId = build_log_id, ErrorId = error)
+ session.add(NewError)
+ session.commit()
+ e_info_id = add_e_info(session, build_log_dict['emerge_info'])
+ NewBuildLogConfig = BuildLogsConfig(BuildLogId = build_log_id, ConfigId = build_dict['config_id'], LogName = build_log_dict['logfilename'], EInfoId = e_info_id)
+ session.add(NewBuildLogConfig)
+ session.flush()
+ log_id = NewBuildLogConfig.LogId
+ session.commit()
+ add_new_hilight(log_id, build_log_dict)
+ if not build_dict['build_useflags'] is None:
+ for use_id, status in build_dict['build_useflags'].items():
+ NewBuildLogUse = BuildLogsUse(BuildLogId = build_log_id, UseId = use_id, Status = status)
+ session.add(NewBuildLogUse)
+ session.flush()
+ session.commit()
+ return build_log_id
+
+ if build_dict['build_job_id'] is None and build_log_id_list == []:
+ build_log_id = build_log_id_no_match(build_dict, build_log_dict)
+ return build_log_id
+ elif build_dict['build_job_id'] is None and build_log_id_list != []:
+ build_log_id, match = build_log_id_match(build_log_id_list, build_dict, build_log_dict)
+ if not match:
+ build_log_id = build_log_id_no_match(build_dict, build_log_dict)
+ return build_log_id
+ elif not build_dict['build_job_id'] is None and build_log_id_list != []:
+ build_log_id, match = build_log_id_match(build_log_id_list, build_dict, build_log_dict)
+ if not match:
+ build_log_id = build_log_id_no_match(build_dict, build_log_dict)
+ del_old_build_jobs(session, build_dict['build_job_id'])
+ return build_log_id
+ elif not build_dict['build_job_id'] is None and build_log_id_list == []:
+ build_log_id = build_log_id_no_match(build_dict, build_log_dict)
+ del_old_build_jobs(session, build_dict['build_job_id'])
+ return build_log_id
+
+def update_fail_times(session, FailInfo):
+ NewBuildJobs = session.query(BuildJobs).filter_by(BuildJobId = FailInfo.BuildJobId).one()
+ NewBuildJobs.TimeStamp = datetime.datetime.utcnow()
+ session.commit()
+
+def get_fail_times(session, build_dict):
+ try:
+ FailInfo = session.query(BuildJobsRedo).filter_by(BuildJobId = build_dict['build_job_id']).filter_by(FailType = build_dict['type_fail']).one()
+ except NoResultFound as e:
+ return False
+ return True
+
+def add_fail_times(session, fail_querue_dict):
+ print(fail_querue_dict)
+ NewBuildJobsRedo = BuildJobsRedo(BuildJobId = fail_querue_dict['build_job_id'], FailType = fail_querue_dict['fail_type'], FailTimes = fail_querue_dict['fail_times'])
+ session.add(NewBuildJobsRedo)
+ session.commit()
+
+# Host Functions
+def update_repo_db(session, repo_list):
+ for repo in repo_list:
+ if not get_repo_info(session, repo):
+ session.add(Repos(Repo = repo))
+ session.commit()
+
+def update_categories_db(session, category):
+ if not get_category_info(session, category):
+ session.add(Categories(Category = category))
+ session.commit()
+
+def get_keyword_id(session, keyword):
+ try:
+ KeywordsInfo = session.query(Keywords).filter_by(Keyword = keyword).one()
+ except NoResultFound as e:
+ return None
+ return KeywordsInfo.KeywordId
+
+def add_new_ebuild_metadata_sql(session, ebuild_id, keywords, restrictions, iuse_list):
+ for restriction in restrictions:
+ if restriction in ["!"]:
+ restriction = restriction[1:]
+ if restriction in ["?"]:
+ restriction = restriction[:1]
+ if restriction != '(' or restriction != ')':
+ try:
+ RestrictionInfo = session.query(Restrictions).filter_by(Restriction = restriction).one()
+ except NoResultFound as e:
+ session.add(Restrictions(Restriction = restriction))
+ session.commit()
+ RestrictionInfo = session.query(Restrictions).filter_by(Restriction = restriction).one()
+ session.add(EbuildsRestrictions(EbuildId = ebuild_id, RestrictionId = RestrictionInfo.RestrictionId))
+ session.commit()
+ for iuse in iuse_list:
+ status = False
+ if iuse[0] in ["+"]:
+ iuse = iuse[1:]
+ status = True
+ elif iuse[0] in ["-"]:
+ iuse = iuse[1:]
+ use_id = get_use_id(session, iuse)
+ if use_id is None:
+ session.add(Uses(Flag = iuse))
+ session.commit()
+ use_id = get_use_id(session, iuse)
+ session.add(EbuildsIUse(EbuildId = ebuild_id, UseId = use_id, Status = status))
+ session.commit()
+ for keyword in keywords:
+ status = 'Stable'
+ if keyword[0] in ["~"]:
+ keyword = keyword[1:]
+ status = 'Unstable'
+ elif keyword[0] in ["-"]:
+ keyword = keyword[1:]
+ status = 'Negative'
+ keyword_id = get_keyword_id(session, keyword)
+ if keyword_id is None:
+ session.add(Keywords(Keyword = keyword))
+ session.commit()
+ keyword_id = get_keyword_id(session, keyword)
+ session.add(EbuildsKeywords(EbuildId = ebuild_id, KeywordId = keyword_id, Status = status))
+ session.commit()
+
+def add_new_ebuild_sql(session, packageDict):
+ ebuild_id_list = []
+ for k, v in packageDict.items():
+ session.add(Ebuilds(PackageId = v['package_id'], Version = v['ebuild_version'], Checksum = v['checksum'], Active = True))
+ session.flush()
+ try:
+ EbuildInfo = session.query(Ebuilds).filter_by(Version = v['ebuild_version']).filter_by(Checksum = v['checksum']).\
+ filter_by(PackageId = v['package_id']).filter_by(Active = True).one()
+ except (MultipleResultsFound) as e:
+ for x in session.query(Ebuilds).filter_by(Version = v['ebuild_version']).filter_by(Checksum = v['checksum']).\
+ filter_by(PackageId = v['package_id']).filter_by(Active = True).all():
+ print(x.EbuildId)
+ # FIXME
+ #x.Checksum = 0
+ #x.Active = False
+ #session.commit()
+ sys.exit()
+ session.add(EbuildsMetadata(EbuildId = EbuildInfo.EbuildId, Revision = v['ebuild_version_revision_tree']))
+ session.commit()
+ ebuild_id_list.append(EbuildInfo.EbuildId)
+ restrictions = []
+ keywords = []
+ iuse = []
+ for i in v['ebuild_version_metadata_tree'][4].split():
+ restrictions.append(i)
+ for i in v['ebuild_version_metadata_tree'][8].split():
+ keywords.append(i)
+ for i in v['ebuild_version_metadata_tree'][10].split():
+ iuse.append(i)
+ add_new_ebuild_metadata_sql(session, EbuildInfo.EbuildId, keywords, restrictions, iuse)
+ return ebuild_id_list
+
+def get_ebuild_id_list(session, package_id):
+ ebuild_id_list = []
+ for EbuildInfo in session.query(Ebuilds).filter_by(PackageId = package_id).filter_by(Active = True).all():
+ ebuild_id_list.append(EbuildInfo.EbuildId)
+ return ebuild_id_list
+
+def get_build_job_all(session, ebuild_id):
+ return session.query(BuildJobs).filter_by(EbuildId = ebuild_id).all()
+
+def add_old_ebuild(session, old_ebuild_list):
+ for ebuild_id in old_ebuild_list:
+ EbuildInfo = session.query(Ebuilds).filter_by(EbuildId = ebuild_id).one()
+ EbuildInfo.Active = False
+ session.commit()
+ build_job_id_list = get_build_job_all(session, ebuild_id)
+ if build_job_id_list != []:
+ for build_job in build_job_id_list:
+ del_old_build_jobs(session, build_job.BuildJobId)
+
+def add_new_package_sql(session, cp, repo):
+ element = cp.split('/')
+ categories = element[0]
+ package = element[1]
+ RepoInfo =get_repo_info(session, repo)
+ repo_id = RepoInfo.RepoId
+ CategoriesInfo = get_category_info(session, categories)
+ category_id = CategoriesInfo.CategoryId
+ session.add(Packages(Package = package, CategoryId = category_id, RepoId = repo_id, Checksum = '0', Active = True))
+ session.commit()
+ PackageInfo = get_package_info(session, categories, package, repo)
+ return PackageInfo.PackageId
+
+def get_package_metadata_sql(session, package_id):
+ try:
+ PackagesMetadataInfo = session.query(PackagesMetadata).filter_by(PackageId = package_id).one()
+ except NoResultFound as e:
+ return False
+ return PackagesMetadataInfo
+
+def update_email_info(session, email):
+ try:
+ EmailInfo = session.query(Emails).filter_by(Email = email).one()
+ except NoResultFound as e:
+ session.add(Emails(Email = email))
+ session.commit()
+ EmailInfo = session.query(Emails).filter_by(Email = email).one()
+ return EmailInfo
+
+def update_package_email_info(session, email_id, package_id):
+ try:
+ PackagesEmailInfo = session.query(PackagesEmails).filter_by(EmailId = email_id).filter_by(PackageId = package_id).one()
+ except NoResultFound as e:
+ session.add(PackagesEmails(EmailId = email_id, PackageId = package_id))
+ session.commit()
+ PackagesEmailInfo = session.query(PackagesEmails).filter_by(EmailId = email_id).filter_by(PackageId = package_id).one()
+ return PackagesEmailInfo
+
+def update_package_metadata(session, package_metadataDict):
+ for k, v in package_metadataDict.items():
+ try:
+ PackagesMetadataInfo = session.query(PackagesMetadata).filter_by(PackageId = k).one()
+ except NoResultFound as e:
+ session.add(PackagesMetadata(PackageId = k, Checksum = v['metadata_xml_checksum']))
+ session.commit()
+ else:
+ PackagesMetadataInfo.Checksum = v['metadata_xml_checksum']
+ session.commit()
+ if v['metadata_xml_email']:
+ for email in v['metadata_xml_email']:
+ EmailInfo = update_email_info(session, email)
+ PackagesEmailInfo = update_package_email_info(session, EmailInfo.EmailId, k)
+
+def update_manifest_sql(session, package_id, manifest_checksum_tree):
+ PackagesInfo = session.query(Packages).filter_by(PackageId = package_id).one()
+ PackagesInfo.Checksum = manifest_checksum_tree
+ session.commit()
+
+def get_package_info_from_package_id(session, package_id):
+ PackageInfo = session.query(Packages).filter_by(PackageId = package_id).one()
+ CategoryInfo = session.query(Categories).filter_by(CategoryId = PackageInfo.CategoryId).one()
+ RepoInfo = session.query(Repos).filter_by(RepoId = PackageInfo.RepoId).one()
+ return PackageInfo, CategoryInfo, RepoInfo
+
+def add_new_build_job(session, ebuild_id, setup_id, use_flagsDict, config_id):
+ NewBuildJobs =BuildJobs(EbuildId = ebuild_id, SetupId = setup_id, ConfigId = config_id, Status = 'Waiting', BuildNow = False, RemoveBin = True)
+ session.add(NewBuildJobs)
+ session.flush()
+ build_job_id = NewBuildJobs.BuildJobId
+ session.commit()
+ for k, v in use_flagsDict.items():
+ use_id = get_use_id(session, k)
+ session.add(BuildJobsUse(BuildJobId = build_job_id, UseId = use_id, Status = v))
+ session.commit()
+
+def get_ebuild_checksums(session, package_id, ebuild_version):
+ ebuild_checksum_list = []
+ try:
+ EbuildInfo = session.query(Ebuilds).filter_by(PackageId = package_id).filter_by(Version = ebuild_version).filter_by(Active = True).one()
+ except NoResultFound as e:
+ return None, False
+ except MultipleResultsFound as e:
+ EbuildInfo2 = session.query(Ebuilds).filter_by(PackageId = package_id).filter_by(Version = ebuild_version).filter_by(Active = True).all()
+ for Ebuild in EbuildInfo2:
+ print("ebuild version checksum")
+ print(ebuild_version)
+ print(Ebuild.Version)
+ print(Ebuild.Checksum)
+ ebuild_checksum_list.append(Ebuild.Checksum)
+ return ebuild_checksum_list, True
+ return EbuildInfo.Checksum, False
+
+def get_ebuild_id_db(session, checksum, package_id):
+ try:
+ EbuildInfos = session.query(Ebuilds).filter_by(PackageId = package_id).filter_by(Checksum = checksum).one()
+ except NoResultFound as e:
+ return None, True
+ except MultipleResultsFound as e:
+ EbuildInfos = session.query(Ebuilds).filter_by(PackageId = package_id).filter_by(Checksum = checksum).all()
+ ebuilds_id = []
+ for EbuildInfo in EbuildInfos:
+ ebuilds_id.append(EbuildInfo.EbuildId)
+ return ebuilds_id, True
+ return EbuildInfos.EbuildId, False
+
+def check_host_updatedb(session):
+ try:
+ JobsInfo = session.query(Jobs).filter_by(Status = 'Done').filter_by(JobType = 'esync').one()
+ except NoResultFound as e:
+ return True
+ return False
diff --git a/tbc/pym/sync.py b/tbc/pym/sync.py
new file mode 100644
index 0000000..a75562f
--- /dev/null
+++ b/tbc/pym/sync.py
@@ -0,0 +1,73 @@
+# Copyright 1998-2015 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from __future__ import print_function
+import portage
+import os
+import errno
+import sys
+import time
+
+from _emerge.main import emerge_main
+from tbc.readconf import get_conf_settings
+from tbc.sqlquerys import get_config_id, add_tbc_logs, get_config_all_info, get_configmetadata_info
+from tbc.updatedb import update_db_main
+from tbc.readconf import read_config_settings
+
+def sync_tree(session):
+ tbc_settings_dict = read_config_settings()
+ _hostname = tbc_settings_dict['hostname']
+ _config = tbc_settings_dict['tbc_config']
+ config_id = get_config_id(session, _config, _hostname)
+ host_config = _hostname +"/" + _config
+ default_config_root = "/var/cache/tbc/" + tbc_settings_dict['tbc_gitreponame'] + "/" + host_config + "/"
+ mysettings = portage.config(config_root = default_config_root)
+ GuestBusy = True
+ log_msg = "Waiting for Guest to be idel"
+ add_tbc_logs(session, log_msg, "info", config_id)
+ guestid_list = []
+ for config in get_config_all_info(session):
+ if not config.Host:
+ guestid_list.append(config.ConfigId)
+ while GuestBusy:
+ Status_list = []
+ for guest_id in guestid_list:
+ ConfigMetadata = get_configmetadata_info(session, guest_id)
+ Status_list.append(ConfigMetadata.Status)
+ if not 'Runing' in Status_list:
+ GuestBusy = False
+ time.sleep(30)
+ try:
+ os.remove(mysettings['PORTDIR'] + "/profiles/config/parent")
+ os.rmdir(mysettings['PORTDIR'] + "/profiles/config")
+ except:
+ pass
+ tmpcmdline = []
+ tmpcmdline.append("--sync")
+ tmpcmdline.append("--quiet")
+ tmpcmdline.append("--config-root=" + default_config_root)
+ log_msg = "Emerge --sync"
+ add_tbc_logs(session, log_msg, "info", config_id)
+ fail_sync = emerge_main(args=tmpcmdline)
+ if fail_sync:
+ log_msg = "Emerge --sync fail!"
+ add_tbc_logs(session, log_msg, "error", config_id)
+ return False
+ else:
+ # Need to add a config dir so we can use profiles/base for reading the tree.
+ # We may allready have the dir on local repo when we sync.
+ try:
+ os.mkdir(mysettings['PORTDIR'] + "/profiles/config", 0o777)
+ with open(mysettings['PORTDIR'] + "/profiles/config/parent", "w") as f:
+ f.write("../base\n")
+ f.close()
+ except:
+ pass
+ log_msg = "Emerge --sync ... Done."
+ add_tbc_logs(session, log_msg, "info", config_id)
+ result = update_db_main(session, config_id)
+ if result:
+ return True
+ else:
+ log_msg = "Updatedb fail"
+ add_tbc_logs(session, log_msg, "info", config_id)
diff --git a/tbc/pym/text.py b/tbc/pym/text.py
new file mode 100644
index 0000000..c78c432
--- /dev/null
+++ b/tbc/pym/text.py
@@ -0,0 +1,49 @@
+# Copyright 1998-2015 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from __future__ import print_function
+import sys
+import re
+import os
+import errno
+from portage.util import grablines
+
+def get_file_text(filename):
+ # Return the filename contents
+ try:
+ textfile = open(filename, encoding='utf-8')
+ except:
+ return "No file", filename
+ text = ""
+ for line in textfile:
+ text += line
+ textfile.close()
+ return text
+
+def get_ebuild_cvs_revision(filename):
+ """Return the ebuild contents"""
+ try:
+ ebuildfile = open(filename, encoding='utf-8')
+ except:
+ return "No Ebuild file there"
+ text = ""
+ dataLines = ebuildfile.readlines()
+ for i in dataLines:
+ text = text + i + " "
+ line2 = dataLines[2]
+ field = line2.split(" ")
+ ebuildfile.close()
+ try:
+ cvs_revision = field[3]
+ except:
+ cvs_revision = ''
+ return cvs_revision
+
+def get_log_text_dict(filename):
+ """Return the log contents as a dict"""
+ logfile_dict = {}
+ index = 1
+ for text_line in grablines(filename):
+ logfile_dict[index] = text_line
+ index = index + 1
+ return logfile_dict, index - 1
diff --git a/tbc/pym/updatedb.py b/tbc/pym/updatedb.py
new file mode 100644
index 0000000..12b94c5
--- /dev/null
+++ b/tbc/pym/updatedb.py
@@ -0,0 +1,135 @@
+# Copyright 1998-2015 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from __future__ import print_function
+import sys
+import os
+import multiprocessing
+import time
+import portage
+from sqlalchemy.orm import scoped_session, sessionmaker
+from tbc.ConnectionManager import NewConnection
+from tbc.sqlquerys import add_tbc_logs, get_package_info, update_repo_db, \
+ update_categories_db, get_configmetadata_info, get_config_all_info, add_new_build_job, \
+ get_config_info
+from tbc.check_setup import check_make_conf
+from tbc.package import tbc_package
+# Get the options from the config file set in tbc.readconf
+from tbc.readconf import get_conf_settings
+
+def init_portage_settings(session, config_id, tbc_settings_dict):
+ # check config setup
+ check_make_conf(session, config_id, tbc_settings_dict)
+ log_msg = "Check configs done"
+ add_tbc_logs(session, log_msg, "info", config_id)
+
+ # Get default config from the configs table and default_config=1
+ host_config = tbc_settings_dict['hostname'] +"/" + tbc_settings_dict['tbc_config']
+ default_config_root = "/var/cache/tbc/" + tbc_settings_dict['tbc_gitreponame'] + "/" + host_config + "/"
+
+ # Set config_root (PORTAGE_CONFIGROOT) to default_config_root
+ mysettings = portage.config(config_root = default_config_root)
+ log_msg = "Setting default config to: %s" % (host_config,)
+ add_tbc_logs(session, log_msg, "info", config_id)
+ return mysettings
+
+def update_cpv_db_pool(mysettings, myportdb, cp, repo, tbc_settings_dict, config_id):
+ session_factory = sessionmaker(bind=NewConnection(tbc_settings_dict))
+ Session = scoped_session(session_factory)
+ session2 = Session()
+ init_package = tbc_package(session2, mysettings, myportdb, config_id, tbc_settings_dict)
+
+ # split the cp to categories and package
+ element = cp.split('/')
+ categories = element[0]
+ package = element[1]
+
+ # update the categories table
+ update_categories_db(session2, categories)
+
+ # Check if we have the cp in the package table
+ PackagesInfo = get_package_info(session2, categories, package, repo)
+ if PackagesInfo:
+ # Update the packages with ebuilds
+ init_package.update_package_db(PackagesInfo.PackageId)
+ else:
+ # Add new package with ebuilds
+ init_package.add_new_package_db(cp, repo)
+ Session.remove()
+
+def update_cpv_db(session, config_id, tbc_settings_dict):
+ GuestBusy = True
+ log_msg = "Waiting for Guest to be idel"
+ add_tbc_logs(session, log_msg, "info", config_id)
+ guestid_list = []
+ for config in get_config_all_info(session):
+ if not config.Host:
+ guestid_list.append(config.ConfigId)
+ while GuestBusy:
+ Status_list = []
+ for guest_id in guestid_list:
+ ConfigMetadata = get_configmetadata_info(session, guest_id)
+ Status_list.append(ConfigMetadata.Status)
+ if not 'Runing' in Status_list:
+ break
+ time.sleep(30)
+
+ mysettings = init_portage_settings(session, config_id, tbc_settings_dict)
+ log_msg = "Checking categories, package, ebuilds"
+ add_tbc_logs(session, log_msg, "info", config_id)
+ new_build_jobs_list = []
+
+ # Setup portdb, package
+ myportdb = portage.portdbapi(mysettings=mysettings)
+ repo_list = ()
+ repos_trees_list = []
+
+ # Use all cores when multiprocessing
+ pool_cores= multiprocessing.cpu_count()
+ pool = multiprocessing.Pool(processes=pool_cores)
+
+ # Will run some update checks and update package if needed
+
+ # Get the repos and update the repos db
+ repo_list = myportdb.getRepositories()
+ update_repo_db(session, repo_list)
+
+ # Close the db for the multiprocessing pool will make new ones
+ # and we don't need this one for some time.
+
+ # Get the rootdirs for the repos
+ repo_trees_list = myportdb.porttrees
+ for repo_dir in repo_trees_list:
+ repo = myportdb.getRepositoryName(repo_dir)
+ repo_dir_list = []
+ repo_dir_list.append(repo_dir)
+
+ # Get the package list from the repo
+ package_list_tree = myportdb.cp_all(trees=repo_dir_list)
+
+ # Run the update package for all package in the list and in a multiprocessing pool
+ for cp in sorted(package_list_tree):
+ pool.apply_async(update_cpv_db_pool, (mysettings, myportdb, cp, repo, tbc_settings_dict, config_id,))
+ # use this when debuging
+ #update_cpv_db_pool(mysettings, myportdb, cp, repo, tbc_settings_dict, config_id)
+
+ #close and join the multiprocessing pools
+ pool.close()
+ pool.join()
+ log_msg = "Checking categories, package and ebuilds ... done"
+ add_tbc_logs(session, log_msg, "info", config_id)
+
+def update_db_main(session, config_id):
+ # Main
+
+ # Logging
+ reader = get_conf_settings()
+ tbc_settings_dict=reader.read_tbc_settings_all()
+ log_msg = "Update db started."
+ add_tbc_logs(session, log_msg, "info", config_id)
+
+ # Update the cpv db
+ update_cpv_db(session, config_id, tbc_settings_dict)
+ log_msg = "Update db ... Done."
+ add_tbc_logs(session, log_msg, "info", config_id)
+ return True
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: tbc/pym/
@ 2015-05-09 15:50 Magnus Granberg
0 siblings, 0 replies; 20+ messages in thread
From: Magnus Granberg @ 2015-05-09 15:50 UTC (permalink / raw
To: gentoo-commits
commit: 902cd7581a4f44f270769547617604d056599bcd
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat May 9 15:49:39 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat May 9 15:49:39 2015 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=902cd758
fix typos from the repoman qa commit
tbc/pym/build_log.py | 2 +-
tbc/pym/jobs.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/tbc/pym/build_log.py b/tbc/pym/build_log.py
index bfbe8ec..f6ceea6 100644
--- a/tbc/pym/build_log.py
+++ b/tbc/pym/build_log.py
@@ -23,7 +23,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
from tbc.repoman_tbc import tbc_repoman
from tbc.text import get_log_text_dict
from tbc.package import tbc_package
-from zobcs.readconf import read_config_settings
+from tbc.readconf import read_config_settings
from tbc.flags import tbc_use_flags
from tbc.ConnectionManager import NewConnection
from tbc.sqlquerys import add_tbc_logs, get_config_id, get_ebuild_id_db, add_new_buildlog, \
diff --git a/tbc/pym/jobs.py b/tbc/pym/jobs.py
index 8559cf5..b75e344 100644
--- a/tbc/pym/jobs.py
+++ b/tbc/pym/jobs.py
@@ -59,7 +59,7 @@ def jobs_main(session, config_id):
update_job_list(session, "Runing", job_id)
log_msg = "Job %s is runing." % (job_id,)
add_tbc_logs(session, log_msg, "info", config_id)
- ifupdate_db_main(session, config_id):
+ if update_db_main(session, config_id):
update_job_list(session, "Done", job_id)
log_msg = "Job %s is done.." % (job_id,)
add_tbc_logs(session, log_msg, "info", config_id)
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: tbc/pym/
@ 2015-05-12 18:37 Magnus Granberg
0 siblings, 0 replies; 20+ messages in thread
From: Magnus Granberg @ 2015-05-12 18:37 UTC (permalink / raw
To: gentoo-commits
commit: 6eeb75c0327219634379666ebe534a71810132a9
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue May 12 18:37:17 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue May 12 18:37:17 2015 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=6eeb75c0
Use git to sync instead of emerge --sync
tbc/pym/jobs.py | 6 +--
tbc/pym/sync.py | 142 +++++++++++++++++++++++++++++++---------------------
tbc/pym/updatedb.py | 84 ++++++++++++++++++-------------
3 files changed, 137 insertions(+), 95 deletions(-)
diff --git a/tbc/pym/jobs.py b/tbc/pym/jobs.py
index b75e344..a805a22 100644
--- a/tbc/pym/jobs.py
+++ b/tbc/pym/jobs.py
@@ -2,7 +2,7 @@
# Distributed under the terms of the GNU General Public License v2
from __future__ import print_function
-from tbc.sync import sync_tree
+from tbc.sync import git_repo_sync_main
#from tbc.buildquerydb import add_buildquery_main, del_buildquery_main
from tbc.updatedb import update_db_main
from tbc.sqlquerys import get_config_id, add_tbc_logs, get_jobs, update_job_list
@@ -47,7 +47,7 @@ def jobs_main(session, config_id):
update_job_list(session, "Runing", job_id)
log_msg = "Job %s is runing." % (job_id,)
add_tbc_logs(session, log_msg, "info", config_id)
- if sync_tree(session) and update_db_main(session, config_id):
+ if update_db_main(session, git_repo_sync_main(session), config_id):
update_job_list(session, "Done", job_id)
log_msg = "Job %s is done.." % (job_id,)
add_tbc_logs(session, log_msg, "info", config_id)
@@ -59,7 +59,7 @@ def jobs_main(session, config_id):
update_job_list(session, "Runing", job_id)
log_msg = "Job %s is runing." % (job_id,)
add_tbc_logs(session, log_msg, "info", config_id)
- if update_db_main(session, config_id):
+ if update_db_main(session, None, config_id):
update_job_list(session, "Done", job_id)
log_msg = "Job %s is done.." % (job_id,)
add_tbc_logs(session, log_msg, "info", config_id)
diff --git a/tbc/pym/sync.py b/tbc/pym/sync.py
index f1bc56a..2622b25 100644
--- a/tbc/pym/sync.py
+++ b/tbc/pym/sync.py
@@ -10,12 +10,49 @@ import time
from pygit2 import Repository, GIT_MERGE_ANALYSIS_FASTFORWARD, GIT_MERGE_ANALYSIS_NORMAL, \
GIT_MERGE_ANALYSIS_UP_TO_DATE
-from _emerge.main import emerge_main
from tbc.sqlquerys import get_config_id, add_tbc_logs, get_config_all_info, get_configmetadata_info
-from tbc.updatedb import update_db_main
from tbc.readconf import read_config_settings
-def sync_tree(session):
+def git_repos_list(session, myportdb):
+ repo_trees_list = myportdb.porttrees
+ for repo_dir in repo_trees_list:
+ repo_dir_list = []
+ repo_dir_list.append(repo_dir)
+ return repo_dir_list
+
+def git_fetch(session, git_repo, config_id):
+ repo = Repository(git_repo)
+ remote = repo.remotes["origin"]
+ remote.fetch()
+ return repo
+
+def git_merge(session, repo, config_id):
+ remote_master_id = repo.lookup_reference('refs/remotes/origin/master').target
+ merge_result, _ = repo.merge_analysis(remote_master_id)
+ if merge_result & GIT_MERGE_ANALYSIS_UP_TO_DATE:
+ log_msg = "Repo is up to date"
+ add_zobcs_logs(session, log_msg, "info", config_id)
+ elif merge_result & GIT_MERGE_ANALYSIS_FASTFORWARD:
+ repo.checkout_tree(repo.get(remote_master_id))
+ master_ref = repo.lookup_reference('refs/heads/master')
+ master_ref.set_target(remote_master_id)
+ repo.head.set_target(remote_master_id)
+ elif merge_result & GIT_MERGE_ANALYSIS_NORMAL:
+ repo.merge(remote_master_id)
+ assert repo.index.conflicts is None, 'Conflicts, ahhhh!'
+ user = repo.default_signature
+ tree = repo.index.write_tree()
+ commit = repo.create_commit('HEAD',
+ user,
+ user,
+ 'Merge!',
+ tree,
+ [repo.head.target, remote_master_id])
+ repo.state_cleanup()
+ else:
+ raise AssertionError('Unknown merge analysis result')
+
+def git_sync_main(session):
tbc_settings_dict = read_config_settings()
_hostname = tbc_settings_dict['hostname']
_config = tbc_settings_dict['tbc_config']
@@ -23,6 +60,7 @@ def sync_tree(session):
host_config = _hostname +"/" + _config
default_config_root = tbc_settings_dict['tbc_gitrepopath'] + "/" + host_config + "/"
mysettings = portage.config(config_root = default_config_root)
+ myportdb = portage.portdbapi(mysettings=mysettings)
GuestBusy = True
log_msg = "Waiting for Guest to be idel"
add_tbc_logs(session, log_msg, "info", config_id)
@@ -37,67 +75,59 @@ def sync_tree(session):
Status_list.append(ConfigMetadata.Status)
if not 'Runing' in Status_list:
GuestBusy = False
- time.sleep(30)
+ else:
+ time.sleep(30)
try:
os.remove(mysettings['PORTDIR'] + "/profiles/config/parent")
os.rmdir(mysettings['PORTDIR'] + "/profiles/config")
except:
pass
- tmpcmdline = []
- tmpcmdline.append("--sync")
- tmpcmdline.append("--quiet")
- tmpcmdline.append("--config-root=" + default_config_root)
- log_msg = "Emerge --sync"
+
+ repo_cp_dict = {}
+ for repo_dir in git_repos_list(session, myportdb):
+ attr = {}
+ repo = git_fetch(session, repo_dir, config_id)
+ remote_master_id = repo.lookup_reference('refs/remotes/origin/master').target
+ merge_result, _ = repo.merge_analysis(remote_master_id)
+ if not merge_result & GIT_MERGE_ANALYSIS_UP_TO_DATE:
+ git_merge(session, repo, config_id)
+ out = repo.diff('HEAD', 'HEAD^')
+ repo_diff = out.patch
+ cp_list = []
+ reponame = myportdb.getRepositoryName(repo_dir)
+ for diff_line in repo_diff.splitlines():
+ if re.search("Manifest", diff_line) and re.search("^diff --git", diff_line):
+ diff_line2 = re.split(' ', re.sub('[a-b]/', '', re.sub('diff --git ', '', diff_line)))
+ if diff_line2[0] == diff_line2[1] or "Manifest" in diff_line2[0]:
+ cp = re.sub('/Manifest', '', diff_line2[0])
+ cp_list.append(cp)
+ else:
+ cp = re.sub('/Manifest', '', diff_line2[1])
+ cp_list.append(cp)
+ attr['cp_list'] = cp_list
+ repo_cp_dict[reponame] = attr
+ else:
+ log_msg = "Repo is up to date"
+ add_tbc_logs(session, log_msg, "info", config_id)
+
+ # Need to add a config dir so we can use profiles/base for reading the tree.
+ # We may allready have the dir on local repo when we sync.
+ try:
+ os.mkdir(mysettings['PORTDIR'] + "/profiles/config", 0o777)
+ with open(mysettings['PORTDIR'] + "/profiles/config/parent", "w") as f:
+ f.write("../base\n")
+ f.close()
+ except:
+ pass
+ log_msg = "Repo sync ... Done."
add_tbc_logs(session, log_msg, "info", config_id)
- fail_sync = emerge_main(args=tmpcmdline)
- if fail_sync:
- log_msg = "Emerge --sync fail!"
- add_tbc_logs(session, log_msg, "error", config_id)
- return False
- else:
- # Need to add a config dir so we can use profiles/base for reading the tree.
- # We may allready have the dir on local repo when we sync.
- try:
- os.mkdir(mysettings['PORTDIR'] + "/profiles/config", 0o777)
- with open(mysettings['PORTDIR'] + "/profiles/config/parent", "w") as f:
- f.write("../base\n")
- f.close()
- except:
- pass
- log_msg = "Emerge --sync ... Done."
- add_tbc_logs(session, log_msg, "info", config_id)
- return True
+ return repo_cp_dict
def git_pull(session, git_repo, config_id):
log_msg = "Git pull"
- add_zobcs_logs(session, log_msg, "info", config_id)
- repo = Repository(git_repo + ".git")
- remote = repo.remotes["origin"]
- remote.fetch()
- remote_master_id = repo.lookup_reference('refs/remotes/origin/master').target
- merge_result, _ = repo.merge_analysis(remote_master_id)
- if merge_result & GIT_MERGE_ANALYSIS_UP_TO_DATE:
- log_msg = "Repo is up to date"
- add_zobcs_logs(session, log_msg, "info", config_id)
- elif merge_result & GIT_MERGE_ANALYSIS_FASTFORWARD:
- repo.checkout_tree(repo.get(remote_master_id))
- master_ref = repo.lookup_reference('refs/heads/master')
- master_ref.set_target(remote_master_id)
- repo.head.set_target(remote_master_id)
- elif merge_result & GIT_MERGE_ANALYSIS_NORMAL:
- repo.merge(remote_master_id)
- assert repo.index.conflicts is None, 'Conflicts, ahhhh!'
- user = repo.default_signature
- tree = repo.index.write_tree()
- commit = repo.create_commit('HEAD',
- user,
- user,
- 'Merge!',
- tree,
- [repo.head.target, remote_master_id])
- repo.state_cleanup()
- else:
- raise AssertionError('Unknown merge analysis result')
+ add_tbc_logs(session, log_msg, "info", config_id)
+ reop = git_fetch(session, git_repo, config_id)
+ git_merge(session, repo, config_id)
log_msg = "Git pull ... Done"
- add_zobcs_logs(session, log_msg, "info", config_id)
+ add_tbc_logs(session, log_msg, "info", config_id)
return True
diff --git a/tbc/pym/updatedb.py b/tbc/pym/updatedb.py
index 90a9e0a..3cbe495 100644
--- a/tbc/pym/updatedb.py
+++ b/tbc/pym/updatedb.py
@@ -15,7 +15,7 @@ from tbc.sqlquerys import add_tbc_logs, get_package_info, update_repo_db, \
from tbc.check_setup import check_make_conf
from tbc.package import tbc_package
# Get the options from the config file set in tbc.readconf
-from tbc.readconf import get_conf_settings
+from tbc.readconf import read_config_settings
def init_portage_settings(session, config_id, tbc_settings_dict):
# check config setup
@@ -74,44 +74,56 @@ def update_cpv_db(session, config_id, tbc_settings_dict):
break
time.sleep(30)
- mysettings = init_portage_settings(session, config_id, tbc_settings_dict)
log_msg = "Checking categories, package, ebuilds"
add_tbc_logs(session, log_msg, "info", config_id)
new_build_jobs_list = []
- # Setup portdb, package
+ # Setup settings, portdb and pool
+ mysettings = init_portage_settings(session, config_id, tbc_settings_dict)
myportdb = portage.portdbapi(mysettings=mysettings)
- repo_list = ()
- repos_trees_list = []
-
+
# Use all cores when multiprocessing
- pool_cores= multiprocessing.cpu_count()
- pool = multiprocessing.Pool(processes=pool_cores)
-
- # Will run some update checks and update package if needed
-
- # Get the repos and update the repos db
- repo_list = myportdb.getRepositories()
- update_repo_db(session, repo_list)
-
- # Close the db for the multiprocessing pool will make new ones
- # and we don't need this one for some time.
-
- # Get the rootdirs for the repos
- repo_trees_list = myportdb.porttrees
- for repo_dir in repo_trees_list:
- repo = myportdb.getRepositoryName(repo_dir)
- repo_dir_list = []
- repo_dir_list.append(repo_dir)
+ pool_cores = multiprocessing.cpu_count()
+ pool = multiprocessing.Pool(processes = pool_cores)
+
+ # Get packages and repo
+ if repo_cp_dict is None:
+ repo_list = []
+ repos_trees_list = []
+
+ # Get the repos and update the repos db
+ repo_list = myportdb.getRepositories()
+ update_repo_db(session, repo_list)
+
+ # Get the rootdirs for the repos
+ repo_trees_list = myportdb.porttrees
+ for repo_dir in repo_trees_list:
+ repo = myportdb.getRepositoryName(repo_dir)
+ repo_dir_list = []
+ repo_dir_list.append(repo_dir)
+
+ # Get the package list from the repo
+ package_list_tree = myportdb.cp_all(trees=repo_dir_list)
+
+ # Run the update package for all package in the list and in a multiprocessing pool
+ for cp in sorted(package_list_tree):
+ pool.apply_async(update_cpv_db_pool, (mysettings, myportdb, cp, repo, zobcs_settings_dict, config_id,))
+ # use this when debuging
+ #update_cpv_db_pool(mysettings, myportdb, cp, repo, zobcs_settings_dict, config_id)
+ else:
+ # Update needed repos and packages in the dict
+ for repo, v in repo_cp_dict.items():
+ # Get the repos and update the repos db
+ repo_list = []
+ repo_list.append(repo)
+ update_repo_db(session, repo_list)
- # Get the package list from the repo
- package_list_tree = myportdb.cp_all(trees=repo_dir_list)
+ # Run the update package for all package in the list and in a multiprocessing pool
+ for cp in v['cp_list']:
+ pool.apply_async(update_cpv_db_pool, (mysettings, myportdb, cp, repo, zobcs_settings_dict, config_id,))
+ # use this when debuging
+ #update_cpv_db_pool(mysettings, myportdb, cp, repo, zobcs_settings_dict, config_id)
- # Run the update package for all package in the list and in a multiprocessing pool
- for cp in sorted(package_list_tree):
- pool.apply_async(update_cpv_db_pool, (mysettings, myportdb, cp, repo, tbc_settings_dict, config_id,))
- # use this when debuging
- #update_cpv_db_pool(mysettings, myportdb, cp, repo, tbc_settings_dict, config_id)
#close and join the multiprocessing pools
pool.close()
@@ -119,17 +131,17 @@ def update_cpv_db(session, config_id, tbc_settings_dict):
log_msg = "Checking categories, package and ebuilds ... done"
add_tbc_logs(session, log_msg, "info", config_id)
-def update_db_main(session, config_id):
+def update_db_main(session, repo_cp_dict, config_id):
# Main
-
+ if repo_cp_dict == {}:
+ return True
# Logging
- reader = get_conf_settings()
- tbc_settings_dict=reader.read_tbc_settings_all()
+ tbc_settings_dict = reader. read_config_settings()
log_msg = "Update db started."
add_tbc_logs(session, log_msg, "info", config_id)
# Update the cpv db
- update_cpv_db(session, config_id, tbc_settings_dict)
+ update_cpv_db(session, repo_cp_dict, config_id, tbc_settings_dict)
log_msg = "Update db ... Done."
add_tbc_logs(session, log_msg, "info", config_id)
return True
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: tbc/pym/
@ 2015-05-15 19:00 Magnus Granberg
0 siblings, 0 replies; 20+ messages in thread
From: Magnus Granberg @ 2015-05-15 19:00 UTC (permalink / raw
To: gentoo-commits
commit: e2fe77b0831451db82d025538228cf54d5f898ea
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Fri May 15 19:00:20 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Fri May 15 19:00:20 2015 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=e2fe77b0
Add text to sync.py and fix some small things in it
tbc/pym/sync.py | 22 +++++++++++++++-------
1 file changed, 15 insertions(+), 7 deletions(-)
diff --git a/tbc/pym/sync.py b/tbc/pym/sync.py
index 2622b25..f9e4cda 100644
--- a/tbc/pym/sync.py
+++ b/tbc/pym/sync.py
@@ -14,19 +14,23 @@ from tbc.sqlquerys import get_config_id, add_tbc_logs, get_config_all_info, get_
from tbc.readconf import read_config_settings
def git_repos_list(session, myportdb):
+ # get repo tree from portage
repo_trees_list = myportdb.porttrees
+ repo_dir_list = []
+ # append repo dirs to a list
for repo_dir in repo_trees_list:
- repo_dir_list = []
repo_dir_list.append(repo_dir)
return repo_dir_list
def git_fetch(session, git_repo, config_id):
+ # setup repo, fetch it and return repo
repo = Repository(git_repo)
remote = repo.remotes["origin"]
remote.fetch()
return repo
def git_merge(session, repo, config_id):
+ # check what of type merge we need to do and do it
remote_master_id = repo.lookup_reference('refs/remotes/origin/master').target
merge_result, _ = repo.merge_analysis(remote_master_id)
if merge_result & GIT_MERGE_ANALYSIS_UP_TO_DATE:
@@ -65,6 +69,7 @@ def git_sync_main(session):
log_msg = "Waiting for Guest to be idel"
add_tbc_logs(session, log_msg, "info", config_id)
guestid_list = []
+ # check if the guests is idel
for config in get_config_all_info(session):
if not config.Host:
guestid_list.append(config.ConfigId)
@@ -77,12 +82,15 @@ def git_sync_main(session):
GuestBusy = False
else:
time.sleep(30)
+ #remove the needed base profile clone
try:
os.remove(mysettings['PORTDIR'] + "/profiles/config/parent")
os.rmdir(mysettings['PORTDIR'] + "/profiles/config")
except:
pass
+ # check git diffs witch Manifests get updated and pass that to a dict
+ # fetch and merge the repo
repo_cp_dict = {}
for repo_dir in git_repos_list(session, myportdb):
attr = {}
@@ -91,11 +99,10 @@ def git_sync_main(session):
merge_result, _ = repo.merge_analysis(remote_master_id)
if not merge_result & GIT_MERGE_ANALYSIS_UP_TO_DATE:
git_merge(session, repo, config_id)
- out = repo.diff('HEAD', 'HEAD^')
- repo_diff = out.patch
+ repo_diff = repo.diff('HEAD', 'HEAD^')
cp_list = []
reponame = myportdb.getRepositoryName(repo_dir)
- for diff_line in repo_diff.splitlines():
+ for diff_line in repo_diff.patch.splitlines():
if re.search("Manifest", diff_line) and re.search("^diff --git", diff_line):
diff_line2 = re.split(' ', re.sub('[a-b]/', '', re.sub('diff --git ', '', diff_line)))
if diff_line2[0] == diff_line2[1] or "Manifest" in diff_line2[0]:
@@ -110,8 +117,7 @@ def git_sync_main(session):
log_msg = "Repo is up to date"
add_tbc_logs(session, log_msg, "info", config_id)
- # Need to add a config dir so we can use profiles/base for reading the tree.
- # We may allready have the dir on local repo when we sync.
+ # Need to add a clone of profiles/base for reading the tree
try:
os.mkdir(mysettings['PORTDIR'] + "/profiles/config", 0o777)
with open(mysettings['PORTDIR'] + "/profiles/config/parent", "w") as f:
@@ -119,14 +125,16 @@ def git_sync_main(session):
f.close()
except:
pass
+
log_msg = "Repo sync ... Done."
add_tbc_logs(session, log_msg, "info", config_id)
return repo_cp_dict
def git_pull(session, git_repo, config_id):
+ # do a gitt pull
log_msg = "Git pull"
add_tbc_logs(session, log_msg, "info", config_id)
- reop = git_fetch(session, git_repo, config_id)
+ repo = git_fetch(session, git_repo, config_id)
git_merge(session, repo, config_id)
log_msg = "Git pull ... Done"
add_tbc_logs(session, log_msg, "info", config_id)
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: tbc/pym/
@ 2015-05-21 21:40 Magnus Granberg
0 siblings, 0 replies; 20+ messages in thread
From: Magnus Granberg @ 2015-05-21 21:40 UTC (permalink / raw
To: gentoo-commits
commit: 186092d7da925bf1a5c59d41510e3029632ca23a
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu May 21 21:39:32 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu May 21 21:39:32 2015 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=186092d7
Fix the regex for cp dict
tbc/pym/sync.py | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/tbc/pym/sync.py b/tbc/pym/sync.py
index f9e4cda..6dab7f3 100644
--- a/tbc/pym/sync.py
+++ b/tbc/pym/sync.py
@@ -7,6 +7,7 @@ import os
import errno
import sys
import time
+import re
from pygit2 import Repository, GIT_MERGE_ANALYSIS_FASTFORWARD, GIT_MERGE_ANALYSIS_NORMAL, \
GIT_MERGE_ANALYSIS_UP_TO_DATE
@@ -104,9 +105,10 @@ def git_sync_main(session):
reponame = myportdb.getRepositoryName(repo_dir)
for diff_line in repo_diff.patch.splitlines():
if re.search("Manifest", diff_line) and re.search("^diff --git", diff_line):
- diff_line2 = re.split(' ', re.sub('[a-b]/', '', re.sub('diff --git ', '', diff_line)))
- if diff_line2[0] == diff_line2[1] or "Manifest" in diff_line2[0]:
- cp = re.sub('/Manifest', '', diff_line2[0])
+ diff_line2 = re.split(' b/', re.sub('diff --git', '', diff_line))
+ diff_line3 = re.sub(' a/', '', diff_line2[0])
+ if diff_line3 == diff_line2[1] or "Manifest" in diff_line3:
+ cp = re.sub('/Manifest', '', diff_line3)
cp_list.append(cp)
else:
cp = re.sub('/Manifest', '', diff_line2[1])
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: tbc/pym/
@ 2015-05-31 21:15 Magnus Granberg
0 siblings, 0 replies; 20+ messages in thread
From: Magnus Granberg @ 2015-05-31 21:15 UTC (permalink / raw
To: gentoo-commits
commit: ba984e1d239c4fcca04ccaa43b879d40cca771fc
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat May 30 22:03:47 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat May 30 22:03:47 2015 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=ba984e1d
add support for test
tbc/pym/build_job.py | 21 +++++++++++++++++----
tbc/pym/package.py | 12 ++++++++----
tbc/pym/sqlquerys.py | 33 +++++++++++++++++++++++----------
3 files changed, 48 insertions(+), 18 deletions(-)
diff --git a/tbc/pym/build_job.py b/tbc/pym/build_job.py
index 8e3a218..25649fb 100644
--- a/tbc/pym/build_job.py
+++ b/tbc/pym/build_job.py
@@ -18,7 +18,7 @@ from tbc.qacheck import check_file_in_manifest
from tbc.main import emerge_main
from tbc.build_log import log_fail_queru
from tbc.actions import load_emerge_config
-from tbc.sqlquerys import add_logs, get_packages_to_build, update_buildjobs_status, is_build_job_done
+from tbc.sqlquerys import add_logs, get_packages_to_build, update_buildjobs_status, is_build_job_done, get_ebuild_restrictions
class build_job_action(object):
@@ -76,9 +76,19 @@ class build_job_action(object):
log_msg = "filetext: %s" % filetext
add_logs(self._session, log_msg, "info", self._config_id)
with open("/etc/portage/package.use/99_autounmask", "a") as f:
- f.write(filetext)
- f.write('\n')
- f.close
+ f.write(filetext)
+ f.write('\n')
+ f.close
+ restrictions_dict = get_ebuild_restrictions(self._session, build_dict['ebuild_id'])
+ if restrictions_dict:
+ if "test" in restrictions_dict:
+ filetext = k + ' ' + 'notest.conf'
+ log_msg = "filetext: %s" % filetext
+ add_logs(self._session, log_msg, "info", self._config_id)
+ with open("/etc/portage/package.env", "a") as f:
+ f.write(filetext)
+ f.write('\n')
+ f.close
log_msg = "build_cpv_list: %s" % (build_cpv_list,)
add_logs(self._session, log_msg, "info", self._config_id)
@@ -121,6 +131,9 @@ class build_job_action(object):
os.remove("/etc/portage/package.use/99_autounmask")
with open("/etc/portage/package.use/99_autounmask", "a") as f:
f.close
+ os.remove("/etc/portage/package.env")
+ with open("/etc/portage/package.env", "a") as f:
+ f.close
except:
pass
diff --git a/tbc/pym/package.py b/tbc/pym/package.py
index ed47955..502ea01 100644
--- a/tbc/pym/package.py
+++ b/tbc/pym/package.py
@@ -14,7 +14,7 @@ from tbc.sqlquerys import add_logs, get_package_info, get_config_info, \
get_package_metadata_sql, update_package_metadata, update_manifest_sql, \
get_package_info_from_package_id, get_config_all_info, add_new_package_sql, \
get_ebuild_checksums, get_ebuild_id_db, get_configmetadata_info, get_setup_info, \
- get_ebuild_info_ebuild_id
+ get_ebuild_info_ebuild_id, get_ebuild_restrictions
class tbc_package(object):
@@ -147,6 +147,10 @@ class tbc_package(object):
# Comper and add the cpv to buildqueue
if build_cpv == k:
+ restrictions_dict = get_ebuild_restrictions(self._session, ebuild_id)
+ if restrictions_dict:
+ if "test" in restrictions_dict and "test" in use_flagsDict:
+ use_flagsDict['test'] = False
add_new_build_job(self._session, ebuild_id, setup_id, use_flagsDict, self._config_id)
# B = Build cpv use-flags config
# FIXME log_msg need a fix to log the use flags corect.
@@ -252,6 +256,8 @@ class tbc_package(object):
log_msg = "N %s:%s" % (cp, repo)
add_logs(self._session, log_msg, "info", self._config_id)
repodir = self._myportdb.getRepositoryPath(repo)
+ mytree = []
+ mytree.append(repodir)
pkgdir = repodir + "/" + cp # Get RepoDIR + cp
manifest_checksum_tree = self.get_manifest_checksum_tree(pkgdir, cp, repo, mytree)
@@ -261,8 +267,6 @@ class tbc_package(object):
package_metadataDict = self.get_package_metadataDict(pkgdir, package_id)
# Get the ebuild list for cp
- mytree = []
- mytree.append(repodir)
ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree)
if ebuild_list_tree == []:
log_msg = "QA: Can't get the ebuilds list. %s:%s" % (cp, repo,)
@@ -363,7 +367,7 @@ class tbc_package(object):
repoman_fail = check_repoman(self._mysettings, self._myportdb, cpv, repo)
if repoman_fail:
log_msg = "Repoman %s:%s ... Fail." % (cpv, repo)
- add_zobcs_logs(self._session, log_msg, "info", self._config_id)
+ add_logs(self._session, log_msg, "info", self._config_id)
# Check if the checksum have change
if ebuild_version_manifest_checksum_db is None:
diff --git a/tbc/pym/sqlquerys.py b/tbc/pym/sqlquerys.py
index c674805..a8f2af4 100644
--- a/tbc/pym/sqlquerys.py
+++ b/tbc/pym/sqlquerys.py
@@ -7,7 +7,7 @@ from tbc.db_mapping import Configs, Logs, ConfigsMetaData, Jobs, BuildJobs, Pack
Uses, ConfigsEmergeOptions, EmergeOptions, HiLight, BuildLogs, BuildLogsConfig, BuildJobsUse, BuildJobsRedo, \
HiLightCss, BuildLogsHiLight, BuildLogsEmergeOptions, BuildLogsErrors, ErrorsInfo, EmergeInfo, BuildLogsUse, \
BuildJobsEmergeOptions, EbuildsMetadata, EbuildsIUse, Restrictions, EbuildsRestrictions, EbuildsKeywords, \
- Keywords, PackagesMetadata, Emails, PackagesEmails, Setups, BuildLogsQA, BuildLogsRepoman
+ Keywords, PackagesMetadata, Emails, PackagesEmails, Setups, BuildLogsQA, BuildLogsRepoman
from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
from sqlalchemy import and_, or_
@@ -304,6 +304,18 @@ def add_fail_times(session, fail_querue_dict):
session.add(NewBuildJobsRedo)
session.commit()
+def check_host_updatedb(session):
+ jobs = False
+ try:
+ JobsInfo = session.query(Jobs).filter_by(Status = 'Done').filter_by(JobType = 'esync').one()
+ except NoResultFound as e:
+ jobs = True
+ try:
+ JobsInfo = session.query(Jobs).filter_by(Status = 'Done').filter_by(JobType = 'updatedb').one()
+ except NoResultFound as e:
+ jobs = True
+ return jobs
+
# Host Functions
def update_repo_db(session, repo_list):
for repo in repo_list:
@@ -520,14 +532,15 @@ def get_ebuild_id_db(session, checksum, package_id):
return ebuilds_id, True
return EbuildInfos.EbuildId, False
-def check_host_updatedb(session):
- jobs = False
+def get_ebuild_restrictions(session, ebuild_id):
+ restrictions = []
try:
- JobsInfo = session.query(Jobs).filter_by(Status = 'Done').filter_by(JobType = 'esync').one()
+ EbuildsRestrictionsInfo = session.query(EbuildsRestrictions).filter_by(EbuildId = ebuild_id).one()
except NoResultFound as e:
- jobs = True
- try:
- JobsInfo = session.query(Jobs).filter_by(Status = 'Done').filter_by(JobType = 'updatedb').one()
- except NoResultFound as e:
- jobs = True
- return jobs
+ return False
+ except MultipleResultsFound as e:
+ EbuildsRestrictionsInfos = session.query(EbuildsRestrictions).filter_by(EbuildId = ebuild_id).all()
+ for EbuildsRestrictionsInfo in EbuildsRestrictionsInfos:
+ restrictions.append(session.query(Restrictions).filter_by(RestrictionId = EbuildsRestrictionsInfo.RestrictionId).one())
+ return restrictions
+ return restrictions.append(session.query(Restrictions).filter_by(RestrictionId = EbuildsRestrictionsInfo.RestrictionId).one())
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: tbc/pym/
@ 2015-05-31 21:15 Magnus Granberg
0 siblings, 0 replies; 20+ messages in thread
From: Magnus Granberg @ 2015-05-31 21:15 UTC (permalink / raw
To: gentoo-commits
commit: 5d9b7179349c0a03b75ee6dd8376ffa3270d9edb
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat May 30 20:15:05 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat May 30 20:15:05 2015 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=5d9b7179
small fixes for add_logs
tbc/pym/package.py | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/tbc/pym/package.py b/tbc/pym/package.py
index 4dac33e..ed47955 100644
--- a/tbc/pym/package.py
+++ b/tbc/pym/package.py
@@ -230,17 +230,16 @@ class tbc_package(object):
package_list_tree =self. _myportdb.cp_all(trees=mytree)
if cp in package_list_tree:
log_msg = "QA: Can't checksum the Manifest file. :%s:%s" % (cp, repo,)
- add_zobcs_logs(self._session, log_msg, "error", self._config_id)
+ add_logs(self._session, log_msg, "error", self._config_id)
log_msg = "C %s:%s ... Fail." % (cp, repo)
- add_zobcs_logs(self._session, log_msg, "error", self._config_id)
+ add_logs(self._session, log_msg, "error", self._config_id)
return "0"
fail_msg = digestcheck(self._mysettings, pkgdir)
if fail_msg:
log_msg = "QA: Manifest file has errors. :%s:%s" % (cp, repo,)
- add_zobcs_logs(self._session, log_msg, "error", self._config_id)
- add_zobcs_logs(self._session, fail_msg, "error", self._config_id)
+ add_logs(self._session, log_msg, "error", self._config_id)
log_msg = "C %s:%s ... Fail." % (cp, repo)
- add_zobcs_logs(self._session, log_msg, "error", self._config_id)
+ add_logs(self._session, log_msg, "error", self._config_id)
return None
return manifest_checksum_tree
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: tbc/pym/
@ 2015-06-06 20:06 Magnus Granberg
0 siblings, 0 replies; 20+ messages in thread
From: Magnus Granberg @ 2015-06-06 20:06 UTC (permalink / raw
To: gentoo-commits
commit: 91feadebbf3440dbd1ac39fec3c7c61c8f5d6d5a
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Jun 6 20:05:38 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Jun 6 20:05:38 2015 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=91feadeb
add support for test in FEATURES
tbc/pym/build_job.py | 27 +++++++++++++++++----------
tbc/pym/qachecks.py | 7 ++++---
2 files changed, 21 insertions(+), 13 deletions(-)
diff --git a/tbc/pym/build_job.py b/tbc/pym/build_job.py
index 25649fb..8b8d393 100644
--- a/tbc/pym/build_job.py
+++ b/tbc/pym/build_job.py
@@ -66,6 +66,7 @@ class build_job_action(object):
def build_procces(self, buildqueru_cpv_dict, build_dict, settings, portdb):
build_cpv_list = []
depclean_fail = True
+ disable_test_features = False
for k, build_use_flags_list in buildqueru_cpv_dict.items():
build_cpv_list.append("=" + k)
if not build_use_flags_list == None:
@@ -79,16 +80,22 @@ class build_job_action(object):
f.write(filetext)
f.write('\n')
f.close
- restrictions_dict = get_ebuild_restrictions(self._session, build_dict['ebuild_id'])
- if restrictions_dict:
- if "test" in restrictions_dict:
- filetext = k + ' ' + 'notest.conf'
- log_msg = "filetext: %s" % filetext
- add_logs(self._session, log_msg, "info", self._config_id)
- with open("/etc/portage/package.env", "a") as f:
- f.write(filetext)
- f.write('\n')
- f.close
+ if not build_dict['build_useflags'] is None:
+ if "test" in build_dict['build_useflags'] and "test" in settings.features:
+ if build_dict['build_useflags']['test'] is False:
+ disable_test_features = True
+ restrictions_dict = get_ebuild_restrictions(self._session, build_dict['ebuild_id'])
+ if restrictions_dict:
+ if "test" in restrictions_dict:
+ disable_test_features = True
+ if disable_test_features:
+ filetext = k + ' ' + 'notest.conf'
+ log_msg = "filetext: %s" % filetext
+ add_logs(self._session, log_msg, "info", self._config_id)
+ with open("/etc/portage/package.env", "a") as f:
+ f.write(filetext)
+ f.write('\n')
+ f.close
log_msg = "build_cpv_list: %s" % (build_cpv_list,)
add_logs(self._session, log_msg, "info", self._config_id)
diff --git a/tbc/pym/qachecks.py b/tbc/pym/qachecks.py
index f1d0fb1..b0d10f8 100644
--- a/tbc/pym/qachecks.py
+++ b/tbc/pym/qachecks.py
@@ -6,6 +6,7 @@ import warnings
import sys
import codecs
from portage import os, _encodings, _unicode_decode
+from portage.checksum import _hash_filter
from portage.exception import DigestException, FileNotFound
from portage.localization import _
from portage.manifest import Manifest
@@ -122,14 +123,14 @@ def check_file_in_manifest(pkgdir, mysettings, portdb, cpv, build_use_flags_list
return "Ebuild file not found."
tree = portdb.getRepositoryPath(repo)
cpv_fetchmap = portdb.getFetchMap(cpv, useflags=build_use_flags_list, mytree=tree)
- self._mysettings.unlock()
+ mysettings.unlock()
try:
portage.fetch(cpv_fetchmap, mysettings, listonly=0, fetchonly=0, locks_in_subdir='.locks', use_locks=1, try_mirrors=1)
except:
- self._mysettings.lock()
+ mysettings.lock()
return "Can't fetch the file."
finally:
- self._mysettings.lock()
+ mysettings.lock()
try:
my_manifest.checkCpvHashes(cpv, checkDistfiles=True, onlyDistfiles=False, checkMiscfiles=True)
except:
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: tbc/pym/
@ 2015-06-12 19:31 Magnus Granberg
0 siblings, 0 replies; 20+ messages in thread
From: Magnus Granberg @ 2015-06-12 19:31 UTC (permalink / raw
To: gentoo-commits
commit: e7b4bfaf66f3a07ed7f32f852e297e45b79382d7
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Jun 11 16:09:22 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Jun 11 16:09:22 2015 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=e7b4bfaf
use git-python instead of pygit2
tbc/pym/sync.py | 81 ++++++++++++++++++++-------------------------------------
1 file changed, 28 insertions(+), 53 deletions(-)
diff --git a/tbc/pym/sync.py b/tbc/pym/sync.py
index cc29f8e..e9a2aca 100644
--- a/tbc/pym/sync.py
+++ b/tbc/pym/sync.py
@@ -8,54 +8,30 @@ import errno
import sys
import time
import re
-from pygit2 import Repository, GIT_MERGE_ANALYSIS_FASTFORWARD, GIT_MERGE_ANALYSIS_NORMAL, \
- GIT_MERGE_ANALYSIS_UP_TO_DATE
+import git
from tbc.sqlquerys import get_config_id, add_logs, get_config_all_info, get_configmetadata_info
from tbc.readconf import read_config_settings
-def git_repos_list(session, myportdb):
- # get repo tree from portage
+def git_repos_list(myportdb):
repo_trees_list = myportdb.porttrees
repo_dir_list = []
- # append repo dirs to a list
for repo_dir in repo_trees_list:
repo_dir_list.append(repo_dir)
return repo_dir_list
-def git_fetch(session, git_repo, config_id):
- # setup repo, fetch it and return repo
- repo = Repository(git_repo)
- remote = repo.remotes["origin"]
- remote.fetch()
- return repo
+def git_fetch(repo):
+ repouptodate = True
+ remote = git.remote.Remote(repo, 'origin')
+ info_list = remote.fetch()
+ local_commit = repo.commit()
+ remote_commit = info_list[0].commit
+ if local_commit.hexsha != remote_commit.hexsha:
+ repouptodate = False
+ return info_list, repouptodate
-def git_merge(session, repo, config_id):
- # check what of type merge we need to do and do it
- remote_master_id = repo.lookup_reference('refs/remotes/origin/master').target
- merge_result, _ = repo.merge_analysis(remote_master_id)
- if merge_result & GIT_MERGE_ANALYSIS_UP_TO_DATE:
- log_msg = "Repo is up to date"
- add_zobcs_logs(session, log_msg, "info", config_id)
- elif merge_result & GIT_MERGE_ANALYSIS_FASTFORWARD:
- repo.checkout_tree(repo.get(remote_master_id))
- master_ref = repo.lookup_reference('refs/heads/master')
- master_ref.set_target(remote_master_id)
- repo.head.set_target(remote_master_id)
- elif merge_result & GIT_MERGE_ANALYSIS_NORMAL:
- repo.merge(remote_master_id)
- assert repo.index.conflicts is None, 'Conflicts, ahhhh!'
- user = repo.default_signature
- tree = repo.index.write_tree()
- commit = repo.create_commit('HEAD',
- user,
- user,
- 'Merge!',
- tree,
- [repo.head.target, remote_master_id])
- repo.state_cleanup()
- else:
- raise AssertionError('Unknown merge analysis result')
+def git_merge(repo, info):
+ repo.git.merge(info.commit)
def git_sync_main(session):
tbc_settings_dict = read_config_settings()
@@ -93,18 +69,15 @@ def git_sync_main(session):
# check git diffs witch Manifests get updated and pass that to a dict
# fetch and merge the repo
repo_cp_dict = {}
- for repo_dir in git_repos_list(session, myportdb):
+ for repo_dir in git_repos_list(myportdb):
+ reponame = myportdb.getRepositoryName(repo_dir)
attr = {}
- repo = git_fetch(session, repo_dir, config_id)
- remote_master_id = repo.lookup_reference('refs/remotes/origin/master').target
- merge_result, _ = repo.merge_analysis(remote_master_id)
- if not merge_result & GIT_MERGE_ANALYSIS_UP_TO_DATE:
- git_merge(session, repo, config_id)
- repo_diff = repo.diff('HEAD', 'HEAD^')
+ repo = git.Repo(repo_dir)
+ info_list, repouptodate = git_fetch(repo)
+ if not repouptodate:
cp_list = []
- reponame = myportdb.getRepositoryName(repo_dir)
- for diff_line in repo_diff.patch.splitlines():
- if re.search("Manifest", diff_line) and re.search("^diff --git", diff_line):
+ for diff_line in repo.git.diff('HEAD^').splitlines():
+ if re.search("^diff --git.*/Manifest", diff_line):
diff_line2 = re.split(' b/', re.sub('diff --git', '', diff_line))
diff_line3 = re.sub(' a/', '', diff_line2[0])
if diff_line3 == diff_line2[1] or "Manifest" in diff_line3:
@@ -115,8 +88,9 @@ def git_sync_main(session):
cp_list.append(cp)
attr['cp_list'] = cp_list
repo_cp_dict[reponame] = attr
+ git_merge(repo, info_list[0])
else:
- log_msg = "Repo is up to date"
+ log_msg = "Repo %s is up to date" % (reponame)
add_logs(session, log_msg, "info", config_id)
# Need to add a clone of profiles/base for reading the tree
@@ -130,14 +104,15 @@ def git_sync_main(session):
log_msg = "Repo sync ... Done."
add_logs(session, log_msg, "info", config_id)
- return repo_cp_dict
+ return repo_cp_dict
-def git_pull(session, git_repo, config_id):
- # do a gitt pull
+def git_pull(session, repo_dir, config_id):
log_msg = "Git pull"
add_logs(session, log_msg, "info", config_id)
- repo = git_fetch(session, git_repo, config_id)
- git_merge(session, repo, config_id)
+ repo = git.Repo(repo_dir)
+ info_list, repouptodate = git_fetch(repo)
+ if not repouptodate:
+ git_merge(repo, info_list[0])
log_msg = "Git pull ... Done"
add_logs(session, log_msg, "info", config_id)
return True
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: tbc/pym/
@ 2015-06-12 19:31 Magnus Granberg
0 siblings, 0 replies; 20+ messages in thread
From: Magnus Granberg @ 2015-06-12 19:31 UTC (permalink / raw
To: gentoo-commits
commit: e9cc80849b10574cd13f0e2ea22df78f830d7cc1
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Fri Jun 12 19:31:29 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Fri Jun 12 19:31:29 2015 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=e9cc8084
Rename git_www to RepoPath
tbc/pym/db_mapping.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tbc/pym/db_mapping.py b/tbc/pym/db_mapping.py
index 31e1b66..b78bd1e 100644
--- a/tbc/pym/db_mapping.py
+++ b/tbc/pym/db_mapping.py
@@ -56,7 +56,7 @@ class ConfigsMetaData(Base):
Updateing = Column('updateing', Boolean, default=False)
Status = Column('status', Enum('Stopped', 'Runing', 'Waiting'))
Auto = Column('auto', Boolean, default=False)
- GitWww = Column('git_www', String(200))
+ RepoPath = Column('repo_path', String(200))
TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow)
__tablename__ = 'configs_metadata'
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: tbc/pym/
@ 2015-06-12 19:31 Magnus Granberg
0 siblings, 0 replies; 20+ messages in thread
From: Magnus Granberg @ 2015-06-12 19:31 UTC (permalink / raw
To: gentoo-commits
commit: 39e30ad6e439888b3097a40b3b693e2193401e30
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Jun 11 15:55:46 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Jun 11 15:55:46 2015 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=39e30ad6
missing strict in digestcheck
tbc/pym/qachecks.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tbc/pym/qachecks.py b/tbc/pym/qachecks.py
index b0d10f8..36d2a31 100644
--- a/tbc/pym/qachecks.py
+++ b/tbc/pym/qachecks.py
@@ -24,7 +24,7 @@ def digestcheck(mysettings, pkgdir):
@rtype: int
@return: 1 on success and 0 on failure
"""
-
+ strict=False
myfiles = []
justmanifest = None
mf = None
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: tbc/pym/
@ 2015-07-15 12:04 Magnus Granberg
0 siblings, 0 replies; 20+ messages in thread
From: Magnus Granberg @ 2015-07-15 12:04 UTC (permalink / raw
To: gentoo-commits
commit: be8a15743d2e39cc96e47e7bd5161a7be4a59fc8
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Wed Jul 15 12:04:18 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Wed Jul 15 12:04:18 2015 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=be8a1574
add Descriptions to ebuild metadata in db
tbc/pym/db_mapping.py | 1 +
tbc/pym/package.py | 2 ++
tbc/pym/sqlquerys.py | 2 +-
3 files changed, 4 insertions(+), 1 deletion(-)
diff --git a/tbc/pym/db_mapping.py b/tbc/pym/db_mapping.py
index b78bd1e..76ff33e 100644
--- a/tbc/pym/db_mapping.py
+++ b/tbc/pym/db_mapping.py
@@ -269,4 +269,5 @@ class EbuildsMetadata(Base):
Id = Column('id', Integer, primary_key=True)
EbuildId = Column('ebuild_id', ForeignKey('ebuilds.ebuild_id'))
Revision = Column('revision', String(30))
+ Descriptions = Column('descriptions', String(200))
__tablename__ = 'ebuilds_metadata'
diff --git a/tbc/pym/package.py b/tbc/pym/package.py
index 502ea01..3e87aa6 100644
--- a/tbc/pym/package.py
+++ b/tbc/pym/package.py
@@ -122,6 +122,7 @@ class tbc_package(object):
attDict['ebuild_version_metadata_tree'] = ebuild_version_metadata_tree
#attDict['ebuild_version_text_tree'] = ebuild_version_text_tree[0]
attDict['ebuild_version_revision_tree'] = ebuild_version_cvs_revision_tree
+ attDict['ebuild_version_descriptions_tree'] = ebuild_version_metadata_tree[7]
return attDict
def add_new_build_job_db(self, ebuild_id_list, packageDict, config_cpv_listDict):
@@ -182,6 +183,7 @@ class tbc_package(object):
log_msg = "Metadata file %s missing Email" % (pkgdir + "/metadata.xml")
add_logs(self._session, log_msg, "qa", self._config_id)
attDict['metadata_xml_email'] = False
+ attDict['metadata_xml_descriptions'] = pkg_md.descriptions()[0]
attDict['metadata_xml_checksum'] = portage.checksum.sha256hash(pkgdir + "/metadata.xml")[0]
#attDict['metadata_xml_text'] = metadata_xml_text_tree
package_metadataDict[package_id] = attDict
diff --git a/tbc/pym/sqlquerys.py b/tbc/pym/sqlquerys.py
index a8f2af4..b897f16 100644
--- a/tbc/pym/sqlquerys.py
+++ b/tbc/pym/sqlquerys.py
@@ -397,7 +397,7 @@ def add_new_ebuild_sql(session, packageDict):
#x.Active = False
#session.commit()
sys.exit()
- session.add(EbuildsMetadata(EbuildId = EbuildInfo.EbuildId, Revision = v['ebuild_version_revision_tree']))
+ session.add(EbuildsMetadata(EbuildId = EbuildInfo.EbuildId, Revision = v['ebuild_version_revision_tree'], Descriptions = v['ebuild_version_descriptions_tree']))
session.commit()
ebuild_id_list.append(EbuildInfo.EbuildId)
restrictions = []
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: tbc/pym/
@ 2015-07-19 11:54 Magnus Granberg
0 siblings, 0 replies; 20+ messages in thread
From: Magnus Granberg @ 2015-07-19 11:54 UTC (permalink / raw
To: gentoo-commits
commit: f0fa7515a57cc273134d89b5a424e8dcca75ba6a
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Jul 19 11:54:13 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Jul 19 11:54:13 2015 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=f0fa7515
change from HEAD^ to origin when we diff to get all Manifests
tbc/pym/sync.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/tbc/pym/sync.py b/tbc/pym/sync.py
index 39d8385..12c4c9f 100644
--- a/tbc/pym/sync.py
+++ b/tbc/pym/sync.py
@@ -73,13 +73,13 @@ def git_sync_main(session):
repo_cp_dict = {}
for repo_dir in git_repos_list(myportdb):
reponame = myportdb.getRepositoryName(repo_dir)
- attr = {}
repo = git.Repo(repo_dir)
info_list, repouptodate = git_fetch(repo)
if not repouptodate:
cp_list = []
- # FiXME We still miss files to update
- for diff_line in repo.git.diff('HEAD^').splitlines():
+ attr = {}
+ # We check for Manifest changes and add the package to a list
+ for diff_line in repo.git.diff('origin').splitlines():
if re.search("^diff --git.*/Manifest", diff_line):
diff_line2 = re.split(' b/', re.sub('diff --git', '', diff_line))
diff_line3 = re.sub(' a/', '', diff_line2[0])
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: tbc/pym/
@ 2015-07-20 15:46 Magnus Granberg
0 siblings, 0 replies; 20+ messages in thread
From: Magnus Granberg @ 2015-07-20 15:46 UTC (permalink / raw
To: gentoo-commits
commit: 932910d804e09718fe1d7e6e7a50d4d943557a66
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Jul 20 15:45:59 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Jul 20 15:45:59 2015 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=932910d8
add CategoriesMetadata
tbc/pym/db_mapping.py | 7 +++++++
tbc/pym/sqlquerys.py | 19 ++++++++++++++++---
tbc/pym/updatedb.py | 15 ++++++++++++++-
3 files changed, 37 insertions(+), 4 deletions(-)
diff --git a/tbc/pym/db_mapping.py b/tbc/pym/db_mapping.py
index 05fccae..cf4a054 100644
--- a/tbc/pym/db_mapping.py
+++ b/tbc/pym/db_mapping.py
@@ -67,6 +67,13 @@ class Categories(Base):
TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow)
__tablename__ = 'categories'
+class CategoriesMetadata(Base):
+ Id = Column('id', Integer, primary_key=True)
+ CategoryId = Column('category_id', Integer, ForeignKey('categories.category_id'))
+ Checksum = Column('checksum', String(100))
+ Descriptions = Column('descriptions', Text)
+ __tablename__ = 'categories_metadata'
+
class Repos(Base):
RepoId = Column('repo_id', Integer, primary_key=True)
Repo = Column('repo', String(100))
diff --git a/tbc/pym/sqlquerys.py b/tbc/pym/sqlquerys.py
index 9d7189d..a521dbc 100644
--- a/tbc/pym/sqlquerys.py
+++ b/tbc/pym/sqlquerys.py
@@ -8,7 +8,7 @@ from tbc.db_mapping import Configs, Logs, ConfigsMetaData, Jobs, BuildJobs, Pack
Uses, ConfigsEmergeOptions, EmergeOptions, HiLight, BuildLogs, BuildLogsConfig, BuildJobsUse, BuildJobsRedo, \
HiLightCss, BuildLogsHiLight, BuildLogsEmergeOptions, BuildLogsErrors, ErrorsInfo, EmergeInfo, BuildLogsUse, \
BuildJobsEmergeOptions, EbuildsMetadata, EbuildsIUse, Restrictions, EbuildsRestrictions, EbuildsKeywords, \
- Keywords, PackagesMetadata, Emails, PackagesEmails, Setups, BuildLogsRepomanQa
+ Keywords, PackagesMetadata, Emails, PackagesEmails, Setups, BuildLogsRepomanQa, CategoriesMetadata
from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
from sqlalchemy import and_, or_
@@ -323,10 +323,23 @@ def update_repo_db(session, repo_list):
session.add(Repos(Repo = repo))
session.commit()
-def update_categories_db(session, category):
- if not get_category_info(session, category):
+def update_categories_db(session, category, categories_metadataDict):
+ CategoryInfo = get_category_info(session, category)
+ if not CategoryInfo:
session.add(Categories(Category = category))
session.commit()
+ CategoryInfo = get_category_info(session, category)
+ try:
+ CategoriesMetadataInfo = session.query(CategoriesMetadata).filter_by(CategoryId = CategoryInfo.CategoryId).one()
+ except NoResultFound as e:
+ NewCategoriesMetadata = CategoriesMetadata(CategoryId = CategoryInfo.CategoryId, Checksum = categories_metadataDict['metadata_xml_checksum'], Descriptions = categories_metadataDict['metadata_xml_descriptions'])
+ session.add(NewCategoriesMetadata)
+ session.commit()
+ return
+ if CategoriesMetadataInfo.Checksum != categories_metadataDict['metadata_xml_checksum']:
+ CategoriesMetadataInfo.Checksum = categories_metadataDict['metadata_xml_checksum']
+ CategoriesMetadataInfo.Descriptions = categories_metadataDict['metadata_xml_descriptions']
+ session.commit()
def get_keyword_id(session, keyword):
try:
diff --git a/tbc/pym/updatedb.py b/tbc/pym/updatedb.py
index 1cacc7d..b8be53f 100644
--- a/tbc/pym/updatedb.py
+++ b/tbc/pym/updatedb.py
@@ -7,6 +7,7 @@ import os
import multiprocessing
import time
import portage
+from portage.xml.metadata import MetaDataXML
from sqlalchemy.orm import scoped_session, sessionmaker
from tbc.ConnectionManager import NewConnection
from tbc.sqlquerys import add_logs, get_package_info, update_repo_db, \
@@ -36,6 +37,15 @@ def init_portage_settings(session, config_id):
add_logs(session, log_msg, "info", config_id)
return mysettings
+def get_categories_metadataDict(pkgdir):
+ # Make categories_metadataDict
+ categories_metadataDict = {}
+ pkg_md = MetaDataXML(pkgdir + "/metadata.xml", None)
+ metadata_xml_descriptions_tree = re.sub('\t', '', pkg_md.descriptions()[0])
+ categories_metadataDict['metadata_xml_descriptions'] = re.sub('\n', '', metadata_xml_descriptions_tree)
+ categories_metadataDict['metadata_xml_checksum'] = portage.checksum.sha256hash(pkgdir + "/metadata.xml")[0]
+ return categories_metadataDict
+
def update_cpv_db_pool(mysettings, myportdb, cp, repo, tbc_settings, config_id):
session_factory = sessionmaker(bind=NewConnection(tbc_settings))
Session = scoped_session(session_factory)
@@ -48,7 +58,10 @@ def update_cpv_db_pool(mysettings, myportdb, cp, repo, tbc_settings, config_id):
package = element[1]
# update the categories table
- update_categories_db(session2, categories)
+ repodir = self._myportdb.getRepositoryPath('gentoo')
+ pkgdir = repodir + "/" + categories
+ categories_metadataDict = get_categories_metadataDict(pkgdir)
+ update_categories_db(session2, categories, categories_metadataDict)
# Check if we have the cp in the package table
PackagesInfo = get_package_info(session2, categories, package, repo)
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: tbc/pym/
@ 2015-07-21 22:42 Magnus Granberg
0 siblings, 0 replies; 20+ messages in thread
From: Magnus Granberg @ 2015-07-21 22:42 UTC (permalink / raw
To: gentoo-commits
commit: 981f86a56468a7561c8804044c238359cb1b7c2d
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Jul 21 22:39:54 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Jul 21 22:39:54 2015 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=981f86a5
add descriptions to package
tbc/pym/db_mapping.py | 3 +--
tbc/pym/package.py | 3 ---
tbc/pym/sqlquerys.py | 5 ++---
3 files changed, 3 insertions(+), 8 deletions(-)
diff --git a/tbc/pym/db_mapping.py b/tbc/pym/db_mapping.py
index 37eaec8..67c2d61 100644
--- a/tbc/pym/db_mapping.py
+++ b/tbc/pym/db_mapping.py
@@ -103,9 +103,8 @@ class PackagesEmails(Base):
class PackagesMetadata(Base):
Id = Column('id', Integer, primary_key=True)
PackageId = Column('package_id', Integer, ForeignKey('packages.package_id'))
- Checksum = Column('checksum', String(100))
- ChecksumChanglog = Column('checksum_chlog', String(100))
Changlog = Column('changlog', Text)
+ Descriptions = Column('descriptions', Text)
__tablename__ = 'packages_metadata'
class Ebuilds(Base):
diff --git a/tbc/pym/package.py b/tbc/pym/package.py
index c8a70db..f834fb2 100644
--- a/tbc/pym/package.py
+++ b/tbc/pym/package.py
@@ -186,7 +186,6 @@ class tbc_package(object):
md_email_list = []
herd = None
pkg_md = MetaDataXML(pkgdir + "/metadata.xml", herd)
- attDict['changelog_checksum'] = portage.checksum.sha256hash(pkgdir + "/ChangeLog")[0]
attDict['changelog_text'] = self.get_changelog_text(pkgdir)
tmp_herds = pkg_md.herds()
if tmp_herds != ():
@@ -202,8 +201,6 @@ class tbc_package(object):
attDict['metadata_xml_email'] = False
metadata_xml_descriptions_tree = re.sub('\t', '', pkg_md.descriptions()[0])
attDict['metadata_xml_descriptions'] = re.sub('\n', '', metadata_xml_descriptions_tree)
- attDict['metadata_xml_checksum'] = portage.checksum.sha256hash(pkgdir + "/metadata.xml")[0]
- #attDict['metadata_xml_text'] = metadata_xml_text_tree
package_metadataDict[package_id] = attDict
return package_metadataDict
diff --git a/tbc/pym/sqlquerys.py b/tbc/pym/sqlquerys.py
index 7a64db7..f7f5a97 100644
--- a/tbc/pym/sqlquerys.py
+++ b/tbc/pym/sqlquerys.py
@@ -490,12 +490,11 @@ def update_package_metadata(session, package_metadataDict):
try:
PackagesMetadataInfo = session.query(PackagesMetadata).filter_by(PackageId = k).one()
except NoResultFound as e:
- session.add(PackagesMetadata(PackageId = k, Checksum = v['metadata_xml_checksum'], ChecksumChanglog = v['changelog_checksum'], Changlog = v['changelog_text']))
+ session.add(PackagesMetadata(PackageId = k, Changlog = v['changelog_text'], Descriptions = v['metadata_xml_descriptions']))
session.commit()
else:
- PackagesMetadataInfo.Checksum = v['metadata_xml_checksum']
- PackagesMetadataInfo.ChecksumChanglog = v['changelog_checksum']
PackagesMetadataInfo.Changlog = v['changelog_text']
+ PackagesMetadataInfo.Descriptions = v['metadata_xml_descriptions']
session.commit()
if v['metadata_xml_email']:
for email in v['metadata_xml_email']:
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: tbc/pym/
@ 2015-07-22 6:22 Magnus Granberg
0 siblings, 0 replies; 20+ messages in thread
From: Magnus Granberg @ 2015-07-22 6:22 UTC (permalink / raw
To: gentoo-commits
commit: 998493de0d2b237f25fdb382e7a0fe570a2cb325
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Wed Jul 22 06:22:02 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Wed Jul 22 06:22:02 2015 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=998493de
take package descriptions from ebuilds
tbc/pym/package.py | 28 +++++++++++++++-------------
1 file changed, 15 insertions(+), 13 deletions(-)
diff --git a/tbc/pym/package.py b/tbc/pym/package.py
index f834fb2..dc4ff3e 100644
--- a/tbc/pym/package.py
+++ b/tbc/pym/package.py
@@ -199,8 +199,6 @@ class tbc_package(object):
log_msg = "Metadata file %s missing Email" % (pkgdir + "/metadata.xml")
add_logs(self._session, log_msg, "qa", self._config_id)
attDict['metadata_xml_email'] = False
- metadata_xml_descriptions_tree = re.sub('\t', '', pkg_md.descriptions()[0])
- attDict['metadata_xml_descriptions'] = re.sub('\n', '', metadata_xml_descriptions_tree)
package_metadataDict[package_id] = attDict
return package_metadataDict
@@ -217,13 +215,9 @@ class tbc_package(object):
old_ebuild_id_list.append(ebuild_id)
if not old_ebuild_id_list == []:
add_old_ebuild(self._session, old_ebuild_id_list)
- PackagesMetadataInfo = get_package_metadata_sql(self._session, package_id)
- if PackagesMetadataInfo:
- package_metadata_checksum_sql = PackagesMetadataInfo.Checksum
- else:
- package_metadata_checksum_sql = None
- if package_metadata_checksum_sql is None or package_metadata_checksum_sql != package_metadataDict[package_id]['metadata_xml_checksum']:
- update_package_metadata(self._session, package_metadataDict)
+
+ # update package metadata
+ update_package_metadata(self._session, package_metadataDict)
# update the cp manifest checksum
update_manifest_sql(self._session, package_id, manifest_checksum_tree)
@@ -299,7 +293,11 @@ class tbc_package(object):
old_ebuild_id_list = []
for cpv in sorted(ebuild_list_tree):
packageDict[cpv] = self.get_packageDict(pkgdir, cpv, repo)
- if packageDict[cpv]['checksum'] == "0":
+
+ # take package descriptions from the ebuilds
+ if package_metadataDict[package_id]['metadata_xml_descriptions'] != packageDict[cpv]['ebuild_version_descriptions_tree']:
+ package_metadataDict[package_id]['metadata_xml_descriptions'] = packageDict[cpv]['ebuild_version_descriptions_tree']
+ if packageDict[cpv]['checksum'] != "0":
repoman_fail = check_repoman(self._mysettings, self._myportdb, cpv, repo)
if repoman_fail:
log_msg = "Repoman %s:%s ... Fail." % (cpv, repo)
@@ -335,7 +333,7 @@ class tbc_package(object):
old_ebuild_id_list = []
ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree)
if ebuild_list_tree == []:
- if manifest_checksum_tree == "0":
+ if manifest_checksum_tree != "0":
old_ebuild_id_list = get_ebuild_id_list(self._session, package_id)
for ebuild_id in old_ebuild_id_list:
EbuildInfo = get_ebuild_info_ebuild_id(self._session, ebuild_id)
@@ -352,6 +350,7 @@ class tbc_package(object):
log_msg = "C %s:%s ... Fail." % (cp, repo)
add_logs(self._session, log_msg, "info", self._config_id)
return
+ package_metadataDict = self.get_package_metadataDict(pkgdir, package_id)
packageDict ={}
new_ebuild_id_list = []
for cpv in sorted(ebuild_list_tree):
@@ -361,7 +360,11 @@ class tbc_package(object):
# Get packageDict for cpv
packageDict[cpv] = self.get_packageDict(pkgdir, cpv, repo)
-
+
+ # take package descriptions from the ebuilds
+ if package_metadataDict[package_id]['metadata_xml_descriptions'] != packageDict[cpv]['ebuild_version_descriptions_tree']:
+ package_metadataDict[package_id]['metadata_xml_descriptions'] = packageDict[cpv]['ebuild_version_descriptions_tree']
+
# Get the checksum of the ebuild in tree and db
ebuild_version_checksum_tree = packageDict[cpv]['checksum']
checksums_db, fail = get_ebuild_checksums(self._session, package_id, ebuild_version_tree)
@@ -403,7 +406,6 @@ class tbc_package(object):
del packageDict[cpv]
ebuild_id , status = get_ebuild_id_db(self._session, ebuild_version_checksum_tree, package_id)
new_ebuild_id_list.append(ebuild_id)
- package_metadataDict = self.get_package_metadataDict(pkgdir, package_id)
self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_checksum_tree)
log_msg = "C %s:%s ... Done." % (cp, repo)
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: tbc/pym/
@ 2015-07-24 0:26 Magnus Granberg
0 siblings, 0 replies; 20+ messages in thread
From: Magnus Granberg @ 2015-07-24 0:26 UTC (permalink / raw
To: gentoo-commits
commit: 63bc5f7ccb24c1d40ffeea6ccb9002f323e75a7b
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Fri Jul 24 00:25:35 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Fri Jul 24 00:25:35 2015 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=63bc5f7c
fix repoman qa and error logning
tbc/pym/build_log.py | 22 +++++++++-------------
tbc/pym/db_mapping.py | 2 --
tbc/pym/package.py | 2 +-
tbc/pym/sqlquerys.py | 7 ++++++-
4 files changed, 16 insertions(+), 17 deletions(-)
diff --git a/tbc/pym/build_log.py b/tbc/pym/build_log.py
index ffa848a..8079443 100644
--- a/tbc/pym/build_log.py
+++ b/tbc/pym/build_log.py
@@ -193,14 +193,14 @@ def get_buildlog_info(session, settings, pkg, build_dict):
error_info_list = get_error_info_list(session)
for k, v in sorted(hilight_dict.items()):
if v['startline'] == v['endline']:
- error_log_list.append(logfile_text_dict[k ])
- if v['hilight_css_id'] == "3" or v['hilight_css_id'] == "4": # qa = 3 and 4
+ error_log_list.append(logfile_text_dict[k])
+ if v['hilight_css_id'] == 3: # qa = 3 and 4
qa_error_list.append(logfile_text_dict[k])
else:
i = k
while i != (v['endline'] + 1):
error_log_list.append(logfile_text_dict[i])
- if v['hilight_css_id'] == "3" or v['hilight_css_id'] == "4": # qa = 3 and 4
+ if v['hilight_css_id'] == 3: # qa = 3 and 4
qa_error_list.append(logfile_text_dict[i])
i = i +1
@@ -208,14 +208,12 @@ def get_buildlog_info(session, settings, pkg, build_dict):
repoman_error_list = check_repoman(settings, myportdb, build_dict['cpv'], pkg.repo)
build_log_dict = {}
build_log_dict['fail'] = False
- build_log_dict['rmqa'] = False
- build_log_dict['others'] = False
if repoman_error_list:
- sum_build_log_list.append("1") # repoman = 1
- build_log_dict['rmqa'] = True
+ sum_build_log_list.append(1) # repoman = 1
+ build_log_dict['fail'] = True
if qa_error_list != []:
- sum_build_log_list.append("2") # qa = 2
- build_log_dict['rmqa'] = True
+ sum_build_log_list.append(2) # qa = 2
+ build_log_dict['fail'] = True
else:
qa_error_list = False
error_search_line = "^ \\* ERROR: "
@@ -299,7 +297,7 @@ def log_fail_queru(session, build_dict, settings):
build_log_dict = {}
error_log_list = []
sum_build_log_list = []
- sum_build_log_list.append("2")
+ sum_build_log_list.append(3) # Others errors
error_log_list.append(build_dict['type_fail'])
build_log_dict['summary_error_list'] = sum_build_log_list
if build_dict['type_fail'] == 'merge fail':
@@ -335,8 +333,6 @@ def log_fail_queru(session, build_dict, settings):
build_log_dict['hilight_dict'] = {}
settings2, trees, tmp = load_emerge_config()
build_log_dict['emerge_info'] = get_emerge_info_id(settings2, trees, session, config_id)
- build_log_dict['others'] = True
- build_log_dict['rmqa'] = False
- build_log_dict['fail'] = False
+ build_log_dict['fail'] = True
log_id = add_new_buildlog(session, build_dict, build_log_dict)
del_old_build_jobs(session, build_dict['build_job_id'])
diff --git a/tbc/pym/db_mapping.py b/tbc/pym/db_mapping.py
index 67c2d61..a296c2e 100644
--- a/tbc/pym/db_mapping.py
+++ b/tbc/pym/db_mapping.py
@@ -183,8 +183,6 @@ class BuildLogs(Base):
BuildLogId = Column('build_log_id', Integer, primary_key=True)
EbuildId = Column('ebuild_id', Integer, ForeignKey('ebuilds.ebuild_id'))
Fail = Column('fail', Boolean, default=False)
- RmQa = Column('rmqa', Boolean, default=False)
- Others = Column('others', Boolean, default=False)
SummeryText = Column('summery_text', Text)
LogHash = Column('log_hash', String(100))
BugId = Column('bug_id', Integer, default=0)
diff --git a/tbc/pym/package.py b/tbc/pym/package.py
index dc4ff3e..cc03f97 100644
--- a/tbc/pym/package.py
+++ b/tbc/pym/package.py
@@ -157,7 +157,7 @@ class tbc_package(object):
log_msg = "B %s:%s USE: %s Setup: %s" % (k, v['repo'], use_flagsDict, setup_id,)
add_logs(self._session, log_msg, "info", self._config_id)
i = i +1
- def get_changelog_text(self,pkgdir):
+ def get_changelog_text(self, pkgdir):
changelog_text_dict, max_text_lines = get_log_text_dict(pkgdir + "/ChangeLog")
spec = 3
spec_tmp = 1
diff --git a/tbc/pym/sqlquerys.py b/tbc/pym/sqlquerys.py
index f7f5a97..8fdf901 100644
--- a/tbc/pym/sqlquerys.py
+++ b/tbc/pym/sqlquerys.py
@@ -229,7 +229,7 @@ def add_new_buildlog(session, build_dict, build_log_dict):
return None, False
def build_log_id_no_match(build_dict, build_log_dict):
- NewBuildLog = BuildLogs(EbuildId = build_dict['ebuild_id'], Fail = build_log_dict['fail'], RmQa = build_log_dict['rmqa'], Others = build_log_dict['others'], SummeryText = build_log_dict['build_error'], LogHash = build_log_dict['log_hash'])
+ NewBuildLog = BuildLogs(EbuildId = build_dict['ebuild_id'], Fail = build_log_dict['fail'], SummeryText = build_log_dict['build_error'], LogHash = build_log_dict['log_hash'])
session.add(NewBuildLog)
session.flush()
build_log_id = NewBuildLog.BuildLogId
@@ -276,12 +276,17 @@ def add_new_buildlog(session, build_dict, build_log_dict):
def add_repoman_qa(session, build_log_dict, log_id):
repoman_error = ""
qa_error = ""
+ error = False
if build_log_dict['repoman_error_list']:
for repoman_text in build_log_dict['repoman_error_list']:
repoman_error = repoman_error + repoman_text
+ error = True
if build_log_dict['qa_error_list']:
for qa_text in build_log_dict['qa_error_list']:
qa_error = qa_error + qa_text
+ error = True
+ repoman_error = repoman_error + qa_error
+ if error:
repoman_error = repoman_error + qa_error
NewBuildLogRepoman = BuildLogsRepomanQa(BuildLogId = log_id, SummeryText = repoman_error)
session.add(NewBuildLogRepoman)
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: tbc/pym/
@ 2015-07-24 1:00 Magnus Granberg
0 siblings, 0 replies; 20+ messages in thread
From: Magnus Granberg @ 2015-07-24 1:00 UTC (permalink / raw
To: gentoo-commits
commit: 16dd837fcde476e2a9890b1bbd5b480bbede1120
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Fri Jul 24 00:59:48 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Fri Jul 24 00:59:48 2015 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=16dd837f
in log_fail_queru() remove summary_error
tbc/pym/build_log.py | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/tbc/pym/build_log.py b/tbc/pym/build_log.py
index 8079443..7c95934 100644
--- a/tbc/pym/build_log.py
+++ b/tbc/pym/build_log.py
@@ -311,9 +311,6 @@ def log_fail_queru(session, build_dict, settings):
build_error = build_error + log_line
build_log_dict['build_error'] = build_error
summary_error = ""
- if sum_build_log_list != []:
- for sum_log_line in sum_build_log_list:
- summary_error = summary_error + " " + sum_log_line
build_log_dict['log_hash'] = '0'
useflagsdict = {}
if build_dict['build_useflags'] == {}:
@@ -335,4 +332,5 @@ def log_fail_queru(session, build_dict, settings):
build_log_dict['emerge_info'] = get_emerge_info_id(settings2, trees, session, config_id)
build_log_dict['fail'] = True
log_id = add_new_buildlog(session, build_dict, build_log_dict)
+
del_old_build_jobs(session, build_dict['build_job_id'])
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: tbc/pym/
@ 2015-07-24 1:28 Magnus Granberg
0 siblings, 0 replies; 20+ messages in thread
From: Magnus Granberg @ 2015-07-24 1:28 UTC (permalink / raw
To: gentoo-commits
commit: 77ac0337fc621fe6e6cd7c57910c03ee14321dc5
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Fri Jul 24 01:27:33 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Fri Jul 24 01:27:33 2015 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=77ac0337
fix some small typos
tbc/pym/package.py | 10 +++++-----
tbc/pym/updatedb.py | 2 +-
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/tbc/pym/package.py b/tbc/pym/package.py
index cc03f97..d861044 100644
--- a/tbc/pym/package.py
+++ b/tbc/pym/package.py
@@ -162,17 +162,17 @@ class tbc_package(object):
spec = 3
spec_tmp = 1
changelog_text_tree = ''
- for index, text_line inchangelog_text_dict.items():
- if re.search('^#', text_line)
+ for index, text_line in changelog_text_dict.items():
+ if re.search('^#', text_line):
changelog_text_tree = changelog_text_tree + text_line
- elif re.search('^\n', text_line) and re.search('^\*', changelog_text_dict[index + 1])
+ elif re.search('^\n', text_line) and re.search('^\*', changelog_text_dict[index + 1]):
changelog_text_tree = changelog_text_tree + text_line
spec_tmp = spec_tmp + 1
spec = spec + 1
- elif re.search('^\n', text_line) and not re.search('^\*', changelog_text_dict[index + 1])
+ elif re.search('^\n', text_line) and not re.search('^\*', changelog_text_dict[index + 1]):
if spec_tmp == spec:
break
- else
+ else:
spec_tmp = spec_tmp + 1
changelog_text_tree = changelog_text_tree + text_line
else:
diff --git a/tbc/pym/updatedb.py b/tbc/pym/updatedb.py
index b8be53f..fb96e13 100644
--- a/tbc/pym/updatedb.py
+++ b/tbc/pym/updatedb.py
@@ -42,7 +42,7 @@ def get_categories_metadataDict(pkgdir):
categories_metadataDict = {}
pkg_md = MetaDataXML(pkgdir + "/metadata.xml", None)
metadata_xml_descriptions_tree = re.sub('\t', '', pkg_md.descriptions()[0])
- categories_metadataDict['metadata_xml_descriptions'] = re.sub('\n', '', metadata_xml_descriptions_tree)
+ categories_metadataDict['metadata_xml_descriptions'] = re.sub('\n', '', metadata_xml_descriptions_tree)
categories_metadataDict['metadata_xml_checksum'] = portage.checksum.sha256hash(pkgdir + "/metadata.xml")[0]
return categories_metadataDict
^ permalink raw reply related [flat|nested] 20+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: tbc/pym/
@ 2015-07-24 1:45 Magnus Granberg
0 siblings, 0 replies; 20+ messages in thread
From: Magnus Granberg @ 2015-07-24 1:45 UTC (permalink / raw
To: gentoo-commits
commit: 17ee8f0921c2dc536c30a6c3f65b4cb95b32afb9
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Fri Jul 24 01:44:26 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Fri Jul 24 01:44:26 2015 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=17ee8f09
missing metadata_xml_descriptions in get_package_metadataDict
tbc/pym/package.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/tbc/pym/package.py b/tbc/pym/package.py
index d861044..9bc6dbc 100644
--- a/tbc/pym/package.py
+++ b/tbc/pym/package.py
@@ -199,6 +199,7 @@ class tbc_package(object):
log_msg = "Metadata file %s missing Email" % (pkgdir + "/metadata.xml")
add_logs(self._session, log_msg, "qa", self._config_id)
attDict['metadata_xml_email'] = False
+ attDict['metadata_xml_descriptions'] = ''
package_metadataDict[package_id] = attDict
return package_metadataDict
^ permalink raw reply related [flat|nested] 20+ messages in thread
end of thread, other threads:[~2015-07-24 1:45 UTC | newest]
Thread overview: 20+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2015-07-24 1:00 [gentoo-commits] proj/tinderbox-cluster:master commit in: tbc/pym/ Magnus Granberg
-- strict thread matches above, loose matches on Subject: below --
2015-07-24 1:45 Magnus Granberg
2015-07-24 1:28 Magnus Granberg
2015-07-24 0:26 Magnus Granberg
2015-07-22 6:22 Magnus Granberg
2015-07-21 22:42 Magnus Granberg
2015-07-20 15:46 Magnus Granberg
2015-07-19 11:54 Magnus Granberg
2015-07-15 12:04 Magnus Granberg
2015-06-12 19:31 Magnus Granberg
2015-06-12 19:31 Magnus Granberg
2015-06-12 19:31 Magnus Granberg
2015-06-06 20:06 Magnus Granberg
2015-05-31 21:15 Magnus Granberg
2015-05-31 21:15 Magnus Granberg
2015-05-21 21:40 Magnus Granberg
2015-05-15 19:00 Magnus Granberg
2015-05-12 18:37 Magnus Granberg
2015-05-09 15:50 Magnus Granberg
2015-04-18 20:27 Magnus Granberg
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox