From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id 4F2901381F3 for ; Sun, 16 Dec 2012 20:45:29 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id DF64921C06D; Sun, 16 Dec 2012 20:45:21 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 4E76921C06D for ; Sun, 16 Dec 2012 20:45:21 +0000 (UTC) Received: from hornbill.gentoo.org (hornbill.gentoo.org [94.100.119.163]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 140A533DB32 for ; Sun, 16 Dec 2012 20:45:20 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by hornbill.gentoo.org (Postfix) with ESMTP id A94B3E543C for ; Sun, 16 Dec 2012 20:45:18 +0000 (UTC) From: "Magnus Granberg" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Magnus Granberg" Message-ID: <1355690698.e386a6810a4c42d91a1faafba6658fdd0cd29030.zorry@gentoo> Subject: [gentoo-commits] dev/zorry:master commit in: gobs/pym/ X-VCS-Repository: dev/zorry X-VCS-Files: gobs/pym/buildquerydb.py gobs/pym/check_setup.py gobs/pym/jobs.py gobs/pym/package.py gobs/pym/pgsql_querys.py X-VCS-Directories: gobs/pym/ X-VCS-Committer: zorry X-VCS-Committer-Name: Magnus Granberg X-VCS-Revision: e386a6810a4c42d91a1faafba6658fdd0cd29030 X-VCS-Branch: master Date: Sun, 16 Dec 2012 20:45:18 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Archives-Salt: 916102f4-94e7-42f5-b07d-dda156d3118c X-Archives-Hash: 2a48ac0998a528892b16aaf77103d14c commit: e386a6810a4c42d91a1faafba6658fdd0cd29030 Author: Magnus Granberg gentoo org> AuthorDate: Sun Dec 16 20:44:58 2012 +0000 Commit: Magnus Granberg gentoo org> CommitDate: Sun Dec 16 20:44:58 2012 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=dev/zorry.git;a=commit;h=e386a681 redo the add and del build jobs --- gobs/pym/buildquerydb.py | 54 +++++++++++++++++++++++++-------------------- gobs/pym/check_setup.py | 21 +++++++++-------- gobs/pym/jobs.py | 6 ++-- gobs/pym/package.py | 31 +++++++------------------ gobs/pym/pgsql_querys.py | 35 ++++++++++++++++++++--------- 5 files changed, 77 insertions(+), 70 deletions(-) diff --git a/gobs/pym/buildquerydb.py b/gobs/pym/buildquerydb.py index abd5ea9..c60129a 100644 --- a/gobs/pym/buildquerydb.py +++ b/gobs/pym/buildquerydb.py @@ -24,8 +24,9 @@ from gobs.package import gobs_package import portage import multiprocessing -def add_cpv_query_pool(mysettings, init_package, config_id, package_line): +def add_cpv_query_pool(mysettings, myportdb, config_id, package_line): conn=CM.getConnection() + init_package = gobs_package(mysettings, myportdb) # FIXME: remove the check for gobs when in tree if package_line != "dev-python/gobs": build_dict = {} @@ -37,20 +38,16 @@ def add_cpv_query_pool(mysettings, init_package, config_id, package_line): package = element[1] log_msg = "C %s/%s" % (categories, package,) add_gobs_logs(conn, log_msg, "info", config_profile) - pkgdir = mysettings['PORTDIR'] + "/" + categories + "/" + package + pkgdir = self._myportdb.getRepositoryPath(repo) + "/" + categories + "/" + package config_id_list = [] config_id_list.append(config_id) - config_cpv_listDict = init_package.config_match_ebuild(categories, package, config_id_list) + config_cpv_listDict = init_package.config_match_ebuild(categories + "/" + package, config_id_list) if config_cpv_listDict != {}: - cpv = categories + "/" + package + "-" + config_cpv_listDict[config_id]['ebuild_version'] - attDict = {} - attDict['categories'] = categories - attDict['package'] = package - attDict['ebuild_version_tree'] = config_cpv_listDict[config_id]['ebuild_version'] - packageDict[cpv] = attDict - build_dict['checksum'] = portage.checksum.sha256hash(pkgdir + "/" + package + "-" + config_cpv_listDict[config_id]['ebuild_version'] + ".ebuild")[0] - build_dict['package_id'] = have_package_db(conn, categories, package)[0] - build_dict['ebuild_version'] = config_cpv_listDict[config_id]['ebuild_version'] + cpv = config_cpv_listDict[config_id]['cpv'] + packageDict['cpv'] = init_package.get_packageDict(pkgdir, cpv, repo) + build_dict['checksum'] = packageDict['cpv']['ebuild_version_checksum_tree'] + build_dict['package_id'] = get_package_id(conn, categories, package, repo) + build_dict['ebuild_version'] = packageDict['cpv']['ebuild_version_tree'] ebuild_id = get_ebuild_id_db_checksum(conn, build_dict) if ebuild_id is not None: ebuild_id_list.append(ebuild_id) @@ -62,20 +59,19 @@ def add_cpv_query_pool(mysettings, init_package, config_id, package_line): def add_buildquery_main(config_id): conn=CM.getConnection() - log_msg = "Adding build querys for: %s" % (config_id,) + config_setup = get_config(conn, config_id) + log_msg = "Adding build jobs for: %s" % (config_setup,) add_gobs_logs(conn, log_msg, "info", config_profile) check_make_conf() log_msg = "Check configs done" add_gobs_logs(conn, log_msg, "info", config_profile) # Get default config from the configs table and default_config=1 - default_config_root = "/var/cache/gobs/" + gobs_settings_dict['gobs_gitreponame'] + "/" + config_id + "/" + default_config_root = "/var/cache/gobs/" + gobs_settings_dict['gobs_gitreponame'] + "/" + config_setup + "/" # Set config_root (PORTAGE_CONFIGROOT) to default_config_root mysettings = portage.config(config_root = default_config_root) myportdb = portage.portdbapi(mysettings=mysettings) init_package = gobs_package(mysettings, myportdb) - # get the cp list - package_list_tree = package_list_tree = myportdb.cp_all() - log_msg = "Setting default config to: %s" % (config_id,) + log_msg = "Setting default config to: %s" % (config_setup) add_gobs_logs(conn, log_msg, "info", config_profile) # Use all exept 2 cores when multiprocessing pool_cores= multiprocessing.cpu_count() @@ -84,11 +80,21 @@ def add_buildquery_main(config_id): else: use_pool_cores = 1 pool = multiprocessing.Pool(processes=use_pool_cores) - for package_line in sorted(package_list_tree): - pool.apply_async(add_cpv_query_pool, (mysettings, init_package, config_id, package_line,)) + + repo_trees_list = myportdb.porttrees + for repo_dir in repo_trees_list: + repo = myportdb.getRepositoryName(repo_dir) + repo_dir_list = [] + repo_dir_list.append(repo_dir) + + # Get the package list from the repo + package_id_list_tree = [] + package_list_tree = myportdb.cp_all(trees=repo_dir_list) + for package_line in sorted(package_list_tree): + pool.apply_async(add_cpv_query_pool, (mysettings, myportdb, config_id, package_line,)) pool.close() pool.join() - log_msg = "Adding build querys for: %s ... Done." % (config_id,) + log_msg = "Adding build jobs for: %s ... Done." % (config_setup,) add_gobs_logs(conn, log_msg, "info", config_profile) CM.putConnection(conn) return True @@ -96,10 +102,10 @@ def add_buildquery_main(config_id): def del_buildquery_main(config_id): log_msg = "Removeing build querys for: %s" % (config_id,) add_gobs_logs(conn, log_msg, "info", config_profile) - querue_id_list = get_queue_id_list_config(conn, config_id) - if querue_id_list is not None: - for querue_id in querue_id_list: - del_old_queue(conn, querue_id) + build_job_id_list = get_build_jobs_id_list_config(conn, config_id) + if build_job_id_list is not None: + for build_job_id in build_job_id_list: + del_old_build_jobs(conn, build_job_id) log_msg = "Removeing build querys for: %s ... Done." % (config_id,) add_gobs_logs(conn, log_msg, "info", config_profile) CM.putConnection(conn) diff --git a/gobs/pym/check_setup.py b/gobs/pym/check_setup.py index 99d1dbb..0856f7f 100644 --- a/gobs/pym/check_setup.py +++ b/gobs/pym/check_setup.py @@ -59,30 +59,31 @@ def check_make_conf(): def check_make_conf_guest(config_profile): conn=CM.getConnection() print('config_profile', config_profile) - make_conf_checksum_db = get_profile_checksum(conn,config_profile) - print('make_conf_checksum_db', make_conf_checksum_db) if make_conf_checksum_db is None: CM.putConnection(conn) return False make_conf_file = "/etc/portage/make.conf" - make_conf_checksum_tree = portage.checksum.sha256hash(make_conf_file)[0] - print('make_conf_checksum_tree', make_conf_checksum_tree) - if make_conf_checksum_tree != make_conf_checksum_db[0]: - CM.putConnection(conn) - return False # Check if we can open the file and close it # Check if we have some error in the file (portage.util.getconfig) # Check if we envorment error with the config (settings.validate) try: - open_make_conf = open(make_conf_file) - open_make_conf.close() - portage.util.getconfig(make_conf_file, tolerant=0, allow_sourcing=False, expand=True) + make_conf_checksum_tree = portage.checksum.sha256hash(make_conf_file)[0] + portage.util.getconfig(make_conf_file, tolerant=0, allow_sourcing=True, expand=True) mysettings = portage.config(config_root = "/") mysettings.validate() # With errors we return false except Exception as e: CM.putConnection(conn) return False + make_conf_checksum_db = get_profile_checksum(conn, config_profile) + if make_conf_checksum_db is None: + CM.putConnection(conn) + return False + print('make_conf_checksum_tree', make_conf_checksum_tree) + print('make_conf_checksum_db', make_conf_checksum_db) + if make_conf_checksum_tree != make_conf_checksum_db: + CM.putConnection(conn) + return False CM.putConnection(conn) return True diff --git a/gobs/pym/jobs.py b/gobs/pym/jobs.py index 11543a2..d68cff2 100644 --- a/gobs/pym/jobs.py +++ b/gobs/pym/jobs.py @@ -22,14 +22,14 @@ def jobs_main(config_profile): CM.putConnection(conn) return for job_id in jobs_id: - job = get_job(conn, job_id) + job, config_id = get_job(conn, job_id) log_msg = "Job: %s Type: %s" % (job_id, job,) add_gobs_logs(conn, log_msg, "info", config_profile) if job == "addbuildquery": update_job_list(conn, "Runing", job_id) log_msg = "Job %s is runing." % (job_id,) add_gobs_logs(conn, log_msg, "info", config_profile) - result = add_buildquery_main(config_profile) + result = add_buildquery_main(config_id) if result is True: update_job_list(conn, "Done", job_id) log_msg = "Job %s is done.." % (job_id,) @@ -42,7 +42,7 @@ def jobs_main(config_profile): update_job_list(conn, "Runing", job_id) log_msg = "Job %s is runing." % (job_id,) add_gobs_logs(conn, log_msg, "info", config_profile) - result = del_buildquery_main(config_profile) + result = del_buildquery_main(config_id) if result is True: update_job_list(conn, "Done", job_id) log_msg = "Job %s is done.." % (job_id,) diff --git a/gobs/pym/package.py b/gobs/pym/package.py index 160a152..5d47fac 100644 --- a/gobs/pym/package.py +++ b/gobs/pym/package.py @@ -80,8 +80,7 @@ class gobs_package(object): ebuild_auxdb_list[i] = '' return ebuild_auxdb_list - def get_packageDict(self, pkgdir, cpv, repo, config_id): - attDict = {} + def get_packageDict(self, pkgdir, cpv, repo): conn=CM.getConnection() #Get categories, package and version from cpv @@ -123,6 +122,7 @@ class gobs_package(object): ebuild_version_checksum_tree = '0' # add the ebuild info to the dict packages + attDict = {} attDict['repo'] = repo attDict['ebuild_version_tree'] = ebuild_version_tree attDict['ebuild_version_checksum_tree']= ebuild_version_checksum_tree @@ -142,7 +142,6 @@ class gobs_package(object): for k, v in config_cpv_listDict.iteritems(): config_id = k build_cpv = v['cpv'] - latest_ebuild_version = v['ebuild_version'] iuse_flags_list = list(set(v['iuse'])) use_enable= v['useflags'] use_disable = list(set(iuse_flags_list).difference(set(use_enable))) @@ -165,8 +164,7 @@ class gobs_package(object): config_setup = get_config(conn, config_id) # FIXME log_msg need a fix to log the use flags corect. - log_msg = "B %s:%s USE: %s %s" % \ - (k, v['repo'], use_enable, config_setup,) + log_msg = "B %s:%s USE: %s %s" % (k, v['repo'], use_flagsDict, config_setup,) add_gobs_logs(conn, log_msg, "info", config_profile) i = i +1 CM.putConnection(conn) @@ -229,7 +227,7 @@ class gobs_package(object): packageDict ={} ebuild_id_list = [] for cpv in sorted(ebuild_list_tree): - packageDict[cpv] = self.get_packageDict(pkgdir, cpv, repo, default_config) + packageDict[cpv] = self.get_packageDict(pkgdir, cpv, repo) # Add new ebuilds to the db ebuild_id_list = add_new_ebuild_sql(conn, package_id, packageDict) @@ -293,28 +291,17 @@ class gobs_package(object): # split out ebuild version ebuild_version_tree = portage.versions.cpv_getversion(cpv) + + # Get packageDict for cpv + packageDict[cpv] = self.get_packageDict(pkgdir, cpv, repo) # Get the checksum of the ebuild in tree and db - # Make a checksum of the ebuild - try: - ebuild_version_checksum_tree = portage.checksum.sha256hash(pkgdir + "/" + package + "-" + ebuild_version_tree + ".ebuild")[0] - except: - ebuild_version_checksum_tree = '0' - manifest_checksum_tree = '0' - log_msg = "QA: Can't checksum the ebuild file. %s on repo %s" % (cpv, repo,) - add_gobs_logs(conn, log_msg, "info", config_profile) - log_msg = "C %s:%s ... Fail." % (cpv, repo) - add_gobs_logs(conn, log_msg, "info", config_profile) + ebuild_version_checksum_tree = packageDict['cpv']['ebuild_version_checksum_tree'] ebuild_version_manifest_checksum_db = get_ebuild_checksum(conn, package_id, ebuild_version_tree) # Check if the checksum have change if ebuild_version_manifest_checksum_db is None or ebuild_version_checksum_tree != ebuild_version_manifest_checksum_db: - # set config to default config - default_config = get_default_config(conn) - - # Get packageDict for ebuild - packageDict[cpv] = self.get_packageDict(pkgdir, cpv, repo, default_config) if ebuild_version_manifest_checksum_db is None: # N = New ebuild log_msg = "N %s:%s" % (cpv, repo,) @@ -328,7 +315,7 @@ class gobs_package(object): old_ebuild_list.append(ebuild_version_tree) add_old_ebuild(conn, package_id, old_ebuild_list) update_active_ebuild_to_fales(conn, package_id, ebuild_version_tree) - # Use packageDictand to update the db + # Use packageDict and to update the db # Add new ebuilds to the db ebuild_id_list = add_new_ebuild_sql(conn, package_id, packageDict) diff --git a/gobs/pym/pgsql_querys.py b/gobs/pym/pgsql_querys.py index 4e882dc..f495c48 100644 --- a/gobs/pym/pgsql_querys.py +++ b/gobs/pym/pgsql_querys.py @@ -9,10 +9,10 @@ def add_gobs_logs(connection, log_msg, log_type, config): connection.commit() # Queryes to handel the jobs table -def get_jobs_id(connection, config_profile): +def get_jobs_id(connection, config_id): cursor = connection.cursor() - sqlQ = "SELECT job_id FROM jobs WHERE status = 'Waiting' AND config_id = (SELECT config_id FROM configs WHERE config = %s)" - cursor.execute(sqlQ, (config_profile,)) + sqlQ = "SELECT job_id FROM jobs WHERE status = 'Waiting' AND config_id = %s" + cursor.execute(sqlQ, (config_id,)) entries = cursor.fetchall() if entries is None: return None @@ -23,10 +23,12 @@ def get_jobs_id(connection, config_profile): def get_job(connection, job_id): cursor = connection.cursor() - sqlQ ='SELECT job FROM jobs WHERE job_id = %s' + sqlQ ='SELECT job, config_id FROM jobs WHERE job_id = %s' cursor.execute(sqlQ, (job_id,)) - job = cursor.fetchone() - return job[0] + entries = cursor.fetchone() + job = entries[0] + config_id = entries[1] + return job, config_id def update_job_list(connection, status, job_id): cursor = connection.cursor() @@ -303,23 +305,34 @@ def get_build_jobs_id_list_config(connection, config_id): sqlQ = 'SELECT build_job_id FROM build_jobs WHERE config_id = %s' cursor.execute(sqlQ, (config_id,)) entries = cursor.fetchall() - return entries + build_jobs_id_list = [] + if not entries == []: + for build_job_id_id in entries: + build_jobs_id_list.append(build_job_id[0]) + else: + build_log_id_list = None + return build_jobs_id_list def del_old_build_jobs(connection, build_job_id): cursor = connection.cursor() sqlQ1 = 'DELETE FROM build_jobs_use WHERE build_job_id = %s' sqlQ2 = 'DELETE FROM build_jobs_retest WHERE build_job_id = %s' sqlQ3 = 'DELETE FROM build_jobs WHERE build_job_id = %s' + sqlQ4 = 'DELETE FROM build_jobs_emerge_options WHERE build_job_id = %s' cursor.execute(sqlQ1, (build_job_id,)) cursor.execute(sqlQ2, (build_job_id,)) + cursor.execute(sqlQ4, (build_job_id,)) cursor.execute(sqlQ3, (build_job_id,)) connection.commit() def get_profile_checksum(connection, config_profile): - cursor = connection.cursor() - sqlQ = "SELECT checksum FROM configs_metadata WHERE active = 'True' AND config_id = (SELECT config_id FROM configs WHERE config = %s) AND auto = 'True'" - cursor.execute(sqlQ, (config_profile,)) - return cursor.fetchone() + cursor = connection.cursor() + sqlQ = "SELECT checksum FROM configs_metadata WHERE active = 'True' AND config_id = (SELECT config_id FROM configs WHERE config = %s) AND auto = 'True'" + cursor.execute(sqlQ, (config_profile,)) + entries = cursor.fetchone() + if entries is None: + return + return entries[0] def get_packages_to_build(connection, config): cursor =connection.cursor()