From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from pigeon.gentoo.org ([208.92.234.80] helo=lists.gentoo.org) by finch.gentoo.org with esmtp (Exim 4.60) (envelope-from ) id 1Qfvyb-0005I5-8I for garchives@archives.gentoo.org; Sun, 10 Jul 2011 15:31:21 +0000 Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 1927921C06C; Sun, 10 Jul 2011 15:31:14 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) by pigeon.gentoo.org (Postfix) with ESMTP id BCD0021C067 for ; Sun, 10 Jul 2011 15:31:13 +0000 (UTC) Received: from pelican.gentoo.org (unknown [66.219.59.40]) (using TLSv1 with cipher ADH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 428921B4068 for ; Sun, 10 Jul 2011 15:31:13 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by pelican.gentoo.org (Postfix) with ESMTP id 8333F8003F for ; Sun, 10 Jul 2011 15:31:12 +0000 (UTC) From: "Magnus Granberg" To: gentoo-commits@lists.gentoo.org Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Magnus Granberg" Message-ID: <2e724a165d596a2cab9b4be8aa78ef3d63edfe4f.zorry@gentoo> Subject: [gentoo-commits] dev/zorry:master commit in: gobs/bin/ X-VCS-Repository: dev/zorry X-VCS-Files: gobs/bin/gobs_buildquerys gobs/bin/gobs_updatedb gobs/bin/portage_hooks X-VCS-Directories: gobs/bin/ X-VCS-Committer: zorry X-VCS-Committer-Name: Magnus Granberg X-VCS-Revision: 2e724a165d596a2cab9b4be8aa78ef3d63edfe4f Date: Sun, 10 Jul 2011 15:31:12 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: quoted-printable X-Archives-Salt: X-Archives-Hash: 1782f46269a4f9c891756ea1e1eac267 commit: 2e724a165d596a2cab9b4be8aa78ef3d63edfe4f Author: Magnus Granberg gentoo org> AuthorDate: Sun Jul 10 15:26:19 2011 +0000 Commit: Magnus Granberg gentoo org> CommitDate: Sun Jul 10 15:26:19 2011 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=3Ddev/zorry.git;a=3D= commit;h=3D2e724a16 Updated the bins files --- gobs/bin/gobs_buildquerys | 181 +++++++++++++++++++++++++++++++++++++++= ++++++ gobs/bin/gobs_updatedb | 57 +++++++++------ gobs/bin/portage_hooks | 78 +++++++++++++++++++ 3 files changed, 293 insertions(+), 23 deletions(-) diff --git a/gobs/bin/gobs_buildquerys b/gobs/bin/gobs_buildquerys new file mode 100755 index 0000000..bc7e320 --- /dev/null +++ b/gobs/bin/gobs_buildquerys @@ -0,0 +1,181 @@ +#!/usr/bin/python + +# Get the options from the config file set in gobs.readconf +from gobs.readconf import get_conf_settings +reader=3Dget_conf_settings() +gobs_settings_dict=3Dreader.read_gobs_settings_all() +# make a CM +from gobs.ConnectionManager import connectionManager +CM=3DconnectionManager(gobs_settings_dict) +#selectively import the pgsql/mysql querys +if CM.getName()=3D=3D'pgsql': + from gobs.querys.pgsql import * +elif CM.getName()=3D=3D'mysql': + from gobs.querys.mysql import * + +from gobs.check_setup import check_configure_guest +from gobs.flags import gobs_use_flags +from gobs.manifest import gobs_manifest +from _emerge.main import emerge_main +import portage +import sys +import os + +def log_qa_error(connection, build_dict, what_error, fail_error, config_= id): + ebuild_id_list =3D [] + qa_error_list =3D [] + repoman_error =3D [] + ebuild_version =3D build_dict['ebuild_version'] + attDict =3D {} + packageDict =3D {} + attDict['ebuild_error'] =3D repoman_error + packageDict[ebuild_version] =3D attDict + qa_error_list.append(fail_error) + fail_times =3D check_fail_times(connection, build_dict['queue_id'], wha= t_error) + if fail_times is None: + add_fail_times(connection, build_dict['queue_id'], what_error) + else: + if fail_times[0] > 5: + return + elif fail_times[0] =3D=3D 5: + ebuild_id_list.append(build_dict['ebuild_id']) + add_qa_repoman(connection, ebuild_id_list, qa_error_list, packageDict= , config_id) + del_old_queue(connection, build_dict['queue_id']) + return + update_fail_times(connection, build_dict['queue_id'], what_error, fail= _times[0]) + +def get_cpv_build_list(connection, build_dict, config_profile): + build_cpv_dict =3D {} + cpv_dict =3D {} + cpv_list =3D "1" + mysettings =3D portage.settings + #myportdb =3D portage.portdbapi(mysettings=3Dmysettings) + myportdb =3D portage.portdb + #porttree =3D portage.portagetree(settings=3Dmysettings) + #trees =3D {"/" : {"porttree":porttree}} + #dep_keys =3D ("DEPEND", "RDEPEND", "PDEPEND") + pkgdir =3D os.path.join(mysettings['PORTDIR'], build_dict['categories']= + "/" + build_dict['package']) + init_manifest =3D gobs_manifest(mysettings, pkgdir) + try: + ebuild_version_checksum_tree =3D portage.checksum.sha256hash(pkgdir+ "= /" + build_dict['package'] + "-" + build_dict['ebuild_version'] + ".ebuil= d")[0] + except: + ebuild_version_checksum_tree =3D None + print ebuild_version_checksum_tree, build_dict['ebuild_checksum'] + ebuild_version_checksum_db =3D have_activ_ebuild_id(connection, build_d= ict['ebuild_id']) + print 'ebuild_version_checksum_db', ebuild_version_checksum_db + if ebuild_version_checksum_db is None: + print "queue_id don't have active ebuild_id" + del_old_queue(connection, build_dict['queue_id']) + return None + if ebuild_version_checksum_tree !=3D build_dict['ebuild_checksum']: + new_queue_id =3D check_revision(connection, build_dict, config_profile= , ebuild_version_checksum_tree) + if new_queue_id is None: + return None + # FIXME use log_qa_error() + if check_fail_times(build_dict['queue_id'], "cpv_checksum") is False: + update_fail_times(build_dict['queue_id'], "cpv_checksum") + else: + add_cpv_fail(build_dict['queue_id'], "cpv_checksum") + return None + build_dict =3D get_package_to_build(connection,config_profile, new_que= ue_id) + categories =3D build_dict['categories'] + package =3D build_dict['package'] + ebuild_version =3D build_dict['ebuild_version'] + ebuild_version_useflags =3D build_dict['build_useflags'] + build_massage =3D build_dict['post_message'] + cpv =3D unicode(categories + "/" + package + "-" + ebuild_version) + cp =3D unicode(categories + "/" + package) + init_flags =3D gobs_use_flags(mysettings, myportdb, cpv) + #mysettings['O'] =3D pkgdir + print portage.getmaskingstatus(cpv, settings=3Dmysettings, portdb=3Dmyp= ortdb) + if portage.getmaskingstatus(cpv, settings=3Dmysettings, portdb=3Dmyport= db) !=3D []: + # FIXME use log_qa_error() + if check_fail_times(build_dict['queue_id'], "cpv_mask") is False: + update_fail_times(build_dict['queue_id'], "cpv_mask") + else: + add_cpv_fail(build_dict['queue_id'], "cpv_mask") + return None + build_use_flags_list =3D init_flags.comper_useflags(build_dict) + manifest_error =3D init_manifest.check_file_in_manifest(myportdb, cpv, = build_dict, build_use_flags_list) + print 'manifest_error', manifest_error + if manifest_error is not None: + what_error =3D "cpv_manifest" + log_qa_error(connection, build_dict, what_error, manifest_error, confi= g_profile) + return None + build_cpv_dict =3D init_flags.get_needed_dep_useflags(build_use_flags_l= ist) + print build_cpv_dict, build_use_flags_list, cpv + build_use_flags_dict =3D {} + if build_use_flags_list is None: + build_use_flags_dict['None'] =3D None + if build_cpv_dict is None: + build_cpv_dict =3D {} + build_cpv_dict[cpv] =3D build_use_flags_dict + else: + build_cpv_dict[cpv] =3D build_use_flags_dict + print build_cpv_dict + return build_cpv_dict + =09 +def main_loop(CM, config_profile): + repeat =3D True + new =3D False + queue_id_list =3D 43280 + #get a connection from the pool + conn=3DCM.getConnection() + while repeat: + build_cpv_dict =3D {} + build_dict =3D {} + if check_configure_guest(conn, config_profile) is not True: + # time.sleep(60) + continue # retunr to the start of the function + else: + build_dict =3D get_package_to_build(conn,config_profile, queue_id_lis= t, new) + print "build_dict['ebuild_id'] build_dict['post_message']", build_dic= t['ebuild_id'], build_dict['post_message'] + if build_dict['ebuild_id'] is None: + if build_dict['post_message'] is None: + # time.sleep(60) + print "build_dict none" + queue_id_list =3D queue_id_list + 1 + print queue_id_list + continue # return to the start of the function + build_cpv_dict =3D get_cpv_build_list(conn, build_dict, config_profil= e) + if build_cpv_dict is None: + queue_id_list =3D queue_id_list + 1 + continue # return to the start of the function + build_cpv_list =3D [] + for k, v in build_cpv_dict.iteritems(): + build_use_flags_list =3D [] + for x, y in v.iteritems(): + if y is True: + build_use_flags_list.append(x) + if y is False: + build_use_flags_list.append("-" + x) + print k, build_use_flags_list + if build_use_flags_list =3D=3D []: + build_cpv_list.append("=3D" + k) + else: + build_use_flags =3D "" + for flags in build_use_flags_list: + build_use_flags =3D build_use_flags + flags + "," + build_cpv_list.append("=3D" + k + "[" + build_use_flags + "]") + print build_cpv_list + argscmd =3D [] + # argscmd.append("--buildpkg") + # argscmd.append("--usepkg") + argscmd.append("--oneshot") + for build_cpv in build_cpv_list: + argscmd.append(build_cpv) + print argscmd + emerge_main(args=3Dargscmd) + sys.exit() + queue_id_list =3D queue_id_list + 1 + CM.putConnection(conn) + +def main(): + # Main + config_profile =3D gobs_settings_dict['gobs_config'] + #we provide the main_loop with the ConnectionManager so we can hand ou= t connections from within the loop + main_loop(CM, config_profile) + connectionManager.closeAllConnections() + +if __name__ =3D=3D "__main__": + main() diff --git a/gobs/bin/gobs_updatedb b/gobs/bin/gobs_updatedb index 582efa6..837cab1 100755 --- a/gobs/bin/gobs_updatedb +++ b/gobs/bin/gobs_updatedb @@ -7,17 +7,31 @@ =20 import sys import os +from threading import Thread + +# Get the options from the config file set in gobs.readconf from gobs.readconf import get_conf_settings -from gobs.initsql import init_sql_db -from gobs.check_setup import check_config -from gobs.arch import arch +reader=3Dget_conf_settings() +gobs_settings_dict=3Dreader.read_gobs_settings_all() +# make a CM +from gobs.ConnectionManager import connectionManager +CM=3DconnectionManager(gobs_settings_dict) +#selectively import the pgsql/mysql querys +if CM.getName()=3D=3D'pgsql': + from gobs.querys.pgsql import * +elif CM.getName()=3D=3D'mysql': + from gobs.querys.mysql import * + +from gobs.check_setup import check_make_conf +from gobs.arch import gobs_arch from gobs.package import gobs_package from gobs.categories import gobs_categories from gobs.old_cpv import gobs_old_cpv from gobs.categories import gobs_categories import portage =20 -def init_portage_settings(database, gobs_settings_dict): +def init_portage_settings(CM, gobs_settings_dict): +=09 """ Get the BASE Setup/Config for portage.settings @type: module @module: The SQL Backend @@ -27,18 +41,19 @@ def init_portage_settings(database, gobs_settings_dic= t): @returns new settings """ # check config setup - init_check =3D check_config(database, gobs_settings_dict) - init_check.check_make_conf() + conn=3DCM.getConnection() + check_make_conf(CM,gobs_settings_dict) print "Check configs done" # Get default config from the configs table and default_config=3D1 - config_id =3D database.get_default_config() # HostConfigDir =3D table= configs id - default_config_root =3D gobs_settings_dict['host_setup_root'] +"config/= " + config_id[0] + "/" + config_id =3D get_default_config(conn) # HostConfigDir =3D table conf= igs id + CM.putConnection(conn); + default_config_root =3D gobs_settings_dict['gobs_root'] +"config/" + co= nfig_id[0] + "/" # Set config_root (PORTAGE_CONFIGROOT) to default_config_root mysettings =3D portage.config(config_root =3D default_config_root) print "Setting default config to:", config_id[0] return mysettings =20 -def update_cpv_db(mysettings, database, gobs_settings_dict): +def update_cpv_db(mysettings, CM, gobs_settings_dict): """Code to update the cpv in the database. @type:settings @parms: portage.settings @@ -50,14 +65,15 @@ def update_cpv_db(mysettings, database, gobs_settings= _dict): print "Checking categories, package, ebuilds" # Setup portdb, gobs_categories, gobs_old_cpv, package myportdb =3D portage.portdbapi(mysettings=3Dmysettings) - init_categories =3D gobs_categories(database, mysettings) - init_old_cpv =3D gobs_old_cpv(database, myportdb, mysettings) - init_package =3D gobs_package(mysettings, database, myportdb, gobs_sett= ings_dict) + init_categories =3D gobs_categories(CM, mysettings) + init_old_cpv =3D gobs_old_cpv(CM, myportdb, mysettings) + init_package =3D gobs_package(mysettings, CM, myportdb, gobs_settings_d= ict) package_id_list_tree =3D [] # Will run some update checks and update package if needed # Get categories/package list from portage package_list_tree =3D myportdb.cp_all() # Run the update package for all package in the list + conn=3DCM.getConnection() for package_line in sorted(package_list_tree): # split the cp to categories and package element =3D package_line.split('/') @@ -65,7 +81,7 @@ def update_cpv_db(mysettings, database, gobs_settings_d= ict): package =3D element[1] print "C", categories + "/" + package # C =3D Checking # Check if we don't have the cp in the package table - package_id =3D database.have_package_db(categories, package) + package_id =3D have_package_db(conn,categories, package) if package_id is None: # Add new package with ebuilds package_id =3D init_package.add_new_package_db(categories, package) @@ -77,6 +93,7 @@ def update_cpv_db(mysettings, database, gobs_settings_d= ict): package_id_list_tree.append(package_id) # Update the metadata for categories init_categories.update_categories_db(categories) + CM.putConnection(conn) # Remove any old ebuild, package and categories init_old_cpv.mark_old_ebuild_db(categories, package, package_id) init_old_cpv.mark_old_package_db(sorted(package_id_list_tree)) @@ -85,19 +102,13 @@ def update_cpv_db(mysettings, database, gobs_setting= s_dict): =20 def main(): # Main - # Get the options from the config file set in gobs.readconf - init_config =3D get_conf_settings() - gobs_settings_dict =3D init_config.read_gobs_settings_all() - # Init the Database - init_sql =3D init_sql_db() - database =3D init_sql.init_sql_backend() # Init settings for the default config - mysettings =3D init_portage_settings(database, gobs_settings_dict) - # Update the arch db if needed - init_arch =3D arch(database) + mysettings =3D init_portage_settings(CM,gobs_settings_dict) + init_arch =3D gobs_arch(CM) init_arch.update_arch_db() # Update the cpv db - update_cpv_db(mysettings, database, gobs_settings_dict) + update_cpv_db(mysettings, CM, gobs_settings_dict) + CM.closeAllConnections() =09 if __name__ =3D=3D "__main__": main() \ No newline at end of file diff --git a/gobs/bin/portage_hooks b/gobs/bin/portage_hooks new file mode 100755 index 0000000..97c2576 --- /dev/null +++ b/gobs/bin/portage_hooks @@ -0,0 +1,78 @@ +#!/usr/bin/python + +# Get the options from the config file set in gobs.readconf +from gobs.readconf import get_conf_settings +reader=3Dget_conf_settings() +gobs_settings_dict=3Dreader.read_gobs_settings_all() +# make a CM +from gobs.ConnectionManager import connectionManager +CM=3DconnectionManager(gobs_settings_dict) +#selectively import the pgsql/mysql querys +if CM.getName()=3D=3D'pgsql': + from gobs.querys.pgsql import * +elif CM.getName()=3D=3D'mysql': + from gobs.querys.mysql import * + +from gobs.package import gobs_package +from gobs.build_log import gobs_buildlog +from gobs.flags import gobs_use_flags +import portage +import os + +def get_build_dict_db(CM, mysettings, config_profile, gobs_settings_dict= ): + conn=3DCM.getConnection() + myportdb =3D portage.portdbapi(mysettings=3Dmysettings) + categories =3D os.environ['CATEGORY'] + package =3D os.environ['PN'] + ebuild_version =3D os.environ['PVR'] + cpv =3D categories + "/" + package + "-" + ebuild_version + init_package =3D gobs_package(mysettings, CM, myportdb, gobs_settings_d= ict) + print "cpv", cpv + package_id =3D have_package_db(conn, categories, package) + print "package_id", package_id + build_dict =3D {} + mybuild_dict =3D {} + build_dict['ebuild_version'] =3D ebuild_version + build_dict['package_id'] =3D package_id + build_dict['cpv'] =3D cpv + build_dict['categories'] =3D categories + build_dict['package'] =3D package + build_dict['config_profile'] =3D config_profile + init_useflags =3D gobs_use_flags(mysettings, myportdb, cpv) + iuse_flags_list, final_use_list =3D init_useflags.get_flags_looked() + print 'final_use_list', final_use_list + if final_use_list !=3D []: + build_dict['build_useflags'] =3D final_use_list + else: + build_dict['build_useflags'] =3D None + print "build_dict['build_useflags']", build_dict['build_useflags'] + pkgdir =3D os.path.join(mysettings['PORTDIR'], categories + "/" + packa= ge) + ebuild_version_checksum_tree =3D portage.checksum.sha256hash(pkgdir+ "/= " + package + "-" + ebuild_version + ".ebuild")[0] + build_dict['checksum'] =3D ebuild_version_checksum_tree + print 'checksum' ,ebuild_version_checksum_tree + ebuild_id =3D get_ebuild_id_db_checksum(conn, build_dict) + print 'ebuild_id in db', ebuild_id + if ebuild_id is None: + print 'have any ebuild', get_ebuild_checksum(conn, package_id, ebuild= _version) + init_package.update_ebuild_db(build_dict) + queue_id =3D check_revision(conn, build_dict, config_profile, ebuild_ve= rsion_checksum_tree) + print "queue_id", queue_id + if queue_id is None: + init_package.add_new_ebuild_buildquery_db_looked(build_dict, config_pr= ofile) + queue_id =3D check_revision(conn, build_dict, config_profile, ebuild_v= ersion_checksum_tree) + mybuild_dict =3D get_package_to_build(conn, config_profile, queue_id, n= ew =3D True) + print 'mybuild_dict', mybuild_dict + return mybuild_dict + +def main(): + # Main + config_profile =3D gobs_settings_dict['gobs_config'] + #we provide the main_loop with the ConnectionManager so we can hand ou= t connections from within the loop + mysettings =3D portage.settings + build_dict =3D get_build_dict_db(CM, mysettings, config_profile, gobs_= settings_dict) + init_buildlog =3D gobs_buildlog(CM ,mysettings, build_dict, config_pro= file) + init_buildlog.add_buildlog_main() + #connectionManager.closeAllConnections() + +if __name__ =3D=3D "__main__": + main() \ No newline at end of file