* [gentoo-commits] dev/zorry:master commit in: buildhost/portage_hook/
@ 2011-04-01 16:42 Magnus Granberg
0 siblings, 0 replies; only message in thread
From: Magnus Granberg @ 2011-04-01 16:42 UTC (permalink / raw
To: gentoo-commits
commit: 6491db2bcdf9350a3cb9f974e590c8e08556a057
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Fri Apr 1 16:05:06 2011 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Fri Apr 1 16:05:06 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=dev/zorry.git;a=commit;h=6491db2b
added the code for the portage hooks
---
buildhost/portage_hook/buildhost_hook.py | 244 +++++++++++
buildhost/portage_hook/sqlbackend.py | 646 ++++++++++++++++++++++++++++++
2 files changed, 890 insertions(+), 0 deletions(-)
diff --git a/buildhost/portage_hook/__init__.py b/buildhost/portage_hook/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/buildhost/portage_hook/buildhost_hook.py b/buildhost/portage_hook/buildhost_hook.py
new file mode 100644
index 0000000..df24e12
--- /dev/null
+++ b/buildhost/portage_hook/buildhost_hook.py
@@ -0,0 +1,244 @@
+# -*- coding: utf-8 -*-
+# written 2010 by Magnus Granberg <zorry@gentoo.org>
+# License: GPL-2
+
+"""core module"""
+import sys
+from StringIO import StringIO
+import os
+import re
+import portage
+import _emerge
+import hashlib
+import errno
+import time
+from xml.etree.ElementTree import ElementTree
+
+class get_conf_settings(object):
+ # open the /etc/buildhost/buildhost.conf file and get the needed
+ # settings for the database
+ def get_conf_settings_from_file():
+ configfile = "/etc/buildhost/buildhost.conf"
+ open_conffile = open(configfile, 'r')
+ dataLines = open_conffile.readlines()
+ for line in dataLines:
+ element = line.split('=')
+ if element[0] == 'SQLBACKEND': # Databas backend (mysql)
+ GetSqlBackend = element[1]
+ if element[0] == 'SQLDB': # Database
+ GetSqlDB = element[1]
+ if element[0] == 'SQLHOST': # Host
+ GetSqlHost = element[1]
+ if element[0] == 'SQLUSER': # User
+ GetSqlUser = element[1]
+ if element[0] == 'SQLPASSWD': # Password
+ GetSqlPasswd = element[1]
+ if element[0] == 'BUILDCONFIG': # Buildhost root
+ GetBuildConfig = element[1]
+ open_conffile.close()
+ return GetSqlBackend.rstrip('\n'), GetSqlDB.rstrip('\n'), GetSqlHost.rstrip('\n'), \
+ GetSqlUser.rstrip('\n'), GetSqlPasswd.rstrip('\n'), GetBuildConfig.rstrip('\n')
+
+ Get_BuildHost_settings = get_conf_settings_from_file()
+ SetSqlBackend = Get_BuildHost_settings[0]
+ SetSqlDB = Get_BuildHost_settings[1]
+ SetSqlHost = Get_BuildHost_settings[2]
+ SetSqlUser = Get_BuildHost_settings[3]
+ SetSqlPasswd = Get_BuildHost_settings[4]
+ SetBuildConfig = Get_BuildHost_settings[5]
+
+def init_sql_backend():
+ from sqlbackend import DatabaseConfig # import DatebaseConfig
+ database = None
+
+ # Setup the database backent to use mysql
+ if DatabaseConfig.mode == 'pgsql':
+ from sqlbackend import PgSQLPackageDB
+ database = PgSQLPackageDB(DatabaseConfig.settings['sql_pgsql'])
+ return database
+
+def get_iuse(portdb, cpv):
+ """Gets the current IUSE flags from the tree
+
+ To be used when a gentoolkit package object is not needed
+ @type: cpv: string
+ @param cpv: cat/pkg-ver
+ @rtype list
+ @returns [] or the list of IUSE flags
+ """
+ return portdb.aux_get(cpv, ["IUSE"])[0].split()
+
+def reduce_flag(flag):
+ """Absolute value function for a USE flag
+
+ @type flag: string
+ @param flag: the use flag to absolute.
+ @rtype: string
+ @return absolute USE flag
+ """
+ if flag[0] in ["+","-"]:
+ return flag[1:]
+ else:
+ return flag
+
+
+def reduce_flags(the_list):
+ """Absolute value function for a USE flag list
+
+ @type the_list: list
+ @param the_list: the use flags to absolute.
+ @rtype: list
+ @return absolute USE flags
+ """
+ r=[]
+ for member in the_list:
+ r.append(reduce_flag(member))
+ return r
+
+
+def filter_flags(settings, use, use_expand_hidden, usemasked, useforced):
+ """Filter function to remove hidden or otherwise not normally
+ visible USE flags from a list.
+
+ @type use: list
+ @param use: the USE flag list to be filtered.
+ @type use_expand_hidden: list
+ @param use_expand_hidden: list of flags hidden.
+ @type usemasked: list
+ @param usemasked: list of masked USE flags.
+ @type useforced: list
+ @param useforced: the forced USE flags.
+ @rtype: list
+ @return the filtered USE flags.
+ """
+ # clean out some environment flags, since they will most probably
+ # be confusing for the user
+ for f in use_expand_hidden:
+ f=f.lower() + "_"
+ for x in use:
+ if f in x:
+ use.remove(x)
+ # clean out any arch's
+ archlist = settings["PORTAGE_ARCHLIST"].split()
+ for a in use[:]:
+ if a in archlist:
+ use.remove(a)
+ # dbl check if any from usemasked or useforced are still there
+ masked = usemasked + useforced
+ for a in use[:]:
+ if a in masked:
+ use.remove(a)
+ return use
+
+def get_all_cpv_use(settings, portdb, cpv):
+ """Uses portage to determine final USE flags and settings for an emerge
+
+ @type cpv: string
+ @param cpv: eg cat/pkg-ver
+ @rtype: lists
+ @return use, use_expand_hidden, usemask, useforce
+ """
+ use = None
+ settings.unlock()
+ try:
+ settings.setcpv(cpv, use_cache=None, mydb=portdb)
+ use = settings['PORTAGE_USE'].split()
+ use_expand_hidden = settings["USE_EXPAND_HIDDEN"].split()
+ usemask = list(settings.usemask)
+ useforce = list(settings.useforce)
+ except KeyError:
+ settings.reset()
+ settings.lock()
+ return [], [], [], []
+ # reset cpv filter
+ settings.reset()
+ settings.lock()
+ return use, use_expand_hidden, usemask, useforce
+
+def get_flags(settings, portdb, cpv):
+ """Retrieves all information needed to filter out hidden, masked, etc.
+ USE flags for a given package.
+
+ @type cpv: string
+ @param cpv: eg. cat/pkg-ver
+ @type final_setting: boolean
+ @param final_setting: used to also determine the final
+ enviroment USE flag settings and return them as well.
+ @rtype: list or list, list
+ @return IUSE or IUSE, final_flags
+ """
+ final_use, use_expand_hidden, usemasked, useforced = get_all_cpv_use(settings, portdb, cpv)
+ iuse_flags = filter_flags(settings, get_iuse(portdb, cpv), use_expand_hidden, usemasked, useforced)
+ #flags = filter_flags(use_flags, use_expand_hidden, usemasked, useforced)
+ final_flags = filter_flags(settings, final_use, use_expand_hidden, usemasked, useforced)
+ return iuse_flags, final_flags
+
+def get_log_text(filename):
+ """Return the log contents as a list"""
+ try:
+ logfile = open(filename)
+ except IOError, oe:
+ if oe.errno not in (errno.ENOENT, ):
+ raise
+ return None
+ text = []
+ dataLines = logfile.readlines()
+ for i in dataLines:
+ text.append(i)
+ return text
+
+def add_new_ebuild_buildquery_db(database, settings, build_dict, ebuild_version_checksum_tree, config_profile):
+ portdb = portage.portdbapi(mysettings=settings)
+ message = None
+ cpv = build_dict['cpv']
+ iuse_flags_list, final_use_list = get_flags(settings, portdb, cpv)
+ iuse = []
+ for iuse_line in iuse_flags_list:
+ iuse.append(reduce_flag(iuse_line))
+ iuse_flags_list2 = list(set(iuse))
+ use_enable = build_dict['build_useflags']
+ use_disable = list(set(iuse_flags_list2).difference(set(use_enable)))
+ use_flagsDict = {}
+ for x in use_enable:
+ use_flagsDict[x] = True
+ for x in use_disable:
+ use_flagsDict[x] = False
+ for u, s in use_flagsDict.iteritems():
+ use_flags_list.append(u)
+ use_enable_list.append(s)
+ ebuild_id = database.get_ebuild_id_db(build_dict, ebuild_version_checksum_tree)
+ database.add_new_package_buildqueue(ebuild_id, config_profile, use_flags_list, use_enable_list, message)
+ return
+def get_buillog_info(settings, mode):
+ logfile_text = get_log_text(settings.get("PORTAGE_LOG_FILE"))
+def get_build_dict_db(settings, database, config_profile):
+ categories = settings['CATEGORY']
+ package = settings['PN']
+ ebuild_version = settings['PVR']
+ cpv = categories + "/" + package + "-" + ebuild_version
+ package_id = database.have_package_db(categories, package)
+ build_dict = {}
+ build_dict['ebuild_version'] = ebuild_version
+ build_dict['package_id'] = package_id
+ build_dict['cpv'] = cpv
+ build_dict['build_useflags'] = settings.get('PORTAGE_BUILT_USE')
+ pkgdir = os.path.join(settings['PORTDIR'], categories + "/" + package)
+ ebuild_version_checksum_tree = portage.checksum.sha256hash(pkgdir+ "/" + package + "-" + ebuild_version + ".ebuild")[0]
+ queue_id = database.check_revision(build_dict, config_profile, ebuild_version_checksum_tree)
+ if queue_id is None:
+ add_new_ebuild_buildquery_db(database, settings, build_dict, ebuild_version_checksum_tree, config_profile)
+ queue_id = database.check_revision(build_dict, config_profile, ebuild_version_checksum_tree)
+ build_dict = database.get_package_to_build(config_profile, queue_id)
+ return build_dict
+
+def main_hook(settings, mode):
+ # Main
+ get_conf_settings.__init__ # Init settings from the configfile
+ database = init_sql_backend() # Init the Database
+ config_profile = get_conf_settings.SetBuildConfig
+ build_dict = get_build_dict_db(settings, database, config_profile)
+ build_log_dict = get_buildlog_info(settings, mode)
+ print settings.get("PORTAGE_LOG_FILE")
+ print settings['PORTAGE_USE'], settings['PORTAGE_BUILDDIR']
+ print build_dict
+
\ No newline at end of file
diff --git a/buildhost/portage_hook/sqlbackend.py b/buildhost/portage_hook/sqlbackend.py
new file mode 100644
index 0000000..399aaa5
--- /dev/null
+++ b/buildhost/portage_hook/sqlbackend.py
@@ -0,0 +1,646 @@
+# -*- coding: utf-8 -*-
+
+import sys, os, re
+
+class DBStateError(Exception):
+ """If the DB is in a state that is not expected, we raise this."""
+ def __init__(self, value):
+ Exception.__init__(self)
+ self.value = value
+ def __str__(self):
+ return repr(self.value)
+
+class DatabaseConfig(object):
+ # No touchy
+ from core import get_conf_settings
+ settings = {}
+ mode = get_conf_settings.SetSqlBackend
+
+ # Settings for MySQL. You need to create one users in
+ # your MySQL database.
+ # The user needs:
+ # DELETE, INSERT, UPDATE, SELECT
+ # Do NOT change these, set the values in the config file /etc/buildhost/buildhost.conf
+ settings['sql_pgsql'] = {}
+ # settings['sql_pgsql']['charset'] = 'utf8'
+ settings['sql_pgsql']['host'] = get_conf_settings.SetSqlHost
+ settings['sql_pgsql']['database'] = get_conf_settings.SetSqlDB
+ settings['sql_pgsql']['user'] = get_conf_settings.SetSqlUser
+ settings['sql_pgsql']['password'] = get_conf_settings.SetSqlPasswd
+
+class SQLPackageDatabase(object):
+ """we have to store our stuff somewhere
+
+ subclass and redefine init to provide
+ at least self.cursor"""
+
+ # These are set by subclasses
+ db = None
+ cursor = None
+ syntax_placeholder = None
+ syntax_autoincrement = None
+
+ sql = {}
+
+ def get_default_config(self, config_profile):
+ cursor = self.conn.cursor()
+ sqlQ = "SELECT make_conf_checksum FROM configs WHERE active = 'True' AND id = %s AND updateing = 'False'"
+ cursor.execute(sqlQ, (config_profile,))
+ return cursor.fetchone()
+
+ def get_package_to_build(self, config_profile, queue_id):
+ cursor = self.conn.cursor()
+ print queue_id, config_profile
+ # get what to build
+ sqlQ1 = "SELECT ebuild_id, post_message FROM buildqueue WHERE config = %s AND extract(epoch from (NOW()) - timestamp) > 7200 AND queue_id = %s"
+ # get use flags to use
+ sqlQ2 = "SELECT useflag, enabled FROM ebuildqueuedwithuses WHERE queue_id = %s"
+ # get ebuild version (v) and package id
+ sqlQ3 = 'SELECT ebuild_version, package_id, ebuild_checksum FROM ebuilds WHERE id = %s'
+ # get categoriy and package (c, p)
+ sqlQ4 ='SELECT category, package_name FROM packages WHERE package_id = %s'
+ build_dict = {}
+ build_useflags_dict = {}
+ cursor.execute(sqlQ1, (config_profile, queue_id))
+ entries = cursor.fetchone()
+ print entries
+ if entries is None:
+ build_dict['ebuild_id'] = None
+ build_dict['post_message'] = None
+ return build_dict
+ build_dict['queue_id'] = queue_id
+ ebuild_id = entries[0]
+ build_dict['ebuild_id'] = ebuild_id
+ build_dict['post_message'] = entries[1]
+ cursor.execute(sqlQ2, (queue_id,))
+ entries = cursor.fetchall()
+ if entries == []:
+ build_useflags = None
+ else:
+ build_useflags = entries
+ cursor.execute(sqlQ3, (ebuild_id,))
+ entries = cursor.fetchall()[0]
+ build_dict['ebuild_version'] = entries[0]
+ build_dict['package_id'] = entries[1]
+ build_dict['ebuild_checksum'] = entries[2]
+ cursor.execute(sqlQ4, (entries[1],))
+ entries = cursor.fetchall()[0]
+ build_dict['categories'] = entries[0]
+ build_dict['package'] = entries[1]
+ if build_useflags is None:
+ build_dict['build_useflags'] = None
+ else:
+ for x in build_useflags:
+ build_useflags_dict[x[0]] = x[1]
+ build_dict['build_useflags'] = build_useflags_dict
+ print "build_dict", build_dict
+ return build_dict
+
+ def check_revision(self, build_dict, config_profile, ebuild_version_checksum_tree):
+ cursor = self.conn.cursor()
+ sqlQ1 = 'SELECT id FROM ebuilds WHERE ebuild_version = %s AND ebuild_checksum = %s AND package_id = %s'
+ sqlQ2 = 'SELECT queue_id FROM buildqueue WHERE ebuild_id = %s AND config = %s'
+ sqlQ3 = "SELECT useflag FROM ebuildqueuedwithuses WHERE queue_id = %s AND enabled = 'True'"
+ cursor.execute(sqlQ1, (build_dict['ebuild_version'], ebuild_version_checksum_tree, build_dict['package_id']))
+ ebuild_id = cursor.fetchone()[0]
+ print ebuild_id
+ if ebuild_id is None:
+ return None
+ cursor.execute(sqlQ2, (ebuild_id, config_profile))
+ queue_id_list = cursor.fetchall()
+ if queue_id_list is None:
+ return None
+ for queue_id in queue_id_list:
+ cursor.execute(sqlQ3, (queue_id,))
+ entries = cursor.fetchall()
+ if entries == []:
+ build_useflags = None
+ else:
+ build_useflags = entries
+ print build_useflags, build_dict['build_useflags']
+ if build_useflags is build_dict['build_useflags']:
+ print queue_id[0]
+ return queue_id[0]
+ return None
+
+ def get_config_list_all(self):
+ cursor = self.conn.cursor()
+ sqlQ = 'SELECT id FROM configs'
+ cursor.execute(sqlQ)
+ return cursor.fetchall()
+
+ def update__make_conf(self, configsDict):
+ cursor = self.conn.cursor()
+
+ sqlQ = 'UPDATE configs SET make_conf_checksum = %s, make_conf_text = %s, active = %s, config_error = %s WHERE id = %s'
+ for k, v in configsDict.iteritems():
+ params = [v['make_conf_checksum_tree'], v['make_conf_text'], v['active'], v['config_error'], k]
+ cursor.execute(sqlQ, params)
+ self.conn.commit()
+
+ def have_package_db(self, categories, package):
+ cursor = self.conn.cursor()
+
+ sqlQ ='SELECT package_id FROM packages WHERE category = %s AND package_name = %s'
+ params = categories, package
+ cursor.execute(sqlQ, params)
+ return cursor.fetchone()
+
+ def get_categories_db(self):
+ cursor = self.conn.cursor()
+ sqlQ =' SELECT category FROM categories'
+ cursor.execute(sqlQ)
+ return cursor.fetchall()
+
+ def get_categories_checksum_db(self, categories):
+ cursor = self.conn.cursor()
+ sqlQ =' SELECT metadata_xml_checksum FROM categories_meta WHERE category = %s'
+ cursor.execute(sqlQ, (categories,))
+ return cursor.fetchone()
+
+ def add_new_categories_meta_sql(self, categories, categories_metadata_xml_checksum_tree, categories_metadata_xml_text_tree):
+ cursor = self.conn.cursor()
+ sqlQ = 'INSERT INTO categories_meta (category, metadata_xml_checksum, metadata_xml_text) VALUES ( %s, %s, %s )'
+ params = categories, categories_metadata_xml_checksum_tree, categories_metadata_xml_text_tree
+ cursor.execute(sqlQ, params)
+ self.conn.commit()
+
+ def update_categories_meta_sql(self, categories, categories_metadata_xml_checksum_tree, categories_metadata_xml_text_tree):
+ cursor = self.conn.cursor()
+ sqlQ ='UPDATE categories_meta SET metadata_xml_checksum = %s, metadata_xml_text = %s WHERE category = %s'
+ params = (categories_metadata_xml_checksum_tree, categories_metadata_xml_text_tree, categories)
+ cursor.execute(sqlQ, params)
+ self.conn.commit()
+
+ def add_new_manifest_sql(self, package_id, get_manifest_text, manifest_checksum_tree):
+ cursor = self.conn.cursor()
+ sqlQ = 'INSERT INTO manifest (package_id, manifest, checksum) VALUES ( %s, %s, %s )'
+ params = package_id, get_manifest_text, manifest_checksum_tree
+ cursor.execute(sqlQ, params)
+ self.conn.commit()
+
+ def add_new_package_metadata(self, package_id, package_metadataDict):
+ cursor = self.conn.cursor()
+ sqlQ = 'SELECT changelog_checksum FROM packages_meta WHERE package_id = %s'
+ cursor.execute(sqlQ, (package_id,))
+ if cursor.fetchone() is None:
+ sqlQ = 'INSERT INTO packages_meta (package_id, changelog_text, changelog_checksum, metadata_text, metadata_checksum) VALUES ( %s, %s, %s, %s, %s )'
+ for k, v in package_metadataDict.iteritems():
+ params = package_id, v['changelog_text'], v['changelog_checksum'], v[' metadata_xml_text'], v['metadata_xml_checksum']
+ cursor.execute(sqlQ, params)
+ self.conn.commit()
+
+ def update_new_package_metadata(self, package_id, package_metadataDict):
+ cursor = self.conn.cursor()
+ sqlQ = 'SELECT changelog_checksum, metadata_checksum FROM packages_meta WHERE package_id = %s'
+ cursor.execute(sqlQ, package_id)
+ entries = cursor.fetchone()
+ changelog_checksum_db = entries[0]
+ metadata_checksum_db = entries[1]
+ for k, v in package_metadataDict.iteritems():
+ if changelog_checksum_db != v['changelog_checksum']:
+ sqlQ = 'UPDATE packages_meta SET changelog_text = %s, changelog_checksum = %s WHERE package_id = %s'
+ params = v['changelog_text'], v['changelog_checksum'], package_id
+ cursor.execute(sqlQ, params)
+ if metadata_checksum_db != v['metadata_xml_checksum']:
+ sqlQ = 'UPDATE packages_meta SET metadata_text = %s, metadata_checksum = %s WHERE package_id = %s'
+ params = v[' metadata_xml_text'], v['metadata_xml_checksum'], package_id
+ cursor.execute(sqlQ, params)
+ self.conn.commit()
+
+ def get_manifest_db(self, package_id):
+ cursor = self.conn.cursor()
+ sqlQ = 'SELECT checksum FROM manifest WHERE package_id = %s'
+ cursor.execute(sqlQ, package_id)
+ return cursor.fetchone()
+
+ def update_manifest_sql(self, package_id, get_manifest_text, manifest_checksum_tree):
+ cursor = self.conn.cursor()
+ sqlQ = 'UPDATE manifest SET checksum = %s, manifest = %s WHERE package_id = %s'
+ params = (manifest_checksum_tree, get_manifest_text, package_id)
+ cursor.execute(sqlQ, params)
+ self.conn.commit()
+
+ def add_new_metadata(self, metadataDict):
+ for k, v in metadataDict.iteritems():
+ cursor = self.conn.cursor()
+ sqlQ = 'SELECT updaterestrictions( %s, %s )'
+ params = k, v['restrictions']
+ cursor.execute(sqlQ, params)
+ sqlQ = 'SELECT updatekeywords( %s, %s )'
+ params = k, v['keyword']
+ cursor.execute(sqlQ, params)
+ sqlQ = 'SELECT updateiuse( %s, %s )'
+ params = k, v['iuse']
+ cursor.execute(sqlQ, params)
+ self.conn.commit()
+
+ def add_new_package_sql(self, packageDict):
+ #lets have a new cursor for each metod as per best practice
+ cursor = self.conn.cursor()
+ sqlQ="SELECT insert_ebuild( %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'True')"
+ ebuild_id_list = []
+ package_id_list = []
+ for k, v in packageDict.iteritems():
+ params = [v['categories'], v['package'], v['ebuild_version_tree'], v['ebuild_version_revision'], v['ebuild_version_checksum_tree'],
+ v['ebuild_version_text'], v['ebuild_version_metadata_tree'][0], v['ebuild_version_metadata_tree'][1],
+ v['ebuild_version_metadata_tree'][12], v['ebuild_version_metadata_tree'][2], v['ebuild_version_metadata_tree'][3],
+ v['ebuild_version_metadata_tree'][5],v['ebuild_version_metadata_tree'][6], v['ebuild_version_metadata_tree'][7],
+ v['ebuild_version_metadata_tree'][9], v['ebuild_version_metadata_tree'][11],
+ v['ebuild_version_metadata_tree'][13],v['ebuild_version_metadata_tree'][14], v['ebuild_version_metadata_tree'][15],
+ v['ebuild_version_metadata_tree'][16]]
+ cursor.execute(sqlQ, params)
+ mid = cursor.fetchone()
+ mid=mid[0]
+ ebuild_id_list.append(mid[1])
+ package_id_list.append(mid[0])
+ self.conn.commit()
+ # add_new_metadata(metadataDict)
+ return ebuild_id_list, package_id_list
+
+ def add_new_ebuild_sql(packageDict, new_ebuild_list):
+ #lets have a new cursor for each metod as per best practice
+ cursor = self.conn.cursor()
+ sqlQ="SELECT insert_ebuild( %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'True')"
+ ebuild_id_list = []
+ package_id_list = []
+ for k, v in packageDict.iteritems():
+ for x in new_ebuild_list:
+ if x == v['ebuild_version_tree']:
+ params = [v['categories'], v['package'], v['ebuild_version_tree'], v['ebuild_version_revision'], v['ebuild_version_checksum_tree'],
+ v['ebuild_version_text'], v['ebuild_version_metadata_tree'][0], v['ebuild_version_metadata_tree'][1],
+ v['ebuild_version_metadata_tree'][12], v['ebuild_version_metadata_tree'][2], v['ebuild_version_metadata_tree'][3],
+ v['ebuild_version_metadata_tree'][5],v['ebuild_version_metadata_tree'][6], v['ebuild_version_metadata_tree'][7],
+ v['ebuild_version_metadata_tree'][9], v['ebuild_version_metadata_tree'][11],
+ v['ebuild_version_metadata_tree'][13],v['ebuild_version_metadata_tree'][14], v['ebuild_version_metadata_tree'][15],
+ v['ebuild_version_metadata_tree'][16]]
+ cursor.execute(sqlQ, params)
+ mid = cursor.fetchone()
+ mid=mid[0]
+ ebuild_id_list.append(mid[1])
+ package_id_list.append(mid[0])
+ self.conn.commit()
+ # add_new_metadata(metadataDict)
+ return ebuild_id_list, package_id_list
+
+ def update_active_ebuild(self, package_id, ebuild_version_tree):
+ cursor = self.conn.cursor()
+ sqlQ ="UPDATE ebuilds SET active = 'False', timestamp = now() WHERE package_id = %s AND ebuild_version = %s AND active = 'True'"
+ cursor.execute(sqlQ, (package_id, ebuild_version_tree))
+ self.conn.commit()
+
+ def get_ebuild_id_db(build_dict, ebuild_version_checksum_tree):
+ cursor = self.conn.cursor()
+ sqlQ = 'SELECT id FROM ebuilds WHERE ebuild_version = %s AND ebuild_checksum = %s AND package_id = %s'
+ cursor.execute(sqlQ, (build_dict['ebuild_version'], ebuild_version_checksum_tree, build_dict['package_id']))
+ ebuild_id = cursor.fetchone()[0]
+ print ebuild_id
+ if ebuild_id is None:
+ return None
+ return ebuild_id
+
+ sql['SELECT_get_cpv_from_ebuild_id'] = """
+ SELECT category, ebuild_name, ebuild_version
+ FROM packages
+ WHERE id = %s
+ """
+ def get_cpv_from_ebuild_id(self, ebuild_id):
+ cursor = self.conn.cursor()
+ sqlQ = 'SELECT package_id FROM ebuild WHERE id = %s'
+ self.cursor.execute(sql, ebuild_id)
+ entries = self.cursor.fetchone()
+ return entries
+
+ def get_cp_from_package_id(self, package_id):
+ cursor = self.conn.cursor()
+ sqlQ = "SELECT ARRAY_TO_STRING(ARRAY[category, package_name] , '/') AS cp FROM packages WHERE package_id = %s"
+ cursor.execute(sqlQ, (package_id,))
+ return cursor.fetchone()
+
+ sql['SELECT_get_keyword_id_db'] = """
+ SELECT id_keyword
+ FROM keywords
+ WHERE ARCH = %s AND stable = %s
+ """
+ def get_keyword_id_db(self, arch, stable):
+ sql = self.sql['SELECT_get_keyword_id_db']
+ self.cursor.execute(sql, (arch, stable))
+ entries = self.cursor.fetchone()
+ return entries
+
+ sql['INSERT_add_new_keywords'] = """
+ INSERT INTO keywordsToEbuild
+ (ebuild_id, id_keyword)
+ VALUES ( %s, %s )
+ """
+ def add_new_keywords(self, ebuild_id, keyword_id):
+ sql = self.sql['INSERT_add_new_keywords']
+ self.cursor.execute(sql, (ebuild_id, keyword_id))
+ self.db.commit()
+
+ sql['SELECT_have_package_buildqueue'] = """
+ SELECT useflags
+ FROM buildqueue
+ WHERE ebuild_id = %s AND config_id = %s
+ """
+ def have_package_buildqueue(self, ebuild_id, config_id):
+ sql = self.sql['SELECT_have_package_buildqueue']
+ params = (ebuild_id[0], config_id)
+ self.cursor.execute(sql, params)
+ entries = self.cursor.fetchone()
+ return entries
+
+ def add_new_package_buildqueue(self, ebuild_id, config_id, iuse_flags_list, use_enable, message):
+ cursor = self.conn.cursor()
+ sqlQ="SELECT insert_buildqueue( %s, %s, %s, %s, %s )"
+ if not iuse_flags_list:
+ iuse_flags_list=None
+ use_enable=None
+ params = ebuild_id, unicode(config_id), iuse_flags_list, use_enable, message
+ cursor.execute(sqlQ, params)
+ self.conn.commit()
+
+ sql['SELECT_get_package_id'] = """
+ SELECT package_id
+ FROM packages
+ WHERE category = %s AND package_name = %s
+ """
+ sql['SELECT_get_ebuild_checksum'] = """
+ SELECT ebuild_checksum
+ FROM ebuild
+ WHERE package_id = %s AND ebuild_version = %s
+ """
+ def get_ebuild_checksum(self, package_id, ebuild_version_tree):
+ cursor = self.conn.cursor()
+ sqlQ = 'SELECT ebuild_checksum FROM ebuilds WHERE package_id = %s AND ebuild_version = %s AND active = TRUE'
+ cursor.execute(sqlQ, (package_id, ebuild_version_tree))
+ entries = cursor.fetchone()
+ if entries is None:
+ return None
+ return entries[0]
+
+ sql['UPDATE_update_ebuild_db'] = """
+ UPDATE packages
+ SET ebuild_checksum = %s, ebuild_text = %s,
+ depend = %s, rdepend = %s, pdepend = %s, slot = %s, src_url = %s, homepage = %s, license = %s,
+ description = %s, keywords = %s, inherited = %s, iuse = %s, required_use = %s, provide = %s,
+ eapi = %s, properties = %s, defined_phases = %s
+ WHERE ebuild_id = %s
+ """
+ def update_ebuild_sql(self, ebuild_id, ebuild_version_checksum_tree,
+ ebuild_version_metadata_tree, ebuild_version_text):
+ sql = self.sql['UPDATE_update_ebuild_db']
+ params = (ebuild_version_checksum_tree[0],
+ ebuild_version_text, ebuild_version_metadata_tree[0], ebuild_version_metadata_tree[1],
+ ebuild_version_metadata_tree[12], ebuild_version_metadata_tree[2], ebuild_version_metadata_tree[3],
+ ebuild_version_metadata_tree[5], ebuild_version_metadata_tree[6], ebuild_version_metadata_tree[7],
+ ebuild_version_metadata_tree[9], ebuild_version_metadata_tree[10], ebuild_version_metadata_tree[11],
+ ebuild_version_metadata_tree[13], ebuild_version_metadata_tree[14], ebuild_version_metadata_tree[15],
+ ebuild_version_metadata_tree[16], ebuild_id)
+ self.cursor.execute(sql, params)
+ self.db.commit()
+
+ sql['SELECT_cp_all_db'] = """
+ SELECT CONCAT_WS('/',categories, package)
+ FROM package
+ WHERE active = '1'
+ """
+ def cp_all_db(self):
+ cursor = self.conn.cursor()
+ sqlQ = "SELECT package_id FROM packages"
+ cursor.execute(sqlQ)
+ return cursor.fetchall()
+
+ def add_old_package(self, old_package_list):
+ cursor = self.conn.cursor()
+ mark_old_list = []
+ sqlQ = "UPDATE ebuilds SET active = 'FALSE', timestamp = NOW() WHERE package_id = %s AND active = 'TRUE' RETURNING package_id"
+ for old_package in old_package_list:
+ cursor.execute(sqlQ, (old_package[0],))
+ entries = cursor.fetchone()
+ if entries is not None:
+ mark_old_list.append(entries[0])
+ self.conn.commit()
+ return mark_old_list
+
+ sql['UPDATE_add_old_categories'] = """
+ UPDATE categories
+ SET active = '0'
+ WHERE categories = %s
+ """
+ def get_old_categories(self, categories_line):
+ cursor = self.conn.cursor()
+ sqlQ = "SELECT package_name FROM packages WHERE category = %s"
+ cursor.execute(sqlQ (categories_line))
+ return cursor.fetchone()
+
+ def del_old_categories(self, real_old_categoriess):
+ cursor = self.conn.cursor()
+ sqlQ = 'DELETE FROM categories categories_meta WHERE category = %s'
+ cursor.execute(sqlQ (real_old_categories))
+ self.conn.commit()
+
+ def add_old_ebuild(self, package_id, old_ebuild_list):
+ cursor = self.conn.cursor()
+ sqlQ1 = "UPDATE ebuilds SET active = 'FALSE' WHERE package_id = %s AND ebuild_version = %s"
+ sqlQ2 = "SELECT id FROM ebuilds WHERE package_id = %s AND ebuild_version = %s AND active = 'TRUE'"
+ sqlQ3 = "SELECT queue_id FROM buildqueue WHERE ebuild_id = %s"
+ sqlQ4 = 'DELETE FROM ebuildqueuedwithuses buildqueue WHERE queue_id = %s'
+ for old_ebuild in old_ebuild_list:
+ cursor.execute(sqlQ2, (package_id, old_ebuild[0]))
+ ebuild_id_list = cursor.fetchall()
+ if ebuild_id_list is not None:
+ for ebuild_id in ebuild_id_list:
+ cursor.execute(sqlQ3, (ebuild_id))
+ queue_id_list = cursor.fetchall()
+ if queue_id_list is not None:
+ for queue_id in queue_id_list:
+ cursor.execute(sqlQ4, (queue_id))
+ cursor.execute(sqlQ1, (package_id, old_ebuild[0]))
+ self.conn.commit()
+
+ sql['SELECT_cp_all_old_db'] = """
+ SELECT CONCAT_WS('/',categories, package)
+ FROM package
+ WHERE active = '0' AND DATEDIFF(NOW(), timestamp) > 60
+ """
+ def cp_all_old_db(self, old_package_id_list):
+ cursor = self.conn.cursor()
+ old_package_list = []
+ for old_package in old_package_id_list:
+ sqlQ = "SELECT package_id FROM ebuilds WHERE package_id = %s AND active = 'FALSE' AND date_part('days', NOW() - timestamp) < 60"
+ cursor.execute(sqlQ, old_package)
+ entries = cursor.fetchone()
+ if entries is None:
+ old_package_list.append(old_package)
+ return old_package_list
+
+ sql['DELETE_del_old_ebuild'] = """
+ DELETE FROM ebuild
+ WHERE categories = %s AND package = %s AND active = '0' AND DATEDIFF(NOW(), timestamp) > 60
+ """
+ def del_old_ebuild(self, ebuild_old_list_db):
+ cursor = self.conn.cursor()
+ sqlQ1 ='SELECT id FROM ebuilds WHERE ebuild_version = %s'
+ sqlQ2 = 'SELECT build_id FROM buildlog WHERE ebuild_id = %s'
+ sqlQ3 = 'DELETE FROM qa_problems repoman_problems ebuildbuildwithuses WHERE build_id = %s'
+ sqlQ4 = 'DELETE FROM ebuildhaveskeywords ebuildhavesiuses ebuildhavesrestrictions buildlog WHERE ebuild_id = %s'
+ for ebuild_version in ebuild_old_list_db:
+ cursor.execute(sqlQ1, (ebuild_version))
+ ebuild_id_list = cursor.fetchall()
+ for ebuild_id in ebuild_id_list:
+ cursor.execute(sqlQ2, ebuild_id)
+ build_id_list = cursor.fetchall()
+ for build_id in build_id_list:
+ cursor.execute(sqlQ3, build_id)
+ cursor.execute(sqlQ4, ebuild_id)
+ self.conn.commit()
+
+ sql['DELETE_del_old_package'] = """
+ DELETE FROM package
+ WHERE categories = %s AND package = %s AND active = '0' AND DATEDIFF(NOW(), timestamp) > 60
+ """
+ def del_old_package(self, package_id_list):
+ cursor = self.conn.cursor()
+ sqlQ1 = 'SELECT id FROM ebuilds WHERE package_id = %s'
+ sqlQ2 = 'SELECT build_id FROM buildlog WHERE ebuild_id = %s'
+ sqlQ3 = 'DELETE FROM qa_problems, repoman_problems, ebuildbuildwithuses WHERE build_id = %s'
+ sqlQ4 = 'DELETE FROM ebuildhaveskeywords, ebuildhavesiuses, ebuildhavesrestrictions, buildlog WHERE ebuild_id = %s'
+ sqlQ5 = 'DELETE FROM ebuilds, manifest, package_meta, packages WHERE package_id = %s'
+ for package_id in package_id_list:
+ cursor.execute(sqlQ1, package_id)
+ ebuild_id_list = cursor.fetchall()
+ for ebuild_id in ebuild_id_list:
+ cursor.execute(sqlQ2, ebuild_id)
+ build_id_list = cursor.fetchall()
+ for build_id in build_id_list:
+ cursor.execute(sqlQ3, build_id)
+ cursor.execute(sqlQ4, ebuild_id)
+ cursor.execute(sqlQ5, package_id)
+ self.conn.commit()
+
+ sql['DELETE_del_old_categories'] = """
+ DELETE FROM categories
+ WHERE categories = %s AND active = '0' AND DATEDIFF(NOW(), timestamp) > 60
+ """
+ def del_old_package(self, categories):
+ sql = self.sql['DELETE_del_old_categories']
+ self.cursor.execute(sql, (categories))
+ self.db.commit()
+
+ def cp_list_db(self, package_id):
+ cursor = self.conn.cursor()
+ sqlQ = "SELECT ebuild_version FROM ebuilds WHERE active = 'TRUE' AND package_id = %s"
+ cursor.execute(sqlQ, (package_id))
+ return cursor.fetchall()
+
+ def cp_list_old_db(self, package_id):
+ cursor = self.conn.cursor()
+ sqlQ ="SELECT id, ebuild_version FROM ebuilds WHERE active = 'FALSE' AND package_id = %s AND date_part('days', NOW() - timestamp) > 60"
+ cursor.execute(sqlQ, package_id)
+ return cursor.fetchall()
+
+ def add_qa_repoman(self, ebuild_id_list, qa_error, packageDict, config_id):
+ ebuild_i = 0
+ cursor = self.conn.cursor()
+ for k, v in packageDict.iteritems():
+ ebuild_id = ebuild_id_list[ebuild_i]
+ sqlQ = 'INSERT INTO buildlog (ebuild_id, config, error_summary, timestamp ) VALUES ( %s, %s, %s, now() ) RETURNING build_id'
+ if v['ebuild_error'] != [] or qa_error != []:
+ if v['ebuild_error'] != [] or qa_error == []:
+ summary = "Repoman"
+ elif v['ebuild_error'] == [] or qa_error != []:
+ summary = "QA"
+ else:
+ summary = "QA:Repoman"
+ params = (ebuild_id, config_id, summary)
+ cursor.execute(sqlQ, params)
+ build_id = cursor.fetchone()
+ if v['ebuild_error'] != []:
+ sqlQ = 'INSERT INTO repoman_problems (problem, build_id ) VALUES ( %s, %s )'
+ for x in v['ebuild_error']:
+ params = (x, build_id)
+ cursor.execute(sqlQ, params)
+ if qa_error != []:
+ sqlQ = 'INSERT INTO qa_problems (problem, build_id ) VALUES ( %s, %s )'
+ for x in qa_error:
+ params = (x, build_id)
+ cursor.execute(sqlQ, params)
+ ebuild_i = ebuild_i +1
+ self.conn.commit()
+
+ sql['SELECT_qa_repoman'] = """
+ SELECT categories, package, ebuild_version, qa, repoman, config, hosts
+ FROM buildlog
+ WHERE ebuild = %sAND build = '' AND config_id = %s
+ """
+ def get_qa_repoman(self, ebuild_id, config_id):
+ sql = self.sql['SELECT_qa_repoman']
+ params = (ebuild_id, config_id)
+ self.cursor.execute(sql, params)
+ entries = self.cursor.fetchall()
+ return entries
+
+ sql['DELETE_del_qa_repoman'] = """
+ DELETE FROM buildlog
+ WHERE ebuild_id AND build = '' AND config_id = %s
+ """
+ def del_qa_repoman(self, ebuild_id, config_id):
+ sql = self.sql['DELETE_del_qa_repoman']
+ self.cursor.execute(sql, (ebuild_id, config_id))
+ self.db.commit()
+
+ sql['UPDATE_update_qa_repoman'] = """
+ UPDATE buildlog
+ SET qa = %s, repoman = %s
+ WHERE ebuild_id = %s AND build = '' AND config_id = %s
+ """
+ def update_qa_repoman(self, ebuild_id, qa_error, repoman_error, config_id):
+ sql = self.sql['UPDATE_update_qa_repoman']
+ self.cursor.execute(sql, (qa_error, repoman_error, ebuild_id, config_id))
+ self.db.commit()
+
+ def get_arch_db(self):
+ cursor = self.conn.cursor()
+ sqlQ = 'SELECT keyword FROM keywords WHERE keyword = %s'
+ cursor.execute(sqlQ, ('ppc',))
+ return cursor.fetchone()
+
+ def add_new_arch_db(self, arch_list):
+ cursor = self.conn.cursor()
+ sqlQ = 'INSERT INTO keywords (keyword) VALUES ( %s )'
+ for arch in arch_list:
+ cursor.execute(sqlQ, (arch,))
+ self.conn.commit()
+
+ sql['INSERT_add_new_restriction_db'] = """
+ INSERT INTO ebuildHavesRestrictions
+ (ebuild_id, restriction)
+ VALUES ( %s, %s )
+ """
+
+ def closeconnection(self):
+ self.conn.close()
+
+class PgSQLPackageDB(SQLPackageDatabase):
+ """override for MySQL backend"""
+
+ syntax_placeholder = "%s"
+
+ def __init__(self, config=None):
+ # Do not complain about correct usage of ** magic
+ # pylint: disable-msg=W0142
+ SQLPackageDatabase.__init__(self)
+
+ if config is None or 'database' not in config:
+ print "No configuration available!"
+ sys.exit(1)
+
+ try:
+ import psycopg2
+ except ImportError:
+ print "Please install a recent version of dev-python/psycopg for Python"
+ sys.exit(1)
+
+ self.conn = psycopg2.connect(**config)
+
^ permalink raw reply related [flat|nested] only message in thread
only message in thread, other threads:[~2011-04-01 16:43 UTC | newest]
Thread overview: (only message) (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2011-04-01 16:42 [gentoo-commits] dev/zorry:master commit in: buildhost/portage_hook/ Magnus Granberg
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox