* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/
@ 2021-01-10 21:41 Magnus Granberg
0 siblings, 0 replies; 9+ messages in thread
From: Magnus Granberg @ 2021-01-10 21:41 UTC (permalink / raw
To: gentoo-commits
commit: 503420b53571da6ee3a2b7db8fa9407337a19162
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Jan 10 21:33:45 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Jan 10 21:33:45 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=503420b5
Update Copyright on db.repositorys.py
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/db/repositorys.py | 18 +++++++++++++++++-
1 file changed, 17 insertions(+), 1 deletion(-)
diff --git a/buildbot_gentoo_ci/db/repositorys.py b/buildbot_gentoo_ci/db/repositorys.py
index d93ee72..4660530 100644
--- a/buildbot_gentoo_ci/db/repositorys.py
+++ b/buildbot_gentoo_ci/db/repositorys.py
@@ -1,5 +1,21 @@
+# This file has parts from Buildbot and is modifyed by Gentoo Authors.
+# Buildbot is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+# Origins: buildbot.db.*
+# Modifyed by Gentoo Authors.
# Copyright 2021 Gentoo Authors
-# Distributed under the terms of the GNU General Public License v2
import sqlalchemy as sa
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/
@ 2021-03-18 23:18 Magnus Granberg
0 siblings, 0 replies; 9+ messages in thread
From: Magnus Granberg @ 2021-03-18 23:18 UTC (permalink / raw
To: gentoo-commits
commit: f06629a2b072f7a93d74a402897c1baaf6010a14
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Mar 18 23:13:07 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Mar 18 23:13:07 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=f06629a2
JSON format don't support datetime format in version data
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/db/versions.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/buildbot_gentoo_ci/db/versions.py b/buildbot_gentoo_ci/db/versions.py
index df9b4d3..85bd105 100644
--- a/buildbot_gentoo_ci/db/versions.py
+++ b/buildbot_gentoo_ci/db/versions.py
@@ -24,7 +24,7 @@ from twisted.internet import defer
from buildbot.db import base
from buildbot.util import epoch2datetime
-
+from buildbot.util import datetime2epoch
class VersionsConnectorComponent(base.DBConnectorComponent):
@defer.inlineCallbacks
@@ -110,5 +110,5 @@ class VersionsConnectorComponent(base.DBConnectorComponent):
file_hash=row.file_hash,
commit_id=row.commit_id,
deleted=row.deleted,
- deleted_at=row.deleted_at
+ deleted_at=datetime2epoch(row.deleted_at)
)
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/
@ 2022-07-28 11:31 Magnus Granberg
0 siblings, 0 replies; 9+ messages in thread
From: Magnus Granberg @ 2022-07-28 11:31 UTC (permalink / raw
To: gentoo-commits
commit: 60ce5fec404770c2dbc58352808397f7e9b87c93
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Jul 28 11:31:07 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Jul 28 11:31:07 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=60ce5fec
Add support for gitlab/github project name in project db
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/db/model.py | 2 ++
buildbot_gentoo_ci/db/projects.py | 1 +
2 files changed, 3 insertions(+)
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index d6e9860..be305f2 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -109,6 +109,8 @@ class Model(base.DBConnectorComponent):
default=lambda: str(uuid.uuid4())),
# project's name
sa.Column('name', sa.String(255), nullable=False),
+ # project name on gitlab/github
+ sa.Column('git_project_name', sa.String(255), nullable=False),
# description of the project
sa.Column('description', sa.Text, nullable=True),
sa.Column('profile', sa.String(255), nullable=False),
diff --git a/buildbot_gentoo_ci/db/projects.py b/buildbot_gentoo_ci/db/projects.py
index 936cd9d..8e2cd6f 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -216,6 +216,7 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
return dict(
uuid=row.uuid,
name=row.name,
+ git_project_name=row.git_project_name,
description=row.description,
profile=row.profile,
profile_repository_uuid=row.profile_repository_uuid,
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/
@ 2022-08-22 22:54 Magnus Granberg
0 siblings, 0 replies; 9+ messages in thread
From: Magnus Granberg @ 2022-08-22 22:54 UTC (permalink / raw
To: gentoo-commits
commit: 9e94e5e0e464ba589d25d60d6891c0a8dee0ecd7
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Aug 22 22:54:39 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Aug 22 22:54:39 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=9e94e5e0
Add Merge Requeste test to projects_repositorys in db
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/db/model.py | 1 +
buildbot_gentoo_ci/db/projects.py | 3 ++-
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index be305f2..3f1dc0a 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -144,6 +144,7 @@ class Model(base.DBConnectorComponent):
sa.Column('pkgcheck', sa.Enum('package','full','none'), default='none'),
sa.Column('build', sa.Boolean, default=False),
sa.Column('test', sa.Boolean, default=False),
+ sa.Column('test_mr', sa.Boolean, default=False),
)
# projects etc/portage settings
diff --git a/buildbot_gentoo_ci/db/projects.py b/buildbot_gentoo_ci/db/projects.py
index 8e2cd6f..9dcfa45 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -241,7 +241,8 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
auto=row.auto,
pkgcheck=pkgcheck,
build=row.build,
- test=row.test
+ test=row.test,
+ test_mr=row.test_mr
)
def _row2dict_projects_workers(self, conn, row):
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/
@ 2022-10-03 1:20 Magnus Granberg
0 siblings, 0 replies; 9+ messages in thread
From: Magnus Granberg @ 2022-10-03 1:20 UTC (permalink / raw
To: gentoo-commits
commit: 4827e44a0e0e6df8786bec353c7f01a39af4e089
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Oct 3 01:20:32 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Oct 3 01:20:32 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=4827e44a
Fix bootstrap_tab in workers db
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/db/workers.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/buildbot_gentoo_ci/db/workers.py b/buildbot_gentoo_ci/db/workers.py
index afc7815..e82a6b2 100644
--- a/buildbot_gentoo_ci/db/workers.py
+++ b/buildbot_gentoo_ci/db/workers.py
@@ -151,7 +151,7 @@ class WorkersConnectorComponent(base.DBConnectorComponent):
name=row.name,
tag=row.tag,
dockerfile=row.dockerfile,
- bootstrap_image_tag=row.bootstrap_image_tag
+ bootstrap_tag=row.bootstrap_tag
)
def _row2dict_worker_config(self, conn, row):
return dict(
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/
@ 2023-03-10 22:36 Magnus Granberg
0 siblings, 0 replies; 9+ messages in thread
From: Magnus Granberg @ 2023-03-10 22:36 UTC (permalink / raw
To: gentoo-commits
commit: 165def4478ff2558075c7dcabdff85b2be31277b
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Fri Mar 10 22:32:06 2023 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Fri Mar 10 22:32:06 2023 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=165def44
Fix 2 bugs in db for email
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/db/packages.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/buildbot_gentoo_ci/db/packages.py b/buildbot_gentoo_ci/db/packages.py
index 79ae685..1f68637 100644
--- a/buildbot_gentoo_ci/db/packages.py
+++ b/buildbot_gentoo_ci/db/packages.py
@@ -210,7 +210,8 @@ class PackagesConnectorComponent(base.DBConnectorComponent):
tbl = self.db.model.packages_emails
conn.execute(tbl.delete(
whereclause=((tbl.c.package_uuid == package_uuid))))
- return self.db.pool.do(thd)
+ res = yield self.db.pool.do(thd)
+ return res
def _row2dict_email(self, conn, row):
return dict(
@@ -222,6 +223,6 @@ class PackagesConnectorComponent(base.DBConnectorComponent):
id=row.id,
email_id=row.email_id,
package_uuid=row.package_uuid,
- type=row.type,
+ mail_type=row.mail_type,
proxied=row.proxied,
)
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/
@ 2023-05-28 9:49 Magnus Granberg
0 siblings, 0 replies; 9+ messages in thread
From: Magnus Granberg @ 2023-05-28 9:49 UTC (permalink / raw
To: gentoo-commits
commit: 2dfba1de696c99c0c03a8c1c6abd878b60f3c7fc
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun May 28 09:48:40 2023 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun May 28 09:48:40 2023 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=2dfba1de
Update db python code to bb3.8
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/db/connector.py | 35 +++++-
buildbot_gentoo_ci/db/model.py | 231 ++++++++++++++++++-------------------
2 files changed, 141 insertions(+), 125 deletions(-)
diff --git a/buildbot_gentoo_ci/db/connector.py b/buildbot_gentoo_ci/db/connector.py
index 7665f84..0cc7884 100644
--- a/buildbot_gentoo_ci/db/connector.py
+++ b/buildbot_gentoo_ci/db/connector.py
@@ -15,7 +15,7 @@
# Copyright Buildbot Team Members
# Origins: buildbot.db.connector.py
# Modifyed by Gentoo Authors.
-# Copyright 2021 Gentoo Authors
+# Copyright 2023 Gentoo Authors
import textwrap
@@ -70,6 +70,10 @@ class DBConnector(service.ReconfigurableServiceMixin,
self.setName('db')
self.basedir = basedir
+ # not configured yet - we don't build an engine until the first
+ # reconfig
+ self.configured_url = None
+
# set up components
self._engine = None # set up in reconfigService
self.pool = None # set up in reconfigService
@@ -88,12 +92,16 @@ class DBConnector(service.ReconfigurableServiceMixin,
self.builds = builds.BuildsConnectorComponent(self)
self.workers = workers.WorkersConnectorComponent(self)
+ self.cleanup_timer = internet.TimerService(self.CLEANUP_PERIOD,
+ self._doCleanup)
+ self.cleanup_timer.clock = self.master.reactor
+ yield self.cleanup_timer.setServiceParent(self)
+
@defer.inlineCallbacks
def setup(self, config, check_version=True, verbose=True):
- db_url = config.db['db_url']
+ db_url = self.configured_url = config.db['db_url']
- log.msg("Setting up database with URL %r"
- % util.stripUrlPassword(db_url))
+ log.msg(f"Setting up database with URL {repr(util.stripUrlPassword(db_url))}")
# set up the engine and pool
self._engine = enginestrategy.create_engine(db_url,
@@ -113,3 +121,22 @@ class DBConnector(service.ReconfigurableServiceMixin,
for l in upgrade_message.format(basedir=self.basedir).split('\n'):
log.msg(l)
raise exceptions.DatabaseNotReadyError()
+
+ def reconfigServiceWithBuildbotConfig(self, new_config):
+ # double-check -- the master ensures this in config checks
+ assert self.configured_url == new_config.db['db_url']
+
+ return super().reconfigServiceWithBuildbotConfig(new_config)
+
+ def _doCleanup(self):
+ """
+ Perform any periodic database cleanup tasks.
+ @returns: Deferred
+ """
+ # pass on this if we're not configured yet
+ if not self.configured_url:
+ return None
+
+ d = self.changes.pruneChanges(self.master.config.changeHorizon)
+ d.addErrback(log.err, 'while pruning changes')
+ return d
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 7ffe0ca..b80281e 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -1,7 +1,7 @@
# This file has parts from Buildbot and is modifyed by Gentoo Authors.
-# Buildbot is free software: you can redistribute it and/or modify it
-# under the terms of the GNU General Public License as published by the
-# Free Software Foundation, version 2.
+# Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
@@ -18,10 +18,10 @@
# Copyright 2023 Gentoo Authors
import uuid
-import migrate
-import migrate.versioning.repository
+
+import alembic
+import alembic.config
import sqlalchemy as sa
-from migrate import exceptions # pylint: disable=ungrouped-imports
from twisted.internet import defer
from twisted.python import log
@@ -29,15 +29,36 @@ from twisted.python import util
from buildbot.db import base
from buildbot.db.migrate_utils import test_unicode
+from buildbot.db.types.json import JsonObject
from buildbot.util import sautils
-try:
- from migrate.versioning.schema import ControlledSchema # pylint: disable=ungrouped-imports
-except ImportError:
- ControlledSchema = None
+
+class UpgradeFromBefore0p9Error(Exception):
+
+ def __init__(self):
+ message = """You are trying to upgrade a buildbot 0.8.x master to buildbot 0.9.x or newer.
+ This is not supported. Please start from a clean database
+ http://docs.buildbot.net/latest/manual/upgrading/0.9-upgrade.html"""
+ # Call the base class constructor with the parameters it needs
+ super().__init__(message)
+
+
+class UpgradeFromBefore3p0Error(Exception):
+
+ def __init__(self):
+ message = """You are trying to upgrade to Buildbot 3.0 or newer from Buildbot 2.x or older.
+ This is only supported via an intermediate upgrade to newest Buildbot 2.10.x that is
+ available. Please first upgrade to 2.10.x and then try to upgrade to this version.
+ http://docs.buildbot.net/latest/manual/upgrading/3.0-upgrade.html"""
+ super().__init__(message)
class Model(base.DBConnectorComponent):
+
+ property_name_length = 256
+ property_source_length = 256
+ hash_length = 40
+
#
# schema
#
@@ -54,9 +75,6 @@ class Model(base.DBConnectorComponent):
#
# * dates are stored as unix timestamps (UTC-ish epoch time)
#
- # * sqlalchemy does not handle sa.Boolean very well on MySQL or Postgres;
- # use sa.SmallInteger instead
-
# Tables related to gentoo-ci-cloud
# -------------------------
@@ -452,35 +470,51 @@ class Model(base.DBConnectorComponent):
# Migration support
# -----------------
- # this is a bit more complicated than might be expected because the first
- # seven database versions were once implemented using a homespun migration
- # system, and we need to support upgrading masters from that system. The
- # old system used a 'version' table, where SQLAlchemy-Migrate uses
- # 'migrate_version'
+ # Buildbot has historically used 3 database migration systems:
+ # - homegrown system that used "version" table to track versions
+ # - SQLAlchemy-migrate that used "migrate_version" table to track versions
+ # - alembic that uses "alembic_version" table to track versions (current)
+ # We need to detect each case and tell the user how to upgrade.
+
+ config_path = util.sibpath(__file__, "migrations/alembic.ini")
+
+ def table_exists(self, conn, table):
+ try:
+ r = conn.execute(f"select * from {table} limit 1")
+ r.close()
+ return True
+ except Exception:
+ return False
+
+ def migrate_get_version(self, conn):
+ r = conn.execute("select version from migrate_version limit 1")
+ version = r.scalar()
+ r.close()
+ return version
- repo_path = util.sibpath(__file__, "migrate")
+ def alembic_get_scripts(self):
+ alembic_config = alembic.config.Config(self.config_path)
+ return alembic.script.ScriptDirectory.from_config(alembic_config)
+
+ def alembic_stamp(self, conn, alembic_scripts, revision):
+ context = alembic.runtime.migration.MigrationContext.configure(conn)
+ context.stamp(alembic_scripts, revision)
@defer.inlineCallbacks
def is_current(self):
- if ControlledSchema is None:
- # this should have been caught earlier by enginestrategy.py with a
- # nicer error message
- raise ImportError("SQLAlchemy/SQLAlchemy-Migrate version conflict")
-
- def thd(engine):
- # we don't even have to look at the old version table - if there's
- # no migrate_version, then we're not up to date.
- repo = migrate.versioning.repository.Repository(self.repo_path)
- repo_version = repo.latest
- try:
- # migrate.api doesn't let us hand in an engine
- schema = ControlledSchema(engine, self.repo_path)
- db_version = schema.version
- except exceptions.DatabaseNotControlledError:
+ def thd(conn):
+ if not self.table_exists(conn, 'alembic_version'):
return False
- return db_version == repo_version
- ret = yield self.db.pool.do_with_engine(thd)
+ alembic_scripts = self.alembic_get_scripts()
+ current_script_rev_head = alembic_scripts.get_current_head()
+
+ context = alembic.runtime.migration.MigrationContext.configure(conn)
+ current_rev = context.get_current_revision()
+
+ return current_rev == current_script_rev_head
+
+ ret = yield self.db.pool.do(thd)
return ret
# returns a Deferred that returns None
@@ -493,94 +527,49 @@ class Model(base.DBConnectorComponent):
@defer.inlineCallbacks
def upgrade(self):
- # here, things are a little tricky. If we have a 'version' table, then
- # we need to version_control the database with the proper version
- # number, drop 'version', and then upgrade. If we have no 'version'
- # table and no 'migrate_version' table, then we need to version_control
- # the database. Otherwise, we just need to upgrade it.
-
- def table_exists(engine, tbl):
- try:
- r = engine.execute("select * from {} limit 1".format(tbl))
- r.close()
- return True
- except Exception:
- return False
+ # the upgrade process must run in a db thread
+ def thd(conn):
+ alembic_scripts = self.alembic_get_scripts()
+ current_script_rev_head = alembic_scripts.get_current_head()
- # http://code.google.com/p/sqlalchemy-migrate/issues/detail?id=100
- # means we cannot use the migrate.versioning.api module. So these
- # methods perform similar wrapping functions to what is done by the API
- # functions, but without disposing of the engine.
- def upgrade(engine):
- schema = ControlledSchema(engine, self.repo_path)
- changeset = schema.changeset(None)
- with sautils.withoutSqliteForeignKeys(engine):
- for version, change in changeset:
- log.msg('migrating schema version {} -> {}'.format(version, version + 1))
- schema.runchange(version, change, 1)
-
- def check_sqlalchemy_migrate_version():
- # sqlalchemy-migrate started including a version number in 0.7; we
- # support back to 0.6.1, but not 0.6. We'll use some discovered
- # differences between 0.6.1 and 0.6 to get that resolution.
- version = getattr(migrate, '__version__', 'old')
- if version == 'old':
- try:
- from migrate.versioning import schemadiff
- if hasattr(schemadiff, 'ColDiff'):
- version = "0.6.1"
- else:
- version = "0.6"
- except Exception:
- version = "0.0"
- version_tup = tuple(map(int, version.split('-', 1)[0].split('.')))
- log.msg("using SQLAlchemy-Migrate version {}".format(version))
- if version_tup < (0, 6, 1):
- raise RuntimeError(("You are using SQLAlchemy-Migrate {}. "
- "The minimum version is 0.6.1.").format(version))
-
- def version_control(engine, version=None):
- ControlledSchema.create(engine, self.repo_path, version)
+ #if self.table_exists(conn, 'version'):
+ # raise UpgradeFromBefore0p9Error()
- # the upgrade process must run in a db thread
- def thd(engine):
- # if the migrate_version table exists, we can just let migrate
- # take care of this process.
- if table_exists(engine, 'migrate_version'):
- r = engine.execute(
- "select version from migrate_version limit 1")
- old_version = r.scalar()
- if old_version < 40:
- raise EightUpgradeError()
- try:
- upgrade(engine)
- except sa.exc.NoSuchTableError as e: # pragma: no cover
- if 'migration_tmp' in str(e):
- log.err('A serious error has been encountered during the upgrade. The '
- 'previous upgrade has been likely interrupted. The database has '
- 'been damaged and automatic recovery is impossible.')
- log.err('If you believe this is an error, please submit a bug to the '
- 'Buildbot project.')
- raise
-
- # if the version table exists, then we can version_control things
- # at that version, drop the version table, and let migrate take
- # care of the rest.
- elif table_exists(engine, 'version'):
- raise EightUpgradeError()
-
- # otherwise, this db is new, so we don't bother using the migration engine
- # and just create the tables, and put the version directly to
- # latest
- else:
- # do some tests before getting started
- test_unicode(engine)
+ if self.table_exists(conn, 'migrate_version'):
+ version = self.migrate_get_version(conn)
+
+ #if version < 40:
+ # raise UpgradeFromBefore0p9Error()
+ last_sqlalchemy_migrate_version = 0
+ if version != last_sqlalchemy_migrate_version:
+ raise UpgradeFromBefore3p0Error()
+
+ self.alembic_stamp(conn, alembic_scripts, alembic_scripts.get_base())
+ conn.execute('drop table migrate_version')
+
+ if not self.table_exists(conn, 'alembic_version'):
log.msg("Initializing empty database")
- Model.metadata.create_all(engine)
- repo = migrate.versioning.repository.Repository(self.repo_path)
- version_control(engine, repo.latest)
+ # Do some tests first
+ test_unicode(conn)
+
+ Model.metadata.create_all(conn)
+ self.alembic_stamp(conn, alembic_scripts, current_script_rev_head)
+ return
+
+ context = alembic.runtime.migration.MigrationContext.configure(conn)
+ current_rev = context.get_current_revision()
+
+ if current_rev == current_script_rev_head:
+ log.msg('Upgrading database: the current database schema is already the newest')
+ return
+
+ log.msg('Upgrading database')
+ with sautils.withoutSqliteForeignKeys(conn):
+ with context.begin_transaction():
+ context.run_migrations()
+
+ log.msg('Upgrading database: done')
- check_sqlalchemy_migrate_version()
- yield self.db.pool.do_with_engine(thd)
+ yield self.db.pool.do(thd)
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/
@ 2023-06-26 21:52 Magnus Granberg
0 siblings, 0 replies; 9+ messages in thread
From: Magnus Granberg @ 2023-06-26 21:52 UTC (permalink / raw
To: gentoo-commits
commit: 73e8b3e7c00b1b2686ae0df6d406181b6c3808a7
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Jun 26 21:49:07 2023 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Jun 26 21:49:07 2023 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=73e8b3e7
Add support to remove ebuilds in db
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/db/versions.py | 40 +++++++++++++++++++++++++++++++++++++++
1 file changed, 40 insertions(+)
diff --git a/buildbot_gentoo_ci/db/versions.py b/buildbot_gentoo_ci/db/versions.py
index 6b534f3..0aaac1b 100644
--- a/buildbot_gentoo_ci/db/versions.py
+++ b/buildbot_gentoo_ci/db/versions.py
@@ -131,6 +131,46 @@ class VersionsConnectorComponent(base.DBConnectorComponent):
res = yield self.db.pool.do(thd)
return res
+ @defer.inlineCallbacks
+ def getEbuildsByPackage(self, p_uuid, deleted=False):
+ def thd(conn):
+ tbl = self.db.model.versions
+ q = tbl.select()
+ q = q.where(tbl.c.deleted == deleted)
+ q = q.where(tbl.c.package_uuid == p_uuid)
+ return [self._row2dict(conn, row)
+ for row in conn.execute(q).fetchall()]
+ res = yield self.db.pool.do(thd)
+ return res
+
+ @defer.inlineCallbacks
+ def removeVersion(self, uuid):
+ def thd(conn, no_recurse=False):
+ tbl = self.db.model.versions
+ q = tbl.delete()
+ q = q.where(tbl.c.uuid == uuid)
+ conn.execute(q)
+ yield self.db.pool.do(thd)
+
+ @defer.inlineCallbacks
+ def removeVersionMetadata(self, version_uuid):
+ def thd(conn, no_recurse=False):
+ tbl = self.db.model.versions_metadata
+ q = tbl.delete()
+ q = q.where(tbl.c.version_uuid == version_uuid)
+ conn.execute(q)
+ yield self.db.pool.do(thd)
+
+ @defer.inlineCallbacks
+ def removeVersionKeyword(self, version_uuid):
+ def thd(conn, no_recurse=False):
+ tbl = self.db.model.versions_keywords
+ q = tbl.delete()
+ q = q.where(tbl.c.version_uuid == version_uuid)
+ conn.execute(q)
+ yield self.db.pool.do(thd)
+
+
def _row2dict(self, conn, row):
return dict(
uuid=row.uuid,
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/
@ 2024-10-30 21:44 Magnus Granberg
0 siblings, 0 replies; 9+ messages in thread
From: Magnus Granberg @ 2024-10-30 21:44 UTC (permalink / raw
To: gentoo-commits
commit: 01d5398cd23ce2bc930694ecfa62408b87bd2ddb
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Wed Oct 30 21:43:44 2024 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Wed Oct 30 21:43:44 2024 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=01d5398c
Update db to support Buildbot 4.1.0
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/db/builds.py | 184 +++++++++++++++++++++++--------------
buildbot_gentoo_ci/db/connector.py | 6 +-
buildbot_gentoo_ci/db/model.py | 55 ++++-------
buildbot_gentoo_ci/db/packages.py | 113 +++++++++++++++--------
buildbot_gentoo_ci/db/versions.py | 174 ++++++++++++++++++++++-------------
5 files changed, 318 insertions(+), 214 deletions(-)
diff --git a/buildbot_gentoo_ci/db/builds.py b/buildbot_gentoo_ci/db/builds.py
index 21adcc0..47413a3 100644
--- a/buildbot_gentoo_ci/db/builds.py
+++ b/buildbot_gentoo_ci/db/builds.py
@@ -15,7 +15,11 @@
# Copyright Buildbot Team Members
# Origins: buildbot.db.*
# Modifyed by Gentoo Authors.
-# Copyright 2023 Gentoo Authors
+# Copyright 2024 Gentoo Authors
+
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
import uuid
import sqlalchemy as sa
@@ -23,94 +27,134 @@ import sqlalchemy as sa
from twisted.internet import defer
from buildbot.db import base
+from buildbot.warnings import warn_deprecated
+
+if TYPE_CHECKING:
+ import datetime
+
+@dataclass
+class ProjectsBuildsModel:
+ id : int
+ build_id : int
+ project_uuid : str
+ version_uuid : str
+ buildbot_build_id : int
+ bug_id : int
+ status : str
+ requested : bool
+ created_at : datetime.datetime | None
+ updated_at : datetime.datetime | None
+ deleted : bool
+ deleted_at: datetime.datetime | None
+
+ # For backward compatibility
+ def __getitem__(self, key: str):
+ warn_deprecated(
+ '4.1.0',
+ (
+ 'VersionsConnectorComponent getVersionByName, '
+ 'getBuildByNumber, getPrevSuccessfulBuild, '
+ 'getBuildsForChange, getBuilds, '
+ '_getRecentBuilds, and _getBuild '
+ 'no longer return Build as dictionnaries. '
+ 'Usage of [] accessor is deprecated: please access the member directly'
+ ),
+ )
+
+ if hasattr(self, key):
+ return getattr(self, key)
+
+ raise KeyError(key)
+
+def _db2data_ProjectsBuilds(model: ProjectsBuildsModel):
+ if model is None:
+ return None
+ return {
+ 'id' : model.id,
+ 'build_id' : model.build_id,
+ 'project_uuid' : model.project_uuid,
+ 'version_uuid' : model.version_uuid,
+ 'buildbot_build_id' : model.buildbot_build_id,
+ 'bug_id' : model.bug_id,
+ 'status' : model.status,
+ 'requested' : model.requested,
+ 'created_at' : model.created_at,
+ 'updated_at' : model.updated_at,
+ 'deleted' : model.deleted,
+ 'deleted_at' : model.deleted_at
+ }
class BuildsConnectorComponent(base.DBConnectorComponent):
- #@defer.inlineCallbacks
def addBuild(self, project_build_data):
- created_at = int(self.master.reactor.seconds())
- def thd(conn, no_recurse=False):
- tbl = self.db.model.projects_builds
+ def thd(conn):
# get the highest current number
- r = conn.execute(sa.select([sa.func.max(tbl.c.build_id)],
- whereclause=(tbl.c.project_uuid == project_build_data['project_uuid'])))
+ tbl = self.db.model.projects_builds
+ r = conn.execute(
+ sa.select(sa.func.max(tbl.c.build_id)).where(tbl.c.project_uuid == project_build_data['project_uuid'])
+ )
number = r.scalar()
new_number = 1 if number is None else number + 1
+ insert_row = {
+ 'project_uuid' : project_build_data['project_uuid'],
+ 'version_uuid' : project_build_data['version_uuid'],
+ 'status' : project_build_data['status'],
+ 'requested' : project_build_data['requested'],
+ 'created_at' : int(self.master.reactor.seconds()),
+ 'buildbot_build_id' : 0,
+ 'build_id' : new_number,
+ 'bug_id' : 0,
+ }
try:
- q = tbl.insert()
- r = conn.execute(q, dict(project_uuid=project_build_data['project_uuid'],
- version_uuid=project_build_data['version_uuid'],
- status=project_build_data['status'],
- requested=project_build_data['requested'],
- created_at=created_at,
- buildbot_build_id=0,
- build_id=new_number,
- bug_id=0
- ))
+ r = conn.execute(tbl.insert(), insert_row)
+ conn.commit()
except (sa.exc.IntegrityError, sa.exc.ProgrammingError):
- id = None
- new_number = None
- else:
- id = r.inserted_primary_key[0]
- return id, new_number
+ conn.rollback()
+ return None, None
+ return r.inserted_primary_key[0], new_number
return self.db.pool.do(thd)
- @defer.inlineCallbacks
def setStatusBuilds(self, id, status):
- updated_at = int(self.master.reactor.seconds())
- def thd(conn, no_recurse=False):
- tbl = self.db.model.projects_builds
- q = tbl.update()
- q = q.where(tbl.c.id == id)
- conn.execute(q, updated_at=updated_at,
- status=status)
- yield self.db.pool.do(thd)
-
- @defer.inlineCallbacks
+ def thd(conn):
+ tbl = self.db.model.projects_builds
+ q = tbl.update().where(tbl.c.id == id)
+ conn.execute(q.values(updated_at=int(self.master.reactor.seconds()), status=status))
+ return self.db.pool.do(thd)
+
def setBuildbotBuildIdBuilds(self, id, buildbot_build_id):
- updated_at = int(self.master.reactor.seconds())
- def thd(conn, no_recurse=False):
- tbl = self.db.model.projects_builds
- q = tbl.update()
- q = q.where(tbl.c.id == id)
- conn.execute(q, updated_at=updated_at,
- buildbot_build_id=buildbot_build_id)
- yield self.db.pool.do(thd)
-
- @defer.inlineCallbacks
+ def thd(conn):
+ tbl = self.db.model.projects_builds
+ q = tbl.update().where(tbl.c.id == id)
+ conn.execute(q.values(updated_at=int(self.master.reactor.seconds()), buildbot_build_id=buildbot_build_id))
+ return self.db.pool.do(thd)
+
def setBugIdBuilds(self, id, bug_id):
- updated_at = int(self.master.reactor.seconds())
- def thd(conn, no_recurse=False):
- tbl = self.db.model.projects_builds
- q = tbl.update()
- q = q.where(tbl.c.id == id)
- conn.execute(q, updated_at=updated_at, bug_id=bug_id)
- yield self.db.pool.do(thd)
-
- @defer.inlineCallbacks
- def getBuildsByVersionUuid(self, uuid):
def thd(conn):
+ tbl = self.db.model.projects_builds
+ q = tbl.update().where(tbl.c.id == id)
+ conn.execute(q.values(updated_at=int(self.master.reactor.seconds()), bug_id=bug_id))
+ return self.db.pool.do(thd)
+
+ def getBuildsByVersionUuid(self, uuid) -> defer.Deferred[list[ProjectsBuildsModel]]:
+ def thd(conn) -> list[ProjectsBuildsModel]:
tbl = self.db.model.projects_builds
q = tbl.select()
q = q.where(tbl.c.version_uuid == uuid)
res = conn.execute(q)
- row = res.fetchone()
- return [self._row2dict(conn, row)
- for row in conn.execute(q).fetchall()]
- res = yield self.db.pool.do(thd)
- return res
-
- @defer.inlineCallbacks
- def removeBuild(self, id):
- def thd(conn, no_recurse=False):
- tbl = self.db.model.projects_builds
- q = tbl.delete()
- q = q.where(tbl.c.id == id)
- conn.execute(q)
- yield self.db.pool.do(thd)
-
- def _row2dict(self, conn, row):
- return dict(
+ return list(self._model_from_row(row) for row in res.fetchall())
+ return self.db.pool.do(thd)
+
+ def removeBuild(self, id: int) -> defer.Deferred[None]:
+ def thd(conn) -> None:
+ tbl = self.db.model.projects_builds
+ q = tbl.delete().where(tbl.c.id == id)
+ res = conn.execute(q)
+ conn.commit()
+ res.close()
+ return self.db.pool.do(thd)
+
+ def _model_from_row(self, row):
+ return ProjectsBuildsModel(
id=row.id,
build_id=row.build_id,
project_uuid=row.project_uuid,
diff --git a/buildbot_gentoo_ci/db/connector.py b/buildbot_gentoo_ci/db/connector.py
index 0cc7884..65d74fc 100644
--- a/buildbot_gentoo_ci/db/connector.py
+++ b/buildbot_gentoo_ci/db/connector.py
@@ -104,10 +104,8 @@ class DBConnector(service.ReconfigurableServiceMixin,
log.msg(f"Setting up database with URL {repr(util.stripUrlPassword(db_url))}")
# set up the engine and pool
- self._engine = enginestrategy.create_engine(db_url,
- basedir=self.basedir)
- self.pool = pool.DBThreadPool(
- self._engine, reactor=self.master.reactor, verbose=verbose)
+ self._engine = enginestrategy.create_engine(db_url, basedir=self.basedir)
+ self.pool = pool.DBThreadPool(self._engine, reactor=self.master.reactor, verbose=verbose)
# make sure the db is up to date, unless specifically asked not to
if check_version:
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 448e67f..d5364c7 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -17,8 +17,11 @@
# Modifyed by Gentoo Authors.
# Copyright 2024 Gentoo Authors
-import uuid
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+import uuid
import alembic
import alembic.config
import sqlalchemy as sa
@@ -32,26 +35,9 @@ from buildbot.db.migrate_utils import test_unicode
from buildbot.db.types.json import JsonObject
from buildbot.util import sautils
-
-class UpgradeFromBefore0p9Error(Exception):
-
- def __init__(self):
- message = """You are trying to upgrade a buildbot 0.8.x master to buildbot 0.9.x or newer.
- This is not supported. Please start from a clean database
- http://docs.buildbot.net/latest/manual/upgrading/0.9-upgrade.html"""
- # Call the base class constructor with the parameters it needs
- super().__init__(message)
-
-
-class UpgradeFromBefore3p0Error(Exception):
-
- def __init__(self):
- message = """You are trying to upgrade to Buildbot 3.0 or newer from Buildbot 2.x or older.
- This is only supported via an intermediate upgrade to newest Buildbot 2.10.x that is
- available. Please first upgrade to 2.10.x and then try to upgrade to this version.
- http://docs.buildbot.net/latest/manual/upgrading/3.0-upgrade.html"""
- super().__init__(message)
-
+if TYPE_CHECKING:
+ from sqlalchemy.engine.base import Connectable as SQLAConnection
+ from sqlalchemy.engine.reflection import Inspector
class Model(base.DBConnectorComponent):
@@ -364,6 +350,7 @@ class Model(base.DBConnectorComponent):
sa.Column('change_id', sa.Integer, nullable=True, default=0),
sa.Column('deleted', sa.Boolean, default=False),
sa.Column('deleted_at', sa.Integer, nullable=True),
+ sa.Column('created_at', sa.Integer, nullable=True),
)
versions_keywords = sautils.Table(
@@ -496,13 +483,9 @@ class Model(base.DBConnectorComponent):
config_path = util.sibpath(__file__, "migrations/alembic.ini")
- def table_exists(self, conn, table):
- try:
- r = conn.execute(f"select * from {table} limit 1")
- r.close()
- return True
- except Exception:
- return False
+ def table_exists(self, conn: SQLAConnection, table: str):
+ inspector: Inspector = sa.inspect(conn.engine)
+ return inspector.has_table(table)
def migrate_get_version(self, conn):
r = conn.execute("select version from migrate_version limit 1")
@@ -550,19 +533,10 @@ class Model(base.DBConnectorComponent):
alembic_scripts = self.alembic_get_scripts()
current_script_rev_head = alembic_scripts.get_current_head()
- #if self.table_exists(conn, 'version'):
- # raise UpgradeFromBefore0p9Error()
-
if self.table_exists(conn, 'migrate_version'):
version = self.migrate_get_version(conn)
- #if version < 40:
- # raise UpgradeFromBefore0p9Error()
-
- last_sqlalchemy_migrate_version = 0
- if version != last_sqlalchemy_migrate_version:
- raise UpgradeFromBefore3p0Error()
-
+ last_sqlalchemy_migrate_version = 58
self.alembic_stamp(conn, alembic_scripts, alembic_scripts.get_base())
conn.execute('drop table migrate_version')
@@ -576,6 +550,11 @@ class Model(base.DBConnectorComponent):
self.alembic_stamp(conn, alembic_scripts, current_script_rev_head)
return
+ def upgrade(rev, context):
+ log.msg(f'Upgrading from {rev} to {current_script_rev_head}')
+ return alembic_scripts._upgrade_revs(current_script_rev_head, rev)
+
+
context = alembic.runtime.migration.MigrationContext.configure(conn)
current_rev = context.get_current_revision()
diff --git a/buildbot_gentoo_ci/db/packages.py b/buildbot_gentoo_ci/db/packages.py
index ad4a244..2e29c02 100644
--- a/buildbot_gentoo_ci/db/packages.py
+++ b/buildbot_gentoo_ci/db/packages.py
@@ -15,7 +15,11 @@
# Copyright Buildbot Team Members
# Origins: buildbot.db.*
# Modifyed by Gentoo Authors.
-# Copyright 2021 Gentoo Authors
+# Copyright 2024 Gentoo Authors
+
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
import uuid
import sqlalchemy as sa
@@ -23,12 +27,42 @@ import sqlalchemy as sa
from twisted.internet import defer
from buildbot.db import base
+from buildbot.warnings import warn_deprecated
+
+if TYPE_CHECKING:
+ import datetime
+
+@dataclass
+class PackageModel:
+ uuid : str
+ name : str
+ category_uuid : str
+ repository_uuid : str
+ deleted : bool
+ deleted_at : datetime.datetime | None
+
+ # For backward compatibility
+ def __getitem__(self, key: str):
+ if hasattr(self, key):
+ return getattr(self, key)
+ raise KeyError(key)
+
+def _db2data_Package(model: PackageModel):
+ if model is None:
+ return None
+ return {
+ 'uuid' : model.uuid,
+ 'name' : model.name,
+ 'category_uuid' : model.category_uuid,
+ 'repository_uuid' : model.repository_uuid,
+ 'deleted' : model.deleted,
+ 'deleted_at' : model.deleted_at,
+ }
class PackagesConnectorComponent(base.DBConnectorComponent):
- @defer.inlineCallbacks
- def getPackageByName(self, name, c_uuid, repo_uuid, deleted=False):
- def thd(conn):
+ def getPackageByName(self, name: str, c_uuid: str, repo_uuid: str, deleted: bool | None = False) -> defer.Deferred[PackageModel | None]:
+ def thd(conn) -> PackageModel | None:
tbl = self.db.model.packages
q = tbl.select()
q = q.where(tbl.c.name == name)
@@ -37,52 +71,54 @@ class PackagesConnectorComponent(base.DBConnectorComponent):
q = q.where(tbl.c.repository_uuid == repo_uuid)
res = conn.execute(q)
row = res.fetchone()
- if not row:
- return None
- return self._row2dict(conn, row)
- res = yield self.db.pool.do(thd)
- return res
+ rv = None
+ if row:
+ rv = self._model_from_row_PackageModel(row)
+ res.close()
+ return rv
+ return self.db.pool.do(thd)
- @defer.inlineCallbacks
- def getPackageByUuid(self, uuid):
- def thd(conn):
+ def getPackageByUuid(self, uuid: str) -> defer.Deferred[PackageModel | None]:
+ def thd(conn) -> PackageModel | None:
tbl = self.db.model.packages
q = tbl.select()
q = q.where(tbl.c.uuid == uuid)
res = conn.execute(q)
row = res.fetchone()
- if not row:
- return None
- return self._row2dict(conn, row)
- res = yield self.db.pool.do(thd)
- return res
+ rv = None
+ if row:
+ rv = self._model_from_row_PackageModel(row)
+ res.close()
+ return rv
+ return self.db.pool.do(thd)
- @defer.inlineCallbacks
- def addPackage(self, name, repository_uuid, category_uuid):
- def thd(conn, no_recurse=False):
+ def addPackage(self, name: str, repository_uuid: str, category_uuid: str) -> defer.Deferred[str]:
+ def thd(conn) -> str:
+ insert_row = {
+ 'name' : name,
+ 'repository_uuid' : repository_uuid,
+ 'category_uuid' : category_uuid,
+ 'deleted_at' : 0,
+ 'deleted' : False,
+ }
try:
- tbl = self.db.model.packages
- q = tbl.insert()
- r = conn.execute(q, dict(name=name,
- repository_uuid=repository_uuid,
- category_uuid=category_uuid))
- except Exception as e:
- print(type(e))
- print(e.args)
- print(e)
- uuid = None
+ r = conn.execute(self.db.model.packages.insert(), insert_row)
+ conn.commit()
+ except (sa.exc.IntegrityError, sa.exc.ProgrammingError):
+ conn.rollback()
+ return None
else:
- uuid = r.inserted_primary_key[0]
- return uuid
- res = yield self.db.pool.do(thd)
- return res
+ return r.inserted_primary_key[0]
+ return self.db.pool.do(thd)
- def _row2dict(self, conn, row):
- return dict(
+ def _model_from_row_PackageModel(self, row):
+ return PackageModel(
uuid=row.uuid,
name=row.name,
repository_uuid=row.repository_uuid,
- category_uuid=row.category_uuid
+ category_uuid=row.category_uuid,
+ deleted=row.deleted,
+ deleted_at=row.deleted_at,
)
@defer.inlineCallbacks
@@ -209,8 +245,7 @@ class PackagesConnectorComponent(base.DBConnectorComponent):
def delPackageEmail(self, package_uuid):
def thd(conn, no_recurse=False):
tbl = self.db.model.packages_emails
- conn.execute(tbl.delete(
- whereclause=((tbl.c.package_uuid == package_uuid))))
+ conn.execute(tbl.delete().where(tbl.c.package_uuid == package_uuid))
res = yield self.db.pool.do(thd)
return res
diff --git a/buildbot_gentoo_ci/db/versions.py b/buildbot_gentoo_ci/db/versions.py
index afb9e52..60946c5 100644
--- a/buildbot_gentoo_ci/db/versions.py
+++ b/buildbot_gentoo_ci/db/versions.py
@@ -15,7 +15,11 @@
# Copyright Buildbot Team Members
# Origins: buildbot.db.*
# Modifyed by Gentoo Authors.
-# Copyright 2021 Gentoo Authors
+# Copyright 2024 Gentoo Authors
+
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
import uuid
import sqlalchemy as sa
@@ -23,11 +27,48 @@ import sqlalchemy as sa
from twisted.internet import defer
from buildbot.db import base
+from buildbot.warnings import warn_deprecated
+
+if TYPE_CHECKING:
+ import datetime
+
+@dataclass
+class VersionModel:
+ uuid: str
+ name: str
+ package_uuid: str
+ file_hash: str
+ commit_id: str
+ change_id: int
+ created_at: datetime.datetime | None
+ deleted: bool
+ deleted_at: datetime.datetime | None
+
+ # For backward compatibility
+ def __getitem__(self, key: str):
+ if hasattr(self, key):
+ return getattr(self, key)
+ raise KeyError(key)
+
+def _db2data_Version(model: VersionModel):
+ if model is None:
+ return None
+ return {
+ 'uuid' : model.uuid,
+ 'name' : model.name,
+ 'package_uuid' : model.package_uuid,
+ 'file_hash' : model.file_hash,
+ 'commit_id' : model.commit_id,
+ 'change_id' : model.change_id,
+ 'deleted' : model.deleted,
+ 'deleted_at' : model.deleted_at,
+ 'created_at' : model.created_at,
+ }
+
class VersionsConnectorComponent(base.DBConnectorComponent):
- @defer.inlineCallbacks
- def getVersionByName(self, name, p_uuid, deleted=False):
- def thd(conn):
+ def getVersionByName(self, name: str, p_uuid: str, deleted: bool | None = False) -> defer.Deferred[VersionModel | None]:
+ def thd(conn) -> VersionModel | None:
tbl = self.db.model.versions
q = tbl.select()
q = q.where(tbl.c.name == name)
@@ -35,56 +76,57 @@ class VersionsConnectorComponent(base.DBConnectorComponent):
q = q.where(tbl.c.deleted == deleted)
res = conn.execute(q)
row = res.fetchone()
- if not row:
- return None
- return self._row2dict(conn, row)
- res = yield self.db.pool.do(thd)
- return res
-
- @defer.inlineCallbacks
- def getVersionByUuid(self, uuid):
- def thd(conn):
+ rv = None
+ if row:
+ rv = self._model_from_row_VersionModel(row)
+ res.close()
+ return rv
+ return self.db.pool.do(thd)
+
+ def getVersionByUuid(self, uuid: str) -> defer.Deferred[VersionModel | None]:
+ def thd(conn) -> VersionModel | None:
tbl = self.db.model.versions
q = tbl.select()
q = q.where(tbl.c.uuid == uuid)
res = conn.execute(q)
row = res.fetchone()
- if not row:
- return None
- return self._row2dict(conn, row)
- res = yield self.db.pool.do(thd)
- return res
-
- @defer.inlineCallbacks
- def addVersion(self, name, package_uuid, file_hash, commit_id, change_id):
- def thd(conn, no_recurse=False):
+ rv = None
+ if row:
+ rv = self._model_from_row_VersionModel(row)
+ res.close()
+ return rv
+ return self.db.pool.do(thd)
+
+ def addVersion(self, name: str, package_uuid: str, file_hash: str, commit_id: int, change_id: int) -> defer.Deferred[str]:
+ def thd(conn) -> str:
+ insert_row = {
+ 'name': name,
+ 'package_uuid' : package_uuid,
+ 'file_hash' : file_hash,
+ 'commit_id' : commit_id,
+ 'change_id' : change_id,
+ 'created_at' : int(self.master.reactor.seconds()),
+ 'deleted_at' : 0,
+ 'deleted' : False,
+ }
try:
- tbl = self.db.model.versions
- q = tbl.insert()
- r = conn.execute(q, dict(name=name,
- package_uuid=package_uuid,
- file_hash=file_hash,
- commit_id=commit_id,
- change_id=change_id))
+ r = conn.execute(self.db.model.versions.insert(), insert_row)
+ conn.commit()
except (sa.exc.IntegrityError, sa.exc.ProgrammingError):
- uuid = None
+ conn.rollback()
+ return None
else:
- uuid = r.inserted_primary_key[0]
- return uuid
- res = yield self.db.pool.do(thd)
- return res
+ return r.inserted_primary_key[0]
+ return self.db.pool.do(thd)
- @defer.inlineCallbacks
def delVersion(self, uuid):
- deleted_at = int(self.master.reactor.seconds())
- def thd(conn, no_recurse=False):
-
- tbl = self.db.model.versions
- q = tbl.update()
- q = q.where(tbl.c.uuid == uuid)
- conn.execute(q, deleted=True,
- deleted_at=deleted_at)
- yield self.db.pool.do(thd)
+ def thd(conn):
+ tbl = self.db.model.versions
+ q = tbl.update().where(tbl.c.uuid == uuid)
+ res = conn.execute(q.values(deleted=True, deleted_at=int(self.master.reactor.seconds())))
+ #conn.commit()
+ res.close()
+ return self.db.pool.do(thd)
@defer.inlineCallbacks
def addKeyword(self, version_uuid, keyword_id, status):
@@ -95,6 +137,7 @@ class VersionsConnectorComponent(base.DBConnectorComponent):
r = conn.execute(q, dict(version_uuid=version_uuid,
keyword_id=keyword_id,
status=status))
+ conn.commit()
except (sa.exc.IntegrityError, sa.exc.ProgrammingError):
uuid = None
else:
@@ -112,6 +155,7 @@ class VersionsConnectorComponent(base.DBConnectorComponent):
r = conn.execute(q, dict(version_uuid=version_uuid,
metadata=metadata,
value=value))
+ #conn.commit()
except (sa.exc.IntegrityError, sa.exc.ProgrammingError):
id = None
else:
@@ -140,41 +184,44 @@ class VersionsConnectorComponent(base.DBConnectorComponent):
q = tbl.select()
q = q.where(tbl.c.deleted == deleted)
q = q.where(tbl.c.package_uuid == p_uuid)
- return [self._row2dict(conn, row)
+ return [self._model_from_row_VersionModel(row)
for row in conn.execute(q).fetchall()]
res = yield self.db.pool.do(thd)
return res
- @defer.inlineCallbacks
- def removeVersion(self, uuid):
- def thd(conn, no_recurse=False):
+ def removeVersion(self, uuid: str) -> defer.Deferred[None]:
+ def thd(conn) -> None:
tbl = self.db.model.versions
q = tbl.delete()
q = q.where(tbl.c.uuid == uuid)
- conn.execute(q)
- yield self.db.pool.do(thd)
+ res = conn.execute(q)
+ conn.commit()
+ res.close()
+ return self.db.pool.do(thd)
- @defer.inlineCallbacks
- def removeVersionMetadata(self, version_uuid):
- def thd(conn, no_recurse=False):
+ def removeVersionMetadata(self, version_uuid: str) -> defer.Deferred[None]:
+ def thd(conn) -> None:
tbl = self.db.model.versions_metadata
q = tbl.delete()
q = q.where(tbl.c.version_uuid == version_uuid)
- conn.execute(q)
- yield self.db.pool.do(thd)
+ res = conn.execute(q)
+ conn.commit()
+ res.close()
+ return self.db.pool.do(thd)
- @defer.inlineCallbacks
- def removeVersionKeyword(self, version_uuid):
- def thd(conn, no_recurse=False):
+ def removeVersionKeyword(self, version_uuid: str) -> defer.Deferred[None]:
+ def thd(conn) -> None:
tbl = self.db.model.versions_keywords
q = tbl.delete()
q = q.where(tbl.c.version_uuid == version_uuid)
- conn.execute(q)
- yield self.db.pool.do(thd)
+ res = conn.execute(q)
+ conn.commit()
+ res.close()
+ return self.db.pool.do(thd)
- def _row2dict(self, conn, row):
- return dict(
+ def _model_from_row_VersionModel(self, row):
+ return VersionModel(
uuid=row.uuid,
name=row.name,
package_uuid=row.package_uuid,
@@ -182,7 +229,8 @@ class VersionsConnectorComponent(base.DBConnectorComponent):
commit_id=row.commit_id,
change_id = row.change_id,
deleted=row.deleted,
- deleted_at=row.deleted_at
+ deleted_at=row.deleted_at,
+ created_at=row.created_at,
)
def _row2dict_version_metadata(self, conn, row):
^ permalink raw reply related [flat|nested] 9+ messages in thread
end of thread, other threads:[~2024-10-30 21:44 UTC | newest]
Thread overview: 9+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2022-07-28 11:31 [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/ Magnus Granberg
-- strict thread matches above, loose matches on Subject: below --
2024-10-30 21:44 Magnus Granberg
2023-06-26 21:52 Magnus Granberg
2023-05-28 9:49 Magnus Granberg
2023-03-10 22:36 Magnus Granberg
2022-10-03 1:20 Magnus Granberg
2022-08-22 22:54 Magnus Granberg
2021-03-18 23:18 Magnus Granberg
2021-01-10 21:41 Magnus Granberg
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox