* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/, buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/
@ 2021-01-24 23:17 Magnus Granberg
0 siblings, 0 replies; 3+ messages in thread
From: Magnus Granberg @ 2021-01-24 23:17 UTC (permalink / raw
To: gentoo-commits
commit: 5cff75736ed859645d9273888b8c8a4a3f66c07f
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Jan 24 23:17:41 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Jan 24 23:17:41 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=5cff7573
Add SetMakeProfile
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/builders.py | 2 +-
buildbot_gentoo_ci/config/buildfactorys.py | 18 ++++++++--
buildbot_gentoo_ci/config/workers.py | 3 +-
buildbot_gentoo_ci/db/model.py | 11 ++++++
buildbot_gentoo_ci/db/projects.py | 20 +++++++++++
buildbot_gentoo_ci/steps/builders.py | 55 ++++++++++++++++++++++++++++--
buildbot_gentoo_ci/steps/update_db.py | 2 +-
7 files changed, 103 insertions(+), 8 deletions(-)
diff --git a/buildbot_gentoo_ci/config/builders.py b/buildbot_gentoo_ci/config/builders.py
index 69dd840..cba9fcc 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -42,7 +42,7 @@ def gentoo_builders(b=[]):
# Use multiplay workers
b.append(util.BuilderConfig(
name='run_build_request',
- workername='updatedb_1',
+ workername='bot-test',
factory=buildfactorys.run_build_request()
)
)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index e0a6634..a6bafb5 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -88,6 +88,20 @@ def build_request_check():
def run_build_request():
f = util.BuildFactory()
# FIXME: 5
- # Check if all is setup on the worker
- f.addStep(builders.SetupBuildWorker())
+ # update repo for the profile
+ f.addStep(builders.UpdateProfileRepo())
+ # Clean and add new /etc/portage
+ f.addStep(buildbot_steps.RemoveDirectory(dir="portage",
+ workdir='/etc/'))
+ f.addStep(buildbot_steps.MakeDirectory(dir="portage",
+ workdir='/etc/'))
+ # setup the profile
+ f.addStep(buildbot_steps.MakeDirectory(dir="make.profile",
+ workdir='/etc/portage/'))
+ f.addStep(builders.SetMakeProfile())
+ # setup repo.conf dir
+ #f.addStep(buildbot_steps.MakeDirectory(dir="repo.conf",
+ workdir='/etc/portage/'))
+ # check if we have all repository's in repos.conf listed in project_repository's
+ # update all repos listed in project_repository's
return f
diff --git a/buildbot_gentoo_ci/config/workers.py b/buildbot_gentoo_ci/config/workers.py
index 3005129..a566b79 100644
--- a/buildbot_gentoo_ci/config/workers.py
+++ b/buildbot_gentoo_ci/config/workers.py
@@ -1,4 +1,4 @@
-# Copyright 2020 Gentoo Authors
+# Copyright 2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from buildbot.plugins import worker
@@ -6,4 +6,5 @@ from buildbot.plugins import worker
def gentoo_workers(w=[]):
# FIXME: Get workers from db
w.append(worker.LocalWorker('updatedb_1'))
+ w.append(worker.Worker('bot-test', 'test1234'))
return w
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index b051b23..0defb0c 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -131,6 +131,17 @@ class Model(base.DBConnectorComponent):
sa.Column('pkgcheck', sa.Boolean, default=False),
)
+ # projects etc/portage settings
+ projects_portage = sautils.Table(
+ "projects_portage", metadata,
+ sa.Column('id', sa.Integer, primary_key=True),
+ sa.Column('project_uuid', sa.String(36),
+ sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
+ nullable=False),
+ sa.Column('directorys', sa.Enum('make.profile'), nullable=False),
+ sa.Column('value', sa.String(255), nullable=False),
+ )
+
keywords = sautils.Table(
"keywords", metadata,
# unique uuid per keyword
diff --git a/buildbot_gentoo_ci/db/projects.py b/buildbot_gentoo_ci/db/projects.py
index 065c9a0..5c3406a 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -79,6 +79,18 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
res = yield self.db.pool.do(thd)
return res
+ @defer.inlineCallbacks
+ def getProjectPortageByUuidAndDirectory(self, uuid, directory):
+ def thd(conn):
+ tbl = self.db.model.projects_portage
+ q = tbl.select()
+ q = q.where(tbl.c.project_uuid == uuid)
+ q = q.where(tbl.c.directorys == directory)
+ return [self._row2dict_projects_portage(conn, row)
+ for row in conn.execute(q).fetchall()]
+ res = yield self.db.pool.do(thd)
+ return res
+
def _row2dict(self, conn, row):
return dict(
uuid=row.uuid,
@@ -102,3 +114,11 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
auto=row.auto,
pkgcheck=row.pkgcheck
)
+
+ def _row2dict_projects_portage(self, conn, row):
+ return dict(
+ id=row.id,
+ project_uuid=row.project_uuid,
+ directorys=row.directorys,
+ value=row.value
+ )
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 29389a1..1c8cbb0 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -1,6 +1,8 @@
# Copyright 2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+import os
+
from twisted.internet import defer
from twisted.python import log
@@ -32,7 +34,6 @@ class TriggerRunBuildRequest(BuildStep):
'cpv' : self.getProperty("cpv"),
'version_data' : self.getProperty("version_data"),
'projectrepository_data' : self.getProperty('projectrepository_data'),
- 'repository_data' : self.getProperty("repository_data"),
'use_data' : self.getProperty("use_data"),
'fullcheck' : self.getProperty("fullcheck"),
}
@@ -79,9 +80,9 @@ class GetProjectRepositoryData(BuildStep):
yield self.build.addStepsAfterCurrentStep([TriggerRunBuildRequest()])
return SUCCESS
-class SetupBuildWorker(BuildStep):
+class UpdateProfileRepo(BuildStep):
- name = 'SetupBuildWorker'
+ name = 'UpdateProfileRepo'
description = 'Running'
descriptionDone = 'Ran'
descriptionSuffix = None
@@ -89,10 +90,58 @@ class SetupBuildWorker(BuildStep):
flunkOnFailure = True
def __init__(self, **kwargs):
+ # set this in config
+ self.portage_repos_path = '/var/db/repos/'
super().__init__(**kwargs)
@defer.inlineCallbacks
def run(self):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
print('build this %s' % self.getProperty("cpv"))
+ self.setProperty('portage_repos_path', self.portage_repos_path, 'portage_repos_path')
+ projectrepository_data = self.getProperty('projectrepository_data')
+ print(projectrepository_data)
+ repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(projectrepository_data['repository_uuid'])
+ project_data = yield self.gentooci.db.projects.getProjectByUuid(projectrepository_data['project_uuid'])
+ self.setProperty('project_data', project_data, 'project_data')
+ self.profile_repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(project_data['profile_repository_uuid'])
+ profile_repository_path = yield os.path.join(self.portage_repos_path, self.profile_repository_data['name'])
+ yield self.build.addStepsAfterCurrentStep([
+ steps.Git(repourl=self.profile_repository_data['mirror_url'],
+ mode='incremental',
+ submodules=True,
+ workdir=os.path.join(profile_repository_path, ''))
+ ])
+ return SUCCESS
+
+class SetMakeProfile(BuildStep):
+
+ name = 'SetMakeProfile'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ portage_repos_path = self.getProperty('portage_repos_path')
+ project_data = self.getProperty('project_data')
+ profile_repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(project_data['profile_repository_uuid'])
+ makeprofiles_paths = []
+ makeprofiles_data = yield self.gentooci.db.projects.getProjectPortageByUuidAndDirectory(project_data['uuid'], 'make.profile')
+ for makeprofile in makeprofiles_data:
+ makeprofile_path = yield os.path.join(portage_repos_path, profile_repository_data['name'], 'profiles', makeprofile['value'], '')
+ makeprofiles_paths.append('../../..' + makeprofile_path)
+ separator = '\n'
+ makeprofile_path_string = separator.join(makeprofiles_paths)
+ yield self.build.addStepsAfterCurrentStep([
+ steps.StringDownload(makeprofile_path_string + separator,
+ workerdest="make.profile/parent",
+ workdir='/etc/portage/')
+ ])
return SUCCESS
diff --git a/buildbot_gentoo_ci/steps/update_db.py b/buildbot_gentoo_ci/steps/update_db.py
index 3817d44..1ffbbff 100644
--- a/buildbot_gentoo_ci/steps/update_db.py
+++ b/buildbot_gentoo_ci/steps/update_db.py
@@ -124,7 +124,7 @@ class CheckCPVGentooCiProject(BuildStep):
def run(self):
#self.cpv_changes = self.getProperty("cpv_changes")
self.cpv_changes = []
- self.cpv_changes.append('dev-lang/python-3.9.1')
+ self.cpv_changes.append('dev-lang/python-3.9.1-r1')
self.cpv_changes.append('dev-python/scrypt-0.8.16')
print(self.cpv_changes)
print(self.getProperty("repository_data"))
^ permalink raw reply related [flat|nested] 3+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/, buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/
@ 2021-04-12 15:41 Magnus Granberg
0 siblings, 0 replies; 3+ messages in thread
From: Magnus Granberg @ 2021-04-12 15:41 UTC (permalink / raw
To: gentoo-commits
commit: 94751f5a25c610be0faa036cfbaf19702c9c05cb
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Apr 12 15:41:03 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Apr 12 15:41:03 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=94751f5a
Move repo update to new builder and change git_change
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/builders.py | 9 ++
buildbot_gentoo_ci/config/buildfactorys.py | 19 ++-
buildbot_gentoo_ci/config/schedulers.py | 53 ++++----
buildbot_gentoo_ci/db/model.py | 1 +
buildbot_gentoo_ci/db/repositorys.py | 13 +-
buildbot_gentoo_ci/steps/portage.py | 30 +++++
buildbot_gentoo_ci/steps/repos.py | 149 +++++++++++++++++++++
buildbot_gentoo_ci/steps/update_db.py | 203 +++++++----------------------
8 files changed, 292 insertions(+), 185 deletions(-)
diff --git a/buildbot_gentoo_ci/config/builders.py b/buildbot_gentoo_ci/config/builders.py
index 25b180c..d022ac7 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -15,6 +15,15 @@ def gentoo_builders(b=[]):
)
)
# FIXME: get workers from db
+ b.append(util.BuilderConfig(
+ name='update_repo_check',
+ workername='updatedb_1',
+ workerbuilddir='builds',
+ collapseRequests=True,
+ factory=buildfactorys.update_repo_check()
+ )
+ )
+ # FIXME: get workers from db
# Use multiplay workers depend on Property(cpv)
# if cp do not match next one, use diffrent worker then
# or first cp have done its buildsteps.
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index 817b324..962eeb0 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -11,6 +11,7 @@ from buildbot_gentoo_ci.steps import version
from buildbot_gentoo_ci.steps import builders
from buildbot_gentoo_ci.steps import portage
from buildbot_gentoo_ci.steps import logs
+from buildbot_gentoo_ci.steps import repos
def update_db_check():
f = util.BuildFactory()
@@ -18,10 +19,10 @@ def update_db_check():
# Get base project data from db
# return profile_repository, project
f.addStep(update_db.GetDataGentooCiProject())
- # Check if needed path is there
- f.addStep(update_db.CheckPath())
# update the repos
- f.addStep(update_db.UpdateRepos())
+ f.addStep(update_db.TriggerUpdateRepositorys())
+ # Check if needed path is there
+ f.addStep(portage.CheckPathLocal())
# setup the profile
f.addStep(portage.SetMakeProfileLocal())
# setup repos.conf dir
@@ -33,6 +34,18 @@ def update_db_check():
f.addStep(update_db.TriggerCheckForCPV())
return f
+def update_repo_check():
+ f = util.BuildFactory()
+ # FIXME: 6
+ # Check if needed path is there
+ f.addStep(repos.CheckPathRepositoryLocal())
+ # update the repos
+ # FIXME:
+ # use doStepIf so we don't need to do step=profile
+ f.addStep(repos.CheckRepository(step='profile'))
+ f.addStep(repos.CheckRepository())
+ return f
+
def update_db_cp():
f = util.BuildFactory()
# FIXME: 2
diff --git a/buildbot_gentoo_ci/config/schedulers.py b/buildbot_gentoo_ci/config/schedulers.py
index 5be8419..107bb3c 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -13,37 +13,37 @@ def builderUpdateDbNames(props):
@util.renderer
def gitUpdateDb(props):
- git_changes = []
- print(props.changes)
- for k in props.changes:
- change_data = {}
- print(k)
- change_data['cpvs'] = []
- for v in k['files']:
- if v.endswith('.ebuild'):
- c = v.split('/')[0]
- pv = v.split('/')[2][:-7]
- cpv = c + '/' + pv
- print(cpv)
- change_data['cpvs'].append(cpv)
- if k['repository'].endswith('.git'):
- for v in k['repository'].split('/'):
- if v.endswith('.git'):
- change_data['repository'] = v[:-4]
- change_data['author'] = k['author']
- change_data['committer'] = k['committer']
- change_data['comments'] = k['comments']
- change_data['revision'] = k['revision']
- git_changes.append(change_data)
- print(git_changes)
- return git_changes
+ k = props.changes[0]
+ change_data = {}
+ print(k)
+ change_data['cpvs'] = []
+ for v in k['files']:
+ if v.endswith('.ebuild'):
+ c = v.split('/')[0]
+ p = v.split('/')[1]
+ pv = v.split('/')[2][:-7]
+ cpv = c + '/' + pv
+ print(cpv)
+ change_data['cp'] = c + '/' + p
+ change_data['cpvs'].append(cpv)
+ if k['repository'].endswith('.git'):
+ for v in k['repository'].split('/'):
+ if v.endswith('.git'):
+ change_data['repository'] = v[:-4]
+ change_data['author'] = k['author']
+ change_data['committer'] = k['committer']
+ change_data['comments'] = k['comments']
+ change_data['revision'] = k['revision']
+ change_data['timestamp'] =k['when_timestamp']
+ print(change_data)
+ return change_data
def gentoo_schedulers():
scheduler_update_db = schedulers.SingleBranchScheduler(
name='scheduler_update_db',
treeStableTimer=0,
properties = {
- 'git_changes' : gitUpdateDb,
+ 'git_change' : gitUpdateDb,
},
builderNames = builderUpdateDbNames,
change_filter=util.ChangeFilter(branch='master'),
@@ -68,6 +68,8 @@ def gentoo_schedulers():
])
update_cpv_data = schedulers.Triggerable(name="update_cpv_data",
builderNames=["update_cpv_data"])
+ update_repo_check = schedulers.Triggerable(name="update_repo_check",
+ builderNames=["update_repo_check"])
update_v_data = schedulers.Triggerable(name="update_v_data",
builderNames=["update_v_data"])
build_request_data = schedulers.Triggerable(name="build_request_data",
@@ -79,6 +81,7 @@ def gentoo_schedulers():
s = []
s.append(test_updatedb)
s.append(scheduler_update_db)
+ s.append(update_repo_check)
s.append(update_cpv_data)
s.append(update_v_data)
s.append(build_request_data)
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 3abc686..54c964f 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -91,6 +91,7 @@ class Model(base.DBConnectorComponent):
sa.Column('poll_interval', sa.Integer, nullable=False, default=600),
sa.Column('poll_random_delay_min', sa.Integer, nullable=False, default=600),
sa.Column('poll_random_delay_max', sa.Integer, nullable=False, default=600),
+ sa.Column('updated_at', sa.Integer, nullable=True),
)
projects = sautils.Table(
diff --git a/buildbot_gentoo_ci/db/repositorys.py b/buildbot_gentoo_ci/db/repositorys.py
index 6a4ef83..8806dd1 100644
--- a/buildbot_gentoo_ci/db/repositorys.py
+++ b/buildbot_gentoo_ci/db/repositorys.py
@@ -80,6 +80,16 @@ class RepositorysConnectorComponent(base.DBConnectorComponent):
res = yield self.db.pool.do(thd)
return res
+ @defer.inlineCallbacks
+ def updateGitPollerTime(self, uuid):
+ updated_at = int(self.master.reactor.seconds())
+ def thd(conn, no_recurse=False):
+ tbl = self.db.model.repositorys_gitpullers
+ q = tbl.update()
+ q = q.where(tbl.c.repository_uuid == uuid)
+ conn.execute(q, updated_at=updated_at)
+ yield self.db.pool.do(thd)
+
def _row2dict(self, conn, row):
return dict(
uuid=row.uuid,
@@ -101,5 +111,6 @@ class RepositorysConnectorComponent(base.DBConnectorComponent):
branches=row.branches,
poll_interval=row.poll_interval,
poll_random_delay_min=row.poll_random_delay_min,
- poll_random_delay_max=row.poll_random_delay_max
+ poll_random_delay_max=row.poll_random_delay_max,
+ updated_at=row.updated_at
)
diff --git a/buildbot_gentoo_ci/steps/portage.py b/buildbot_gentoo_ci/steps/portage.py
index 7822af1..2308617 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -353,6 +353,36 @@ class SetEnvDefault(BuildStep):
yield self.build.addStepsAfterCurrentStep(aftersteps_list)
return SUCCESS
+class CheckPathLocal(BuildStep):
+
+ name = 'CheckPathLocal'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run(self):
+ self.portage_path = yield os.path.join('etc', 'portage')
+ self.profile_path = yield os.path.join(self.portage_path, 'make.profile')
+ self.repos_path = yield os.path.join(self.portage_path, 'repos.conf')
+ print(os.getcwd())
+ print(self.getProperty("builddir"))
+ yield os.chdir(self.getProperty("builddir"))
+ print(os.getcwd())
+ for x in [
+ self.portage_path,
+ self.profile_path,
+ self.repos_path,
+ ]:
+ if not os.path.isdir(x):
+ os.makedirs(x)
+ return SUCCESS
+
class SetMakeProfileLocal(BuildStep):
name = 'SetMakeProfileLocal'
diff --git a/buildbot_gentoo_ci/steps/repos.py b/buildbot_gentoo_ci/steps/repos.py
new file mode 100644
index 0000000..bc06e45
--- /dev/null
+++ b/buildbot_gentoo_ci/steps/repos.py
@@ -0,0 +1,149 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import os
+import pygit2
+
+from twisted.internet import defer
+
+from buildbot.process.buildstep import BuildStep
+from buildbot.process.results import SUCCESS
+from buildbot.process.results import FAILURE
+from buildbot.process.results import SKIPPED
+from buildbot.plugins import steps
+from buildbot.config import error as config_error
+
+class CheckPathRepositoryLocal(BuildStep):
+
+ name = 'CheckPathRepositoryLocal'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ descriptionSuffix = None
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ # self.repository_basedir = self.gentooci.config.project['mirror_repository_basedir']
+ repository_basedir = '/home/repos2/'
+ self.setProperty("repository_basedir", repository_basedir, 'repository_basedir')
+ if os.path.isdir(repository_basedir):
+ return SUCCESS
+ return FAILURE
+
+class CheckRepository(BuildStep):
+
+ name = 'CheckRepository'
+ description = 'Running'
+ descriptionDone = 'Ran'
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def __init__(self, step=None, **kwargs):
+ self.step = step
+ super().__init__(**kwargs)
+
+ # Origin: https://github.com/MichaelBoselowitz/pygit2-examples/blob/master/examples.py#L54
+ # Modifyed by Gentoo Authors.
+ def gitPull(self, repo, remote_name='origin', branch='master'):
+ for remote in repo.remotes:
+ if remote.name == remote_name:
+ remote.fetch()
+ remote_master_id = repo.lookup_reference('refs/remotes/origin/%s' % (branch)).target
+ print(remote_master_id)
+ merge_result, _ = repo.merge_analysis(remote_master_id)
+ print(merge_result)
+ # Up to date, do nothing
+ if merge_result & pygit2.GIT_MERGE_ANALYSIS_UP_TO_DATE:
+ print('UP_TO_DATE')
+ return None
+ # We can just fastforward
+ elif merge_result & pygit2.GIT_MERGE_ANALYSIS_FASTFORWARD:
+ print('FASTFORWARD')
+ repo.checkout_tree(repo.get(remote_master_id))
+ try:
+ master_ref = repo.lookup_reference('refs/heads/%s' % (branch))
+ master_ref.set_target(remote_master_id)
+ except KeyError:
+ repo.create_branch(branch, repo.get(remote_master_id))
+ repo.head.set_target(remote_master_id)
+ return True
+ elif merge_result & pygit2.GIT_MERGE_ANALYSIS_NORMAL:
+ print('NORMAL')
+ repo.merge(remote_master_id)
+ if repo.index.conflicts is not None:
+ for conflict in repo.index.conflicts:
+ print('Conflicts found in:', conflict[0].path)
+ raise AssertionError('Conflicts, ahhhhh!!')
+
+ user = repo.default_signature
+ tree = repo.index.write_tree()
+ commit = repo.create_commit('HEAD',
+ user,
+ user,
+ 'Merge!',
+ tree,
+ [repo.head.target, remote_master_id])
+ # We need to do this or git CLI will think we are still merging.
+ repo.state_cleanup()
+ return True
+ else:
+ raise AssertionError('Unknown merge analysis result')
+ return True
+
+ @defer.inlineCallbacks
+ def setchmod(self, path):
+ for root, dirs, files in os.walk(path):
+ for d in dirs:
+ yield os.chmod(os.path.join(root, d), 0o0755)
+ for f in files:
+ yield os.chmod(os.path.join(root, f), 0o0644)
+
+ @defer.inlineCallbacks
+ def checkRepos(self, repository_data):
+ repository_path = yield os.path.join(self.getProperty("repository_basedir"), repository_data['name'])
+ repo_path = yield pygit2.discover_repository(repository_path)
+ print(repo_path)
+ if repo_path is None:
+ yield pygit2.clone_repository(repository_data['mirror_url'], repository_path)
+ success = True
+ else:
+ repo = yield pygit2.Repository(repo_path)
+ commit = repo.get(repo.head.target)
+ success = yield self.gitPull(repo)
+ print(commit.hex)
+ print(commit.commit_time)
+ # chmod needed for ebuilds metadata portage.GetAuxMetadata step
+ yield self.setchmod(repository_path)
+ return success
+
+ @defer.inlineCallbacks
+ def run(self):
+ #FIXME: # move som of it to buildfactory update_repo_check
+ if self.step == 'profile':
+ if self.getProperty("profile_repository_uuid") == self.getProperty("repository_uuid"):
+ return SKIPPED
+ repository_uuid = self.getProperty("profile_repository_uuid")
+ else:
+ repository_uuid = self.getProperty("repository_uuid")
+ #---
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(repository_uuid)
+ self.descriptionSuffix = repository_data['name']
+ if repository_data['type'] == 'gitpuller':
+ Poller_data = yield self.gentooci.db.repositorys.getGitPollerByUuid(repository_uuid)
+ print(Poller_data['updated_at'])
+ print(self.getProperty("commit_time"))
+ if Poller_data['updated_at'] > self.getProperty("commit_time"):
+ return SKIPPED
+ success = yield self.checkRepos(repository_data)
+ if success is None:
+ return SKIPPED
+ if not success:
+ return FAILURE
+ if repository_data['type'] == 'gitpuller':
+ yield self.gentooci.db.repositorys.updateGitPollerTime(repository_uuid)
+ return SUCCESS
diff --git a/buildbot_gentoo_ci/steps/update_db.py b/buildbot_gentoo_ci/steps/update_db.py
index 3594c1b..b19d0e7 100644
--- a/buildbot_gentoo_ci/steps/update_db.py
+++ b/buildbot_gentoo_ci/steps/update_db.py
@@ -1,12 +1,7 @@
# Copyright 2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
-import os
-import pygit2
-
-from portage import config as portage_config
from portage.versions import catpkgsplit
-from portage.util import getconfig
from twisted.internet import defer
from twisted.python import log
@@ -34,7 +29,7 @@ class GetDataGentooCiProject(BuildStep):
return FAILURE
print(self.project_data)
print(self.profile_repository_data)
- print(self.getProperty("git_changes"))
+ print(self.getProperty("git_change"))
print(self.getProperty("repository"))
repository = False
self.repository_data = False
@@ -49,133 +44,31 @@ class GetDataGentooCiProject(BuildStep):
self.setProperty("repository_data", self.repository_data, 'repository_data')
return SUCCESS
-class CheckPath(BuildStep):
-
- name = 'CheckPath'
- description = 'Running'
- descriptionDone = 'Ran'
- descriptionSuffix = None
- haltOnFailure = True
- flunkOnFailure = True
-
+class TriggerUpdateRepositorys(BuildStep):
def __init__(self, **kwargs):
super().__init__(**kwargs)
- @defer.inlineCallbacks
- def run(self):
- self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
- self.repository_basedir = self.gentooci.config.project['repository_basedir']
- self.portage_path = yield os.path.join('etc', 'portage')
- self.profile_path = yield os.path.join(self.portage_path, 'make.profile')
- self.repos_path = yield os.path.join(self.portage_path, 'repos.conf')
- print(os.getcwd())
- print(self.getProperty("builddir"))
- yield os.chdir(self.getProperty("builddir"))
- success = True
- print(os.getcwd())
- for x in [
- self.portage_path,
- self.profile_path,
- self.repos_path,
- self.repository_basedir
- ]:
- if not os.path.isdir(x):
- os.makedirs(x)
- return SUCCESS
-
-class UpdateRepos(BuildStep):
-
- name = 'UpdateRepos'
+ name = 'TriggerUpdateRepositorys'
description = 'Running'
descriptionDone = 'Ran'
descriptionSuffix = None
haltOnFailure = True
flunkOnFailure = True
- def __init__(self, **kwargs):
- super().__init__(**kwargs)
-
- # Origin: https://github.com/MichaelBoselowitz/pygit2-examples/blob/master/examples.py#L54
- # Modifyed by Gentoo Authors.
- def gitPull(self, repo, remote_name='origin', branch='master'):
- for remote in repo.remotes:
- if remote.name == remote_name:
- remote.fetch()
- remote_master_id = repo.lookup_reference('refs/remotes/origin/%s' % (branch)).target
- print(remote_master_id)
- merge_result, _ = repo.merge_analysis(remote_master_id)
- print(merge_result)
- # Up to date, do nothing
- if merge_result & pygit2.GIT_MERGE_ANALYSIS_UP_TO_DATE:
- print('UP_TO_DATE')
- return
- # We can just fastforward
- elif merge_result & pygit2.GIT_MERGE_ANALYSIS_FASTFORWARD:
- print('FASTFORWARD')
- repo.checkout_tree(repo.get(remote_master_id))
- try:
- master_ref = repo.lookup_reference('refs/heads/%s' % (branch))
- master_ref.set_target(remote_master_id)
- except KeyError:
- repo.create_branch(branch, repo.get(remote_master_id))
- repo.head.set_target(remote_master_id)
- elif merge_result & pygit2.GIT_MERGE_ANALYSIS_NORMAL:
- print('NORMAL')
- repo.merge(remote_master_id)
- if repo.index.conflicts is not None:
- for conflict in repo.index.conflicts:
- print('Conflicts found in:', conflict[0].path)
- raise AssertionError('Conflicts, ahhhhh!!')
-
- user = repo.default_signature
- tree = repo.index.write_tree()
- commit = repo.create_commit('HEAD',
- user,
- user,
- 'Merge!',
- tree,
- [repo.head.target, remote_master_id])
- # We need to do this or git CLI will think we are still merging.
- repo.state_cleanup()
- else:
- raise AssertionError('Unknown merge analysis result')
-
- @defer.inlineCallbacks
- def setchmod(self, path):
- for root, dirs, files in os.walk(path):
- for d in dirs:
- yield os.chmod(os.path.join(root, d), 0o0755)
- for f in files:
- yield os.chmod(os.path.join(root, f), 0o0644)
-
@defer.inlineCallbacks
def run(self):
- #FIXME check HEAD agenst local and worker local tree so we don't gitpull evrytime
- self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
- #self.repository_basedir = self.gentooci.config.project['repository_basedir']
- self.repository_basedir = yield os.path.join('/home', 'repos2')
- self.profile_repository_path = yield os.path.join(self.repository_basedir, self.getProperty("profile_repository_data")['name'])
- repo_path = yield pygit2.discover_repository(self.profile_repository_path)
- print(repo_path)
- if repo_path is None:
- yield pygit2.clone_repository(self.getProperty("profile_repository_data")['mirror_url'], self.profile_repository_path)
- else:
- repo = yield pygit2.Repository(repo_path)
- yield self.gitPull(repo)
- # chmod and chown
- yield self.setchmod(self.profile_repository_path)
- #yield os.chown(self.profile_repository_path, 'buildbot', 'portage')
- if self.getProperty("profile_repository_data")['name'] != self.getProperty("repository_data")['name']:
- self.repository_path = yield os.path.join(self.repository_basedir, self.getProperty("repository_data")['name'])
- repo_path = yield pygit2.discover_repository(self.repository_path)
- if repo_path is None:
- yield pygit2.clone_repository(self.getProperty("profile_repository_data")['mirror_url'], self.repository_path)
- else:
- repo = yield pygit2.Repository(repo_path)
- yield self.gitPull(repo)
- # chmod and chown
- yield self.setchmod(self.profile_repository_path)
- #yield os.chown(self.repository_path, 'buildbot', 'portage')
+ yield self.build.addStepsAfterCurrentStep([
+ steps.Trigger(
+ schedulerNames=['update_repo_check'],
+ waitForFinish=True,
+ updateSourceStamp=False,
+ set_properties={
+ 'profile_repository_uuid' : self.getProperty("profile_repository_data")['uuid'],
+ 'repository_uuid' : self.getProperty("repository_data")['uuid'],
+ 'commit_time' : self.getProperty("git_change")['timestamp'],
+ }
+ )
+ ])
return SUCCESS
class TriggerCheckForCPV(BuildStep):
@@ -191,43 +84,41 @@ class TriggerCheckForCPV(BuildStep):
@defer.inlineCallbacks
def run(self):
- self.git_changes = self.getProperty("git_changes")
- print(self.git_changes)
- # check if git_change is a string or a list
- if not isinstance(self.git_changes, list):
+ change_data = self.getProperty("git_change")
+ # check if git_change is a dict
+ if not isinstance(change_data, dict):
return FAILURE
addStepUpdateCPVData = []
- for change_data in self.git_changes:
- # make a trigger for all cpv in the list
- for cpv in change_data['cpvs']:
- self.success = True
- if change_data['repository'] != self.getProperty("repository_data")['name']:
- log.msg("%s don't match" % change_data['repository'])
- self.success = False
- # Trigger cpv builds and update db if we are working with ebuilds
- # check that cpv is valied
- if catpkgsplit(cpv) is None:
- log.msg("%s is not vaild package name" % cpv)
- self.success = False
- if self.success:
- revision_data = {}
- revision_data['author'] = change_data['author']
- revision_data['committer'] = change_data['committer']
- revision_data['comments'] = change_data['comments']
- revision_data['revision'] = change_data['revision']
- addStepUpdateCPVData.append(
- steps.Trigger(
- schedulerNames=['update_cpv_data'],
- waitForFinish=False,
- updateSourceStamp=False,
- set_properties={
- 'cpv' : cpv,
- 'project_data' : self.getProperty("project_data"),
- 'repository_data' : self.getProperty("repository_data"),
- 'revision_data' : revision_data,
- }
- )
+ # make a trigger for all cpv in the list
+ for cpv in change_data['cpvs']:
+ self.success = True
+ if change_data['repository'] != self.getProperty("repository_data")['name']:
+ log.msg("%s don't match" % change_data['repository'])
+ self.success = False
+ # Trigger cpv builds and update db if we are working with ebuilds
+ # check that cpv is valied
+ if catpkgsplit(cpv) is None:
+ log.msg("%s is not vaild package name" % cpv)
+ self.success = False
+ if self.success:
+ revision_data = {}
+ revision_data['author'] = change_data['author']
+ revision_data['committer'] = change_data['committer']
+ revision_data['comments'] = change_data['comments']
+ revision_data['revision'] = change_data['revision']
+ addStepUpdateCPVData.append(
+ steps.Trigger(
+ schedulerNames=['update_cpv_data'],
+ waitForFinish=False,
+ updateSourceStamp=False,
+ set_properties={
+ 'cpv' : cpv,
+ 'project_data' : self.getProperty("project_data"),
+ 'repository_data' : self.getProperty("repository_data"),
+ 'revision_data' : revision_data,
+ }
)
+ )
print(addStepUpdateCPVData)
yield self.build.addStepsAfterCurrentStep(addStepUpdateCPVData)
if self.success is False:
^ permalink raw reply related [flat|nested] 3+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/, buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/
@ 2021-09-25 19:51 Magnus Granberg
0 siblings, 0 replies; 3+ messages in thread
From: Magnus Granberg @ 2021-09-25 19:51 UTC (permalink / raw
To: gentoo-commits
commit: dfb70ebe084176e56f39b71545b9bd71196b1c93
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Sep 25 19:51:56 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Sep 25 19:51:56 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=dfb70ebe
Add support for package.use and exclude package to build
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/config/buildfactorys.py | 10 +++------
buildbot_gentoo_ci/db/model.py | 10 ++++-----
buildbot_gentoo_ci/db/projects.py | 24 +++++++++++++++------
buildbot_gentoo_ci/steps/builders.py | 16 +++++++++++++-
buildbot_gentoo_ci/steps/portage.py | 34 ++++++++++++++++++------------
5 files changed, 62 insertions(+), 32 deletions(-)
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index d2e195c..41c3171 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -152,13 +152,9 @@ def run_build_request():
f.addStep(builders.RunEmerge(step='match'))
# Add the needed steps for build
f.addStep(builders.RunBuild())
- # clean up the worker
- # look at the log to see if we need to do stuff
- # run pre-depclean and depclean if set
- f.addStep(builders.RunEmerge(step='pre-depclean'))
- # run preserved-libs and depclean
- f.addStep(builders.RunEmerge(step='preserved-libs'))
- f.addStep(builders.RunEmerge(step='depclean'))
+ # run eclean pkg and dist
+ #f.addStep(builders.RunEclean(step='pkg')
+ #f.addStep(builders.RunEclean(step='dist')
return f
def parse_build_log():
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index e2cbc8d..12d5ee9 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -178,15 +178,15 @@ class Model(base.DBConnectorComponent):
)
# projects etc/portage/package.* settings
- projects_portage_package = sautils.Table(
- "projects_portage_package", metadata,
+ projects_portages_package = sautils.Table(
+ "projects_portages_package", metadata,
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('project_uuid', sa.String(36),
sa.ForeignKey('projects.uuid', ondelete='CASCADE'),
nullable=False),
- sa.Column('directorys', sa.Enum('use', 'accept_keywords', 'env'), nullable=False),
- sa.Column('value1', sa.String(255), nullable=False),
- sa.Column('value2', sa.String(255), nullable=True),
+ sa.Column('directory', sa.Enum('use', 'accept_keywords', 'env', 'exclude'), nullable=False),
+ sa.Column('package', sa.String(255), nullable=False),
+ sa.Column('value', sa.String(255), nullable=True),
)
projects_emerge_options = sautils.Table(
diff --git a/buildbot_gentoo_ci/db/projects.py b/buildbot_gentoo_ci/db/projects.py
index 408450e..48d4288 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -152,6 +152,18 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
res = yield self.db.pool.do(thd)
return res
+ @defer.inlineCallbacks
+ def getProjectPortagePackageByUuidAndExclude(self, uuid):
+ def thd(conn):
+ tbl = self.db.model.projects_portages_package
+ q = tbl.select()
+ q = q.where(tbl.c.project_uuid == uuid)
+ q = q.where(tbl.c.directory == 'exclude')
+ return [self._row2dict_projects_portages_package(conn, row)
+ for row in conn.execute(q).fetchall()]
+ res = yield self.db.pool.do(thd)
+ return res
+
@defer.inlineCallbacks
def getProjectEmergeOptionsByUuid(self, uuid):
def thd(conn):
@@ -263,16 +275,16 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
)
def _row2dict_projects_portages_package(self, conn, row):
- if row.value2 == '':
- value2 = None
+ if row.value == '':
+ value = None
else:
- value2 = row.value2
+ value = row.value
return dict(
id=row.id,
project_uuid=row.project_uuid,
- directorys=row.directorys,
- value1=row.value1,
- value2=value2
+ directory=row.directory,
+ package=row.package,
+ value=value
)
def _row2dict_projects_emerge_options(self, conn, row):
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 7732f39..4f2b1b6 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -13,6 +13,7 @@ from twisted.python import log
from buildbot.process.buildstep import BuildStep
from buildbot.process.results import SUCCESS
from buildbot.process.results import FAILURE
+from buildbot.process.results import SKIPPED
from buildbot.plugins import steps
def PersOutputOfEmerge(rc, stdout, stderr):
@@ -451,9 +452,19 @@ class RunEmerge(BuildStep):
aftersteps_list.append(CheckDepcleanLogs('depclean'))
if self.step == 'match':
+ packages_excludes = yield self.gentooci.db.projects.getProjectPortagePackageByUuidAndExclude(self.getProperty('project_data')['uuid'])
cpv = self.getProperty("cpv")
c = yield catpkgsplit(cpv)[0]
p = yield catpkgsplit(cpv)[1]
+ # Check if package is on the exclude list
+ if packages_excludes != []:
+ for package_exclude in packages_excludes:
+ if '/' not in package_exclude:
+ if package_exclude == p:
+ return SKIPPED
+ else:
+ if package_exclude == c + '/' + p:
+ return SKIPPED
shell_commad_list.append('-pO')
# don't use bin for match
shell_commad_list.append('--usepkg=n')
@@ -900,11 +911,14 @@ class RunBuild(BuildStep):
if not self.getProperty('cpv_build'):
#FIXME:
# trigger pars_build_log if we have any logs to check
- return SUCCESS
+ return SKIPPED
aftersteps_list = []
aftersteps_list.append(RunEmergeInfo())
aftersteps_list.append(RunEmerge(step='pre-build'))
aftersteps_list.append(RunEmerge(step='build'))
+ aftersteps_list.append(RunEmerge(step='pre-depclean'))
+ aftersteps_list.append(RunEmerge(step='preserved-libs'))
+ aftersteps_list.append(RunEmerge(step='depclean'))
self.setProperty('depclean', False, 'depclean')
self.setProperty('preserved_libs', False, 'preserved-libs')
yield self.build.addStepsAfterCurrentStep(aftersteps_list)
diff --git a/buildbot_gentoo_ci/steps/portage.py b/buildbot_gentoo_ci/steps/portage.py
index 03552b6..94323ba 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -252,20 +252,28 @@ class SetPackageDefault(BuildStep):
@defer.inlineCallbacks
def run(self):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
- project_data = self.getProperty('project_data')
- aftersteps_list = []
- packagedir_list = []
- packagedir_list.append('env')
- packagedir_list.append('use')
- #FIXME:
- # get list what dir we need to make from db
- # create the dirs
- for packagedir in packagedir_list:
- aftersteps_list.append(steps.MakeDirectory(dir='package.' + packagedir,
- workdir='/etc/portage/'))
- #FIXME:
+ separator1 = '\n'
+ separator2 = ' '
+ self.aftersteps_list = []
+ #FIXME: accept_keywords, env
# add the needed package.* settings from db
- yield self.build.addStepsAfterCurrentStep(aftersteps_list)
+ package_conf_use_list = []
+ package_settings = yield self.gentooci.db.projects.getProjectPortagePackageByUuid(self.getProperty('project_data')['uuid'])
+ for package_setting in package_settings:
+ if package_setting['directory'] == 'use':
+ package_conf_use_list.append(separator2.join(package_setting['package'],package_setting['value']))
+ if package_conf_use_list != []:
+ package_conf_use_string = separator1.join(package_conf_use_list)
+ self.aftersteps_list.append(
+ steps.StringDownload(package_conf_use_string + separator1,
+ workerdest='default.conf',
+ workdir='/etc/portage/package.use/'
+ )
+ )
+ # create the dir
+ aftersteps_list.append(steps.MakeDirectory(dir='package.use',
+ workdir='/etc/portage/'))
+ yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
return SUCCESS
class SetEnvDefault(BuildStep):
^ permalink raw reply related [flat|nested] 3+ messages in thread
end of thread, other threads:[~2021-09-25 19:51 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2021-09-25 19:51 [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/db/, buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/config/ Magnus Granberg
-- strict thread matches above, loose matches on Subject: below --
2021-04-12 15:41 Magnus Granberg
2021-01-24 23:17 Magnus Granberg
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox