Lubuntu/ci-tooling
8
0
Fork 0
mirror of https://github.com/lubuntu-team/ci-tooling.git synced 2025-05-03 22:01:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Massive cleanup of jobgenerator to allow for more flexible metadata.

Browse Source
This commit is contained in:
Simon Quigley 2020-06-05 12:32:57 -05:00
parent 12de78635b
commit 5f198dc770
2 changed files with 134 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

259
ci/jobgenerator.py
View File

@ -62,6 +62,22 @@ class Generator:
"ci.conf") "ci.conf")
with open(config_file) as metadata_conf_file: with open(config_file) as metadata_conf_file:
metadata_conf = yaml_load(metadata_conf_file, Loader=CLoader) metadata_conf = yaml_load(metadata_conf_file, Loader=CLoader)
# Load all of the active config files and replace the given patch
# with the data from those files
active_configs = {}
for conf in metadata_conf["active_configs"]:
conf_path = path.join(metadata_loc, metadata_repo_name,
conf)
# Replace the string with a dict having all the data
with open(conf_path) as conf_file:
conf_loaded = yaml_load(conf_file, Loader=CLoader)
active_configs[conf.replace(".conf", "")] = conf_loaded
# Since metadata_conf["active_configs"] is a list, we have to use
# a separate dict to store the new data until here
metadata_conf["active_configs"] = active_configs
finally: finally:
if metadata_loc: if metadata_loc:
rmtree(metadata_loc) rmtree(metadata_loc)
@ -78,26 +94,38 @@ class Generator:
shorter and cleaner configuration files. shorter and cleaner configuration files.
""" """
metadata_conf = self.clone_metadata() mdata_conf = self.clone_metadata()
metadata_req_keys = ["name", "packaging_url", mdata_req_keys = ["name", "packaging_url", "packaging_branch",
"packaging_branch_unstable", "upload_target", "releases", "default_branch",
"packaging_branch_stable", "type", "upstream_url", "upstream_branch"]
"upload_target_unstable", "upload_target_stable", mdata_opt_keys = ["upstream_url", "upstream_branch"]
"releases", "default_branch"] mdata_sub_keys = {"NAME": "name"}
metadata_opt_keys = ["upstream_url", "upstream_branch"]
for package in metadata_conf["repositories"]: for config in mdata_conf["active_configs"]:
# Load defaults in if they're not there, ignore the optional ones # Get the data, not just the key
for mkey in metadata_req_keys: config = mdata_conf["active_configs"][config]
if mkey not in package and mkey in metadata_conf["default"]:
package[mkey] = metadata_conf["default"][mkey]
# Don't proceed if any of the keys in the config are invalid
for mkey in package:
if mkey not in metadata_req_keys and mkey not in \
metadata_opt_keys:
raise ValueError("Invalid key present:", mkey)
return metadata_conf["repositories"] # If the config is a merger job, don't bother with it
if config["default"]["type"] == "merger":
continue
for package in config["repositories"]:
# Load defaults in if they're not there, ignore the optionals
for mkey in mdata_req_keys:
if mkey not in package and mkey in config["default"]:
package[mkey] = config["default"][mkey]
# Don't proceed if any of the keys in the config are invalid
for mkey in package:
if mkey not in mdata_req_keys and mkey not in \
mdata_opt_keys:
raise ValueError("Invalid key present:", mkey)
# Substitute keys in
for skey in mdata_sub_keys:
if skey in package[mkey]:
nkey = package[mkey]
nkey = nkey.replace(skey, mdata_sub_keys[skey])
package[mkey] = nkey
return mdata_conf
@timer.run("Auth to Jenkins") @timer.run("Auth to Jenkins")
def auth_jenkins_server(self): def auth_jenkins_server(self):
@ -138,38 +166,43 @@ class Generator:
if data is not None: if data is not None:
url = data["packaging_url"] url = data["packaging_url"]
u_branch = data["packaging_branch_unstable"] branch = data["packaging_branch"]
s_branch = data["packaging_branch_stable"] upload_target = data["upload_target"]
u_upload_target = data["upload_target_unstable"]
s_upload_target = data["upload_target_stable"]
elif job_type != "release-mgmt": elif job_type != "release-mgmt":
raise AttributeError("Data cannot be empty, cannot parse job data.") raise AttributeError("Data cannot be empty, cannot parse job data.")
if job_type.startswith("package"): if job_type.startswith("package"):
upstream = data["upstream_url"] upstream = data["upstream_url"]
package_config = template.render(PACKAGING_URL=url, package_config = template.render(PACKAGING_URL=url,
PACKAGING_BRANCH_U=u_branch, PACKAGING_BRANCH=branch,
PACKAGING_BRANCH_S=s_branch,
UPSTREAM_URL=upstream, UPSTREAM_URL=upstream,
NAME=data["name"], NAME=data["name"],
RELEASE=data["release"], RELEASE=data["release"],
UPLOAD_TARGET_U=u_upload_target, UPLOAD_TARGET=upload_target)
UPLOAD_TARGET_S=s_upload_target)
elif job_type == "merger": elif job_type == "merger":
default_branch = data["default_branch"] # Cascading merges
# HACKY HACKY HACKY cascade = ""
# If we can't push to it, the merger job is useless # Iterate on each value
if not "phab.lubuntu.me" in url: cascading = data["cascade"]
with open(path.join("templates", "useless-merger.xml")) as f: for i in range(len(cascading)):
package_config = "" # The default branch is first, we know this exists
for text in f.readlines(): if i == 0:
package_config += text cascade += "git checkout %s\n" % cascading[0]
else: continue
package_config = template.render(PACKAGING_URL=url, c = cascading[i]
PACKAGING_BRANCH_U=u_branch, # Create branch if it doesn't exist, check it out
PACKAGING_BRANCH_S=s_branch, cascade += "git branch -a | egrep \"remotes/origin/"
NAME=data["name"], cascade += "%s\" && git checkout %s || git " % (c, c)
DEFAULT_BRANCH=default_branch) cascade += "checkout -b %s\n" % c
# Fast-forward merge the previous branch in
cascade += "git merge --ff-only %s\n" % cascading[i-1]
# Push this branch
cascade += "git push --set-upstream origin %s\n" % c
package_config = template.render(PACKAGING_URL=url,
MERGE_COMMANDS=cascade,
NAME=data["name"],
DEFAULT_BRANCH=cascading[0])
elif job_type == "release-mgmt": elif job_type == "release-mgmt":
package_config = template.render() package_config = template.render()
else: else:
@ -177,29 +210,25 @@ class Generator:
return package_config return package_config
@timer.run("Get existing jobs") @timer.run("Create jobs and add to views")
def get_existing_jenkins_jobs(self, server): def create_jenkins_job(self, server, config, name, view):
"""This returns a tuple of all existing Jenkins jobs """This interacts with the Jenkins API to create the job"""
This is separated into a different function to make the code slightly print("Creating %s..." % name)
more efficient and clean. With generators being difficult to work with
and the need for several high-volume variables, this makes sense.
"""
# Get the generator object with the jobs and create an empty list if name in server.keys():
s_jobs = server.get_jobs() job = server.get_job(name)
jobs = [] job.update_config(config)
else:
job = server.create_job(name, str(config))
if view in server.views:
view = server.views[view]
else:
view = server.views.create(view)
# The list from the server is in the following format: # Only add to the view if it's not already in there
# [('JOBNAME', <jenkinsapi.job.Job JOBNAME>)] if not job_name in server.views[view]:
# We only want JOBNAME, so let's put that in jobs view.add_job(job_name)
for job_name in s_jobs:
jobs.append(job_name[0])
# Make sure jobs is a tuple
jobs = tuple(jobs)
return jobs
def create_jenkins_jobs(self): def create_jenkins_jobs(self):
"""Interface with Jenkins to create the jobs required """Interface with Jenkins to create the jobs required
@ -215,92 +244,70 @@ class Generator:
""" """
# Authenticate to the Jenkins server # Authenticate to the Jenkins server
print("Authenticated to Jenkins...")
server = self.auth_jenkins_server() server = self.auth_jenkins_server()
# Parse the metadata # Parse the metadata
print("Parsing the metadata...")
metadata = self.parse_metadata() metadata = self.parse_metadata()
# Get a list of existing jobs
jobs = self.get_existing_jenkins_jobs(server)
total_rel = set() total_rel = set()
for package in metadata: configs = {"merger": [], "stable": [], "unstable": []}
timer.start("Merger job creation") # Sort config names into different categories
# Create the merger jobs first for config in metadata["active_configs"]:
job_name = "merger_" + package["name"] config_name = config
package_config = self.load_config("merger", package) config = metadata["active_configs"][config]
# TODO: This is duplicate code, and it should be consolidated for config_type in configs:
if job_name in jobs: if config["default"]["type"] == config_type:
job = server.get_job(job_name) configs[config_type].append(metadata["active_configs"][config_name].copy())
job.update_config(package_config)
else:
job = server.create_job(job_name, str(package_config))
if "merger" in server.views:
view = server.views["merger"]
else:
view = server.views.create("merger")
view.add_job(job_name)
timer.stop("Merger job creation")
timer.start("Release job creation")
for release in package["releases"]:
# Add the release to the total release set, which is used to
# generate the management jobs
total_rel.add(release)
# Load the config given the current data # Create the merger jobs first
package["release"] = release for config in configs["merger"]:
for jobtype in ["unstable", "stable"]: parent = metadata["active_configs"][config["default"]["parent"]]
job_name = release + "_" + jobtype + "_" + package["name"] for package in parent["repositories"]:
package_config = self.load_config("package-" + jobtype, package["cascade"] = config["default"]["cascade"]
package) name = "merger_" + package["name"]
if job_name in jobs: p_config = self.load_config("merger", package)
job = server.get_job(job_name) self.create_jenkins_job(server, p_config, name, "merger")
job.update_config(str(package_config))
else:
job = server.create_job(job_name, str(package_config))
viewname = release + " " + jobtype # Create the package jobs
if viewname in server.views: for job_type in ["stable", "unstable"]:
view = server.views[viewname] # This is the actual loop
else: for config in configs[job_type]:
view = server.views.create(viewname) # Loop on the individual packages
for package in config["repositories"]:
# Loop on each release
for release in package["releases"]:
# Add the release to the total release set, which is
# used to generate the management jobs
total_rel.add(release)
view.add_job(job_name) package["release"] = release
timer.stop("Release job creation") # Get the package config from the template
p_config = self.load_config("package-" + job_type,
package)
name = "%s_%s_%s" % (release, job_type,
package["name"])
view_name = release + " " + job_type
# Actually create the job
self.create_jenkins_job(server, p_config, name,
view_name)
timer.start("Management job creation")
# From here on out, the same template is used # From here on out, the same template is used
package_config = self.load_config("release-mgmt") p_config = self.load_config("release-mgmt")
# Generate a management job for every release, stable and unstable # Generate a management job for every release, stable and unstable
for release in total_rel: for release in total_rel:
for jobtype in ["unstable", "stable"]: for jobtype in ["stable", "unstable"]:
job_name = "mgmt_build_" + release + "_" + jobtype job_name = "mgmt_build_" + release + "_" + jobtype
if job_name in jobs: self.create_jenkins_job(server, p_config, job_name, "mgmt")
job = server.get_job(job_name)
job.update_config(package_config)
else:
job = server.create_job(job_name, str(package_config))
# The mgmt view should be the first view created, we don't
# have to create it if it doesn't exist because that's a
# Huge Problem anyway
view = server.views["mgmt"]
view.add_job(job_name)
# Generate one last merger management job # Generate one last merger management job
if "merger" in jobs: self.create_jenkins_job(server, p_config, "merger", "mgmt")
job = server.get_job("merger")
job.update_config(package_config)
else:
job = server.create_job("merger", str(package_config))
view = server.views["mgmt"]
view.add_job("merger")
timer.stop("Management job creation")
if __name__ == "__main__": if __name__ == "__main__":

14
templates/merger.xml
View File

@ -12,12 +12,6 @@
</hudson.plugins.git.UserRemoteConfig> </hudson.plugins.git.UserRemoteConfig>
</userRemoteConfigs> </userRemoteConfigs>
<branches> <branches>
<hudson.plugins.git.BranchSpec>
<name>*/{{ PACKAGING_BRANCH_U }}</name>
</hudson.plugins.git.BranchSpec>
<hudson.plugins.git.BranchSpec>
<name>*/{{ PACKAGING_BRANCH_S }}</name>
</hudson.plugins.git.BranchSpec>
<hudson.plugins.git.BranchSpec> <hudson.plugins.git.BranchSpec>
<name>*/{{ DEFAULT_BRANCH }}</name> <name>*/{{ DEFAULT_BRANCH }}</name>
</hudson.plugins.git.BranchSpec> </hudson.plugins.git.BranchSpec>
@ -55,13 +49,7 @@
<hudson.tasks.Shell> <hudson.tasks.Shell>
<command> <command>
cd {{ NAME }} cd {{ NAME }}
git checkout {{ DEFAULT_BRANCH }} {{ MERGE_COMMANDS }}
git branch -a | egrep "remotes/origin/{{ PACKAGING_BRANCH_S }}" &amp;&amp; git checkout {{ PACKAGING_BRANCH_S }} || git checkout -b {{ PACKAGING_BRANCH_S }}
git merge --ff-only {{ DEFAULT_BRANCH }}
git push --set-upstream origin {{ PACKAGING_BRANCH_S }}
git branch -a | egrep "remotes/origin/{{ PACKAGING_BRANCH_U }}" &amp;&amp; git checkout {{ PACKAGING_BRANCH_U }} || git checkout -b {{ PACKAGING_BRANCH_U }}
git merge --ff-only {{ PACKAGING_BRANCH_S }}
git push --set-upstream origin {{ PACKAGING_BRANCH_U }}
</command> </command>
</hudson.tasks.Shell> </hudson.tasks.Shell>
</builders> </builders>