buildsrht: Use GraphQL to submit builds

Use the GraphQL API to submit builds so that job creation webhooks are
delivered.
This commit is contained in:
Adnan Maolood 2022-06-13 16:52:23 -04:00 committed by Drew DeVault
parent 0a2552096c
commit 470032569e
4 changed files with 113 additions and 120 deletions

View File

@ -3,9 +3,10 @@ from srht.api import paginated_response
from srht.config import cfg
from srht.database import db
from srht.flask import csrf_bypass
from srht.graphql import exec_gql
from srht.validation import Validation
from srht.oauth import oauth, current_token
from buildsrht.runner import queue_build, requires_payment
from buildsrht.runner import requires_payment
from buildsrht.types import Artifact, Job, JobStatus, Task, JobGroup
from buildsrht.types import Trigger, TriggerType, TriggerCondition
from buildsrht.manifest import Manifest
@ -40,53 +41,67 @@ def jobs_POST():
"Manifest must be less than {} bytes".format(max_len),
field="manifest")
note = valid.optional("note", cls=str)
read = valid.optional("access:read", ["*"], list)
write = valid.optional("access:write", [current_token.user.username], list)
secrets = valid.optional("secrets", cls=bool, default=True)
secrets = valid.optional("secrets", cls=bool)
tags = valid.optional("tags", [], list)
valid.expect(not valid.ok or all(re.match(r"^[A-Za-z0-9_.-]+$", tag) for tag in tags),
"Invalid tag name, tags must use lowercase alphanumeric characters, underscores, dashes, or dots",
field="tags")
triggers = valid.optional("triggers", list(), list)
execute = valid.optional("execute", True, bool)
execute = valid.optional("execute", cls=bool)
if not valid.ok:
return valid.response
try:
manifest = Manifest(yaml.safe_load(_manifest))
except Exception as ex:
valid.error(str(ex))
resp = exec_gql("builds.sr.ht", """
mutation SubmitBuild(
$manifest: String!,
$note: String,
$tags: [String!],
$secrets: Boolean,
$execute: Boolean,
) {
submit(
manifest: $manifest,
note: $note,
tags: $tags,
secrets: $secrets,
execute: $execute,
) {
id
log {
fullURL
}
tasks {
name
status
log {
fullURL
}
}
note
runner
tags
owner {
canonical_name: canonicalName
... on User {
name: username
}
}
}
}
""", user=current_token.user, valid=valid, manifest=_manifest, note=note, tags=tags, secrets=secrets, execute=execute)
if not valid.ok:
return valid.response
# TODO: access controls
job = Job(current_token.user, _manifest)
job.image = manifest.image
job.note = note
if tags:
job.tags = "/".join(tags)
job.secrets = secrets
db.session.add(job)
db.session.flush()
for task in manifest.tasks:
t = Task(job, task.name)
db.session.add(t)
db.session.flush() # assigns IDs for ordering purposes
for index, trigger in enumerate(triggers):
_valid = Validation(trigger)
action = _valid.require("action", TriggerType)
condition = _valid.require("condition", TriggerCondition)
if not _valid.ok:
_valid.copy(valid, "triggers[{}]".format(index))
return valid.response
# TODO: Validate details based on trigger type
t = Trigger(job)
t.trigger_type = action
t.condition = condition
t.details = json.dumps(trigger)
db.session.add(t)
if execute:
queue_build(job, manifest) # commits the session
else:
db.session.commit()
return job.to_dict()
resp = resp["submit"]
if resp["log"]:
resp["setup_log"] = resp["log"]["fullURL"]
del resp["log"]
resp["tags"] = "/".join(resp["tags"])
for task in resp["tasks"]:
task["status"] = task["status"].lower()
if task["log"]:
task["log"] = task["log"]["fullURL"]
return resp
@api.route("/api/jobs/<int:job_id>")
@oauth("jobs:read")
@ -134,7 +149,11 @@ def jobs_by_id_start_POST(job_id):
{ "reason": "This job is already {}".format(reason_map.get(job.status)) }
]
}, 400
queue_build(job, Manifest(yaml.safe_load(job.manifest)))
exec_gql("builds.sr.ht", """
mutation StartJob($jobId: Int!) {
start(jobID: $jobId) { id }
}
""", user=current_token.user, jobId=job.id)
return { }
@api.route("/api/jobs/<int:job_id>/cancel", methods=["POST"])
@ -151,7 +170,6 @@ def jobs_by_id_cancel_POST(job_id):
@api.route("/api/job-group", methods=["POST"])
@oauth("jobs:write")
def job_group_POST():
# TODO: implement starting the job group right away
valid = Validation(request)
jobs = valid.require("jobs")
valid.expect(not jobs or isinstance(jobs, list) and all(isinstance(j, int) for j in jobs),
@ -165,49 +183,45 @@ def job_group_POST():
if not valid.ok:
return valid.response
job_group = JobGroup()
job_group.note = note
job_group.owner_id = current_token.user_id
db.session.add(job_group)
db.session.flush()
triggers = [{
"type": trigger["action"].upper(),
"condition": trigger["condition"].upper(),
"email": {
"to": trigger["to"],
"cc": trigger.get("cc"),
"inReplyTo": trigger.get("in_reply_to"),
} if trigger["action"] == "email" else None,
"webhook": {
"url": trigger["url"],
} if trigger["action"] == "webhook" else None,
} for trigger in triggers]
for job_id in jobs:
job = Job.query.filter(Job.id == job_id).one_or_none()
valid.expect(job, f"Job ID {job_id} not found")
if not job:
continue
valid.expect(job.status == JobStatus.pending,
f"Job ID {job.id} has already been started; submit jobs with execute=false to create groups")
valid.expect(job.owner_id == current_token.user_id,
f"Job ID {job_id} is not owned by you")
valid.expect(not job.job_group_id,
f"Job ID {job_id} is already assigned to a job group")
job.job_group_id = job_group.id
job_group.jobs.append(job)
for index, trigger in enumerate(triggers):
_valid = Validation(trigger)
action = _valid.require("action", TriggerType)
condition = _valid.require("condition", TriggerCondition)
if not _valid.ok:
_valid.copy(valid, "triggers[{}]".format(index))
return valid.response
t = Trigger(job_group)
t.trigger_type = action
t.condition = condition
t.details = json.dumps(trigger)
db.session.add(t)
resp = exec_gql("builds.sr.ht", """
mutation CreateJobGroup($jobIds: [Int!]!, $triggers: [TriggerInput!], $execute: Boolean, $note: String) {
createGroup(jobIds: $jobIds, triggers: $triggers, execute: $execute, note: $note) {
id
note
owner {
canonical_name: canonicalName
... on User {
name: username
}
}
jobs {
id
status
}
}
}
""", user=current_token.user, valid=valid, jobIds=jobs, triggers=triggers, execute=execute, note=note)
if not valid.ok:
return valid.response
db.session.commit()
if execute:
for job in job_group.jobs:
queue_build(job, Manifest(yaml.safe_load(job.manifest)))
db.session.commit()
return job_group.to_dict()
resp = resp["createGroup"]
for job in resp["jobs"]:
job["status"] = job["status"].lower()
return resp
@api.route("/api/job-group/<int:job_group_id>")
@oauth("jobs:read")

View File

@ -1,7 +1,7 @@
from ansi2html import Ansi2HTMLConverter
from buildsrht.manifest import Manifest
from buildsrht.rss import generate_feed
from buildsrht.runner import queue_build, requires_payment
from buildsrht.runner import submit_build, requires_payment
from buildsrht.search import apply_search
from buildsrht.types import Job, JobStatus, Task, TaskStatus, User
from datetime import datetime, timedelta
@ -249,17 +249,8 @@ def submit_POST():
except Exception as ex:
valid.error(str(ex), field="manifest")
return render_template("submit.html", **valid.kwargs)
job = Job(current_user, _manifest)
job.image = manifest.image
job.note = note
db.session.add(job)
db.session.flush()
for task in manifest.tasks:
t = Task(job, task.name)
db.session.add(t)
db.session.flush() # assigns IDs for ordering purposes
queue_build(job, manifest) # commits the session
return redirect("/~" + current_user.username + "/job/" + str(job.id))
job_id = submit_build(current_user, _manifest, note=note)
return redirect("/~" + current_user.username + "/job/" + str(job_id))
@jobs.route("/cancel/<int:job_id>", methods=["POST"])
@loginrequired

View File

@ -4,6 +4,7 @@ from datetime import datetime
from srht.config import cfg
from srht.database import db
from srht.email import send_email
from srht.graphql import exec_gql
from srht.oauth import UserType
from srht.metrics import RedisQueueCollector
from prometheus_client import Counter
@ -22,20 +23,15 @@ runner = Celery('builds', broker=builds_broker, config_source={
builds_queue_metrics_collector = RedisQueueCollector(builds_broker, "buildsrht_builds", "Number of builds currently in queue")
builds_submitted = Counter("buildsrht_builds_submited", "Number of builds submitted")
def queue_build(job, manifest):
from buildsrht.types import JobStatus
job.status = JobStatus.queued
db.session.commit()
# crypto mining attempt
# pretend to accept it and let the admins know
sample = json.dumps(manifest.to_dict())
if "xuirig" in sample or "miner" in sample or "selci" in sample:
send_email(f"User {job.owner.canonical_name} attempted to submit cryptocurrency mining job #{job.id}",
cfg("sr.ht", "owner-email"),
"Cryptocurrency mining attempt on builds.sr.ht")
else:
builds_submitted.inc()
run_build.delay(job.id, manifest.to_dict())
def submit_build(user, manifest, note=None, tags=[]):
resp = exec_gql("builds.sr.ht", """
mutation SubmitBuild($manifest: String!, $tags: [String!], $note: String) {
submit(manifest: $manifest, tags: $tags, note: $note) {
id
}
}
""", user=user, manifest=manifest, note=note, tags=tags)
return resp["submit"]["id"]
def requires_payment(user):
if allow_free:

View File

@ -1,6 +1,6 @@
#!/usr/bin/env python3
from buildsrht.manifest import Manifest
from buildsrht.runner import queue_build
from buildsrht.runner import submit_build
from buildsrht.types import Job, Task, User
from getopt import getopt, GetoptError
from srht.config import cfg, get_origin
@ -43,18 +43,10 @@ if cmd[0] == "submit":
manifest = Manifest(yaml.safe_load(_manifest))
except Exception as ex:
fail(str(ex))
job = Job(user, _manifest)
job.image = manifest.image
job.note = " ".join([y for x, y in opts if x == "-n"]) or None
job.tags = "/".join([y for x, y in opts if x == "-t"]) or None
db.session.add(job)
db.session.flush()
for task in manifest.tasks:
t = Task(job, task.name)
db.session.add(t)
db.session.flush() # assigns IDs for ordering purposes
queue_build(job, manifest) # commits the session
url = f"{get_origin('builds.sr.ht', external=True)}/~{username}/job/{job.id}"
note = " ".join([y for x, y in opts if x == "-n"]) or None
tags = [y for x, y in opts if x == "-t"] or None
job_id = submit_build(user, _manifest, note=note, tags=tags)
url = f"{get_origin('builds.sr.ht', external=True)}/~{username}/job/{job_id}"
print(url)
else:
fail(f"Unknown command {cmd[0]}")