bots: Let make-checkout produce a standalone, temporary directory
Closes #12426
This commit is contained in:
parent
ba04dc970c
commit
c0d69d8ea6
|
@ -103,6 +103,7 @@ depcomp
|
||||||
/bots/images/*.partial
|
/bots/images/*.partial
|
||||||
/bots/images/*.xz
|
/bots/images/*.xz
|
||||||
/bots/images/*.img
|
/bots/images/*.img
|
||||||
|
/bots/make-checkout-workdir
|
||||||
/test/images/*
|
/test/images/*
|
||||||
/test/verify/naughty-*/*
|
/test/verify/naughty-*/*
|
||||||
/test/container-probe*
|
/test/container-probe*
|
||||||
|
|
|
@ -163,11 +163,11 @@ def tasks_for_issues(issues_data):
|
||||||
checklist = github.Checklist(issue["body"])
|
checklist = github.Checklist(issue["body"])
|
||||||
for item, checked in checklist.items.items():
|
for item, checked in checklist.items.items():
|
||||||
if not checked:
|
if not checked:
|
||||||
results.append((item, issue))
|
results.append((item, issue, api.repo))
|
||||||
break
|
break
|
||||||
return results
|
return results
|
||||||
|
|
||||||
def output_task(command, issue, verbose):
|
def output_task(command, issue, repo, verbose):
|
||||||
name, unused, context = command.partition(" ")
|
name, unused, context = command.partition(" ")
|
||||||
if name not in NAMES:
|
if name not in NAMES:
|
||||||
return None
|
return None
|
||||||
|
@ -182,7 +182,9 @@ def output_task(command, issue, verbose):
|
||||||
# `--issues-data` should also be able to receive pull_request events, in that
|
# `--issues-data` should also be able to receive pull_request events, in that
|
||||||
# case pull_request won't be present in the object, but commits will be
|
# case pull_request won't be present in the object, but commits will be
|
||||||
if "pull_request" in issue or "commits" in issue:
|
if "pull_request" in issue or "commits" in issue:
|
||||||
checkout += "bots/make-checkout --verbose pull/{issue}/head && "
|
checkout += "bots/make-checkout --verbose --repo {repo} pull/{issue}/head && "
|
||||||
|
else:
|
||||||
|
checkout += "bots/make-checkout --verbose --repo {repo} master && "
|
||||||
|
|
||||||
if verbose:
|
if verbose:
|
||||||
return "issue-{issue} {name} {context} {priority}".format(
|
return "issue-{issue} {name} {context} {priority}".format(
|
||||||
|
@ -194,11 +196,12 @@ def output_task(command, issue, verbose):
|
||||||
else:
|
else:
|
||||||
if context:
|
if context:
|
||||||
context = pipes.quote(context)
|
context = pipes.quote(context)
|
||||||
return (checkout + cmd).format(
|
return (checkout + "cd bots/make-checkout-workdir && " + cmd + " ; cd ../..").format(
|
||||||
issue=int(number),
|
issue=int(number),
|
||||||
priority=distributed_queue.MAX_PRIORITY,
|
priority=distributed_queue.MAX_PRIORITY,
|
||||||
name=name,
|
name=name,
|
||||||
context=context,
|
context=context,
|
||||||
|
repo=repo,
|
||||||
)
|
)
|
||||||
|
|
||||||
def queue_task(channel, result):
|
def queue_task(channel, result):
|
||||||
|
@ -216,8 +219,8 @@ def scan(issues_data, verbose):
|
||||||
results = [ ]
|
results = [ ]
|
||||||
|
|
||||||
# Now go through each fixture
|
# Now go through each fixture
|
||||||
for (command, issue) in tasks_for_issues(issues_data):
|
for (command, issue, repo) in tasks_for_issues(issues_data):
|
||||||
result = output_task(command, issue, verbose)
|
result = output_task(command, issue, repo, verbose)
|
||||||
if result is not None:
|
if result is not None:
|
||||||
results.append(result)
|
results.append(result)
|
||||||
|
|
||||||
|
|
|
@ -19,6 +19,7 @@
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import os
|
import os
|
||||||
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
from task import github
|
from task import github
|
||||||
|
@ -30,6 +31,7 @@ sys.dont_write_bytecode = True
|
||||||
|
|
||||||
BOTS = os.path.dirname(__file__)
|
BOTS = os.path.dirname(__file__)
|
||||||
BASE = os.path.normpath(os.path.join(BOTS, ".."))
|
BASE = os.path.normpath(os.path.join(BOTS, ".."))
|
||||||
|
TARGET_DIR = os.path.join(BOTS, "make-checkout-workdir")
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
parser = argparse.ArgumentParser(description="Fetch and checkout specific revision")
|
parser = argparse.ArgumentParser(description="Fetch and checkout specific revision")
|
||||||
|
@ -46,54 +48,33 @@ def main():
|
||||||
|
|
||||||
api = github.GitHub(repo=opts.repo)
|
api = github.GitHub(repo=opts.repo)
|
||||||
|
|
||||||
def execute(*args):
|
def execute(*args, cwd=BASE, error_on_fail=True):
|
||||||
|
output = None
|
||||||
if opts.verbose:
|
if opts.verbose:
|
||||||
sys.stderr.write("+ " + " ".join(args) + "\n")
|
sys.stderr.write("+ " + " ".join(args) + "\n")
|
||||||
try:
|
try:
|
||||||
output = subprocess.check_output(args, cwd=BASE, universal_newlines=True)
|
output = subprocess.check_output(args, cwd=cwd, universal_newlines=True)
|
||||||
except subprocess.CalledProcessError as ex:
|
except subprocess.CalledProcessError as ex:
|
||||||
sys.exit(ex.returncode)
|
sys.exit(ex.returncode if error_on_fail else 0)
|
||||||
if opts.verbose and output:
|
finally:
|
||||||
sys.stderr.write("> " + output + "\n")
|
if opts.verbose and output:
|
||||||
|
sys.stderr.write("> " + output + "\n")
|
||||||
return output
|
return output
|
||||||
|
|
||||||
# do this before fetching the actual test repo, so that FETCH_HEAD always points to that at the end
|
if os.path.exists(TARGET_DIR):
|
||||||
if opts.bots_ref:
|
shutil.rmtree(TARGET_DIR)
|
||||||
execute("git", "fetch", "origin", opts.bots_ref)
|
|
||||||
bots_ref = execute("git", "rev-parse", "FETCH_HEAD").strip()
|
|
||||||
else:
|
|
||||||
bots_ref = "origin/master"
|
|
||||||
|
|
||||||
# Testing external project, firstly we needs to add remote
|
cache = os.getenv('XDG_CACHE_HOME', ".")
|
||||||
if api.repo != "cockpit-project/cockpit":
|
execute("git", "clone", "--reference-if-able", "{}/{}".format(cache, api.repo), "https://github.com/{}".format(api.repo), TARGET_DIR)
|
||||||
if "test\n" in subprocess.check_output([ "git", "remote" ], universal_newlines=True):
|
execute("git", "fetch", "origin", opts.ref, cwd=TARGET_DIR, error_on_fail=False)
|
||||||
execute("git", "remote", "remove", "test")
|
execute("git", "checkout", "--detach", opts.revision, cwd=TARGET_DIR, error_on_fail=False)
|
||||||
execute("git", "remote", "add", "test", "https://github.com/" + api.repo)
|
|
||||||
|
|
||||||
# if the pr was updated while the command was in the queue, the revision
|
# get bots/
|
||||||
# won't be tied to the ref anymore, and it won't exist locally, so exit
|
|
||||||
# silently
|
|
||||||
output = None
|
|
||||||
try:
|
|
||||||
args = ["git", "fetch", "test" if api.repo != "cockpit-project/cockpit" else "origin", opts.ref]
|
|
||||||
if opts.verbose:
|
|
||||||
sys.stderr.write("+ {0}\n".format(" ".join(args)))
|
|
||||||
output = subprocess.check_output(args, cwd=BASE, universal_newlines=True)
|
|
||||||
args = ["git", "checkout", "--detach", opts.revision]
|
|
||||||
if opts.verbose:
|
|
||||||
sys.stderr.write("+ {0}\n".format(" ".join(args)))
|
|
||||||
output = subprocess.check_output(args, cwd=BASE, universal_newlines=True)
|
|
||||||
except subprocess.CalledProcessError:
|
|
||||||
return 0
|
|
||||||
finally:
|
|
||||||
if opts.verbose and output:
|
|
||||||
sys.stderr.write("> " + output + "\n")
|
|
||||||
|
|
||||||
# If the bots directory doesn't exist in this branch or repo, check it out from master
|
|
||||||
if api.repo != "cockpit-project/cockpit" or (opts.base and opts.base != "master"):
|
if api.repo != "cockpit-project/cockpit" or (opts.base and opts.base != "master"):
|
||||||
sys.stderr.write("Checking out bots directory from Cockpit %s ...\n" % (opts.bots_ref or bots_ref))
|
execute("git", "remote", "add", "bots_origin", "https://github.com/cockpit-project/cockpit", cwd=TARGET_DIR)
|
||||||
execute("git", "checkout", "--force", bots_ref, "--", "bots/")
|
execute("git", "fetch", "--depth=1", "bots_origin", opts.bots_ref or "master", cwd=TARGET_DIR)
|
||||||
|
bots_ref = execute("git", "rev-parse", "FETCH_HEAD", cwd=TARGET_DIR).strip()
|
||||||
|
execute("git", "checkout", "--force", bots_ref or "bots_origin/master", "--", "bots/", cwd=TARGET_DIR, error_on_fail=False)
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
sys.exit(main())
|
sys.exit(main())
|
||||||
|
|
|
@ -37,8 +37,6 @@ DEVNULL = open("/dev/null", "r+")
|
||||||
def main():
|
def main():
|
||||||
parser = argparse.ArgumentParser(description='Run integration tests')
|
parser = argparse.ArgumentParser(description='Run integration tests')
|
||||||
parser.add_argument('--rebase', help="Rebase onto the specific branch before testing")
|
parser.add_argument('--rebase', help="Rebase onto the specific branch before testing")
|
||||||
parser.add_argument('--remote', help="The Git remote to use (instead of 'origin')",
|
|
||||||
default="origin")
|
|
||||||
parser.add_argument('-o', "--offline", action='store_true',
|
parser.add_argument('-o', "--offline", action='store_true',
|
||||||
help="Work offline, don''t fetch new data from origin for rebase")
|
help="Work offline, don''t fetch new data from origin for rebase")
|
||||||
parser.add_argument('--publish', dest='publish', default=os.environ.get("TEST_PUBLISH", ""),
|
parser.add_argument('--publish', dest='publish', default=os.environ.get("TEST_PUBLISH", ""),
|
||||||
|
@ -58,8 +56,9 @@ def main():
|
||||||
test_branch = os.environ.get("TEST_BRANCH")
|
test_branch = os.environ.get("TEST_BRANCH")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
task = PullTask(name, revision, opts.context, opts.rebase, opts.remote,
|
task = PullTask(name, revision, opts.context, opts.rebase,
|
||||||
test_project=test_project, test_branch=test_branch, html_logs=opts.html_logs)
|
test_project=test_project, test_branch=test_branch,
|
||||||
|
html_logs=opts.html_logs)
|
||||||
ret = task.run(opts)
|
ret = task.run(opts)
|
||||||
except RuntimeError as ex:
|
except RuntimeError as ex:
|
||||||
ret = str(ex)
|
ret = str(ex)
|
||||||
|
@ -70,12 +69,11 @@ def main():
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
class PullTask(object):
|
class PullTask(object):
|
||||||
def __init__(self, name, revision, context, base, remote, test_project, test_branch, html_logs):
|
def __init__(self, name, revision, context, base, test_project, test_branch, html_logs):
|
||||||
self.name = name
|
self.name = name
|
||||||
self.revision = revision
|
self.revision = revision
|
||||||
self.context = context
|
self.context = context
|
||||||
self.base = base
|
self.base = base
|
||||||
self.remote = remote
|
|
||||||
self.test_project = test_project
|
self.test_project = test_project
|
||||||
self.test_branch = test_branch
|
self.test_branch = test_branch
|
||||||
self.html_logs = html_logs
|
self.html_logs = html_logs
|
||||||
|
@ -193,8 +191,8 @@ class PullTask(object):
|
||||||
|
|
||||||
# Include information about which base we're testing against
|
# Include information about which base we're testing against
|
||||||
if self.base:
|
if self.base:
|
||||||
subprocess.check_call([ "git", "fetch", self.remote, self.base ])
|
subprocess.check_call([ "git", "fetch", "origin", self.base ])
|
||||||
commit = subprocess.check_output([ "git", "rev-parse", self.remote + "/" + self.base ],
|
commit = subprocess.check_output([ "git", "rev-parse", "origin/" + self.base ],
|
||||||
universal_newlines=True).strip()
|
universal_newlines=True).strip()
|
||||||
status["base"] = commit
|
status["base"] = commit
|
||||||
|
|
||||||
|
@ -206,7 +204,7 @@ class PullTask(object):
|
||||||
self.sink = sink.Sink(host, identifier, status)
|
self.sink = sink.Sink(host, identifier, status)
|
||||||
|
|
||||||
def rebase(self):
|
def rebase(self):
|
||||||
remote_base = self.remote + "/" + self.base
|
remote_base = "origin" + "/" + self.base
|
||||||
|
|
||||||
# Rebase this branch onto the base, but only if it's not already an ancestor
|
# Rebase this branch onto the base, but only if it's not already an ancestor
|
||||||
try:
|
try:
|
||||||
|
@ -271,7 +269,7 @@ class PullTask(object):
|
||||||
|
|
||||||
# Retrieve information about our base branch and master (for bots/)
|
# Retrieve information about our base branch and master (for bots/)
|
||||||
if self.base and not opts.offline:
|
if self.base and not opts.offline:
|
||||||
subprocess.check_call([ "git", "fetch", self.remote, self.base, "master" ])
|
subprocess.check_call([ "git", "fetch", "origin", self.base, "master" ])
|
||||||
|
|
||||||
# Clean out the test directory
|
# Clean out the test directory
|
||||||
subprocess.check_call([ "git", "clean", "-d", "--force", "--quiet", "-x", "--", "test/" ])
|
subprocess.check_call([ "git", "clean", "-d", "--force", "--quiet", "-x", "--", "test/" ])
|
||||||
|
|
|
@ -157,12 +157,10 @@ def tests_invoke(priority, name, number, revision, ref, context, base, original_
|
||||||
|
|
||||||
if bots_ref:
|
if bots_ref:
|
||||||
checkout += " --bots-ref={bots_ref}"
|
checkout += " --bots-ref={bots_ref}"
|
||||||
else:
|
|
||||||
# we are checking the external repo on a PR of that external repo
|
|
||||||
cmd = "GITHUB_BASE={repo} " + cmd
|
|
||||||
|
|
||||||
if repo != "cockpit-project/cockpit":
|
# The repo of this test differs from the PR's repo
|
||||||
cmd = cmd + " --remote=test"
|
if options.repo != repo:
|
||||||
|
cmd = "GITHUB_BASE={github_base} " + cmd
|
||||||
|
|
||||||
if repo in REPO_EXTRA_INVOKE_OPTIONS:
|
if repo in REPO_EXTRA_INVOKE_OPTIONS:
|
||||||
cmd += " " + " ".join(REPO_EXTRA_INVOKE_OPTIONS[repo])
|
cmd += " " + " ".join(REPO_EXTRA_INVOKE_OPTIONS[repo])
|
||||||
|
@ -179,7 +177,7 @@ def tests_invoke(priority, name, number, revision, ref, context, base, original_
|
||||||
# we are testing the repo itself, checkout revision from the PR
|
# we are testing the repo itself, checkout revision from the PR
|
||||||
checkout += " {ref} {revision} && "
|
checkout += " {ref} {revision} && "
|
||||||
|
|
||||||
return (checkout + cmd).format(
|
return (checkout + "cd bots/make-checkout-workdir && " + cmd + " ; cd ../..").format(
|
||||||
priority=priority,
|
priority=priority,
|
||||||
name=pipes.quote(name),
|
name=pipes.quote(name),
|
||||||
revision=pipes.quote(revision),
|
revision=pipes.quote(revision),
|
||||||
|
@ -190,6 +188,7 @@ def tests_invoke(priority, name, number, revision, ref, context, base, original_
|
||||||
current=current,
|
current=current,
|
||||||
pull_number=number,
|
pull_number=number,
|
||||||
repo=pipes.quote(repo),
|
repo=pipes.quote(repo),
|
||||||
|
github_base=pipes.quote(options.repo),
|
||||||
)
|
)
|
||||||
|
|
||||||
def queue_test(priority, name, number, revision, ref, context, base, original_base, repo, bots_ref, channel, options):
|
def queue_test(priority, name, number, revision, ref, context, base, original_base, repo, bots_ref, channel, options):
|
||||||
|
|
Loading…
Reference in New Issue