Submit patches to builds.sr.ht

hubsrht/blueprints/webhooks.py

@@ -2,9 +2,10 @@ import email
 import json
 from datetime import datetime
 from flask import Blueprint, request, current_app
+from hubsrht.builds import submit_patchset
+from hubsrht.services import todo
 from hubsrht.types import Event, EventType, MailingList, SourceRepo, RepoType
 from hubsrht.types import Tracker, User, Visibility
-from hubsrht.services import todo
 from srht.config import get_origin
 from srht.crypto import verify_request_signature
 from srht.database import db
@@ -163,7 +164,6 @@ def mailing_list(list_id):
         return f"Deleted local:{ml.id}/remote:{ml.remote_id}. Thanks!", 200
     elif event == "post:received":
         event = Event()
-        print(payload)
         sender = payload["sender"]
         if sender:
             sender = current_app.oauth_service.lookup_user(sender['name'])
@@ -192,8 +192,11 @@ def mailing_list(list_id):
         db.session.commit()
         return "Thanks!"
     elif event == "patchset:received":
-        # TODO?
-        return "Thanks!"
+        build_ids = submit_patchset(ml, payload)
+        if build_ids:
+            return f"Submitted builds #{build_ids}. Thanks!"
+        else:
+            return "Thanks!"
     else:
         raise NotImplementedError()
 

hubsrht/builds.py

@@ -0,0 +1,79 @@
+import yaml
+import email.utils
+from srht.config import get_origin
+from hubsrht.services import builds, git
+from hubsrht.types import SourceRepo, RepoType
+from sqlalchemy import func
+
+def submit_patchset(ml, payload):
+    if not get_origin("builds.sr.ht", default=None):
+        return None
+    from buildsrht.manifest import Manifest, Task
+    from buildsrht.manifest import Trigger, TriggerAction, TriggerCondition
+
+    project = ml.project
+    subject = payload["subject"]
+    prefix = payload["prefix"]
+    # TODO: More sophisticated matching is possible
+    # - test if patch is applicable to a repo; see the following:
+    #   https://github.com/libgit2/pygit2/pull/1019
+    # Will be useful for mailing lists shared by many repositories
+    repo = (SourceRepo.query
+            .filter(SourceRepo.project_id == project.id)
+            .filter(func.lower(SourceRepo.name) == prefix.lower())).one_or_none()
+    if repo.repo_type != RepoType.git:
+        # TODO: support for hg.sr.ht
+        return None
+    manifests = git.get_manifests(repo.owner, repo.remote_id)
+    if not manifests:
+        return None
+    # TODO: Add UI to lists.sr.ht indicating build status
+    ids = []
+    for key, value in manifests.items():
+        manifest = Manifest(yaml.safe_load(value))
+        # TODO: https://todo.sr.ht/~sircmpwn/builds.sr.ht/291
+        task = Task({
+            "_apply_patch": f"""echo Applying patch from lists.sr.ht
+git config --global user.name 'builds.sr.ht'
+git config --global user.email builds@sr.ht
+cd {repo.name}
+curl -s {ml.url()}/patches/{payload["id"]}/mbox >/tmp/{payload["id"]}.patch
+git am -3 /tmp/{payload["id"]}.patch"""
+        })
+        manifest.tasks.insert(0, task)
+
+        trigger = next((t for t in manifest.triggers
+            if t.action == TriggerAction.email), None)
+        if not trigger:
+            trigger = Trigger({
+                "action": TriggerAction.email,
+                "condition": TriggerCondition.always,
+            })
+            manifest.triggers.append(trigger)
+        trigger.condition = TriggerCondition.always
+
+        addrs = email.utils.getaddresses(trigger.attrs.get("to", ""))
+        submitter = email.utils.parseaddr(payload["submitter"])
+        if submitter not in addrs:
+            addrs.append(submitter)
+        trigger.attrs["to"] = ", ".join([email.utils.formataddr(a) for a in addrs])
+
+        cc = email.utils.getaddresses(trigger.attrs.get("cc", ""))
+        if not ml.posting_addr() in cc:
+            cc.append(('', ml.posting_addr()))
+        trigger.attrs["cc"] = ", ".join([email.utils.formataddr(a) for a in cc])
+
+        trigger.attrs["in_reply_to"] = payload["message_id"]
+
+        version = payload["version"]
+        if version == 1:
+            version = ""
+        else:
+            version = f" v{version}"
+        b = builds.submit_build(project.owner, manifest,
+        f"""[{subject}][0]{version} from [{submitter[0]}][1]
+
+[0]: {ml.url()}/patches/{payload["id"]}
+[1]: mailto:{submitter[1]}""", tags=[repo.name, "patches", key])
+        ids.append(b["id"])
+    return ids

hubsrht/services.py

@@ -1,5 +1,6 @@
 import os.path
 import requests
+import yaml
 from abc import ABC
 from flask import url_for
 from jinja2 import Markup, escape
@@ -11,6 +12,7 @@ _gitsrht = get_origin("git.sr.ht", external=True, default=None)
 _hgsrht = get_origin("hg.sr.ht", external=True, default=None)
 _listsrht = get_origin("lists.sr.ht", external=True, default=None)
 _todosrht = get_origin("todo.sr.ht", external=True, default=None)
+_buildsrht = get_origin("builds.sr.ht", default=None)
 origin = get_origin("hub.sr.ht")
 
 readme_names = ["README.md", "README.markdown", "README"]
@@ -34,13 +36,38 @@ class SrhtService(ABC):
             headers=get_authorization(user),
             json=payload)
         if r.status_code == 400:
-            for error in r.json()["errors"]:
-                valid.error(error["reason"], field=error.get("field"))
+            if valid:
+                for error in r.json()["errors"]:
+                    valid.error(error["reason"], field=error.get("field"))
             return None
-        elif r.status_code != 201:
+        elif r.status_code not in [200, 201]:
             raise Exception(r.text)
         return r.json()
 
+manifests_query = """
+query Manifests($repoId: Int!) {
+  repository(id: $repoId) {
+    multiple: path(path:".builds") {
+      object {
+        ... on Tree {
+          entries {
+            results {
+              name
+              object { ... on TextBlob { text } }
+            }
+          }
+        }
+      }
+    },
+    single: path(path:".build.yml") {
+      object {
+        ... on TextBlob { text }
+      }
+    }
+  }
+}
+"""
+
 class GitService(SrhtService):
     def __init__(self):
         super().__init__()
@@ -69,6 +96,23 @@ class GitService(SrhtService):
             return format_readme(r.text, readme_name, link_prefix)
         return format_readme("")
 
+    def get_manifests(self, user, repo_id):
+        r = self.post(user, None, f"{_gitsrht}/query", {
+            "query": manifests_query,
+            "variables": {
+                "repoId": repo_id,
+            },
+        })
+        manifests = dict()
+        if r["data"]["repository"]["multiple"]:
+            raise NotImplemented() # TODO
+        elif r["data"]["repository"]["single"]:
+            manifests[".build.yml"] = r["data"]["repository"]["single"]\
+                    ["object"]["text"]
+        else:
+            return None
+        return manifests
+
     def create_repo(self, user, valid, visibility):
         name = valid.require("name")
         description = valid.require("description")
@@ -307,7 +351,17 @@ class TodoService(SrhtService):
             "description": description,
         })
 
+class BuildService(SrhtService):
+    def submit_build(self, user, manifest, note, tags):
+        return self.post(user, None, f"{_buildsrht}/api/jobs", {
+            "manifest": yaml.dump(manifest.to_dict(), default_flow_style=False),
+            "tags": tags,
+            "note": note,
+            "secrets": False,
+        })
+
 git = GitService()
 hg = HgService()
 lists = ListService()
 todo = TodoService()
+builds = BuildService()

hubsrht/types/mailinglist.py

@@ -3,6 +3,7 @@ import sqlalchemy_utils as sau
 from hubsrht.types import Visibility
 from srht.config import get_origin
 from srht.database import Base
+from urllib.parse import urlparse
 
 _listsrht = get_origin("lists.sr.ht", external=True, default=None)
 
@@ -31,3 +32,7 @@ class MailingList(Base):
 
     def url(self):
         return f"{_listsrht}/{self.owner.canonical_name}/{self.name}"
+
+    def posting_addr(self):
+        p = urlparse(_listsrht)
+        return f"{self.owner.canonical_name}/{self.name}@{p.netloc}"