Rewrite gitsrht-update-hook in Go
This commit is contained in:
Drew DeVault
parent
d2cd785d65
commit
84afd9d7b0
|
|
@ -211,7 +211,7 @@ func main() {
|
|||
push := uuid.New()
|
||||
shellCommand := fmt.Sprintf("%s '%d' '%s' '%s'",
|
||||
shell, userId, username, b64key)
|
||||
fmt.Printf(`restrict,command="%s",environment="SRHT_UID=%d",`+
|
||||
fmt.Printf(`restrict,command="%s",`+
|
||||
`environment="SRHT_PUSH=%s" %s %s %s`+"\n",
|
||||
shellCommand, userId, push.String(), keyType, b64key, username)
|
||||
shellCommand, push.String(), keyType, b64key, username)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,45 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from srht.config import cfg
|
||||
from configparser import ConfigParser
|
||||
from datetime import datetime, timedelta
|
||||
from gitsrht.submit import do_post_update
|
||||
from scmsrht.redis import redis
|
||||
|
||||
op = sys.argv[0]
|
||||
origin = cfg("git.sr.ht", "origin")
|
||||
|
||||
if op == "hooks/update":
|
||||
# Stash updated refs for later processing
|
||||
refname = sys.argv[1]
|
||||
old = sys.argv[2]
|
||||
new = sys.argv[3]
|
||||
|
||||
push_uuid = os.environ.get("SRHT_PUSH")
|
||||
if not push_uuid:
|
||||
sys.exit(0)
|
||||
redis.setex(f"update.{push_uuid}.{refname}",
|
||||
timedelta(minutes=10), f"{old}:{new}")
|
||||
|
||||
if op == "hooks/post-update":
|
||||
refs = sys.argv[1:]
|
||||
|
||||
config = ConfigParser()
|
||||
with open("config") as f:
|
||||
config.read_file(f)
|
||||
|
||||
context = json.loads(os.environ.get("SRHT_PUSH_CTX"))
|
||||
repo = context["repo"]
|
||||
|
||||
if repo["visibility"] == "autocreated":
|
||||
print("\n\t\033[93mNOTICE\033[0m")
|
||||
print("\tWe saved your changes, but this repository does not exist.")
|
||||
print("\tClick here to create it:")
|
||||
print()
|
||||
print("\t{}/create?name={}".format(origin, repo["name"]))
|
||||
print()
|
||||
print("\tYour changes will be discarded in 20 minutes.\n")
|
||||
|
||||
do_post_update(context, refs)
|
||||
|
|
@ -0,0 +1 @@
|
|||
gitsrht-update-hook
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
module git.sr.ht/~sircmpwn/git.sr.ht/gitsrht-update-hook
|
||||
|
||||
go 1.13
|
||||
|
||||
require (
|
||||
github.com/go-redis/redis v6.15.6+incompatible
|
||||
github.com/google/uuid v1.1.1
|
||||
github.com/lib/pq v1.2.0
|
||||
github.com/mattn/go-runewidth v0.0.6
|
||||
github.com/microcosm-cc/bluemonday v1.0.2
|
||||
github.com/pkg/errors v0.8.1
|
||||
github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec
|
||||
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4
|
||||
gopkg.in/src-d/go-git.v4 v4.13.1
|
||||
gopkg.in/yaml.v2 v2.2.7
|
||||
)
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
|
||||
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
|
||||
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
|
||||
github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
|
||||
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
|
||||
github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
|
||||
github.com/go-redis/redis v6.15.6+incompatible h1:H9evprGPLI8+ci7fxQx6WNZHJSb7be8FqJQRhdQZ5Sg=
|
||||
github.com/go-redis/redis v6.15.6+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
|
||||
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
|
||||
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY=
|
||||
github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
|
||||
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/mattn/go-runewidth v0.0.6 h1:V2iyH+aX9C5fsYCpK60U8BYIvmhqxuOL3JZcqc1NB7k=
|
||||
github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
|
||||
github.com/microcosm-cc/bluemonday v1.0.2 h1:5lPfLTTAvAbtS0VqT+94yOtFnGfUWYyx0+iToC3Os3s=
|
||||
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
|
||||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
|
||||
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4=
|
||||
github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec h1:DGmKwyZwEB8dI7tbLt/I/gQuP559o/0FrAkHKlQM/Ks=
|
||||
github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec/go.mod h1:owBmyHYMLkxyrugmfwE/DLJyW8Ro9mkphwuVErQ0iUw=
|
||||
github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70=
|
||||
github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
|
||||
golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
|
||||
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
|
||||
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e h1:D5TXcfTk7xF7hvieo4QErS3qqCB4teTffacDWr7CI+0=
|
||||
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg=
|
||||
gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98=
|
||||
gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g=
|
||||
gopkg.in/src-d/go-git.v4 v4.13.1 h1:SRtFyV8Kxc0UP7aCHcijOMQGPxHSmMOPrzulQWolkYE=
|
||||
gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8=
|
||||
gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
|
||||
gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
|
||||
gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo=
|
||||
gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/vaughan0/go-ini"
|
||||
)
|
||||
|
||||
var (
|
||||
buildOrigin string
|
||||
config ini.File
|
||||
logger *log.Logger
|
||||
origin string
|
||||
pgcs string
|
||||
)
|
||||
|
||||
func main() {
|
||||
log.SetFlags(0)
|
||||
// The update hook is run on the update and post-update git hooks, and also
|
||||
// runs a third stage directly. The first two stages are performance
|
||||
// critical and take place while the user is blocked at their terminal. The
|
||||
// third stage is done in the background.
|
||||
if os.Args[0] == "hooks/update" {
|
||||
update()
|
||||
} else if os.Args[0] == "hooks/post-update" {
|
||||
postUpdate()
|
||||
} else if os.Args[0] == "stage-3" {
|
||||
stage3()
|
||||
} else {
|
||||
log.Fatalf("Unknown git hook %s", os.Args[0])
|
||||
}
|
||||
}
|
||||
|
||||
func init() {
|
||||
logf, err := os.OpenFile("/var/log/gitsrht-update-hook",
|
||||
os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
|
||||
if err != nil {
|
||||
log.Printf("Warning: unable to open log file: %v "+
|
||||
"(using stderr instead)", err)
|
||||
logger = log.New(os.Stderr, os.Args[0]+" ", log.LstdFlags)
|
||||
} else {
|
||||
logger = log.New(logf, os.Args[0]+" ", log.LstdFlags)
|
||||
}
|
||||
|
||||
for _, path := range []string{"../config.ini", "/etc/sr.ht/config.ini"} {
|
||||
config, err = ini.LoadFile(path)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
logger.Fatalf("Failed to load config file: %v", err)
|
||||
}
|
||||
|
||||
var ok bool
|
||||
origin, ok = config.Get("git.sr.ht", "origin")
|
||||
if !ok {
|
||||
logger.Fatalf("No origin configured for git.sr.ht")
|
||||
}
|
||||
pgcs, ok = config.Get("git.sr.ht", "connection-string")
|
||||
if !ok {
|
||||
logger.Fatalf("No connection string configured for git.sr.ht: %v", err)
|
||||
}
|
||||
|
||||
buildOrigin, _ = config.Get("builds.sr.ht", "origin") // Optional
|
||||
}
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
package main
|
||||
|
||||
// TODO: Move this into builds.sr.ht
|
||||
|
||||
import (
|
||||
"gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
type Manifest struct {
|
||||
Arch *string `yaml:"arch",omitempty`
|
||||
Environment map[string]interface{} `yaml:"environment",omitempty`
|
||||
Image string `yaml:"image"`
|
||||
Packages []string `yaml:"packages",omitempty`
|
||||
Repositories map[string]string `yaml:"repositories",omitempty`
|
||||
Secrets []string `yaml:"secrets",omitempty`
|
||||
Shell bool `yaml:"shell",omitempty`
|
||||
Sources []string `yaml:"sources",omitempty`
|
||||
Tasks []map[string]string `yaml:"tasks"`
|
||||
Triggers []map[string]interface{} `yaml:"triggers",omitempty`
|
||||
}
|
||||
|
||||
func ManifestFromYAML(src string) (Manifest, error) {
|
||||
var m Manifest
|
||||
if err := yaml.Unmarshal([]byte(src), &m); err != nil {
|
||||
return m, err
|
||||
}
|
||||
// XXX: We could do validation here, but builds.sr.ht will also catch it
|
||||
// for us later so it's not especially important to
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (manifest Manifest) ToYAML() (string, error) {
|
||||
bytes, err := yaml.Marshal(&manifest)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(bytes), nil
|
||||
}
|
||||
|
|
@ -0,0 +1,273 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
|
||||
goredis "github.com/go-redis/redis"
|
||||
_ "github.com/lib/pq"
|
||||
"gopkg.in/src-d/go-git.v4"
|
||||
"gopkg.in/src-d/go-git.v4/plumbing"
|
||||
"gopkg.in/src-d/go-git.v4/plumbing/object"
|
||||
)
|
||||
|
||||
func printAutocreateInfo(context PushContext) {
|
||||
log.Println("\n\t\033[93mNOTICE\033[0m")
|
||||
log.Println("\tWe saved your changes, but this repository does not exist.")
|
||||
log.Println("\tClick here to create it:")
|
||||
log.Println()
|
||||
log.Printf("\t%s/create?name=%s", origin, context.Repo.Name)
|
||||
log.Println()
|
||||
log.Println("\tYour changes will be discarded in 20 minutes.")
|
||||
log.Println()
|
||||
}
|
||||
|
||||
type DbInfo struct {
|
||||
RepoId int
|
||||
RepoName string
|
||||
Visibility string
|
||||
OwnerUsername string
|
||||
OwnerToken string
|
||||
AsyncWebhooks []WebhookSubscription
|
||||
SyncWebhooks []WebhookSubscription
|
||||
}
|
||||
|
||||
func fetchInfoForPush(db *sql.DB, repoId int) (DbInfo, error) {
|
||||
var dbinfo DbInfo = DbInfo{RepoId: repoId}
|
||||
|
||||
// With this query, we:
|
||||
// 1. Fetch the owner's username and OAuth token
|
||||
// 2. Fetch the repository's name and visibility
|
||||
// 3. Update the repository's mtime
|
||||
// 4. Determine how many webhooks this repo has: if there are zero sync
|
||||
// webhooks then we can defer looking them up until after we've sent the
|
||||
// user on their way.
|
||||
query, err := db.Prepare(`
|
||||
UPDATE repository repo
|
||||
SET updated = NOW() AT TIME ZONE 'UTC'
|
||||
FROM (
|
||||
SELECT "user".username, "user".oauth_token
|
||||
FROM "user"
|
||||
JOIN repository r ON r.owner_id = "user".id
|
||||
WHERE r.id = $1
|
||||
) AS owner, (
|
||||
SELECT
|
||||
COUNT(*) FILTER(WHERE rws.sync = true) sync_count,
|
||||
COUNT(*) FILTER(WHERE rws.sync = false) async_count
|
||||
FROM repo_webhook_subscription rws
|
||||
WHERE rws.repo_id = $1 AND rws.events LIKE '%repo:post-update%'
|
||||
) AS webhooks
|
||||
WHERE repo.id = $1
|
||||
RETURNING
|
||||
repo.name,
|
||||
repo.visibility,
|
||||
owner.username,
|
||||
owner.oauth_token,
|
||||
webhooks.sync_count,
|
||||
webhooks.async_count;
|
||||
`)
|
||||
if err != nil {
|
||||
return dbinfo, err
|
||||
}
|
||||
defer query.Close()
|
||||
|
||||
var nasync, nsync int
|
||||
if err = query.QueryRow(repoId).Scan(&dbinfo.RepoName, &dbinfo.Visibility,
|
||||
&dbinfo.OwnerUsername, &dbinfo.OwnerToken,
|
||||
&nsync, &nasync); err != nil {
|
||||
|
||||
return dbinfo, err
|
||||
}
|
||||
|
||||
dbinfo.AsyncWebhooks = make([]WebhookSubscription, nasync)
|
||||
dbinfo.SyncWebhooks = make([]WebhookSubscription, nsync)
|
||||
if nsync == 0 {
|
||||
// Don't fetch webhooks, we don't need to waste the user's time
|
||||
return dbinfo, nil
|
||||
}
|
||||
|
||||
var rows *sql.Rows
|
||||
if rows, err = db.Query(`
|
||||
SELECT id, url, events
|
||||
FROM repo_webhook_subscription rws
|
||||
WHERE rws.repo_id = $1
|
||||
AND rws.events LIKE '%repo:post-update%'
|
||||
AND rws.sync = true
|
||||
`, repoId); err != nil {
|
||||
|
||||
return dbinfo, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for i := 0; rows.Next(); i++ {
|
||||
var whs WebhookSubscription
|
||||
if err = rows.Scan(&whs.Id, &whs.Url, &whs.Events); err != nil {
|
||||
return dbinfo, err
|
||||
}
|
||||
dbinfo.SyncWebhooks[i] = whs
|
||||
}
|
||||
|
||||
return dbinfo, nil
|
||||
}
|
||||
|
||||
func postUpdate() {
|
||||
var context PushContext
|
||||
refs := os.Args[1:]
|
||||
|
||||
contextJson, ctxOk := os.LookupEnv("SRHT_PUSH_CTX")
|
||||
pushUuid, pushOk := os.LookupEnv("SRHT_PUSH")
|
||||
if !ctxOk || !pushOk {
|
||||
logger.Fatal("Missing required variables in environment, " +
|
||||
"configuration error?")
|
||||
}
|
||||
|
||||
logger.Printf("Running post-update for push %s", pushUuid)
|
||||
|
||||
if err := json.Unmarshal([]byte(contextJson), &context); err != nil {
|
||||
logger.Fatalf("unmarshal SRHT_PUSH_CTX: %v", err)
|
||||
}
|
||||
|
||||
if context.Repo.Visibility == "autocreated" {
|
||||
printAutocreateInfo(context)
|
||||
}
|
||||
|
||||
payload := WebhookPayload{
|
||||
Push: pushUuid,
|
||||
Pusher: context.User,
|
||||
Refs: make([]UpdatedRef, len(refs)),
|
||||
}
|
||||
|
||||
oids := make(map[string]interface{})
|
||||
repo, err := git.PlainOpen(context.Repo.Path)
|
||||
if err != nil {
|
||||
logger.Fatalf("git.PlainOpen: %v", err)
|
||||
}
|
||||
|
||||
db, err := sql.Open("postgres", pgcs)
|
||||
if err != nil {
|
||||
logger.Fatalf("Failed to open a database connection: %v", err)
|
||||
}
|
||||
|
||||
dbinfo, err := fetchInfoForPush(db, context.Repo.Id)
|
||||
if err != nil {
|
||||
logger.Fatalf("Failed to fetch info from database: %v", err)
|
||||
}
|
||||
|
||||
redis := goredis.NewClient(&goredis.Options{Addr: "localhost:6379"})
|
||||
for i, refname := range refs {
|
||||
var oldref, newref string
|
||||
var oldobj, newobj object.Object
|
||||
updateKey := fmt.Sprintf("update.%s.%s", pushUuid, refname)
|
||||
update, err := redis.Get(updateKey).Result()
|
||||
if update == "" || err != nil {
|
||||
logger.Println("redis.Get: missing key")
|
||||
continue
|
||||
} else {
|
||||
parts := strings.Split(update, ":")
|
||||
oldref = parts[0]
|
||||
newref = parts[1]
|
||||
}
|
||||
oldobj, err = repo.Object(plumbing.AnyObject, plumbing.NewHash(oldref))
|
||||
if err == plumbing.ErrObjectNotFound {
|
||||
logger.Printf("old object %s not found", oldref)
|
||||
continue
|
||||
}
|
||||
newobj, err = repo.Object(plumbing.AnyObject, plumbing.NewHash(newref))
|
||||
if err == plumbing.ErrObjectNotFound {
|
||||
logger.Printf("new object %s not found", newref)
|
||||
continue
|
||||
}
|
||||
|
||||
var atag *AnnotatedTag = nil
|
||||
if tag, ok := newobj.(*object.Tag); ok {
|
||||
atag = &AnnotatedTag{
|
||||
Name: tag.Name,
|
||||
Message: tag.Message,
|
||||
}
|
||||
newobj, err = repo.CommitObject(tag.Target)
|
||||
if err != nil {
|
||||
logger.Println("unresolvable annotated tag")
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
oldcommit, ok := oldobj.(*object.Commit)
|
||||
if !ok {
|
||||
logger.Println("Skipping non-commit old ref")
|
||||
continue
|
||||
}
|
||||
commit, ok := newobj.(*object.Commit)
|
||||
if !ok {
|
||||
logger.Println("Skipping non-commit new ref")
|
||||
continue
|
||||
}
|
||||
|
||||
payload.Refs[i] = UpdatedRef{
|
||||
Tag: atag,
|
||||
Name: refname,
|
||||
Old: GitCommitToWebhookCommit(oldcommit),
|
||||
New: GitCommitToWebhookCommit(commit),
|
||||
}
|
||||
|
||||
if _, ok := oids[commit.Hash.String()]; ok {
|
||||
continue
|
||||
}
|
||||
oids[commit.Hash.String()] = nil
|
||||
|
||||
if buildOrigin != "" {
|
||||
submitter := GitBuildSubmitter{
|
||||
BuildOrigin: buildOrigin,
|
||||
Commit: commit,
|
||||
GitOrigin: origin,
|
||||
OwnerName: dbinfo.OwnerUsername,
|
||||
OwnerToken: dbinfo.OwnerToken,
|
||||
RepoName: dbinfo.RepoName,
|
||||
Repository: repo,
|
||||
Visibility: dbinfo.Visibility,
|
||||
}
|
||||
results, err := SubmitBuild(submitter)
|
||||
if err != nil {
|
||||
log.Fatalf("Error submitting build job: %v", err)
|
||||
}
|
||||
if len(results) == 0 {
|
||||
continue
|
||||
} else if len(results) == 1 {
|
||||
log.Println("\033[1mBuild started:\033[0m")
|
||||
} else {
|
||||
log.Println("\033[1mBuilds started:\033[0m")
|
||||
}
|
||||
logger.Printf("Submitted %d builds for %s",
|
||||
len(results), refname)
|
||||
for _, result := range results {
|
||||
log.Printf("\033[94m%s\033[0m [%s]", result.Url, result.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
payloadBytes, err := json.Marshal(&payload)
|
||||
if err != nil {
|
||||
logger.Fatalf("Failed to marshal webhook payload: %v", err)
|
||||
}
|
||||
|
||||
deliveries := deliverWebhooks(dbinfo.SyncWebhooks, payloadBytes)
|
||||
deliveriesJson, err := json.Marshal(deliveries)
|
||||
if err != nil {
|
||||
logger.Fatalf("Failed to marshal webhook deliveries: %v", err)
|
||||
}
|
||||
|
||||
hook, ok := config.Get("git.sr.ht", "post-update-script")
|
||||
if !ok {
|
||||
logger.Fatal("No post-update script configured, cannot run stage 3")
|
||||
}
|
||||
|
||||
// Run stage 3 asyncronously - the last few tasks can be done without
|
||||
// blocking the pusher's terminal.
|
||||
stage3 := exec.Command(hook, string(deliveriesJson), string(payloadBytes))
|
||||
stage3.Args[0] = "stage-3"
|
||||
stage3.Start()
|
||||
}
|
||||
|
|
@ -0,0 +1,83 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"os"
|
||||
|
||||
_ "github.com/lib/pq"
|
||||
)
|
||||
|
||||
func stage3() {
|
||||
var context PushContext
|
||||
contextJson, ctxOk := os.LookupEnv("SRHT_PUSH_CTX")
|
||||
pushUuid, pushOk := os.LookupEnv("SRHT_PUSH")
|
||||
if !ctxOk || !pushOk {
|
||||
logger.Fatal("Missing required variables in environment, " +
|
||||
"configuration error?")
|
||||
}
|
||||
|
||||
logger.Printf("Running stage 3 for push %s", pushUuid)
|
||||
|
||||
if err := json.Unmarshal([]byte(contextJson), &context); err != nil {
|
||||
logger.Fatalf("unmarshal SRHT_PUSH_CTX: %v", err)
|
||||
}
|
||||
|
||||
db, err := sql.Open("postgres", pgcs)
|
||||
if err != nil {
|
||||
logger.Fatalf("Failed to open a database connection: %v", err)
|
||||
}
|
||||
|
||||
var subscriptions []WebhookSubscription
|
||||
var deliveries []WebhookDelivery
|
||||
if err := json.Unmarshal([]byte(os.Args[1]), &deliveries); err != nil {
|
||||
logger.Fatalf("Unable to unmarhsal delivery array: %v", err)
|
||||
}
|
||||
payload := []byte(os.Args[2])
|
||||
|
||||
var rows *sql.Rows
|
||||
if rows, err = db.Query(`
|
||||
SELECT id, url, events
|
||||
FROM repo_webhook_subscription rws
|
||||
WHERE rws.repo_id = $1
|
||||
AND rws.events LIKE '%repo:post-update%'
|
||||
AND rws.sync = false`, context.Repo.Id); err != nil {
|
||||
logger.Fatalf("Error fetching webhooks: %v", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for i := 0; rows.Next(); i++ {
|
||||
var whs WebhookSubscription
|
||||
if err = rows.Scan(&whs.Id, &whs.Url, &whs.Events); err != nil {
|
||||
logger.Fatalf("Scanning webhook rows: %v", err)
|
||||
}
|
||||
subscriptions = append(subscriptions, whs)
|
||||
}
|
||||
|
||||
deliveries = append(deliveries, deliverWebhooks(subscriptions, payload)...)
|
||||
for _, delivery := range deliveries {
|
||||
if _, err := db.Exec(`
|
||||
INSERT INTO repo_webhook_delivery (
|
||||
uuid,
|
||||
created,
|
||||
event,
|
||||
url,
|
||||
payload,
|
||||
payload_headers,
|
||||
response,
|
||||
response_status,
|
||||
response_headers,
|
||||
subscription_id
|
||||
) VALUES (
|
||||
$1, NOW() AT TIME ZONE 'UTC', 'repo:post-update',
|
||||
$2, $3, $4, $5, $6, $7
|
||||
);
|
||||
`, delivery.UUID, delivery.Url,
|
||||
delivery.Payload, delivery.Headers,
|
||||
delivery.Response, delivery.ResponseStatus, delivery.ResponseHeaders,
|
||||
delivery.SubscriptionId); err != nil {
|
||||
|
||||
logger.Fatalf("Error inserting webhook delivery: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,265 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"path"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/microcosm-cc/bluemonday"
|
||||
"github.com/pkg/errors"
|
||||
"gopkg.in/src-d/go-git.v4"
|
||||
"gopkg.in/src-d/go-git.v4/plumbing/object"
|
||||
)
|
||||
|
||||
type BuildSubmitter interface {
|
||||
// Return a list of build manifests and their names
|
||||
FindManifests() (map[string]string, error)
|
||||
// Get builds.sr.ht origin
|
||||
GetBuildsOrigin() string
|
||||
// Get builds.sr.ht OAuth token
|
||||
GetOauthToken() string
|
||||
// Get a checkout-able string to append to matching source URLs
|
||||
GetCommitId() string
|
||||
// Get the build note which corresponds to this commit
|
||||
GetCommitNote() string
|
||||
// Get the clone URL for this repository
|
||||
GetCloneUrl() string
|
||||
// Get the name of the repository
|
||||
GetRepoName() string
|
||||
// Get the name of the repository owner
|
||||
GetOwnerName() string
|
||||
}
|
||||
|
||||
// SQL notes
|
||||
//
|
||||
// We need:
|
||||
// - The repo ID
|
||||
// - The repo name & visibility
|
||||
// - The owner's username & canonical name
|
||||
// - The owner's OAuth token & scopes
|
||||
// - A list of affected webhooks
|
||||
type GitBuildSubmitter struct {
|
||||
BuildOrigin string
|
||||
Commit *object.Commit
|
||||
GitOrigin string
|
||||
OwnerName string
|
||||
OwnerToken string
|
||||
RepoName string
|
||||
Repository *git.Repository
|
||||
Visibility string
|
||||
}
|
||||
|
||||
func (submitter GitBuildSubmitter) FindManifests() (map[string]string, error) {
|
||||
tree, err := submitter.Repository.TreeObject(submitter.Commit.TreeHash)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "lookup tree failed")
|
||||
}
|
||||
|
||||
var files []*object.File
|
||||
file, err := tree.File(".build.yml")
|
||||
if err == nil {
|
||||
files = append(files, file)
|
||||
} else {
|
||||
subtree, err := tree.Tree(".builds")
|
||||
if err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
entries := subtree.Files()
|
||||
for {
|
||||
file, err = entries.Next()
|
||||
if file == nil || err != nil {
|
||||
break;
|
||||
}
|
||||
if strings.HasSuffix(file.Name, ".yml") {
|
||||
files = append(files, file)
|
||||
}
|
||||
}
|
||||
if err != io.EOF {
|
||||
return nil, errors.Wrap(err, "EOF finding build manifest")
|
||||
}
|
||||
}
|
||||
|
||||
manifests := make(map[string]string)
|
||||
for _, file := range files {
|
||||
var (
|
||||
reader io.Reader
|
||||
content []byte
|
||||
)
|
||||
if reader, err = file.Reader(); err != nil {
|
||||
return nil, errors.Wrapf(err, "creating reader for %s", file.Name)
|
||||
}
|
||||
if content, err = ioutil.ReadAll(reader); err != nil {
|
||||
return nil, errors.Wrap(err, "reading build manifest")
|
||||
}
|
||||
if !utf8.Valid(content) {
|
||||
return nil, errors.Wrap(err, "manifest is not valid UTF-8 file")
|
||||
}
|
||||
manifests[file.Name] = string(content)
|
||||
}
|
||||
return manifests, nil
|
||||
}
|
||||
|
||||
func (submitter GitBuildSubmitter) GetBuildsOrigin() string {
|
||||
return submitter.BuildOrigin
|
||||
}
|
||||
|
||||
func (submitter GitBuildSubmitter) GetOauthToken() string {
|
||||
return submitter.OwnerToken
|
||||
}
|
||||
|
||||
func (submitter GitBuildSubmitter) GetCommitId() string {
|
||||
return submitter.Commit.Hash.String()
|
||||
}
|
||||
|
||||
func firstLine(text string) string {
|
||||
buf := bytes.NewBufferString(text)
|
||||
scanner := bufio.NewScanner(buf)
|
||||
if !scanner.Scan() {
|
||||
return ""
|
||||
}
|
||||
return scanner.Text()
|
||||
}
|
||||
|
||||
func (submitter GitBuildSubmitter) GetCommitNote() string {
|
||||
policy := bluemonday.StrictPolicy()
|
||||
commitUrl := fmt.Sprintf("%s/%s/%s/commit/%s", submitter.GitOrigin,
|
||||
submitter.OwnerName, submitter.RepoName,
|
||||
submitter.GetCommitId())
|
||||
return fmt.Sprintf(`[%s](%s) — [%s](mailto:%s)\n\n<pre>%s</pre>`,
|
||||
submitter.GetCommitId()[:7], commitUrl,
|
||||
submitter.Commit.Author.Name, submitter.Commit.Author.Email,
|
||||
policy.Sanitize(firstLine(submitter.Commit.Message)))
|
||||
}
|
||||
|
||||
func (submitter GitBuildSubmitter) GetCloneUrl() string {
|
||||
if submitter.Visibility == "private" {
|
||||
origin := strings.ReplaceAll(submitter.GitOrigin, "http://", "")
|
||||
origin = strings.ReplaceAll(origin, "https://", "")
|
||||
// Use SSH URL
|
||||
return fmt.Sprintf("git+ssh://git@%s/~%s/%s", origin,
|
||||
submitter.OwnerName, submitter.RepoName)
|
||||
} else {
|
||||
// Use HTTP(s) URL
|
||||
return fmt.Sprintf("%s/~%s/%s", submitter.GitOrigin,
|
||||
submitter.OwnerName, submitter.RepoName)
|
||||
}
|
||||
}
|
||||
|
||||
func (submitter GitBuildSubmitter) GetRepoName() string {
|
||||
return submitter.RepoName
|
||||
}
|
||||
|
||||
func (submitter GitBuildSubmitter) GetOwnerName() string {
|
||||
return submitter.OwnerName
|
||||
}
|
||||
|
||||
type BuildSubmission struct {
|
||||
// TODO: Move errors into this struct and set up per-submission error
|
||||
// tracking
|
||||
Name string
|
||||
Url string
|
||||
}
|
||||
|
||||
// TODO: Move this to scm.sr.ht
|
||||
func SubmitBuild(submitter BuildSubmitter) ([]BuildSubmission, error) {
|
||||
manifests, err := submitter.FindManifests()
|
||||
if err != nil || manifests == nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var results []BuildSubmission
|
||||
for name, contents := range manifests {
|
||||
manifest, err := ManifestFromYAML(contents)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, name)
|
||||
}
|
||||
autoSetupManifest(submitter, &manifest)
|
||||
|
||||
yaml, err := manifest.ToYAML()
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, name)
|
||||
}
|
||||
|
||||
client := &http.Client{}
|
||||
|
||||
submission := struct {
|
||||
Manifest string `json:"manifest"`
|
||||
Tags []string `json:"tags"`
|
||||
}{
|
||||
Manifest: yaml,
|
||||
Tags: []string{submitter.GetRepoName(), name},
|
||||
}
|
||||
bodyBytes, err := json.Marshal(&submission)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "preparing job")
|
||||
}
|
||||
body := bytes.NewBuffer(bodyBytes)
|
||||
|
||||
req, err := http.NewRequest("POST", fmt.Sprintf("%s/api/jobs",
|
||||
submitter.GetBuildsOrigin()), body)
|
||||
req.Header.Add("Authorization", fmt.Sprintf("token %s",
|
||||
submitter.GetOauthToken()))
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "job submission")
|
||||
}
|
||||
|
||||
if resp.StatusCode == 403 {
|
||||
return nil, errors.New("builds.sr.ht returned 403\n" +
|
||||
"Log out and back into the website to authorize " +
|
||||
"builds integration.")
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
respBytes, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "read response")
|
||||
}
|
||||
|
||||
if resp.StatusCode == 400 {
|
||||
return nil, errors.New(fmt.Sprintf(
|
||||
"builds.sr.ht returned %d\n", resp.StatusCode) +
|
||||
string(respBytes))
|
||||
}
|
||||
|
||||
var job struct {
|
||||
Id int `json:"id"`
|
||||
}
|
||||
err = json.Unmarshal(respBytes, &job)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "interpret response")
|
||||
}
|
||||
|
||||
results = append(results, BuildSubmission{
|
||||
Name: name,
|
||||
Url: fmt.Sprintf("%s/~%s/job/%d",
|
||||
submitter.GetBuildsOrigin(),
|
||||
submitter.GetOwnerName(),
|
||||
job.Id),
|
||||
})
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func autoSetupManifest(submitter BuildSubmitter, manifest *Manifest) {
|
||||
var hasSelf bool
|
||||
cloneUrl := submitter.GetCloneUrl() + "#" + submitter.GetCommitId()
|
||||
for i, src := range manifest.Sources {
|
||||
if path.Base(src) == submitter.GetRepoName() {
|
||||
manifest.Sources[i] = cloneUrl
|
||||
hasSelf = true
|
||||
}
|
||||
}
|
||||
if !hasSelf {
|
||||
manifest.Sources = append(manifest.Sources, cloneUrl)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,106 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"io/ioutil"
|
||||
|
||||
"gopkg.in/src-d/go-git.v4/plumbing"
|
||||
"gopkg.in/src-d/go-git.v4/plumbing/object"
|
||||
)
|
||||
|
||||
type RepoContext struct {
|
||||
Id int `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Path string `json:"path"`
|
||||
Visibility string `json:"visibility"`
|
||||
}
|
||||
|
||||
type UserContext struct {
|
||||
CanonicalName string `json:"canonical_name"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
type PushContext struct {
|
||||
Repo RepoContext `json:"repo"`
|
||||
User UserContext `json:"user"`
|
||||
}
|
||||
|
||||
type AnnotatedTag struct {
|
||||
Name string `json:"name"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
type CommitSignature struct {
|
||||
Data string `json:"data"`
|
||||
Signature string `json:"signature"`
|
||||
}
|
||||
|
||||
type CommitAuthorship struct {
|
||||
Email string `json:"email"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// See gitsrht/blueprints/api.py
|
||||
type Commit struct {
|
||||
Id string `json:"id"`
|
||||
Message string `json:"message"`
|
||||
Parents []string `json:"parents"`
|
||||
ShortId string `json:"short_id"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
Tree string `json:"tree"`
|
||||
|
||||
Author CommitAuthorship `json:"author"`
|
||||
Committer CommitAuthorship `json:"committer"`
|
||||
Signature *CommitSignature `json:"signature"`
|
||||
}
|
||||
|
||||
type UpdatedRef struct {
|
||||
Tag *AnnotatedTag `json:"annotated_tag",omitempty`
|
||||
Name string `json:"name"`
|
||||
Old *Commit `json:"old"`
|
||||
New *Commit `json:"commit"`
|
||||
}
|
||||
|
||||
type WebhookPayload struct {
|
||||
Push string `json:"push"`
|
||||
Pusher UserContext `json:"pusher"`
|
||||
Refs []UpdatedRef `json:"refs"`
|
||||
}
|
||||
|
||||
func GitCommitToWebhookCommit(c *object.Commit) *Commit {
|
||||
parents := make([]string, len(c.ParentHashes))
|
||||
for i, p := range c.ParentHashes {
|
||||
parents[i] = p.String()
|
||||
}
|
||||
|
||||
var signature *CommitSignature = nil
|
||||
if c.PGPSignature != "" {
|
||||
encoded := &plumbing.MemoryObject{}
|
||||
c.EncodeWithoutSignature(encoded)
|
||||
reader, _ := encoded.Reader()
|
||||
data, _ := ioutil.ReadAll(ioutil.NopCloser(reader))
|
||||
signature = &CommitSignature{
|
||||
Data: base64.StdEncoding.EncodeToString(data),
|
||||
Signature: base64.StdEncoding.EncodeToString([]byte(c.PGPSignature)),
|
||||
}
|
||||
}
|
||||
|
||||
return &Commit{
|
||||
Id: c.Hash.String(),
|
||||
Message: c.Message,
|
||||
Parents: parents,
|
||||
ShortId: c.Hash.String()[:7],
|
||||
Timestamp: c.Author.When.Format("2006-01-02T15:04:05-07:00"),
|
||||
Tree: c.TreeHash.String(),
|
||||
Author: CommitAuthorship{
|
||||
// TODO: Add timestamp
|
||||
Name: c.Author.Name,
|
||||
Email: c.Author.Email,
|
||||
},
|
||||
Committer: CommitAuthorship{
|
||||
Name: c.Committer.Name,
|
||||
Email: c.Committer.Email,
|
||||
},
|
||||
Signature: signature,
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
goredis "github.com/go-redis/redis"
|
||||
)
|
||||
|
||||
// XXX: This is run once for every single ref that's pushed. If someone pushes
|
||||
// lots of refs, it might be expensive. Needs to be tested.
|
||||
func update() {
|
||||
var (
|
||||
refname string = os.Args[1]
|
||||
oldref string = os.Args[2]
|
||||
newref string = os.Args[3]
|
||||
)
|
||||
pushUuid, ok := os.LookupEnv("SRHT_PUSH")
|
||||
if !ok {
|
||||
logger.Fatal("Missing SRHT_PUSH in environment, configuration error?")
|
||||
}
|
||||
logger.Printf("Running update for push %s", pushUuid)
|
||||
|
||||
redis := goredis.NewClient(&goredis.Options{Addr: "localhost:6379"})
|
||||
redis.Set(fmt.Sprintf("update.%s.%s", pushUuid, refname),
|
||||
fmt.Sprintf("%s:%s", oldref, newref), 10*time.Minute)
|
||||
}
|
||||
|
|
@ -0,0 +1,137 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/mattn/go-runewidth"
|
||||
"golang.org/x/crypto/ed25519"
|
||||
)
|
||||
|
||||
var (
|
||||
privkey ed25519.PrivateKey
|
||||
)
|
||||
|
||||
type WebhookSubscription struct {
|
||||
Id int
|
||||
Url string
|
||||
Events string
|
||||
}
|
||||
|
||||
// Note: unlike normal sr.ht services, we don't add webhook deliveries to the
|
||||
// database until after the HTTP request has been completed, to reduce time
|
||||
// spent blocking the user's terminal.
|
||||
type WebhookDelivery struct {
|
||||
Headers string
|
||||
Payload string
|
||||
Response string
|
||||
ResponseHeaders string
|
||||
ResponseStatus int
|
||||
SubscriptionId int
|
||||
UUID string
|
||||
Url string
|
||||
}
|
||||
|
||||
func initWebhookKey() {
|
||||
b64key, ok := config.Get("webhooks", "private-key")
|
||||
if !ok {
|
||||
logger.Fatalf("No webhook key configured")
|
||||
}
|
||||
seed, err := base64.StdEncoding.DecodeString(b64key)
|
||||
if err != nil {
|
||||
logger.Fatalf("base64 decode webhooks private key: %v", err)
|
||||
}
|
||||
privkey = ed25519.NewKeyFromSeed(seed)
|
||||
}
|
||||
|
||||
func deliverWebhooks(subs []WebhookSubscription,
|
||||
payload []byte) []WebhookDelivery {
|
||||
|
||||
var deliveries []WebhookDelivery
|
||||
initWebhookKey()
|
||||
client := &http.Client{Timeout: 5 * time.Second}
|
||||
|
||||
for _, sub := range subs {
|
||||
var (
|
||||
nonceSeed []byte
|
||||
nonceHex []byte
|
||||
)
|
||||
_, err := rand.Read(nonceSeed)
|
||||
if err != nil {
|
||||
logger.Fatalf("generate nonce: %v", err)
|
||||
}
|
||||
hex.Encode(nonceHex, nonceSeed)
|
||||
signature := ed25519.Sign(privkey, append(payload, nonceHex...))
|
||||
|
||||
deliveryUuid := uuid.New().String()
|
||||
body := bytes.NewBuffer(payload)
|
||||
req, err := http.NewRequest("POST", sub.Url, body)
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("X-Webhook-Event", "repo:post-update")
|
||||
req.Header.Add("X-Webhook-Delivery", deliveryUuid)
|
||||
req.Header.Add("X-Payload-Nonce", string(nonceHex))
|
||||
req.Header.Add("X-Payload-Signature",
|
||||
base64.StdEncoding.EncodeToString(signature))
|
||||
|
||||
var requestHeaders bytes.Buffer
|
||||
for name, values := range req.Header {
|
||||
requestHeaders.WriteString(fmt.Sprintf("%s: %s\n",
|
||||
name, strings.Join(values, ", ")))
|
||||
}
|
||||
|
||||
delivery := WebhookDelivery{
|
||||
Headers: requestHeaders.String(),
|
||||
Payload: string(payload),
|
||||
ResponseStatus: -1,
|
||||
SubscriptionId: sub.Id,
|
||||
UUID: deliveryUuid,
|
||||
Url: sub.Url,
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
delivery.Response = fmt.Sprintf("Error sending webhook: %v")
|
||||
log.Printf(delivery.Response)
|
||||
deliveries = append(deliveries, delivery)
|
||||
continue
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
respBody, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
delivery.Response = fmt.Sprintf("Error reading webhook "+
|
||||
"response: %v", err)
|
||||
log.Printf(delivery.Response)
|
||||
deliveries = append(deliveries, delivery)
|
||||
continue
|
||||
}
|
||||
if !utf8.Valid(respBody) {
|
||||
delivery.Response = "Webhook response is not valid UTF-8"
|
||||
log.Printf(delivery.Response)
|
||||
deliveries = append(deliveries, delivery)
|
||||
continue
|
||||
}
|
||||
log.Println(runewidth.Truncate(string(respBody), 1024, "..."))
|
||||
|
||||
var responseHeaders bytes.Buffer
|
||||
for name, values := range resp.Header {
|
||||
responseHeaders.WriteString(fmt.Sprintf("%s: %s\n",
|
||||
name, strings.Join(values, ", ")))
|
||||
}
|
||||
|
||||
delivery.ResponseHeaders = responseHeaders.String()
|
||||
delivery.Response = string(respBody)[:65535]
|
||||
deliveries = append(deliveries, delivery)
|
||||
}
|
||||
|
||||
return deliveries
|
||||
}
|
||||
|
|
@ -17,6 +17,7 @@ from srht.validation import Validation
|
|||
|
||||
data = Blueprint("api.data", __name__)
|
||||
|
||||
# See also gitsrht-update-hook/types.go
|
||||
def commit_to_dict(c):
|
||||
return {
|
||||
"id": str(c.id),
|
||||
|
|
|
|||
|
|
@ -1,178 +0,0 @@
|
|||
import html
|
||||
import os
|
||||
import re
|
||||
from pygit2 import Repository as GitRepository, Commit, Tag
|
||||
from gitsrht.blueprints.api import commit_to_dict
|
||||
from gitsrht.types import User, Repository
|
||||
from scmsrht.redis import redis
|
||||
from scmsrht.repos import RepoVisibility
|
||||
from scmsrht.submit import BuildSubmitterBase
|
||||
from gitsrht.webhooks import RepoWebhook
|
||||
from srht.config import cfg, get_origin
|
||||
from srht.database import db
|
||||
from urllib.parse import urlparse
|
||||
|
||||
builds_sr_ht = cfg("builds.sr.ht", "origin", None)
|
||||
git_sr_ht = get_origin("git.sr.ht", external=True)
|
||||
|
||||
def first_line(text):
|
||||
try:
|
||||
i = text.index("\n")
|
||||
except ValueError:
|
||||
return text + "\n"
|
||||
else:
|
||||
return text[:i + 1]
|
||||
|
||||
class GitBuildSubmitter(BuildSubmitterBase):
|
||||
def __init__(self, repo, git_repo):
|
||||
super().__init__(git_sr_ht, 'git', repo)
|
||||
self.git_repo = git_repo
|
||||
|
||||
def find_manifests(self, commit):
|
||||
manifest_blobs = dict()
|
||||
if ".build.yml" in commit.tree:
|
||||
build_yml = commit.tree[".build.yml"]
|
||||
if build_yml.type == 'blob':
|
||||
manifest_blobs[".build.yml"] = build_yml
|
||||
elif ".builds" in commit.tree:
|
||||
build_dir = commit.tree[".builds"]
|
||||
if build_dir.type == 'tree':
|
||||
manifest_blobs.update(
|
||||
{
|
||||
blob.name: blob
|
||||
for blob in self.git_repo.get(build_dir.id)
|
||||
if blob.type == 'blob' and (
|
||||
blob.name.endswith('.yml')
|
||||
or blob.name.endswith('.yaml')
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
manifests = {}
|
||||
for name, blob in manifest_blobs.items():
|
||||
m = self.git_repo.get(blob.id).data.decode()
|
||||
manifests[name] = m
|
||||
return manifests
|
||||
|
||||
def get_commit_id(self, commit):
|
||||
return str(commit.id)
|
||||
|
||||
def get_commit_note(self, commit):
|
||||
return "[{}]({}) — [{}](mailto:{})\n\n{}".format(
|
||||
str(commit.id)[:7],
|
||||
"{}/{}/{}/commit/{}".format(
|
||||
git_sr_ht,
|
||||
"~" + self.repo.owner.username,
|
||||
self.repo.name,
|
||||
str(commit.id)),
|
||||
commit.author.name,
|
||||
commit.author.email,
|
||||
"<pre>" + html.escape(first_line(commit.message)) + "</pre>",
|
||||
)
|
||||
|
||||
def get_clone_url(self):
|
||||
origin = get_origin("git.sr.ht", external=True)
|
||||
owner_name = self.repo.owner.canonical_name
|
||||
repo_name = self.repo.name
|
||||
if self.repo.visibility == RepoVisibility.private:
|
||||
# Use SSH URL
|
||||
origin = origin.replace("http://", "").replace("https://", "")
|
||||
return f"git+ssh://git@{origin}/{owner_name}/{repo_name}"
|
||||
else:
|
||||
# Use http(s) URL
|
||||
return f"{origin}/{owner_name}/{repo_name}"
|
||||
|
||||
# https://stackoverflow.com/a/14693789
|
||||
ansi_escape = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]')
|
||||
|
||||
def do_post_update(context, refs):
|
||||
global db
|
||||
# TODO: we shouldn't need this once we move most of this shit to the
|
||||
# internal API
|
||||
if not hasattr(db, "session"):
|
||||
import gitsrht.types
|
||||
from srht.database import DbSession
|
||||
db = DbSession(cfg("git.sr.ht", "connection-string"))
|
||||
db.init()
|
||||
|
||||
uid = os.environ.get("SRHT_UID")
|
||||
push = os.environ.get("SRHT_PUSH")
|
||||
user = context["user"]
|
||||
repo = context["repo"]
|
||||
|
||||
payload = {
|
||||
"push": push,
|
||||
"pusher": user,
|
||||
"refs": list(),
|
||||
}
|
||||
|
||||
git_repo = GitRepository(repo["path"])
|
||||
oids = set()
|
||||
for ref in refs:
|
||||
update = redis.get(f"update.{push}.{ref}")
|
||||
if update:
|
||||
old, new = update.decode().split(":")
|
||||
old = git_repo.get(old)
|
||||
new = git_repo.get(new)
|
||||
update = dict()
|
||||
if isinstance(new, Tag):
|
||||
update.update({
|
||||
"annotated_tag": {
|
||||
"name": new.name,
|
||||
"message": new.message,
|
||||
},
|
||||
})
|
||||
new = git_repo.get(new.target)
|
||||
update.update({
|
||||
"name": ref,
|
||||
"old": commit_to_dict(old) if old else None,
|
||||
"new": commit_to_dict(new) if new else None,
|
||||
})
|
||||
payload["refs"].append(update)
|
||||
|
||||
try:
|
||||
if re.match(r"^[0-9a-z]{40}$", ref): # commit
|
||||
commit = git_repo.get(ref)
|
||||
elif ref.startswith("refs/"): # ref
|
||||
target_id = git_repo.lookup_reference(ref).target
|
||||
commit = git_repo.get(target_id)
|
||||
if isinstance(commit, Tag):
|
||||
commit = git_repo.get(commit.target)
|
||||
else:
|
||||
continue
|
||||
if not isinstance(commit, Commit):
|
||||
continue
|
||||
if commit.id in oids:
|
||||
continue
|
||||
oids.add(commit.id)
|
||||
except:
|
||||
continue
|
||||
|
||||
if builds_sr_ht:
|
||||
# TODO: move this to internal API
|
||||
r = Repository.query.get(repo["id"])
|
||||
s = GitBuildSubmitter(r, git_repo)
|
||||
res = s.submit(commit)
|
||||
if res.status != 'skipped':
|
||||
res.printmsgs()
|
||||
|
||||
# TODO: get these from internal API
|
||||
# sync webhooks
|
||||
for resp in RepoWebhook.deliver(RepoWebhook.Events.repo_post_update, payload,
|
||||
RepoWebhook.Subscription.repo_id == repo["id"],
|
||||
RepoWebhook.Subscription.sync,
|
||||
delay=False):
|
||||
if resp == None:
|
||||
# TODO: Add details?
|
||||
print("Error submitting webhook")
|
||||
continue
|
||||
if resp.status_code != 200:
|
||||
print(f"Webhook returned status {resp.status_code}")
|
||||
try:
|
||||
print(ansi_escape.sub('', resp.text))
|
||||
except:
|
||||
print("Unable to decode webhook response")
|
||||
# async webhooks
|
||||
RepoWebhook.deliver(RepoWebhook.Events.repo_post_update, payload,
|
||||
RepoWebhook.Subscription.repo_id == repo["id"],
|
||||
RepoWebhook.Subscription.sync == False)
|
||||
|
|
@ -12,9 +12,14 @@ def migrate(path, link):
|
|||
if not os.path.exists(path) \
|
||||
or not os.path.islink(path) \
|
||||
or os.readlink(path) != link:
|
||||
if os.path.exists(path):
|
||||
try:
|
||||
os.remove(path)
|
||||
os.symlink(link, path)
|
||||
except:
|
||||
pass
|
||||
try:
|
||||
os.symlink(link, path)
|
||||
except:
|
||||
pass
|
||||
return True
|
||||
return False
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue