api: Implement tracker dump signature verification

Changes the ticket and comment signature format to remove useless
fields.

Also updates the frontend to use the GraphQL API for tracker dump
imports.
This commit is contained in:
Adnan Maolood 2022-11-08 07:16:22 -05:00 committed by Drew DeVault
parent 9f4e4b5a0b
commit fc70ae13a4
5 changed files with 264 additions and 337 deletions

View File

@ -7,65 +7,51 @@ import (
"errors" "errors"
"fmt" "fmt"
"io" "io"
"log"
"strings" "strings"
"time" "time"
"git.sr.ht/~sircmpwn/core-go/crypto"
"git.sr.ht/~sircmpwn/core-go/database" "git.sr.ht/~sircmpwn/core-go/database"
"git.sr.ht/~sircmpwn/todo.sr.ht/api/graph/model" "git.sr.ht/~sircmpwn/todo.sr.ht/api/graph/model"
"git.sr.ht/~sircmpwn/todo.sr.ht/api/loaders" "git.sr.ht/~sircmpwn/todo.sr.ht/api/loaders"
) )
type TrackerDump struct { type TrackerDump struct {
Owner Owner `json:"owner"` ID int `json:"id"`
Name string `json:"name"` Owner User `json:"owner"`
Labels []Label `json:"labels"` Created time.Time `json:"created"`
Tickets []Ticket `json:"tickets"` Updated time.Time `json:"updated"`
} Name string `json:"name"`
Description string `json:"description"`
type Owner struct { Labels []Label `json:"labels"`
CanonicalName string `json:"canonical_name"` Tickets []Ticket `json:"tickets"`
Name string `json:"name"`
Email string `json:"email"`
URL string `json:"url"`
Location string `json:"location"`
Bio string `json:"bio"`
} }
type Label struct { type Label struct {
Name string `json:"name"` ID int `json:"id"`
Colors struct { Created time.Time `json:"created"`
Background string `json:"background"` Name string `json:"name"`
Foreground string `json:"text"` BackgroundColor string `json:"background_color"`
} `json:"colors"` ForegroundColor string `json:"foreground_color"`
Created time.Time `json:"created"`
Tracker Tracker `json:"tracker"`
}
type Tracker struct {
ID int `json:"id"`
Owner User `json:"owner"`
Created time.Time `json:"created"`
Updated time.Time `json:"updated"`
Name string `json:"name"`
} }
type Ticket struct { type Ticket struct {
ID int `json:"id"` ID int `json:"id"`
Ref string `json:"ref"` Created time.Time `json:"created"`
Tracker Tracker `json:"tracker"` Updated time.Time `json:"updated"`
Subject string `json:"title"` Submitter Participant `json:"submitter"`
Created time.Time `json:"created"` Ref string `json:"ref"`
Updated time.Time `json:"updated"` Subject string `json:"subject"`
Submitter *Participant `json:"submitter"` // null in shorter ticket dicts Body string `json:"body"`
Body string `json:"description"` Status string `json:"status"`
Status string `json:"status"` Resolution string `json:"resolution"`
Resolution string `json:"resolution"` Labels []string `json:"labels"`
Labels []string `json:"labels"` Assignees []User `json:"assignees"`
Assignees []User `json:"assignees"` Upstream string `json:"upstream"`
Upstream string `json:"upstream"` Signature string `json:"X-Payload-Signature"`
Signature string `json:"X-Payload-Signature"` Nonce string `json:"X-Payload-Nonce"`
Nonce string `json:"X-Payload-Nonce"` Events []Event `json:"events"`
Events []Event `json:"events"`
} }
type Event struct { type Event struct {
@ -76,8 +62,7 @@ type Event struct {
OldResolution *string `json:"old_resolution"` OldResolution *string `json:"old_resolution"`
NewStatus *string `json:"new_status"` NewStatus *string `json:"new_status"`
NewResolution *string `json:"new_resolution"` NewResolution *string `json:"new_resolution"`
User *Participant `json:"user"` Participant *Participant `json:"participant"`
Ticket *Ticket `json:"ticket"`
Comment *Comment `json:"comment"` Comment *Comment `json:"comment"`
Label *string `json:"label"` Label *string `json:"label"`
ByUser *Participant `json:"by_user"` ByUser *Participant `json:"by_user"`
@ -89,6 +74,7 @@ type Event struct {
type Participant struct { type Participant struct {
Type string `json:"type"` Type string `json:"type"`
UserID int `json:"user_id"`
CanonicalName string `json:"canonical_name"` CanonicalName string `json:"canonical_name"`
Name string `json:"name"` Name string `json:"name"`
Address string `json:"address"` Address string `json:"address"`
@ -97,15 +83,33 @@ type Participant struct {
} }
type User struct { type User struct {
ID int `json:"id"`
CanonicalName string `json:"canonical_name"` CanonicalName string `json:"canonical_name"`
Name string `json:"name"` Name string `json:"name"`
} }
type Comment struct { type Comment struct {
ID int `json:"id"` ID int `json:"id"`
Created time.Time `json:"created"` Created time.Time `json:"created"`
Submitter Participant `json:"submitter"` Author Participant `json:"author"`
Text string `json:"text"` Text string `json:"text"`
}
type TicketSignatureData struct {
TrackerID int `json:"tracker_id"`
TicketID int `json:"ticket_id"`
Subject string `json:"subject"`
Body string `json:"body"`
SubmitterID int `json:"submitter_id"`
Upstream string `json:"upstream"`
}
type CommentSignatureData struct {
TrackerID int `json:"tracker_id"`
TicketID int `json:"ticket_id"`
Comment string `json:"comment"`
AuthorID int `json:"author_id"`
Upstream string `json:"upstream"`
} }
func importParticipant(ctx context.Context, part Participant, upstream, ourUpstream string) (int, error) { func importParticipant(ctx context.Context, part Participant, upstream, ourUpstream string) (int, error) {
@ -192,29 +196,9 @@ func importTrackerDump(ctx context.Context, trackerID int, dump io.Reader, ourUp
return err return err
} }
// Create labels
labelIDs := map[string]int{}
if err := database.WithTx(ctx, nil, func(tx *sql.Tx) error {
for _, label := range tracker.Labels {
row := tx.QueryRowContext(ctx, `
INSERT INTO label (
created, updated, tracker_id, name, color, text_color
) VALUES (
$1, $1, $2, $3, $4, $5
) RETURNING id
`, label.Created, trackerID, label.Name, label.Colors.Background, label.Colors.Foreground)
var labelID int
if err := row.Scan(&labelID); err != nil {
return err
}
labelIDs[label.Name] = labelID
}
return nil
}); err != nil {
return err
}
defer func() { defer func() {
r := recover()
if err := database.WithTx(ctx, nil, func(tx *sql.Tx) error { if err := database.WithTx(ctx, nil, func(tx *sql.Tx) error {
_, err := tx.ExecContext(ctx, ` _, err := tx.ExecContext(ctx, `
UPDATE tracker UPDATE tracker
@ -225,9 +209,31 @@ func importTrackerDump(ctx context.Context, trackerID int, dump io.Reader, ourUp
}); err != nil { }); err != nil {
panic(err) panic(err)
} }
if r != nil {
panic(r)
}
}() }()
if err := database.WithTx(ctx, nil, func(tx *sql.Tx) error { if err := database.WithTx(ctx, nil, func(tx *sql.Tx) error {
// Create labels
labelIDs := map[string]int{}
for _, label := range tracker.Labels {
row := tx.QueryRowContext(ctx, `
INSERT INTO label (
created, updated, tracker_id, name, color, text_color
) VALUES (
$1, $1, $2, $3, $4, $5
) RETURNING id
`, label.Created, trackerID, label.Name,
label.BackgroundColor, label.ForegroundColor)
var labelID int
if err := row.Scan(&labelID); err != nil {
return err
}
labelIDs[label.Name] = labelID
}
var nextTicketID int var nextTicketID int
row := tx.QueryRowContext(ctx, row := tx.QueryRowContext(ctx,
`SELECT next_ticket_id FROM tracker WHERE id = $1`, `SELECT next_ticket_id FROM tracker WHERE id = $1`,
@ -244,11 +250,32 @@ func importTrackerDump(ctx context.Context, trackerID int, dump io.Reader, ourUp
var maxTicketID int var maxTicketID int
for _, ticket := range tracker.Tickets { for _, ticket := range tracker.Tickets {
submitterID, err := importParticipant(ctx, *ticket.Submitter, ticket.Upstream, ourUpstream) submitterID, err := importParticipant(ctx, ticket.Submitter, ticket.Upstream, ourUpstream)
if err != nil { if err != nil {
return err return err
} }
ticketAuthenticity := model.AUTH_UNAUTHENTICATED
if ticket.Upstream == ourUpstream && ticket.Submitter.Type == "user" {
sigdata := TicketSignatureData{
TrackerID: tracker.ID,
TicketID: ticket.ID,
Subject: ticket.Subject,
Body: ticket.Body,
SubmitterID: ticket.Submitter.UserID,
Upstream: ticket.Upstream,
}
payload, err := json.Marshal(sigdata)
if err != nil {
panic(err)
}
if crypto.VerifyWebhook(payload, ticket.Nonce, ticket.Signature) {
ticketAuthenticity = model.AUTH_AUTHENTIC
} else {
ticketAuthenticity = model.AUTH_TAMPERED
}
}
// Compute the max ticket ID. We can't use the number of tickets as // Compute the max ticket ID. We can't use the number of tickets as
// the next ticket ID because that won't include deleted tickets // the next ticket ID because that won't include deleted tickets
if ticket.ID > maxTicketID { if ticket.ID > maxTicketID {
@ -268,9 +295,9 @@ func importTrackerDump(ctx context.Context, trackerID int, dump io.Reader, ourUp
RETURNING id RETURNING id
`, ticket.Created, ticket.Updated, trackerID, ticket.ID, `, ticket.Created, ticket.Updated, trackerID, ticket.ID,
submitterID, ticket.Subject, ticket.Body, submitterID, ticket.Subject, ticket.Body,
model.TicketStatus(strings.ToUpper(ticket.Status)).ToInt(), model.TicketStatus(ticket.Status).ToInt(),
model.TicketResolution(strings.ToUpper(ticket.Resolution)).ToInt(), model.TicketResolution(ticket.Resolution).ToInt(),
model.AUTH_UNAUTHENTICATED) ticketAuthenticity)
var ticketPKID int var ticketPKID int
if err := row.Scan(&ticketPKID); err != nil { if err := row.Scan(&ticketPKID); err != nil {
return err return err
@ -307,26 +334,51 @@ func importTrackerDump(ctx context.Context, trackerID int, dump io.Reader, ourUp
var eventType int var eventType int
for _, etype := range event.EventType { for _, etype := range event.EventType {
etype := model.EventType(etype)
if !etype.IsValid() {
fmt.Errorf("failed to import ticket #%d: invalid event type %s", ticket.ID, etype)
}
eventType |= eventTypeMap[etype] eventType |= eventTypeMap[etype]
} }
if eventType == 0 { if eventType == 0 {
return fmt.Errorf("failed to import ticket #%d: invalid ticket event", ticket.ID, eventType) return fmt.Errorf("failed to import ticket #%d: invalid ticket event", ticket.ID, eventType)
} }
if event.User != nil { if event.Participant != nil {
userPartID, err := importParticipant(ctx, *event.User, event.Upstream, ourUpstream) eventPartID, err := importParticipant(ctx, *event.Participant, event.Upstream, ourUpstream)
if err != nil { if err != nil {
return err return err
} }
partID = &userPartID partID = &eventPartID
} }
if eventType&model.EVENT_COMMENT != 0 { if eventType&model.EVENT_COMMENT != 0 {
submitterID, err := importParticipant(ctx, event.Comment.Submitter, event.Upstream, ourUpstream) authorID, err := importParticipant(ctx, event.Comment.Author, event.Upstream, ourUpstream)
if err != nil { if err != nil {
return err return err
} }
commentAuthenticity := model.AUTH_UNAUTHENTICATED
if event.Upstream == ourUpstream && event.Comment.Author.Type == "user" {
log.Println(event.Comment.Author.UserID)
sigdata := CommentSignatureData{
TrackerID: tracker.ID,
TicketID: ticket.ID,
Comment: event.Comment.Text,
AuthorID: event.Comment.Author.UserID,
Upstream: event.Upstream,
}
payload, err := json.Marshal(sigdata)
if err != nil {
panic(err)
}
if crypto.VerifyWebhook(payload, event.Nonce, event.Signature) {
commentAuthenticity = model.AUTH_AUTHENTIC
} else {
commentAuthenticity = model.AUTH_TAMPERED
}
}
row := tx.QueryRowContext(ctx, ` row := tx.QueryRowContext(ctx, `
INSERT INTO ticket_comment ( INSERT INTO ticket_comment (
created, updated, submitter_id, ticket_id, text, created, updated, submitter_id, ticket_id, text,
@ -334,8 +386,8 @@ func importTrackerDump(ctx context.Context, trackerID int, dump io.Reader, ourUp
) VALUES ( ) VALUES (
$1, $1, $2, $3, $4, $5 $1, $1, $2, $3, $4, $5
) RETURNING id ) RETURNING id
`, event.Comment.Created, submitterID, ticketPKID, event.Comment.Text, `, event.Comment.Created, authorID, ticketPKID, event.Comment.Text,
model.AUTH_UNAUTHENTICATED) commentAuthenticity)
var _commentID int var _commentID int
if err := row.Scan(&_commentID); err != nil { if err := row.Scan(&_commentID); err != nil {
return err return err
@ -453,14 +505,14 @@ func convertResolutionToInt(resolution *string) *int {
return &resolutionInt return &resolutionInt
} }
var eventTypeMap = map[string]int{ var eventTypeMap = map[model.EventType]int{
"created": model.EVENT_CREATED, model.EventTypeCreated: model.EVENT_CREATED,
"comment": model.EVENT_COMMENT, model.EventTypeComment: model.EVENT_COMMENT,
"status_change": model.EVENT_STATUS_CHANGE, model.EventTypeStatusChange: model.EVENT_STATUS_CHANGE,
"label_added": model.EVENT_LABEL_ADDED, model.EventTypeLabelAdded: model.EVENT_LABEL_ADDED,
"label_removed": model.EVENT_LABEL_REMOVED, model.EventTypeLabelRemoved: model.EVENT_LABEL_REMOVED,
"assigned_user": model.EVENT_ASSIGNED_USER, model.EventTypeAssignedUser: model.EVENT_ASSIGNED_USER,
"unassigned_user": model.EVENT_UNASSIGNED_USER, model.EventTypeUnassignedUser: model.EVENT_UNASSIGNED_USER,
"user_mentioned": model.EVENT_USER_MENTIONED, model.EventTypeUserMentioned: model.EVENT_USER_MENTIONED,
"ticket_mentioned": model.EVENT_TICKET_MENTIONED, model.EventTypeTicketMentioned: model.EVENT_TICKET_MENTIONED,
} }

View File

@ -9,7 +9,7 @@ from srht.crypto import sign_payload
from srht.database import db from srht.database import db
from srht.oauth import current_user, loginrequired from srht.oauth import current_user, loginrequired
from srht.flask import date_handler, session from srht.flask import date_handler, session
from srht.graphql import exec_gql from srht.graphql import exec_gql, GraphQLOperation, GraphQLUpload
from srht.validation import Validation from srht.validation import Validation
from tempfile import NamedTemporaryFile from tempfile import NamedTemporaryFile
from todosrht.access import get_tracker from todosrht.access import get_tracker
@ -17,7 +17,6 @@ from todosrht.trackers import get_recent_users
from todosrht.types import Event, EventType, Ticket, TicketAccess, Visibility from todosrht.types import Event, EventType, Ticket, TicketAccess, Visibility
from todosrht.types import ParticipantType, UserAccess, User from todosrht.types import ParticipantType, UserAccess, User
from todosrht.urls import tracker_url from todosrht.urls import tracker_url
from todosrht.tracker_import import tracker_import
settings = Blueprint("settings", __name__) settings = Blueprint("settings", __name__)
@ -243,20 +242,57 @@ def export_POST(owner, name):
if current_user.id != tracker.owner_id: if current_user.id != tracker.owner_id:
abort(403) abort(403)
upstream = get_origin("todo.sr.ht", external=True)
def participant_to_dict(self):
if self.participant_type == ParticipantType.user:
return {
"type": "user",
"user_id": self.user.id,
"canonical_name": self.user.canonical_name,
"name": self.user.username,
}
elif self.participant_type == ParticipantType.email:
return {
"type": "email",
"address": self.email,
"name": self.email_name,
}
elif self.participant_type == ParticipantType.external:
return {
"type": "external",
"external_id": self.external_id,
"external_url": self.external_url,
}
assert False
dump = list() dump = list()
tickets = Ticket.query.filter(Ticket.tracker_id == tracker.id).all() tickets = Ticket.query.filter(Ticket.tracker_id == tracker.id).all()
for ticket in tickets: for ticket in tickets:
td = ticket.to_dict() td = {
td["upstream"] = get_origin("todo.sr.ht", external=True) "id": ticket.scoped_id,
"created": ticket.created,
"updated": ticket.updated,
"submitter": participant_to_dict(ticket.submitter),
"ref": ticket.ref(),
"subject": ticket.title,
"body": ticket.description,
"status": ticket.status.name.upper(),
"resolution": ticket.resolution.name.upper(),
"labels": [l.name for l in ticket.labels],
"assignees": [u.to_dict(short=True) for u in ticket.assigned_users],
}
td["upstream"] = upstream
if ticket.submitter.participant_type == ParticipantType.user: if ticket.submitter.participant_type == ParticipantType.user:
sigdata = OrderedDict({ sigdata = OrderedDict({
"description": ticket.description, "tracker_id": tracker.id,
"ref": ticket.ref(), "ticket_id": ticket.scoped_id,
"submitter": ticket.submitter.user.canonical_name, "subject": ticket.title,
"title": ticket.title, "body": ticket.description,
"upstream": get_origin("todo.sr.ht", external=True), "submitter_id": ticket.submitter.user.id,
"upstream": upstream,
}) })
sigdata = json.dumps(sigdata) sigdata = json.dumps(sigdata, separators=(',',':'))
signature = sign_payload(sigdata) signature = sign_payload(sigdata)
td.update(signature) td.update(signature)
@ -264,27 +300,64 @@ def export_POST(owner, name):
if any(events): if any(events):
td["events"] = list() td["events"] = list()
for event in events: for event in events:
ev = event.to_dict() ev = {
ev["upstream"] = get_origin("todo.sr.ht", external=True) "id": event.id,
"created": event.created,
"event_type": [t.name.upper() for t in EventType if t in event.event_type],
"old_status": event.old_status.name.upper()
if event.old_status else None,
"old_resolution": event.old_resolution.name.upper()
if event.old_resolution else None,
"new_status": event.new_status.name.upper()
if event.new_status else None,
"new_resolution": event.new_resolution.name.upper()
if event.new_resolution else None,
"participant": participant_to_dict(event.participant)
if event.participant else None,
"ticket_id": event.ticket.scoped_id
if event.ticket else None,
"comment": {
"id": event.comment.id,
"created": event.comment.created,
"author": participant_to_dict(event.comment.submitter),
"text": event.comment.text,
} if event.comment else None,
"label": event.label.name if event.label else None,
"by_user": participant_to_dict(event.by_participant)
if event.by_participant else None,
"from_ticket_id": event.from_ticket.scoped_id
if event.from_ticket else None,
}
ev["upstream"] = upstream
if (EventType.comment in event.event_type if (EventType.comment in event.event_type
and event.participant.participant_type == ParticipantType.user): and event.participant.participant_type == ParticipantType.user):
sigdata = OrderedDict({ sigdata = OrderedDict({
"tracker_id": tracker.id,
"ticket_id": ticket.scoped_id,
"comment": event.comment.text, "comment": event.comment.text,
"id": event.id, "author_id": event.comment.submitter.user.id,
"ticket": event.ticket.ref(), "upstream": upstream
"user": event.participant.user.canonical_name,
"upstream": get_origin("todo.sr.ht", external=True),
}) })
sigdata = json.dumps(sigdata) sigdata = json.dumps(sigdata, separators=(',',':'))
signature = sign_payload(sigdata) signature = sign_payload(sigdata)
ev.update(signature) ev.update(signature)
td["events"].append(ev) td["events"].append(ev)
dump.append(td) dump.append(td)
dump = json.dumps({ dump = json.dumps({
"owner": tracker.owner.to_dict(), "id": tracker.id,
"owner": tracker.owner.to_dict(short=True),
"created": tracker.created,
"updated": tracker.updated,
"name": tracker.name, "name": tracker.name,
"labels": [l.to_dict() for l in tracker.labels], "description": tracker.description,
"labels": [{
"id": l.id,
"created": l.created,
"name": l.name,
"background_color": l.color,
"foreground_color": l.text_color,
} for l in tracker.labels],
"tickets": dump, "tickets": dump,
}, default=date_handler) }, default=date_handler)
with NamedTemporaryFile() as ntf: with NamedTemporaryFile() as ntf:
@ -304,22 +377,32 @@ def import_POST(owner, name):
if current_user.id != tracker.owner_id: if current_user.id != tracker.owner_id:
abort(403) abort(403)
dump = request.files.get("dump")
valid = Validation(request) valid = Validation(request)
dump = request.files.get("dump")
valid.expect(dump is not None, valid.expect(dump is not None,
"Tracker dump file is required", field="dump") "Tracker dump file is required", field="dump")
if not valid.ok: if not valid.ok:
return render_template("tracker-import-export.html", return render_template("tracker-import-export.html",
view="import/export", tracker=tracker, **valid.kwargs) view="import/export", tracker=tracker, **valid.kwargs)
try: op = GraphQLOperation("""
dump = dump.stream.read() mutation ImportTrackerDump($trackerId: Int!, $dump: Upload!) {
dump = gzip.decompress(dump) importTrackerDump(trackerId: $trackerId, dump: $dump)
dump = json.loads(dump) }
except: """)
abort(400)
tracker_import.delay(dump, tracker.id) dump = GraphQLUpload(
dump.filename,
dump.stream,
"application/octet-stream",
)
op.var("trackerId", tracker.id)
op.var("dump", dump)
op.execute("todo.sr.ht", valid=valid)
if not valid.ok:
return render_template("tracker-import-export.html",
view="import/export", tracker=tracker, **valid.kwargs)
tracker.import_in_progress = True
db.session.commit()
return redirect(tracker_url(tracker)) return redirect(tracker_url(tracker))

View File

@ -36,20 +36,21 @@
after preparing the following JSON data as UTF-8: after preparing the following JSON data as UTF-8:
</p> </p>
<pre>{ <pre>{
"description": <strong>$description</strong>, "tracker_id": <strong>$trackerId</strong>,
"ref": "{{tracker.ref()}}#<strong>$id</strong>", "ticket_id": <strong>$ticketId</strong>,
"submitter": "~<strong>$username</strong>", "subject": <strong>$subject</strong>,
"title": <strong>$title</strong>, "body": <strong>$body</strong>,
"submitter_id": "<strong>$userId</strong>",
"upstream": "{{get_origin('todo.sr.ht', external=True)}}", "upstream": "{{get_origin('todo.sr.ht', external=True)}}",
}</pre> }</pre>
<p> <p>
Comments use the same process with the following JSON data as UTF-8: Comments use the same process with the following JSON data as UTF-8:
</p> </p>
<pre>{ <pre>{
"tracker_id": <strong>$trackerId</strong>,
"ticket_id": <strong>$ticketId</strong>,
"comment": <strong>$comment</strong>, "comment": <strong>$comment</strong>,
"id": <strong>$id</strong>, "author_id": "<strong>$userId</strong>",
"ticket": "{{tracker.ref()}}#<strong>$id</strong>",
"user": "~<strong>$username</strong>",
"upstream": "{{get_origin('todo.sr.ht', external=True)}}", "upstream": "{{get_origin('todo.sr.ht', external=True)}}",
}</pre> }</pre>
<p> <p>

View File

@ -1,207 +0,0 @@
import json
from collections import OrderedDict
from datetime import datetime, timezone
from srht.crypto import verify_payload
from srht.config import get_origin
from srht.database import db
from todosrht.tickets import submit_ticket
from todosrht.tickets import get_participant_for_email
from todosrht.tickets import get_participant_for_external
from todosrht.tickets import get_participant_for_user
from todosrht.types import Event, EventType, Tracker, Ticket, TicketComment
from todosrht.types import ParticipantType, User
from todosrht.types import Label, TicketLabel
from todosrht.types import TicketStatus, TicketResolution
from todosrht.types import TicketAuthenticity
from todosrht.webhooks import worker
our_upstream = get_origin("todo.sr.ht", external=True)
def _parse_date(date):
date = datetime.fromisoformat(date)
date = date.astimezone(timezone.utc).replace(tzinfo=None)
return date
def _import_participant(pdata, upstream):
if pdata["type"] == "user":
if upstream == our_upstream:
user = User.query.filter(User.username == pdata["name"]).first()
else:
user = None
if user:
submitter = get_participant_for_user(user)
else:
submitter = get_participant_for_external(
pdata["canonical_name"],
upstream + "/" + pdata["canonical_name"])
elif pdata["type"] == "email":
submitter = get_participant_for_email(pdata["address"], pdata["name"])
elif pdata["type"] == "external":
submitter = get_participant_for_external(
pdata["external_id"], pdata["external_url"])
return submitter
def _import_comment(ticket, event, edata):
cdata = edata["comment"]
comment = TicketComment()
submitter = _import_participant(cdata["submitter"], edata["upstream"])
comment.submitter_id = submitter.id
comment.ticket_id = ticket.id
comment.text = cdata["text"]
comment.authenticity = TicketAuthenticity.unauthenticated
comment.created = _parse_date(cdata["created"])
comment.updated = comment.created
comment._no_autoupdate = True
signature, nonce = edata.get("X-Payload-Signature"), edata.get("X-Payload-Nonce")
if (signature and nonce
and edata["upstream"] == our_upstream
and submitter.participant_type == ParticipantType.user):
# TODO: Validate signatures from trusted third-parties
sigdata = OrderedDict({
"comment": comment.text,
"id": edata["id"], # not important to verify this
"ticket": edata["ticket"]["ref"], # not important to verify this
"user": submitter.user.canonical_name,
"upstream": edata["upstream"],
})
sigdata = json.dumps(sigdata)
if verify_payload(sigdata, signature, nonce):
comment.authenticity = TicketAuthenticity.authentic
else:
comment.authenticity = TicketAuthenticity.tampered
ticket.comment_count += 1
db.session.add(comment)
db.session.flush()
event.comment_id = comment.id
def _tracker_import(dump, tracker):
ldict = dict()
for ldata in dump["labels"]:
label = Label()
label.tracker_id = tracker.id
label.name = ldata["name"]
label.color = ldata["colors"]["background"]
label.text_color = ldata["colors"]["text"]
db.session.add(label)
db.session.flush()
ldict[label.name] = label.id
tickets = sorted(dump["tickets"], key=lambda t: t["id"])
for tdata in tickets:
for field in [
"id", "title", "created", "description", "status", "resolution",
"labels", "assignees", "upstream", "events", "submitter",
]:
if not field in tdata:
print("Invalid ticket data")
continue
ticket = Ticket.query.filter(
Ticket.tracker_id == tracker.id,
Ticket.scoped_id == tdata["id"]).one_or_none()
if ticket:
print(f"Ticket {tdata['id']} already imported - skipping")
continue
submitter = _import_participant(tdata["submitter"], tdata["upstream"])
ticket = submit_ticket(tracker, submitter,
tdata["title"], tdata["description"], importing=True)
try:
created = _parse_date(tdata["created"])
except ValueError:
created = datetime.utcnow()
ticket._no_autoupdate = True
ticket.created = created
ticket.updated = created
ticket.status = TicketStatus[tdata["status"]]
ticket.resolution = TicketResolution[tdata["resolution"]]
ticket.authenticity = TicketAuthenticity.unauthenticated
for label in tdata["labels"]:
tl = TicketLabel()
tl.ticket_id = ticket.id
tl.label_id = ldict.get(label)
tl.user_id = tracker.owner_id
db.session.add(tl)
# TODO: assignees
signature, nonce = tdata.get("X-Payload-Signature"), tdata.get("X-Payload-Nonce")
if (signature and nonce
and tdata["upstream"] == our_upstream
and submitter.participant_type == ParticipantType.user):
# TODO: Validate signatures from trusted third-parties
sigdata = OrderedDict({
"description": ticket.description,
"ref": tdata["ref"], # not important to verify this
"submitter": ticket.submitter.user.canonical_name,
"title": ticket.title,
"upstream": tdata["upstream"],
})
sigdata = json.dumps(sigdata)
if verify_payload(sigdata, signature, nonce):
ticket.authenticity = TicketAuthenticity.authentic
else:
ticket.authenticity = TicketAuthenticity.tampered
for edata in tdata.get("events", []):
for field in [
"created", "event_type", "old_status", "new_status",
"old_resolution", "new_resolution", "user", "ticket",
"comment", "label", "by_user", "from_ticket", "upstream",
]:
if not field in edata:
print("Invalid ticket event")
return
event = Event()
for etype in edata["event_type"]:
if event.event_type == None:
event.event_type = EventType[etype]
else:
event.event_type |= EventType[etype]
if event.event_type == None:
print("Invalid ticket event")
continue
if EventType.comment in event.event_type:
_import_comment(ticket, event, edata)
if EventType.status_change in event.event_type:
if edata["old_status"]:
event.old_status = TicketStatus[edata["old_status"]]
if edata["new_status"]:
event.new_status = TicketStatus[edata["new_status"]]
if EventType.label_added in event.event_type:
event.label_id = ldict.get(edata["label"])
if not event.label_id:
continue
if EventType.label_removed in event.event_type:
event.label_id = ldict.get(edata["label"])
if not event.label_id:
continue
if EventType.assigned_user in event.event_type:
by_participant = _import_participant(
edata["by_user"], edata["upstream"])
event.by_participant_id = by_participant.id
if EventType.unassigned_user in event.event_type:
by_participant = _import_participant(
edata["by_user"], edata["upstream"])
event.by_participant_id = by_participant.id
if EventType.user_mentioned in event.event_type:
continue # Magic event type, do not import
if EventType.ticket_mentioned in event.event_type:
continue # TODO: Could reference tickets imported in later iters
event.created = _parse_date(edata["created"])
event.updated = event.created
event._no_autoupdate = True
event.ticket_id = ticket.id
participant = _import_participant(edata["user"], edata["upstream"])
event.participant_id = participant.id
db.session.add(event)
print(f"Imported {ticket.ref()}")
@worker.task
def tracker_import(dump, tracker_id):
tracker = Tracker.query.get(tracker_id)
try:
_tracker_import(dump, tracker)
except:
# TODO: Tell user that the import failed?
db.session.rollback()
tracker = Tracker.query.get(tracker_id)
raise
finally:
tracker.import_in_progress = False
db.session.commit()

View File

@ -15,8 +15,6 @@ webhooks_broker = cfg("todo.sr.ht", "webhooks")
worker = make_worker(broker=webhooks_broker) worker = make_worker(broker=webhooks_broker)
webhook_metrics_collector = RedisQueueCollector(webhooks_broker, "srht_webhooks", "Webhook queue length") webhook_metrics_collector = RedisQueueCollector(webhooks_broker, "srht_webhooks", "Webhook queue length")
import todosrht.tracker_import
class UserWebhook(CeleryWebhook): class UserWebhook(CeleryWebhook):
events = [ events = [
Event("tracker:create", "trackers:read"), Event("tracker:create", "trackers:read"),