From fc70ae13a47f976dfa773586daee982be6231654 Mon Sep 17 00:00:00 2001 From: Adnan Maolood Date: Tue, 8 Nov 2022 07:16:22 -0500 Subject: [PATCH] api: Implement tracker dump signature verification Changes the ticket and comment signature format to remove useless fields. Also updates the frontend to use the GraphQL API for tracker dump imports. --- api/trackers/import.go | 236 +++++++++++------- todosrht/blueprints/settings.py | 141 ++++++++--- todosrht/templates/tracker-import-export.html | 15 +- todosrht/tracker_import.py | 207 --------------- todosrht/webhooks.py | 2 - 5 files changed, 264 insertions(+), 337 deletions(-) delete mode 100644 todosrht/tracker_import.py diff --git a/api/trackers/import.go b/api/trackers/import.go index 51cc0a0..4f2ab0c 100644 --- a/api/trackers/import.go +++ b/api/trackers/import.go @@ -7,65 +7,51 @@ import ( "errors" "fmt" "io" + "log" "strings" "time" + "git.sr.ht/~sircmpwn/core-go/crypto" "git.sr.ht/~sircmpwn/core-go/database" "git.sr.ht/~sircmpwn/todo.sr.ht/api/graph/model" "git.sr.ht/~sircmpwn/todo.sr.ht/api/loaders" ) type TrackerDump struct { - Owner Owner `json:"owner"` - Name string `json:"name"` - Labels []Label `json:"labels"` - Tickets []Ticket `json:"tickets"` -} - -type Owner struct { - CanonicalName string `json:"canonical_name"` - Name string `json:"name"` - Email string `json:"email"` - URL string `json:"url"` - Location string `json:"location"` - Bio string `json:"bio"` + ID int `json:"id"` + Owner User `json:"owner"` + Created time.Time `json:"created"` + Updated time.Time `json:"updated"` + Name string `json:"name"` + Description string `json:"description"` + Labels []Label `json:"labels"` + Tickets []Ticket `json:"tickets"` } type Label struct { - Name string `json:"name"` - Colors struct { - Background string `json:"background"` - Foreground string `json:"text"` - } `json:"colors"` - Created time.Time `json:"created"` - Tracker Tracker `json:"tracker"` -} - -type Tracker struct { - ID int `json:"id"` - Owner User `json:"owner"` - Created time.Time `json:"created"` - Updated time.Time `json:"updated"` - Name string `json:"name"` + ID int `json:"id"` + Created time.Time `json:"created"` + Name string `json:"name"` + BackgroundColor string `json:"background_color"` + ForegroundColor string `json:"foreground_color"` } type Ticket struct { - ID int `json:"id"` - Ref string `json:"ref"` - Tracker Tracker `json:"tracker"` - Subject string `json:"title"` - Created time.Time `json:"created"` - Updated time.Time `json:"updated"` - Submitter *Participant `json:"submitter"` // null in shorter ticket dicts - Body string `json:"description"` - Status string `json:"status"` - Resolution string `json:"resolution"` - Labels []string `json:"labels"` - Assignees []User `json:"assignees"` - Upstream string `json:"upstream"` - Signature string `json:"X-Payload-Signature"` - Nonce string `json:"X-Payload-Nonce"` - Events []Event `json:"events"` + ID int `json:"id"` + Created time.Time `json:"created"` + Updated time.Time `json:"updated"` + Submitter Participant `json:"submitter"` + Ref string `json:"ref"` + Subject string `json:"subject"` + Body string `json:"body"` + Status string `json:"status"` + Resolution string `json:"resolution"` + Labels []string `json:"labels"` + Assignees []User `json:"assignees"` + Upstream string `json:"upstream"` + Signature string `json:"X-Payload-Signature"` + Nonce string `json:"X-Payload-Nonce"` + Events []Event `json:"events"` } type Event struct { @@ -76,8 +62,7 @@ type Event struct { OldResolution *string `json:"old_resolution"` NewStatus *string `json:"new_status"` NewResolution *string `json:"new_resolution"` - User *Participant `json:"user"` - Ticket *Ticket `json:"ticket"` + Participant *Participant `json:"participant"` Comment *Comment `json:"comment"` Label *string `json:"label"` ByUser *Participant `json:"by_user"` @@ -89,6 +74,7 @@ type Event struct { type Participant struct { Type string `json:"type"` + UserID int `json:"user_id"` CanonicalName string `json:"canonical_name"` Name string `json:"name"` Address string `json:"address"` @@ -97,15 +83,33 @@ type Participant struct { } type User struct { + ID int `json:"id"` CanonicalName string `json:"canonical_name"` Name string `json:"name"` } type Comment struct { - ID int `json:"id"` - Created time.Time `json:"created"` - Submitter Participant `json:"submitter"` - Text string `json:"text"` + ID int `json:"id"` + Created time.Time `json:"created"` + Author Participant `json:"author"` + Text string `json:"text"` +} + +type TicketSignatureData struct { + TrackerID int `json:"tracker_id"` + TicketID int `json:"ticket_id"` + Subject string `json:"subject"` + Body string `json:"body"` + SubmitterID int `json:"submitter_id"` + Upstream string `json:"upstream"` +} + +type CommentSignatureData struct { + TrackerID int `json:"tracker_id"` + TicketID int `json:"ticket_id"` + Comment string `json:"comment"` + AuthorID int `json:"author_id"` + Upstream string `json:"upstream"` } func importParticipant(ctx context.Context, part Participant, upstream, ourUpstream string) (int, error) { @@ -192,29 +196,9 @@ func importTrackerDump(ctx context.Context, trackerID int, dump io.Reader, ourUp return err } - // Create labels - labelIDs := map[string]int{} - if err := database.WithTx(ctx, nil, func(tx *sql.Tx) error { - for _, label := range tracker.Labels { - row := tx.QueryRowContext(ctx, ` - INSERT INTO label ( - created, updated, tracker_id, name, color, text_color - ) VALUES ( - $1, $1, $2, $3, $4, $5 - ) RETURNING id - `, label.Created, trackerID, label.Name, label.Colors.Background, label.Colors.Foreground) - var labelID int - if err := row.Scan(&labelID); err != nil { - return err - } - labelIDs[label.Name] = labelID - } - return nil - }); err != nil { - return err - } - defer func() { + r := recover() + if err := database.WithTx(ctx, nil, func(tx *sql.Tx) error { _, err := tx.ExecContext(ctx, ` UPDATE tracker @@ -225,9 +209,31 @@ func importTrackerDump(ctx context.Context, trackerID int, dump io.Reader, ourUp }); err != nil { panic(err) } + + if r != nil { + panic(r) + } }() if err := database.WithTx(ctx, nil, func(tx *sql.Tx) error { + // Create labels + labelIDs := map[string]int{} + for _, label := range tracker.Labels { + row := tx.QueryRowContext(ctx, ` + INSERT INTO label ( + created, updated, tracker_id, name, color, text_color + ) VALUES ( + $1, $1, $2, $3, $4, $5 + ) RETURNING id + `, label.Created, trackerID, label.Name, + label.BackgroundColor, label.ForegroundColor) + var labelID int + if err := row.Scan(&labelID); err != nil { + return err + } + labelIDs[label.Name] = labelID + } + var nextTicketID int row := tx.QueryRowContext(ctx, `SELECT next_ticket_id FROM tracker WHERE id = $1`, @@ -244,11 +250,32 @@ func importTrackerDump(ctx context.Context, trackerID int, dump io.Reader, ourUp var maxTicketID int for _, ticket := range tracker.Tickets { - submitterID, err := importParticipant(ctx, *ticket.Submitter, ticket.Upstream, ourUpstream) + submitterID, err := importParticipant(ctx, ticket.Submitter, ticket.Upstream, ourUpstream) if err != nil { return err } + ticketAuthenticity := model.AUTH_UNAUTHENTICATED + if ticket.Upstream == ourUpstream && ticket.Submitter.Type == "user" { + sigdata := TicketSignatureData{ + TrackerID: tracker.ID, + TicketID: ticket.ID, + Subject: ticket.Subject, + Body: ticket.Body, + SubmitterID: ticket.Submitter.UserID, + Upstream: ticket.Upstream, + } + payload, err := json.Marshal(sigdata) + if err != nil { + panic(err) + } + if crypto.VerifyWebhook(payload, ticket.Nonce, ticket.Signature) { + ticketAuthenticity = model.AUTH_AUTHENTIC + } else { + ticketAuthenticity = model.AUTH_TAMPERED + } + } + // Compute the max ticket ID. We can't use the number of tickets as // the next ticket ID because that won't include deleted tickets if ticket.ID > maxTicketID { @@ -268,9 +295,9 @@ func importTrackerDump(ctx context.Context, trackerID int, dump io.Reader, ourUp RETURNING id `, ticket.Created, ticket.Updated, trackerID, ticket.ID, submitterID, ticket.Subject, ticket.Body, - model.TicketStatus(strings.ToUpper(ticket.Status)).ToInt(), - model.TicketResolution(strings.ToUpper(ticket.Resolution)).ToInt(), - model.AUTH_UNAUTHENTICATED) + model.TicketStatus(ticket.Status).ToInt(), + model.TicketResolution(ticket.Resolution).ToInt(), + ticketAuthenticity) var ticketPKID int if err := row.Scan(&ticketPKID); err != nil { return err @@ -307,26 +334,51 @@ func importTrackerDump(ctx context.Context, trackerID int, dump io.Reader, ourUp var eventType int for _, etype := range event.EventType { + etype := model.EventType(etype) + if !etype.IsValid() { + fmt.Errorf("failed to import ticket #%d: invalid event type %s", ticket.ID, etype) + } eventType |= eventTypeMap[etype] } if eventType == 0 { return fmt.Errorf("failed to import ticket #%d: invalid ticket event", ticket.ID, eventType) } - if event.User != nil { - userPartID, err := importParticipant(ctx, *event.User, event.Upstream, ourUpstream) + if event.Participant != nil { + eventPartID, err := importParticipant(ctx, *event.Participant, event.Upstream, ourUpstream) if err != nil { return err } - partID = &userPartID + partID = &eventPartID } if eventType&model.EVENT_COMMENT != 0 { - submitterID, err := importParticipant(ctx, event.Comment.Submitter, event.Upstream, ourUpstream) + authorID, err := importParticipant(ctx, event.Comment.Author, event.Upstream, ourUpstream) if err != nil { return err } + commentAuthenticity := model.AUTH_UNAUTHENTICATED + if event.Upstream == ourUpstream && event.Comment.Author.Type == "user" { + log.Println(event.Comment.Author.UserID) + sigdata := CommentSignatureData{ + TrackerID: tracker.ID, + TicketID: ticket.ID, + Comment: event.Comment.Text, + AuthorID: event.Comment.Author.UserID, + Upstream: event.Upstream, + } + payload, err := json.Marshal(sigdata) + if err != nil { + panic(err) + } + if crypto.VerifyWebhook(payload, event.Nonce, event.Signature) { + commentAuthenticity = model.AUTH_AUTHENTIC + } else { + commentAuthenticity = model.AUTH_TAMPERED + } + } + row := tx.QueryRowContext(ctx, ` INSERT INTO ticket_comment ( created, updated, submitter_id, ticket_id, text, @@ -334,8 +386,8 @@ func importTrackerDump(ctx context.Context, trackerID int, dump io.Reader, ourUp ) VALUES ( $1, $1, $2, $3, $4, $5 ) RETURNING id - `, event.Comment.Created, submitterID, ticketPKID, event.Comment.Text, - model.AUTH_UNAUTHENTICATED) + `, event.Comment.Created, authorID, ticketPKID, event.Comment.Text, + commentAuthenticity) var _commentID int if err := row.Scan(&_commentID); err != nil { return err @@ -453,14 +505,14 @@ func convertResolutionToInt(resolution *string) *int { return &resolutionInt } -var eventTypeMap = map[string]int{ - "created": model.EVENT_CREATED, - "comment": model.EVENT_COMMENT, - "status_change": model.EVENT_STATUS_CHANGE, - "label_added": model.EVENT_LABEL_ADDED, - "label_removed": model.EVENT_LABEL_REMOVED, - "assigned_user": model.EVENT_ASSIGNED_USER, - "unassigned_user": model.EVENT_UNASSIGNED_USER, - "user_mentioned": model.EVENT_USER_MENTIONED, - "ticket_mentioned": model.EVENT_TICKET_MENTIONED, +var eventTypeMap = map[model.EventType]int{ + model.EventTypeCreated: model.EVENT_CREATED, + model.EventTypeComment: model.EVENT_COMMENT, + model.EventTypeStatusChange: model.EVENT_STATUS_CHANGE, + model.EventTypeLabelAdded: model.EVENT_LABEL_ADDED, + model.EventTypeLabelRemoved: model.EVENT_LABEL_REMOVED, + model.EventTypeAssignedUser: model.EVENT_ASSIGNED_USER, + model.EventTypeUnassignedUser: model.EVENT_UNASSIGNED_USER, + model.EventTypeUserMentioned: model.EVENT_USER_MENTIONED, + model.EventTypeTicketMentioned: model.EVENT_TICKET_MENTIONED, } diff --git a/todosrht/blueprints/settings.py b/todosrht/blueprints/settings.py index 1579ba6..5c1abc5 100644 --- a/todosrht/blueprints/settings.py +++ b/todosrht/blueprints/settings.py @@ -9,7 +9,7 @@ from srht.crypto import sign_payload from srht.database import db from srht.oauth import current_user, loginrequired from srht.flask import date_handler, session -from srht.graphql import exec_gql +from srht.graphql import exec_gql, GraphQLOperation, GraphQLUpload from srht.validation import Validation from tempfile import NamedTemporaryFile from todosrht.access import get_tracker @@ -17,7 +17,6 @@ from todosrht.trackers import get_recent_users from todosrht.types import Event, EventType, Ticket, TicketAccess, Visibility from todosrht.types import ParticipantType, UserAccess, User from todosrht.urls import tracker_url -from todosrht.tracker_import import tracker_import settings = Blueprint("settings", __name__) @@ -243,20 +242,57 @@ def export_POST(owner, name): if current_user.id != tracker.owner_id: abort(403) + upstream = get_origin("todo.sr.ht", external=True) + + def participant_to_dict(self): + if self.participant_type == ParticipantType.user: + return { + "type": "user", + "user_id": self.user.id, + "canonical_name": self.user.canonical_name, + "name": self.user.username, + } + elif self.participant_type == ParticipantType.email: + return { + "type": "email", + "address": self.email, + "name": self.email_name, + } + elif self.participant_type == ParticipantType.external: + return { + "type": "external", + "external_id": self.external_id, + "external_url": self.external_url, + } + assert False + dump = list() tickets = Ticket.query.filter(Ticket.tracker_id == tracker.id).all() for ticket in tickets: - td = ticket.to_dict() - td["upstream"] = get_origin("todo.sr.ht", external=True) + td = { + "id": ticket.scoped_id, + "created": ticket.created, + "updated": ticket.updated, + "submitter": participant_to_dict(ticket.submitter), + "ref": ticket.ref(), + "subject": ticket.title, + "body": ticket.description, + "status": ticket.status.name.upper(), + "resolution": ticket.resolution.name.upper(), + "labels": [l.name for l in ticket.labels], + "assignees": [u.to_dict(short=True) for u in ticket.assigned_users], + } + td["upstream"] = upstream if ticket.submitter.participant_type == ParticipantType.user: sigdata = OrderedDict({ - "description": ticket.description, - "ref": ticket.ref(), - "submitter": ticket.submitter.user.canonical_name, - "title": ticket.title, - "upstream": get_origin("todo.sr.ht", external=True), + "tracker_id": tracker.id, + "ticket_id": ticket.scoped_id, + "subject": ticket.title, + "body": ticket.description, + "submitter_id": ticket.submitter.user.id, + "upstream": upstream, }) - sigdata = json.dumps(sigdata) + sigdata = json.dumps(sigdata, separators=(',',':')) signature = sign_payload(sigdata) td.update(signature) @@ -264,27 +300,64 @@ def export_POST(owner, name): if any(events): td["events"] = list() for event in events: - ev = event.to_dict() - ev["upstream"] = get_origin("todo.sr.ht", external=True) + ev = { + "id": event.id, + "created": event.created, + "event_type": [t.name.upper() for t in EventType if t in event.event_type], + "old_status": event.old_status.name.upper() + if event.old_status else None, + "old_resolution": event.old_resolution.name.upper() + if event.old_resolution else None, + "new_status": event.new_status.name.upper() + if event.new_status else None, + "new_resolution": event.new_resolution.name.upper() + if event.new_resolution else None, + "participant": participant_to_dict(event.participant) + if event.participant else None, + "ticket_id": event.ticket.scoped_id + if event.ticket else None, + "comment": { + "id": event.comment.id, + "created": event.comment.created, + "author": participant_to_dict(event.comment.submitter), + "text": event.comment.text, + } if event.comment else None, + "label": event.label.name if event.label else None, + "by_user": participant_to_dict(event.by_participant) + if event.by_participant else None, + "from_ticket_id": event.from_ticket.scoped_id + if event.from_ticket else None, + } + ev["upstream"] = upstream if (EventType.comment in event.event_type and event.participant.participant_type == ParticipantType.user): sigdata = OrderedDict({ + "tracker_id": tracker.id, + "ticket_id": ticket.scoped_id, "comment": event.comment.text, - "id": event.id, - "ticket": event.ticket.ref(), - "user": event.participant.user.canonical_name, - "upstream": get_origin("todo.sr.ht", external=True), + "author_id": event.comment.submitter.user.id, + "upstream": upstream }) - sigdata = json.dumps(sigdata) + sigdata = json.dumps(sigdata, separators=(',',':')) signature = sign_payload(sigdata) ev.update(signature) td["events"].append(ev) dump.append(td) dump = json.dumps({ - "owner": tracker.owner.to_dict(), + "id": tracker.id, + "owner": tracker.owner.to_dict(short=True), + "created": tracker.created, + "updated": tracker.updated, "name": tracker.name, - "labels": [l.to_dict() for l in tracker.labels], + "description": tracker.description, + "labels": [{ + "id": l.id, + "created": l.created, + "name": l.name, + "background_color": l.color, + "foreground_color": l.text_color, + } for l in tracker.labels], "tickets": dump, }, default=date_handler) with NamedTemporaryFile() as ntf: @@ -304,22 +377,32 @@ def import_POST(owner, name): if current_user.id != tracker.owner_id: abort(403) - dump = request.files.get("dump") valid = Validation(request) + dump = request.files.get("dump") valid.expect(dump is not None, "Tracker dump file is required", field="dump") + if not valid.ok: return render_template("tracker-import-export.html", view="import/export", tracker=tracker, **valid.kwargs) - try: - dump = dump.stream.read() - dump = gzip.decompress(dump) - dump = json.loads(dump) - except: - abort(400) - tracker_import.delay(dump, tracker.id) + op = GraphQLOperation(""" + mutation ImportTrackerDump($trackerId: Int!, $dump: Upload!) { + importTrackerDump(trackerId: $trackerId, dump: $dump) + } + """) + + dump = GraphQLUpload( + dump.filename, + dump.stream, + "application/octet-stream", + ) + op.var("trackerId", tracker.id) + op.var("dump", dump) + op.execute("todo.sr.ht", valid=valid) + + if not valid.ok: + return render_template("tracker-import-export.html", + view="import/export", tracker=tracker, **valid.kwargs) - tracker.import_in_progress = True - db.session.commit() return redirect(tracker_url(tracker)) diff --git a/todosrht/templates/tracker-import-export.html b/todosrht/templates/tracker-import-export.html index edd162b..0e43e69 100644 --- a/todosrht/templates/tracker-import-export.html +++ b/todosrht/templates/tracker-import-export.html @@ -36,20 +36,21 @@ after preparing the following JSON data as UTF-8: 

{
-  "description": $description,
-  "ref": "{{tracker.ref()}}#$id",
-  "submitter": "~$username",
-  "title": $title,
+  "tracker_id": $trackerId,
+  "ticket_id": $ticketId,
+  "subject": $subject,
+  "body": $body,
+  "submitter_id": "$userId",
   "upstream": "{{get_origin('todo.sr.ht', external=True)}}",
 }

Comments use the same process with the following JSON data as UTF-8: 

{
+  "tracker_id": $trackerId,
+  "ticket_id": $ticketId,
   "comment": $comment,
-  "id": $id,
-  "ticket": "{{tracker.ref()}}#$id",
-  "user": "~$username",
+  "author_id": "$userId",
   "upstream": "{{get_origin('todo.sr.ht', external=True)}}",
 }

diff --git a/todosrht/tracker_import.py b/todosrht/tracker_import.py deleted file mode 100644 index e68a59f..0000000 --- a/todosrht/tracker_import.py +++ /dev/null @@ -1,207 +0,0 @@ -import json -from collections import OrderedDict -from datetime import datetime, timezone -from srht.crypto import verify_payload -from srht.config import get_origin -from srht.database import db -from todosrht.tickets import submit_ticket -from todosrht.tickets import get_participant_for_email -from todosrht.tickets import get_participant_for_external -from todosrht.tickets import get_participant_for_user -from todosrht.types import Event, EventType, Tracker, Ticket, TicketComment -from todosrht.types import ParticipantType, User -from todosrht.types import Label, TicketLabel -from todosrht.types import TicketStatus, TicketResolution -from todosrht.types import TicketAuthenticity -from todosrht.webhooks import worker - -our_upstream = get_origin("todo.sr.ht", external=True) - -def _parse_date(date): - date = datetime.fromisoformat(date) - date = date.astimezone(timezone.utc).replace(tzinfo=None) - return date - -def _import_participant(pdata, upstream): - if pdata["type"] == "user": - if upstream == our_upstream: - user = User.query.filter(User.username == pdata["name"]).first() - else: - user = None - - if user: - submitter = get_participant_for_user(user) - else: - submitter = get_participant_for_external( - pdata["canonical_name"], - upstream + "/" + pdata["canonical_name"]) - elif pdata["type"] == "email": - submitter = get_participant_for_email(pdata["address"], pdata["name"]) - elif pdata["type"] == "external": - submitter = get_participant_for_external( - pdata["external_id"], pdata["external_url"]) - return submitter - -def _import_comment(ticket, event, edata): - cdata = edata["comment"] - comment = TicketComment() - submitter = _import_participant(cdata["submitter"], edata["upstream"]) - comment.submitter_id = submitter.id - comment.ticket_id = ticket.id - comment.text = cdata["text"] - comment.authenticity = TicketAuthenticity.unauthenticated - comment.created = _parse_date(cdata["created"]) - comment.updated = comment.created - comment._no_autoupdate = True - signature, nonce = edata.get("X-Payload-Signature"), edata.get("X-Payload-Nonce") - if (signature and nonce - and edata["upstream"] == our_upstream - and submitter.participant_type == ParticipantType.user): - # TODO: Validate signatures from trusted third-parties - sigdata = OrderedDict({ - "comment": comment.text, - "id": edata["id"], # not important to verify this - "ticket": edata["ticket"]["ref"], # not important to verify this - "user": submitter.user.canonical_name, - "upstream": edata["upstream"], - }) - sigdata = json.dumps(sigdata) - if verify_payload(sigdata, signature, nonce): - comment.authenticity = TicketAuthenticity.authentic - else: - comment.authenticity = TicketAuthenticity.tampered - ticket.comment_count += 1 - db.session.add(comment) - db.session.flush() - event.comment_id = comment.id - -def _tracker_import(dump, tracker): - ldict = dict() - for ldata in dump["labels"]: - label = Label() - label.tracker_id = tracker.id - label.name = ldata["name"] - label.color = ldata["colors"]["background"] - label.text_color = ldata["colors"]["text"] - db.session.add(label) - db.session.flush() - ldict[label.name] = label.id - tickets = sorted(dump["tickets"], key=lambda t: t["id"]) - for tdata in tickets: - for field in [ - "id", "title", "created", "description", "status", "resolution", - "labels", "assignees", "upstream", "events", "submitter", - ]: - if not field in tdata: - print("Invalid ticket data") - continue - ticket = Ticket.query.filter( - Ticket.tracker_id == tracker.id, - Ticket.scoped_id == tdata["id"]).one_or_none() - if ticket: - print(f"Ticket {tdata['id']} already imported - skipping") - continue - submitter = _import_participant(tdata["submitter"], tdata["upstream"]) - ticket = submit_ticket(tracker, submitter, - tdata["title"], tdata["description"], importing=True) - try: - created = _parse_date(tdata["created"]) - except ValueError: - created = datetime.utcnow() - ticket._no_autoupdate = True - ticket.created = created - ticket.updated = created - ticket.status = TicketStatus[tdata["status"]] - ticket.resolution = TicketResolution[tdata["resolution"]] - ticket.authenticity = TicketAuthenticity.unauthenticated - for label in tdata["labels"]: - tl = TicketLabel() - tl.ticket_id = ticket.id - tl.label_id = ldict.get(label) - tl.user_id = tracker.owner_id - db.session.add(tl) - # TODO: assignees - signature, nonce = tdata.get("X-Payload-Signature"), tdata.get("X-Payload-Nonce") - if (signature and nonce - and tdata["upstream"] == our_upstream - and submitter.participant_type == ParticipantType.user): - # TODO: Validate signatures from trusted third-parties - sigdata = OrderedDict({ - "description": ticket.description, - "ref": tdata["ref"], # not important to verify this - "submitter": ticket.submitter.user.canonical_name, - "title": ticket.title, - "upstream": tdata["upstream"], - }) - sigdata = json.dumps(sigdata) - if verify_payload(sigdata, signature, nonce): - ticket.authenticity = TicketAuthenticity.authentic - else: - ticket.authenticity = TicketAuthenticity.tampered - for edata in tdata.get("events", []): - for field in [ - "created", "event_type", "old_status", "new_status", - "old_resolution", "new_resolution", "user", "ticket", - "comment", "label", "by_user", "from_ticket", "upstream", - ]: - if not field in edata: - print("Invalid ticket event") - return - event = Event() - for etype in edata["event_type"]: - if event.event_type == None: - event.event_type = EventType[etype] - else: - event.event_type |= EventType[etype] - if event.event_type == None: - print("Invalid ticket event") - continue - if EventType.comment in event.event_type: - _import_comment(ticket, event, edata) - if EventType.status_change in event.event_type: - if edata["old_status"]: - event.old_status = TicketStatus[edata["old_status"]] - if edata["new_status"]: - event.new_status = TicketStatus[edata["new_status"]] - if EventType.label_added in event.event_type: - event.label_id = ldict.get(edata["label"]) - if not event.label_id: - continue - if EventType.label_removed in event.event_type: - event.label_id = ldict.get(edata["label"]) - if not event.label_id: - continue - if EventType.assigned_user in event.event_type: - by_participant = _import_participant( - edata["by_user"], edata["upstream"]) - event.by_participant_id = by_participant.id - if EventType.unassigned_user in event.event_type: - by_participant = _import_participant( - edata["by_user"], edata["upstream"]) - event.by_participant_id = by_participant.id - if EventType.user_mentioned in event.event_type: - continue # Magic event type, do not import - if EventType.ticket_mentioned in event.event_type: - continue # TODO: Could reference tickets imported in later iters - event.created = _parse_date(edata["created"]) - event.updated = event.created - event._no_autoupdate = True - event.ticket_id = ticket.id - participant = _import_participant(edata["user"], edata["upstream"]) - event.participant_id = participant.id - db.session.add(event) - print(f"Imported {ticket.ref()}") - -@worker.task -def tracker_import(dump, tracker_id): - tracker = Tracker.query.get(tracker_id) - try: - _tracker_import(dump, tracker) - except: - # TODO: Tell user that the import failed? - db.session.rollback() - tracker = Tracker.query.get(tracker_id) - raise - finally: - tracker.import_in_progress = False - db.session.commit() diff --git a/todosrht/webhooks.py b/todosrht/webhooks.py index eb8e08a..5a06bab 100644 --- a/todosrht/webhooks.py +++ b/todosrht/webhooks.py @@ -15,8 +15,6 @@ webhooks_broker = cfg("todo.sr.ht", "webhooks") worker = make_worker(broker=webhooks_broker) webhook_metrics_collector = RedisQueueCollector(webhooks_broker, "srht_webhooks", "Webhook queue length") -import todosrht.tracker_import - class UserWebhook(CeleryWebhook): events = [ Event("tracker:create", "trackers:read"),