api/graph: implement tracker export

This commit is contained in:
Simon Ser 2023-08-02 08:52:58 +00:00 committed by Drew DeVault
parent 366713a3c2
commit c2a87f590d
7 changed files with 540 additions and 47 deletions

View File

@ -57,6 +57,9 @@ models:
Cursor:
model:
- git.sr.ht/~sircmpwn/core-go/model.Cursor
URL:
model:
- git.sr.ht/~sircmpwn/todo.sr.ht/api/graph/model.URL
Filter:
model:
- git.sr.ht/~sircmpwn/core-go/model.Filter

34
api/graph/model/url.go Normal file
View File

@ -0,0 +1,34 @@
package model
import (
"encoding/json"
"fmt"
"io"
"net/url"
)
// XXX: gqlgen bug prevents us from using type URL *url.URL
type URL struct {
*url.URL
}
func (u *URL) UnmarshalGQL(v interface{}) error {
raw, ok := v.(string)
if !ok {
return fmt.Errorf("Mail format is a base64-encoded string")
}
parsed, err := url.Parse(raw)
if err != nil {
return err
}
u.URL = parsed
return nil
}
func (u URL) MarshalGQL(w io.Writer) {
data, err := json.Marshal(u.URL.String())
if err != nil {
panic(err)
}
w.Write(data)
}

View File

@ -3111,8 +3111,14 @@ func (r *trackerResolver) Acls(ctx context.Context, obj *model.Tracker, cursor *
}
// Export is the resolver for the export field.
func (r *trackerResolver) Export(ctx context.Context, obj *model.Tracker) (string, error) {
panic(fmt.Errorf("not implemented")) // TODO
func (r *trackerResolver) Export(ctx context.Context, obj *model.Tracker) (*model.URL, error) {
origin := config.GetOrigin(config.ForContext(ctx), "todo.sr.ht", true)
uri := fmt.Sprintf("%s/query/tracker/%d.json.gz", origin, obj.ID)
url, err := url.Parse(uri)
if err != nil {
panic(err)
}
return &model.URL{url}, nil
}
// Webhooks is the resolver for the webhooks field.

View File

@ -2,12 +2,16 @@ package main
import (
"context"
"log"
"net/http"
"strconv"
"git.sr.ht/~sircmpwn/core-go/config"
"git.sr.ht/~sircmpwn/core-go/server"
"git.sr.ht/~sircmpwn/core-go/webhooks"
work "git.sr.ht/~sircmpwn/dowork"
"github.com/99designs/gqlgen/graphql"
"github.com/go-chi/chi"
"git.sr.ht/~sircmpwn/todo.sr.ht/api/account"
"git.sr.ht/~sircmpwn/todo.sr.ht/api/graph"
@ -40,7 +44,7 @@ func main() {
webhookQueue := webhooks.NewQueue(schema)
legacyWebhooks := webhooks.NewLegacyQueue()
server.NewServer("todo.sr.ht", appConfig).
gsrv := server.NewServer("todo.sr.ht", appConfig).
WithDefaultMiddleware().
WithMiddleware(
loaders.Middleware,
@ -55,6 +59,23 @@ func main() {
trackersQueue,
webhookQueue.Queue,
legacyWebhooks.Queue,
).
Run()
)
gsrv.Router().Get("/query/tracker/{id}.json.gz", func(w http.ResponseWriter, r *http.Request) {
id, err := strconv.Atoi(chi.URLParam(r, "id"))
if err != nil {
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte("Invalid tracker ID\r\n"))
return
}
w.Header().Set("Content-Type", "application/gzip")
w.Header().Set("Content-Disposition", `attachment; filename="tracker.json.gz"`)
if err := trackers.ExportDump(r.Context(), id, w); err != nil {
log.Printf("Tracker export failed: %v", err)
w.WriteHeader(http.StatusInternalServerError)
}
})
gsrv.Run()
}

426
api/trackers/export.go Normal file
View File

@ -0,0 +1,426 @@
package trackers
import (
"compress/gzip"
"context"
"database/sql"
"encoding/json"
"fmt"
"io"
"git.sr.ht/~sircmpwn/core-go/auth"
"git.sr.ht/~sircmpwn/core-go/config"
"git.sr.ht/~sircmpwn/core-go/crypto"
"git.sr.ht/~sircmpwn/core-go/database"
sq "github.com/Masterminds/squirrel"
"git.sr.ht/~sircmpwn/todo.sr.ht/api/graph/model"
"git.sr.ht/~sircmpwn/todo.sr.ht/api/loaders"
)
func ExportDump(ctx context.Context, trackerID int, w io.Writer) error {
tracker, err := loaders.ForContext(ctx).TrackersByID.Load(trackerID)
if err != nil {
return err
}
owner, err := loaders.ForContext(ctx).UsersByID.Load(tracker.OwnerID)
if err != nil {
return err
}
trackerDump := TrackerDump{
ID: tracker.ID,
Owner: User{
ID: owner.ID,
CanonicalName: owner.CanonicalName(),
Name: owner.Username,
},
Created: tracker.Created,
Updated: tracker.Updated,
Name: tracker.Name,
Description: convertNullString(tracker.Description),
}
if err := database.WithTx(ctx, &sql.TxOptions{
ReadOnly: true,
}, func(tx *sql.Tx) error {
trackerDump.Labels, err = exportLabels(ctx, tx, tracker.ID)
if err != nil {
return err
}
var pkids []int
trackerDump.Tickets, pkids, err = exportTickets(ctx, tx, tracker)
if err != nil {
return err
}
for i := range trackerDump.Tickets {
ticket := &trackerDump.Tickets[i]
ticket.Labels, err = exportTicketLabels(ctx, tx, pkids[i])
if err != nil {
return err
}
ticket.Assignees, err = exportTicketAssignees(ctx, tx, pkids[i])
if err != nil {
return err
}
ticket.Events, err = exportTicketEvents(ctx, tx, pkids[i])
if err != nil {
return err
}
}
return nil
}); err != nil {
return err
}
signDump(&trackerDump)
gw := gzip.NewWriter(w)
if err := json.NewEncoder(gw).Encode(&trackerDump); err != nil {
return err
}
return gw.Close()
}
func signDump(tracker *TrackerDump) {
for i := range tracker.Tickets {
ticket := &tracker.Tickets[i]
if ticket.Submitter.Type == "user" {
signTicket(ticket, tracker.ID)
}
for j := range ticket.Events {
event := &ticket.Events[j]
if event.Participant.Type == "user" && event.Comment != nil {
signCommentEvent(event, tracker.ID, ticket.ID)
}
}
}
}
func signTicket(ticket *Ticket, trackerID int) {
sigdata := TicketSignatureData{
TrackerID: trackerID,
TicketID: ticket.ID,
Subject: ticket.Subject,
Body: ticket.Body,
SubmitterID: ticket.Submitter.UserID,
Upstream: ticket.Upstream,
}
payload, err := json.Marshal(sigdata)
if err != nil {
panic(err)
}
ticket.Nonce, ticket.Signature = crypto.SignWebhook(payload)
}
func signCommentEvent(event *Event, trackerID, ticketID int) {
sigdata := CommentSignatureData{
TrackerID: trackerID,
TicketID: ticketID,
Comment: event.Comment.Text,
AuthorID: event.Comment.Author.UserID,
Upstream: event.Upstream,
}
payload, err := json.Marshal(sigdata)
if err != nil {
panic(err)
}
event.Nonce, event.Signature = crypto.SignWebhook(payload)
}
func exportLabels(ctx context.Context, tx *sql.Tx, trackerID int) ([]Label, error) {
label := (&model.Label{}).As(`l`)
query := database.
Select(ctx, label).
From(`label l`).
Where(`l.tracker_id = ?`, trackerID)
rows, err := query.RunWith(tx).QueryContext(ctx)
if err != nil {
return nil, err
}
defer rows.Close()
var l []Label
for rows.Next() {
var label model.Label
if err := rows.Scan(database.Scan(ctx, &label)...); err != nil {
return nil, err
}
l = append(l, Label{
ID: label.ID,
Created: label.Created,
Name: label.Name,
BackgroundColor: label.BackgroundColor,
ForegroundColor: label.ForegroundColor,
})
}
return l, nil
}
func exportTickets(ctx context.Context, tx *sql.Tx, tracker *model.Tracker) ([]Ticket, []int, error) {
ticket := (&model.Ticket{}).As(`tk`)
var query sq.SelectBuilder
if tracker.CanBrowse() {
query = database.
Select(ctx, ticket).
From(`ticket tk`).
Where(`tk.tracker_id = ?`, tracker.ID)
} else {
user := auth.ForContext(ctx)
query = database.
Select(ctx, ticket).
From(`ticket tk`).
Join(`participant p ON p.user_id = ?`, user.UserID).
Where(sq.And{
sq.Expr(`tk.tracker_id = ?`, tracker.ID),
sq.Expr(`tk.submitter_id = p.id`),
})
}
rows, err := query.RunWith(tx).QueryContext(ctx)
if err != nil {
return nil, nil, err
}
defer rows.Close()
origin := config.GetOrigin(config.ForContext(ctx), "todo.sr.ht", true)
var (
l []Ticket
pkids []int
)
for rows.Next() {
var ticket model.Ticket
if err := rows.Scan(database.Scan(ctx, &ticket)...); err != nil {
return nil, nil, err
}
submitter, err := loaders.ForContext(ctx).EntitiesByParticipantID.Load(ticket.SubmitterID)
if err != nil {
return nil, nil, err
}
l = append(l, Ticket{
ID: ticket.ID,
Created: ticket.Created,
Updated: ticket.Updated,
Submitter: *exportParticipant(submitter),
Ref: ticket.Ref(),
Subject: ticket.Subject,
Body: convertNullString(ticket.Body),
Status: string(ticket.Status()),
Resolution: string(ticket.Resolution()),
Upstream: origin,
})
pkids = append(pkids, ticket.PKID)
}
return l, pkids, nil
}
func exportTicketEvents(ctx context.Context, tx *sql.Tx, ticketID int) ([]Event, error) {
event := (&model.Event{}).As(`ev`)
query := database.
Select(ctx, event).
From(`event ev`).
Where(`ev.ticket_id = ?`, ticketID)
rows, err := query.RunWith(tx).QueryContext(ctx)
if err != nil {
return nil, err
}
defer rows.Close()
origin := config.GetOrigin(config.ForContext(ctx), "todo.sr.ht", true)
var l []Event
for rows.Next() {
var ev model.Event
if err := rows.Scan(database.Scan(ctx, &ev)...); err != nil {
return nil, err
}
evDump := Event{
ID: ev.ID,
Created: ev.Created,
Upstream: origin,
}
if ev.EventType&model.EVENT_CREATED != 0 {
evDump.EventType = append(evDump.EventType, string(model.EventTypeCreated))
}
if ev.EventType&model.EVENT_COMMENT != 0 {
evDump.EventType = append(evDump.EventType, string(model.EventTypeComment))
}
if ev.EventType&model.EVENT_STATUS_CHANGE != 0 {
evDump.EventType = append(evDump.EventType, string(model.EventTypeStatusChange))
evDump.OldStatus = string(model.TicketStatusFromInt(*ev.OldStatus))
evDump.NewStatus = string(model.TicketStatusFromInt(*ev.NewStatus))
evDump.OldResolution = string(model.TicketResolutionFromInt(*ev.OldResolution))
evDump.NewResolution = string(model.TicketResolutionFromInt(*ev.NewResolution))
}
if ev.EventType&model.EVENT_LABEL_ADDED != 0 {
evDump.EventType = append(evDump.EventType, string(model.EventTypeLabelAdded))
}
if ev.EventType&model.EVENT_LABEL_REMOVED != 0 {
evDump.EventType = append(evDump.EventType, string(model.EventTypeLabelRemoved))
}
if ev.EventType&model.EVENT_ASSIGNED_USER != 0 {
evDump.EventType = append(evDump.EventType, string(model.EventTypeAssignedUser))
}
if ev.EventType&model.EVENT_UNASSIGNED_USER != 0 {
evDump.EventType = append(evDump.EventType, string(model.EventTypeUnassignedUser))
}
if ev.EventType&model.EVENT_USER_MENTIONED != 0 {
evDump.EventType = append(evDump.EventType, string(model.EventTypeUserMentioned))
}
if ev.EventType&model.EVENT_TICKET_MENTIONED != 0 {
evDump.EventType = append(evDump.EventType, string(model.EventTypeTicketMentioned))
}
if ev.ParticipantID != nil {
entity, err := loaders.ForContext(ctx).EntitiesByParticipantID.Load(*ev.ParticipantID)
if err != nil {
return nil, err
}
evDump.Participant = exportParticipant(entity)
}
if ev.EventType&model.EVENT_COMMENT != 0 {
comment, err := loaders.ForContext(ctx).CommentsByIDUnsafe.Load(*ev.CommentID)
if err != nil {
return nil, err
}
author, err := loaders.ForContext(ctx).EntitiesByParticipantID.Load(*ev.ParticipantID)
if err != nil {
return nil, err
}
evDump.Comment = &Comment{
ID: comment.Database.ID,
Created: ev.Created,
Author: *exportParticipant(author),
Text: comment.Database.Text,
}
}
if ev.EventType&(model.EVENT_LABEL_ADDED|model.EVENT_LABEL_REMOVED) != 0 {
name, err := exportLabelName(ctx, *ev.LabelID)
if err != nil {
return nil, err
}
evDump.Label = &name
}
if ev.EventType&(model.EVENT_ASSIGNED_USER|model.EVENT_UNASSIGNED_USER) != 0 {
byUser, err := loaders.ForContext(ctx).EntitiesByParticipantID.Load(*ev.ByParticipantID)
if err != nil {
return nil, err
}
evDump.ByUser = exportParticipant(byUser)
}
// TODO: populate FromTicket
l = append(l, evDump)
}
return l, nil
}
func exportLabelName(ctx context.Context, labelID int) (string, error) {
var name string
err := database.WithTx(ctx, &sql.TxOptions{
ReadOnly: true,
}, func(tx *sql.Tx) error {
return tx.QueryRowContext(ctx, `SELECT name FROM label WHERE id = $1`, labelID).Scan(&name)
})
return name, err
}
func exportTicketAssignees(ctx context.Context, tx *sql.Tx, ticketID int) ([]User, error) {
user := (&model.User{}).As(`u`)
query := database.
Select(ctx, user).
From(`ticket_assignee ta`).
Join(`"user" u ON ta.assignee_id = u.id`).
Where(`ta.ticket_id = ?`, ticketID)
rows, err := query.RunWith(tx).QueryContext(ctx)
if err != nil {
return nil, err
}
defer rows.Close()
var l []User
for rows.Next() {
var user model.User
if err := rows.Scan(database.Scan(ctx, &user)...); err != nil {
return nil, err
}
l = append(l, User{
ID: user.ID,
CanonicalName: user.CanonicalName(),
Name: user.Username,
})
}
return l, nil
}
func exportTicketLabels(ctx context.Context, tx *sql.Tx, ticketID int) ([]string, error) {
rows, err := tx.Query(`SELECT l.name
FROM label l
JOIN ticket_label tl ON tl.label_id = l.id
WHERE tl.ticket_id = $1`, ticketID)
if err != nil {
return nil, err
}
defer rows.Close()
var l []string
for rows.Next() {
var name string
if err := rows.Scan(&name); err != nil {
return nil, err
}
l = append(l, name)
}
return l, nil
}
func exportParticipant(e model.Entity) *Participant {
switch e := e.(type) {
case *model.User:
return &Participant{
Type: "user",
UserID: e.ID,
CanonicalName: e.CanonicalName(),
Name: e.Username,
}
case *model.EmailAddress:
return &Participant{
Type: "email",
Address: e.Mailbox,
Name: convertNullString(e.Name),
}
case *model.ExternalUser:
return &Participant{
Type: "external",
ExternalID: e.ExternalID,
ExternalURL: convertNullString(e.ExternalURL),
}
}
panic(fmt.Errorf("unknown entity type %T", e))
}
func convertNullString(ns *string) string {
if ns == nil {
return ""
}
return *ns
}

View File

@ -23,9 +23,9 @@ type TrackerDump struct {
Created time.Time `json:"created"`
Updated time.Time `json:"updated"`
Name string `json:"name"`
Description string `json:"description"`
Labels []Label `json:"labels"`
Tickets []Ticket `json:"tickets"`
Description string `json:"description,omitempty"`
Labels []Label `json:"labels,omitempty"`
Tickets []Ticket `json:"tickets,omitempty"`
}
type Label struct {
@ -43,43 +43,43 @@ type Ticket struct {
Submitter Participant `json:"submitter"`
Ref string `json:"ref"`
Subject string `json:"subject"`
Body string `json:"body"`
Body string `json:"body,omitempty"`
Status string `json:"status"`
Resolution string `json:"resolution"`
Labels []string `json:"labels"`
Assignees []User `json:"assignees"`
Labels []string `json:"labels,omitempty"`
Assignees []User `json:"assignees,omitempty"`
Upstream string `json:"upstream"`
Signature string `json:"X-Payload-Signature"`
Nonce string `json:"X-Payload-Nonce"`
Events []Event `json:"events"`
Signature string `json:"X-Payload-Signature,omitempty"`
Nonce string `json:"X-Payload-Nonce,omitempty"`
Events []Event `json:"events,omitempty"`
}
type Event struct {
ID int `json:"id"`
Created time.Time `json:"created"`
EventType []string `json:"event_type"`
OldStatus *string `json:"old_status"`
OldResolution *string `json:"old_resolution"`
NewStatus *string `json:"new_status"`
NewResolution *string `json:"new_resolution"`
Participant *Participant `json:"participant"`
Comment *Comment `json:"comment"`
Label *string `json:"label"`
ByUser *Participant `json:"by_user"`
FromTicket *Ticket `json:"from_ticket"`
OldStatus string `json:"old_status,omitempty"`
OldResolution string `json:"old_resolution,omitempty"`
NewStatus string `json:"new_status,omitempty"`
NewResolution string `json:"new_resolution,omitempty"`
Participant *Participant `json:"participant,omitempty"`
Comment *Comment `json:"comment,omitempty"`
Label *string `json:"label,omitempty"`
ByUser *Participant `json:"by_user,omitempty"`
FromTicket *Ticket `json:"from_ticket,omitempty"`
Upstream string `json:"upstream"`
Signature string `json:"X-Payload-Signature"`
Nonce string `json:"X-Payload-Nonce"`
Signature string `json:"X-Payload-Signature,omitempty"`
Nonce string `json:"X-Payload-Nonce,omitempty"`
}
type Participant struct {
Type string `json:"type"`
UserID int `json:"user_id"`
CanonicalName string `json:"canonical_name"`
Name string `json:"name"`
Address string `json:"address"`
ExternalID string `json:"external_id"`
ExternalURL string `json:"external_url"`
UserID int `json:"user_id,omitempty"`
CanonicalName string `json:"canonical_name,omitempty"`
Name string `json:"name,omitempty"`
Address string `json:"address,omitempty"`
ExternalID string `json:"external_id,omitempty"`
ExternalURL string `json:"external_url,omitempty"`
}
type User struct {
@ -465,39 +465,41 @@ func importTrackerDump(ctx context.Context, trackerID int, dump io.Reader, ourUp
return nil
}
func convertStatus(status *string) *model.TicketStatus {
if status == nil {
func convertStatus(status string) *model.TicketStatus {
if status == "" {
return nil
}
*status = strings.ToUpper(*status)
return (*model.TicketStatus)(status)
status = strings.ToUpper(status)
modelStatus := model.TicketStatus(status)
return &modelStatus
}
func convertStatusToInt(status *string) *int {
if status == nil {
func convertStatusToInt(status string) *int {
if status == "" {
statusInt := model.STATUS_REPORTED
return &statusInt
}
*status = strings.ToUpper(*status)
statusInt := (model.TicketStatus)(*status).ToInt()
status = strings.ToUpper(status)
statusInt := model.TicketStatus(status).ToInt()
return &statusInt
}
func convertResolution(resolution *string) *model.TicketResolution {
if resolution == nil {
func convertResolution(resolution string) *model.TicketResolution {
if resolution == "" {
return nil
}
*resolution = strings.ToUpper(*resolution)
return (*model.TicketResolution)(resolution)
resolution = strings.ToUpper(resolution)
modelRes := model.TicketResolution(resolution)
return &modelRes
}
func convertResolutionToInt(resolution *string) *int {
if resolution == nil {
func convertResolutionToInt(resolution string) *int {
if resolution == "" {
resolutionInt := model.RESOLVED_UNRESOLVED
return &resolutionInt
}
*resolution = strings.ToUpper(*resolution)
resolutionInt := (model.TicketResolution)(*resolution).ToInt()
resolution = strings.ToUpper(resolution)
resolutionInt := model.TicketResolution(resolution).ToInt()
return &resolutionInt
}

1
go.mod
View File

@ -8,6 +8,7 @@ require (
github.com/99designs/gqlgen v0.17.20
github.com/Masterminds/squirrel v1.4.0
github.com/emersion/go-message v0.15.0
github.com/go-chi/chi v4.1.2+incompatible
github.com/google/uuid v1.3.0
github.com/kr/text v0.2.0 // indirect
github.com/lib/pq v1.8.0