Implement ticket import

This commit is contained in:
Drew DeVault 2020-01-09 12:19:44 -05:00
parent 45fb4cf3e4
commit fe08cd4447
14 changed files with 474 additions and 29 deletions

View File

@ -0,0 +1,28 @@
"""Add authenticity column to tickets & comments
Revision ID: 4b32d0e0603d
Revises: 4631a2317dd0
Create Date: 2020-01-09 09:39:26.614899
"""
# revision identifiers, used by Alembic.
revision = '4b32d0e0603d'
down_revision = '4631a2317dd0'
from alembic import op
import sqlalchemy as sa
def upgrade():
op.add_column("ticket", sa.Column("authenticity", sa.Integer,
nullable=False, server_default="0"))
op.add_column("ticket_comment", sa.Column("authenticity", sa.Integer,
nullable=False, server_default="0"))
op.add_column("tracker", sa.Column("import_in_progress", sa.Boolean,
nullable=False, server_default="f"))
def downgrade():
op.drop_column("ticket", "authenticity")
op.drop_column("ticket_comment", "authenticity")
op.drop_column("tracker", "import_in_progress")

View File

@ -0,0 +1,104 @@
"""Add cascade to tracker & ticket webhooks
Revision ID: 61e241dc978c
Revises: 4b32d0e0603d
Create Date: 2020-01-09 10:38:42.203433
"""
# revision identifiers, used by Alembic.
revision = '61e241dc978c'
down_revision = '4b32d0e0603d'
from alembic import op
import sqlalchemy as sa
def upgrade():
op.drop_constraint(
constraint_name="tracker_webhook_subscription_tracker_id_fkey",
table_name="tracker_webhook_subscription",
type_="foreignkey")
op.create_foreign_key(
constraint_name="tracker_webhook_subscription_tracker_id_fkey",
source_table="tracker_webhook_subscription",
referent_table="tracker",
local_cols=["tracker_id"],
remote_cols=["id"],
ondelete="CASCADE")
op.drop_constraint(
constraint_name="ticket_webhook_subscription_ticket_id_fkey",
table_name="ticket_webhook_subscription",
type_="foreignkey")
op.create_foreign_key(
constraint_name="ticket_webhook_subscription_ticket_id_fkey",
source_table="ticket_webhook_subscription",
referent_table="ticket",
local_cols=["ticket_id"],
remote_cols=["id"],
ondelete="CASCADE")
op.drop_constraint(
constraint_name="tracker_webhook_delivery_subscription_id_fkey",
table_name="tracker_webhook_delivery",
type_="foreignkey")
op.create_foreign_key(
constraint_name="tracker_webhook_delivery_subscription_id_fkey",
source_table="tracker_webhook_delivery",
referent_table="tracker_webhook_subscription",
local_cols=["subscription_id"],
remote_cols=["id"],
ondelete="CASCADE")
op.drop_constraint(
constraint_name="ticket_webhook_delivery_subscription_id_fkey",
table_name="ticket_webhook_delivery",
type_="foreignkey")
op.create_foreign_key(
constraint_name="ticket_webhook_delivery_subscription_id_fkey",
source_table="ticket_webhook_delivery",
referent_table="ticket_webhook_subscription",
local_cols=["subscription_id"],
remote_cols=["id"],
ondelete="CASCADE")
def downgrade():
op.drop_constraint(
constraint_name="tracker_webhook_subscription_tracker_id_fkey",
table_name="tracker_webhook_subscription",
type_="foreignkey")
op.create_foreign_key(
constraint_name="tracker_webhook_subscription_tracker_id_fkey",
source_table="tracker_webhook_subscription",
referent_table="tracker",
local_cols=["tracker_id"],
remote_cols=["id"])
op.drop_constraint(
constraint_name="ticket_webhook_subscription_ticket_id_fkey",
table_name="ticket_webhook_subscription",
type_="foreignkey")
op.create_foreign_key(
constraint_name="ticket_webhook_subscription_ticket_id_fkey",
source_table="ticket_webhook_subscription",
referent_table="ticket",
local_cols=["ticket_id"],
remote_cols=["id"])
op.drop_constraint(
constraint_name="tracker_webhook_delivery_subscription_id_fkey",
table_name="tracker_webhook_delivery",
type_="foreignkey")
op.create_foreign_key(
constraint_name="tracker_webhook_delivery_subscription_id_fkey",
source_table="tracker_webhook_delivery",
referent_table="tracker_webhook_subscription",
local_cols=["subscription_id"],
remote_cols=["id"])
op.drop_constraint(
constraint_name="ticket_webhook_delivery_subscription_id_fkey",
table_name="ticket_webhook_delivery",
type_="foreignkey")
op.create_foreign_key(
constraint_name="ticket_webhook_delivery_subscription_id_fkey",
source_table="ticket_webhook_delivery",
referent_table="ticket_webhook_subscription",
local_cols=["subscription_id"],
remote_cols=["id"])

View File

@ -17,6 +17,7 @@ from todosrht.types import Event, EventType, Ticket, TicketAccess
from todosrht.types import ParticipantType, UserAccess, User
from todosrht.urls import tracker_url
from todosrht.webhooks import UserWebhook
from todosrht.tracker_import import tracker_import
settings = Blueprint("settings", __name__)
@ -273,7 +274,12 @@ def export_POST(owner, name):
td["events"].append(ev)
dump.append(td)
dump = json.dumps(dump, default=date_handler)
dump = json.dumps({
"owner": tracker.owner.to_dict(),
"name": tracker.name,
"labels": [l.to_dict() for l in tracker.labels],
"tickets": dump,
}, default=date_handler)
with NamedTemporaryFile() as ntf:
ntf.write(gzip.compress(dump.encode()))
f = open(ntf.name, "rb")
@ -281,3 +287,29 @@ def export_POST(owner, name):
return send_file(f, as_attachment=True,
attachment_filename=f"{tracker.owner.username}-{tracker.name}.json.gz",
mimetype="application/gzip")
@settings.route("/<owner>/<name>/settings/import", methods=["POST"])
@loginrequired
def import_POST(owner, name):
tracker, access = get_tracker(owner, name)
if not tracker:
abort(404)
if current_user.id != tracker.owner_id:
abort(403)
dump = request.files.get("dump")
valid = Validation(request)
valid.expect(dump is not None,
"Tracker dump file is required", field="dump")
if not valid.ok:
return render_template("tracker-import-export.html",
view="import/export", tracker=tracker, **valid.kwargs)
dump = dump.stream.read()
dump = gzip.decompress(dump)
dump = json.loads(dump)
tracker_import.delay(dump, tracker.id)
tracker.import_in_progress = True
db.session.commit()
return redirect(tracker_url(tracker))

View File

@ -95,9 +95,21 @@
</dd>
<dt class="col-md-3">Submitter</dt>
<dd class="col-md-9">
<a id="submitter-field" href="{{ ticket.submitter|participant_url }}">
{{ ticket.submitter }}
</a>
<a
id="submitter-field"
href="{{ ticket.submitter|participant_url }}"
>{{ ticket.submitter }}</a>
{% if ticket.authenticity.name == "unauthenticated" %}
<span
class="text-danger"
title="This ticket was imported from an external source and its authenticity cannot be guaranteed."
>(unverified)</span>
{% elif ticket.authenticity.name == "tampered" %}
<span
class="text-danger"
title="This ticket has been edited by a third-party - its contents are not genuine."
>(edited)</span>
{% endif %}
</dd>
<dt class="col-md-3">Assigned to</dt>
<dd id="assignee-field" class="col-md-9">
@ -240,6 +252,19 @@
<a href="{{ event.participant|participant_url }}">
{{ event.participant }}
</a>
{% if EventType.comment in event.event_type %}
{% if event.comment.authenticity.name == "unauthenticated" %}
<span
class="text-danger"
title="This ticket was imported from an external source and its authenticity cannot be guaranteed."
>(unverified)</span>
{% elif event.comment.authenticity.name == "tampered" %}
<span
class="text-danger"
title="This ticket has been edited by a third-party - its contents are not genuine."
>(edited)</span>
{% endif %}
{% endif %}
{% endif %}
{% if EventType.status_change in event.event_type %}
<strong class="text-success">

View File

@ -64,9 +64,12 @@
{{icon('caret-right')}}
</button>
</form>
{# TODO
<form method="POST" action="{{url_for('settings.import_POST',
owner=tracker.owner.canonical_name, name=tracker.name)}}">
<form
method="POST"
action="{{url_for('settings.import_POST',
owner=tracker.owner.canonical_name, name=tracker.name)}}"
enctype="multipart/form-data"
>
{{csrf_token()}}
<h3>Import tracker dump</h3>
<p>
@ -74,14 +77,17 @@
tool on this or another todo.sr.ht instance.
</p>
<div class="form-group">
<input type="file" name="dump" class="form-control" />
<input
type="file"
name="dump"
class="form-control {{valid.cls('dump')}}" />
{{valid.summary("dump")}}
</div>
<button type="submit" class="btn btn-primary">
Import tracker data
{{icon('caret-right')}}
</button>
</form>
#}
</div>
</div>
{% endblock %}

View File

@ -144,6 +144,11 @@
{% endif %}
</div>
<div class="col-md-8">
{% if tracker.import_in_progress %}
<div class="alert alert-primary">
An import operation is currently in progress.
</div>
{% endif %}
<form style="margin-bottom: 0.5rem">
<input
name="search"

View File

@ -445,7 +445,7 @@ def _send_new_ticket_notification(subscription, ticket):
notify(subscription, "new_ticket", subject,
headers=headers, ticket=ticket, ticket_url=ticket_url(ticket))
def submit_ticket(tracker, submitter, title, description):
def submit_ticket(tracker, submitter, title, description, importing=False):
ticket = Ticket(
submitter=submitter,
tracker=tracker,
@ -465,21 +465,22 @@ def submit_ticket(tracker, submitter, title, description):
db.session.flush()
# Subscribe submitter to the ticket if not already subscribed to the tracker
get_or_create_subscription(ticket, submitter)
if not importing:
get_or_create_subscription(ticket, submitter)
# Send notifications
for sub in tracker.subscriptions:
_create_event_notification(sub.participant, event)
if sub.participant != submitter:
_send_new_ticket_notification(sub, ticket)
# Send notifications
for sub in tracker.subscriptions:
_create_event_notification(sub.participant, event)
if sub.participant != submitter:
_send_new_ticket_notification(sub, ticket)
notified_users = [sub.participant for sub in tracker.subscriptions]
_handle_mentions(
ticket,
ticket.submitter,
ticket.description,
notified_users,
)
notified_users = [sub.participant for sub in tracker.subscriptions]
_handle_mentions(
ticket,
ticket.submitter,
ticket.description,
notified_users,
)
db.session.commit()
db.session.commit()
return ticket

208
todosrht/tracker_import.py Normal file
View File

@ -0,0 +1,208 @@
import json
from collections import OrderedDict
from datetime import datetime, timezone
from srht.crypto import verify_payload
from srht.config import get_origin
from srht.database import db
from todosrht.tickets import submit_ticket
from todosrht.tickets import get_participant_for_email
from todosrht.tickets import get_participant_for_external
from todosrht.tickets import get_participant_for_user
from todosrht.types import Event, EventType, Tracker, Ticket, TicketComment
from todosrht.types import ParticipantType, User
from todosrht.types import Label, TicketLabel
from todosrht.types import TicketStatus, TicketResolution
from todosrht.types import TicketAuthenticity
from todosrht.webhooks import worker
our_upstream = get_origin("todo.sr.ht", external=True)
def _parse_date(date):
if "+" in date:
date = date[:date.index("+")]
date = datetime.strptime(date, "%Y-%m-%dT%H:%M:%S")
date = date.astimezone(timezone.utc).replace(tzinfo=None)
return date
def _import_participant(pdata, upstream):
if pdata["type"] == "user":
if upstream == our_upstream:
user = User.query.filter(User.username == pdata["name"]).first()
else:
user = None
if user:
submitter = get_participant_for_user(user)
else:
submitter = get_participant_for_external(
pdata["canonical_name"],
upstream + "/" + pdata["canonical_name"])
elif pdata["type"] == "email":
submitter = get_participant_for_email(pdata["address"], pdata["name"])
elif pdata["type"] == "external":
submitter = get_participant_for_external(
pdata["external_id"], pdata["external_url"])
return submitter
def _import_comment(ticket, event, edata):
cdata = edata["comment"]
comment = TicketComment()
submitter = _import_participant(cdata["submitter"], edata["upstream"])
comment.submitter_id = submitter.id
comment.ticket_id = ticket.id
comment.text = cdata["text"]
comment.authenticity = TicketAuthenticity.unauthenticated
comment.created = _parse_date(cdata["created"])
comment.updated = comment.created
comment._no_autoupdate = True
signature, nonce = edata.get("X-Payload-Signature"), edata.get("X-Payload-Nonce")
if (signature and nonce
and edata["upstream"] == our_upstream
and submitter.participant_type == ParticipantType.user):
# TODO: Validate signatures from trusted third-parties
sigdata = OrderedDict({
"comment": comment.text,
"id": edata["id"], # not important to verify this
"ticket": edata["ticket"]["ref"], # not important to verify this
"user": submitter.user.canonical_name,
"upstream": edata["upstream"],
})
sigdata = json.dumps(sigdata)
if verify_payload(sigdata, signature, nonce):
comment.authenticity = TicketAuthenticity.authentic
else:
comment.authenticity = TicketAuthenticity.tampered
db.session.add(comment)
db.session.flush()
event.comment_id = comment.id
def _tracker_import(dump, tracker):
ldict = dict()
for ldata in dump["labels"]:
label = Label()
label.tracker_id = tracker.id
label.name = ldata["name"]
label.color = ldata["colors"]["background"]
label.text_color = ldata["colors"]["text"]
db.session.add(label)
db.session.flush()
ldict[label.name] = label.id
tickets = sorted(dump["tickets"], key=lambda t: t["id"])
for tdata in tickets:
for field in [
"id", "title", "created", "description", "status", "resolution",
"labels", "assignees", "upstream", "events", "submitter",
]:
if not field in tdata:
print("Invalid ticket data")
continue
ticket = Ticket.query.filter(
Ticket.tracker_id == tracker.id,
Ticket.scoped_id == tdata["id"]).one_or_none()
if ticket:
print(f"Ticket {tdata['id']} already imported - skipping")
continue
submitter = _import_participant(tdata["submitter"], tdata["upstream"])
ticket = submit_ticket(tracker, submitter,
tdata["title"], tdata["description"], importing=True)
try:
created = _parse_date(tdata["created"])
except ValueError:
created = datetime.utcnow()
ticket._no_autoupdate = True
ticket.created = created
ticket.updated = created
ticket.status = TicketStatus[tdata["status"]]
ticket.resolution = TicketResolution[tdata["resolution"]]
ticket.authenticity = TicketAuthenticity.unauthenticated
for label in tdata["labels"]:
tl = TicketLabel()
tl.ticket_id = ticket.id
tl.label_id = ldict.get(label)
tl.user_id = tracker.owner_id
db.session.add(tl)
# TODO: assignees
signature, nonce = tdata.get("X-Payload-Signature"), tdata.get("X-Payload-Nonce")
if (signature and nonce
and tdata["upstream"] == our_upstream
and submitter.participant_type == ParticipantType.user):
# TODO: Validate signatures from trusted third-parties
sigdata = OrderedDict({
"description": ticket.description,
"ref": tdata["ref"], # not important to verify this
"submitter": ticket.submitter.user.canonical_name,
"title": ticket.title,
"upstream": tdata["upstream"],
})
sigdata = json.dumps(sigdata)
if verify_payload(sigdata, signature, nonce):
ticket.authenticity = TicketAuthenticity.authentic
else:
ticket.authenticity = TicketAuthenticity.tampered
for edata in tdata["events"]:
for field in [
"created", "event_type", "old_status", "new_status",
"old_resolution", "new_resolution", "user", "ticket",
"comment", "label", "by_user", "from_ticket", "upstream",
]:
if not field in edata:
print("Invalid ticket event")
return
event = Event()
for etype in edata["event_type"]:
if event.event_type == None:
event.event_type = EventType[etype]
else:
event.event_type |= EventType[etype]
if event.event_type == None:
print("Invalid ticket event")
continue
if EventType.comment in event.event_type:
_import_comment(ticket, event, edata)
if EventType.status_change in event.event_type:
if edata["old_status"]:
event.old_status = TicketStatus[edata["old_status"]]
if edata["new_status"]:
event.new_status = TicketStatus[edata["new_status"]]
if EventType.label_added in event.event_type:
event.label_id = ldict.get(edata["label"])
if not event.label_id:
continue
if EventType.label_removed in event.event_type:
event.label_id = ldict.get(edata["label"])
if not event.label_id:
continue
if EventType.assigned_user in event.event_type:
by_participant = _import_participant(
edata["by_user"], edata["upstream"])
event.by_participant_id = by_participant.id
if EventType.unassigned_user in event.event_type:
by_participant = _import_participant(
edata["by_user"], edata["upstream"])
event.by_participant_id = by_participant.id
if EventType.user_mentioned in event.event_type:
continue # Magic event type, do not import
if EventType.ticket_mentioned in event.event_type:
continue # TODO: Could reference tickets imported in later iters
event.created = _parse_date(edata["created"])
event.updated = event.created
event._no_autoupdate = True
event.ticket_id = ticket.id
participant = _import_participant(edata["user"], edata["upstream"])
event.participant_id = participant.id
db.session.add(event)
print(f"Imported {ticket.ref()}")
@worker.task
def tracker_import(dump, tracker_id):
tracker = Tracker.query.get(tracker_id)
try:
_tracker_import(dump, tracker)
except:
# TODO: Tell user that the import failed?
db.session.rollback()
tracker = Tracker.query.get(tracker_id)
raise
finally:
tracker.import_in_progress = False
db.session.commit()

View File

@ -10,6 +10,7 @@ class OAuthToken(Base, ExternalOAuthTokenMixin):
from todosrht.types.ticketaccess import TicketAccess
from todosrht.types.ticketstatus import TicketStatus, TicketResolution
from todosrht.types.ticketstatus import TicketAuthenticity
from todosrht.types.event import Event, EventType, EventNotification
from todosrht.types.label import Label, TicketLabel

View File

@ -1,7 +1,9 @@
import sqlalchemy as sa
import sqlalchemy_utils as sau
from srht.database import Base
from srht.flagtype import FlagType
from todosrht.types import TicketAccess, TicketStatus, TicketResolution
from todosrht.types import TicketAuthenticity
class Ticket(Base):
__tablename__ = 'ticket'
@ -64,6 +66,16 @@ class Ticket(Base):
secondary="ticket_assignee",
foreign_keys="[TicketAssignee.ticket_id,TicketAssignee.assignee_id]")
authenticity = sa.Column(
sau.ChoiceType(TicketAuthenticity, impl=sa.Integer()),
nullable=False, server_default="0")
"""
The authenticity of the ticket. Tickets submitted by logged-in users are
considered authentic. Tickets which have been exported and re-imported are
considered authentic if the signature validates, unauthenticated if no
signature is present, or tampered if the signature does not validate.
"""
def ref(self, short=False, email=False):
if short:
return "#" + str(self.scoped_id)

View File

@ -1,7 +1,9 @@
import sqlalchemy as sa
import sqlalchemy_utils as sau
from srht.database import Base
from srht.flagtype import FlagType
from todosrht.types import TicketAccess, TicketStatus, TicketResolution
from todosrht.types import TicketAuthenticity
class TicketComment(Base):
__tablename__ = 'ticket_comment'
@ -21,6 +23,16 @@ class TicketComment(Base):
text = sa.Column(sa.Unicode(16384))
authenticity = sa.Column(
sau.ChoiceType(TicketAuthenticity, impl=sa.Integer()),
nullable=False, server_default="0")
"""
The authenticity of the comment. Comments submitted by logged-in users are
considered authentic. Comments which have been exported and re-imported are
considered authentic if the signature validates, unauthenticated if no
signature is present, or tampered if the signature does not validate.
"""
def to_dict(self, short=False):
return {
"id": self.id,

View File

@ -1,4 +1,4 @@
from enum import IntFlag
from enum import IntFlag, Enum
class TicketStatus(IntFlag):
reported = 0
@ -16,3 +16,8 @@ class TicketResolution(IntFlag):
invalid = 16
duplicate = 32
not_our_bug = 64
class TicketAuthenticity(Enum):
authentic = 0
unauthenticated = 1
tampered = 2

View File

@ -55,6 +55,9 @@ class Tracker(Base):
default=TicketAccess.browse + TicketAccess.submit + TicketAccess.comment)
"""Permissions granted to anonymous (non-logged in) users"""
import_in_progress = sa.Column(sa.Boolean,
nullable=False, server_default='f')
@staticmethod
def create_from_request(request, user):
valid = Validation(request)

View File

@ -11,6 +11,8 @@ import sqlalchemy as sa
worker = make_worker(broker=cfg("todo.sr.ht", "webhooks"))
import todosrht.tracker_import
class UserWebhook(CeleryWebhook):
events = [
Event("tracker:create", "trackers:read"),
@ -28,8 +30,8 @@ class TrackerWebhook(CeleryWebhook):
]
tracker_id = sa.Column(sa.Integer,
sa.ForeignKey('tracker.id'), nullable=False)
tracker = sa.orm.relationship('Tracker')
sa.ForeignKey('tracker.id', ondelete="CASCADE"), nullable=False)
tracker = sa.orm.relationship('Tracker', cascade="all, delete-orphan")
class TicketWebhook(CeleryWebhook):
events = [
@ -38,5 +40,6 @@ class TicketWebhook(CeleryWebhook):
]
ticket_id = sa.Column(sa.Integer,
sa.ForeignKey('ticket.id'), nullable=False)
ticket = sa.orm.relationship('Ticket')
sa.ForeignKey('ticket.id', ondelete="CASCADE"), nullable=False)
ticket = sa.orm.relationship('Ticket', cascade="all, delete-orphan")