{
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "type": "dashboard"
      }
    ]
  },
  "editable": true,
  "gnetId": null,
  "graphTooltip": 0,
  "id": 2,
  "iteration": 1603378842428,
  "links": [],
  "panels": [
    {
      "datasource": "Datanommer",
      "description": "",
      "fieldConfig": {
        "defaults": {
          "custom": {}
        },
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 18,
        "x": 0,
        "y": 0
      },
      "id": 39,
      "options": {
        "content": "# Monitor-gating analysis\n\nAnalysis of runs of the https://pagure.io/fedora-ci/monitor-gating project over time.\n\nNotes on triaging problems here: https://hackmd.io/3NWpzzutTDu8CXFrIT1JqQ?both         ",
        "mode": "markdown"
      },
      "pluginVersion": "7.1.0",
      "targets": [
        {
          "format": "time_series",
          "group": [],
          "metricColumn": "none",
          "rawQuery": false,
          "rawSql": "SELECT\n  $__time(time_column),\n  value1\nFROM\n  metric_table\nWHERE\n  $__timeFilter(time_column)\n",
          "refId": "A",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "column"
              }
            ]
          ],
          "timeColumn": "time",
          "where": [
            {
              "name": "$__timeFilter",
              "params": [],
              "type": "macro"
            }
          ]
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Monitor-gating analysis",
      "type": "text"
    },
    {
      "alert": {
        "alertRuleTags": {},
        "conditions": [
          {
            "evaluator": {
              "params": [
                15
              ],
              "type": "lt"
            },
            "operator": {
              "type": "and"
            },
            "query": {
              "params": [
                "A",
                "5m",
                "now"
              ]
            },
            "reducer": {
              "params": [],
              "type": "max"
            },
            "type": "query"
          }
        ],
        "executionErrorState": "alerting",
        "for": "10m",
        "frequency": "10m",
        "handler": 1,
        "message": "Less than 10 runs in the last day.",
        "name": "Tests per day alert",
        "noDataState": "keep_state",
        "notifications": [
          {
            "uid": "l7OKaKpGk"
          }
        ]
      },
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "Datanommer",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": null,
            "filterable": false
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 18,
        "y": 0
      },
      "hiddenSeries": false,
      "id": 41,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.2.1",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "format": "time_series",
          "group": [],
          "metricColumn": "none",
          "rawQuery": true,
          "rawSql": "select now() as time, count(*) from messages where topic like '%gating%end%' and \"timestamp\" > now() - interval '1 day'\r\n",
          "refId": "A",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "column"
              }
            ]
          ],
          "timeColumn": "time",
          "where": [
            {
              "name": "$__timeFilter",
              "params": [],
              "type": "macro"
            }
          ]
        }
      ],
      "thresholds": [
        {
          "colorMode": "critical",
          "fill": true,
          "line": true,
          "op": "lt",
          "value": 15
        }
      ],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Test runs last 24 hours",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "alert": {
        "alertRuleTags": {},
        "conditions": [
          {
            "evaluator": {
              "params": [
                5
              ],
              "type": "gt"
            },
            "operator": {
              "type": "and"
            },
            "query": {
              "params": [
                "A",
                "5m",
                "now"
              ]
            },
            "reducer": {
              "params": [],
              "type": "avg"
            },
            "type": "query"
          }
        ],
        "executionErrorState": "alerting",
        "for": "5m",
        "frequency": "1m",
        "handler": 1,
        "message": "More than 5 failures",
        "name": "Failures last 24 hours alert",
        "noDataState": "no_data",
        "notifications": []
      },
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "Datanommer",
      "description": "",
      "fieldConfig": {
        "defaults": {
          "custom": {}
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 21,
        "y": 0
      },
      "hiddenSeries": false,
      "id": 43,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.2.1",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "format": "time_series",
          "group": [],
          "metricColumn": "none",
          "rawQuery": true,
          "rawSql": "select now() as time, count(*) from messages where topic like '%gating%end%fail' and \"timestamp\" > now() - interval '1 day'\r\n",
          "refId": "A",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "column"
              }
            ]
          ],
          "timeColumn": "time",
          "where": [
            {
              "name": "$__timeFilter",
              "params": [],
              "type": "macro"
            }
          ]
        }
      ],
      "thresholds": [
        {
          "colorMode": "critical",
          "fill": true,
          "line": true,
          "op": "gt",
          "value": 5
        }
      ],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Failures last 24 hours",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "datasource": "Datanommer",
      "description": "Percentage of successful runs (as noted by monitor-gating sending org.fedoraproject.prod.monitor-gating.$test.end.success message) out of all of org.fedoraproject.prod.monitor-gating.$test.end.* messages ",
      "fieldConfig": {
        "defaults": {
          "custom": {},
          "mappings": [
            {
              "from": "",
              "id": 1,
              "text": "",
              "to": "",
              "type": 1
            }
          ],
          "max": 100,
          "min": 0,
          "thresholds": {
            "mode": "percentage",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "green",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 3,
        "x": 0,
        "y": 5
      },
      "id": 8,
      "options": {
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "mean"
          ],
          "fields": "/^percent_ci_done$/",
          "values": true
        },
        "showThresholdLabels": true,
        "showThresholdMarkers": true
      },
      "pluginVersion": "7.2.1",
      "targets": [
        {
          "format": "table",
          "group": [],
          "metricColumn": "none",
          "rawQuery": true,
          "rawSql": "with c as (select count(*) as finished_gating_runs from messages where topic like '%gating%$test%.end.%' and $__timeFilter(\"timestamp\")),\r\ns as (select count(*) as ci_completed from messages where topic like '%gating%$test%.end%succe%'  and $__timeFilter(\"timestamp\"))\r\nselect (s.ci_completed*100)/c.finished_gating_runs as percent_ci_done from c,s\r\n",
          "refId": "A",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "column"
              }
            ]
          ],
          "timeColumn": "time",
          "where": [
            {
              "name": "$__timeFilter",
              "params": [],
              "type": "macro"
            }
          ]
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Full Run Success in %",
      "type": "gauge"
    },
    {
      "datasource": "Datanommer",
      "description": "Percentage of runs that reached the step in the build where the package was successfully cloned, its version was bumped and it was pushed again.",
      "fieldConfig": {
        "defaults": {
          "custom": {},
          "mappings": [],
          "max": 100,
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "green",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 3,
        "x": 3,
        "y": 5
      },
      "id": 9,
      "options": {
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "mean"
          ],
          "fields": "/^percent_ci_done$/",
          "values": false
        },
        "showThresholdLabels": true,
        "showThresholdMarkers": true
      },
      "pluginVersion": "7.2.1",
      "targets": [
        {
          "format": "table",
          "group": [],
          "metricColumn": "none",
          "rawQuery": true,
          "rawSql": "with t as (select * from messages where topic like 'org.fedoraproject.prod.monitor-gating.$test%' and $__timeFilter(\"timestamp\")),\r\nc as (select count(*) as finished_gating_runs from t where topic like '%end%'),\r\ns as (select count(*) as ci_completed from t where (topic like '%succeeded' or ((topic like '%failed' or topic like '%error') and _msg::jsonb#>>'{output_text}' ~ '.*Pushing changes\\s+\\[DONE\\].*')))\r\nselect ((s.ci_completed)*100)/c.finished_gating_runs as percent_ci_done from c,s",
          "refId": "A",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "column"
              }
            ]
          ],
          "timeColumn": "time",
          "where": [
            {
              "name": "$__timeFilter",
              "params": [],
              "type": "macro"
            }
          ]
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Pagure Success in %",
      "type": "gauge"
    },
    {
      "datasource": "Datanommer",
      "description": "Percentage of runs that reached the step of Koji successfully finishing the package build.",
      "fieldConfig": {
        "defaults": {
          "custom": {},
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "green",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 3,
        "x": 6,
        "y": 5
      },
      "id": 11,
      "options": {
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "mean"
          ],
          "fields": "/^percent_ci_done$/",
          "values": false
        },
        "showThresholdLabels": true,
        "showThresholdMarkers": true
      },
      "pluginVersion": "7.2.1",
      "targets": [
        {
          "format": "table",
          "group": [],
          "metricColumn": "none",
          "rawQuery": true,
          "rawSql": "with t as (select * from messages where topic like 'org.fedoraproject.prod.monitor-gating.$test%' and $__timeFilter(\"timestamp\")),\r\nc as (select count(*) as finished_gating_runs from t where topic like '%end%'),\r\ns as (select count(*) as ci_completed from t where (topic like '%succeeded' or ((topic like '%failed' or topic like '%error') and _msg::jsonb#>>'{output_text}' ~ '.*uilding the package.+\\[DONE\\]')))\r\nselect ((s.ci_completed)*100)/c.finished_gating_runs as percent_ci_done from c,s",
          "refId": "A",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "column"
              }
            ]
          ],
          "timeColumn": "time",
          "where": [
            {
              "name": "$__timeFilter",
              "params": [],
              "type": "macro"
            }
          ]
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Koji Success in %",
      "type": "gauge"
    },
    {
      "datasource": "Datanommer",
      "description": "Percentage of runs that successfuly reached the step of initial creation of the update in Bodhi",
      "fieldConfig": {
        "defaults": {
          "custom": {},
          "mappings": [],
          "max": 100,
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "green",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 3,
        "x": 9,
        "y": 5
      },
      "id": 10,
      "options": {
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "mean"
          ],
          "fields": "/^percent_ci_done$/",
          "values": false
        },
        "showThresholdLabels": true,
        "showThresholdMarkers": true
      },
      "pluginVersion": "7.2.1",
      "targets": [
        {
          "format": "table",
          "group": [],
          "metricColumn": "none",
          "rawQuery": true,
          "rawSql": "with t as (select * from messages where topic like 'org.fedoraproject.prod.monitor-gating.$test%' and $__timeFilter(\"timestamp\")),\r\nc as (select count(*) as finished_gating_runs from t where topic like '%end%'),\r\ns as (select count(*) as ci_completed from t where (topic like '%succeeded' or ((topic like '%failed' or topic like '%error') and _msg::jsonb#>>'{output_text}' ~ 'bodhi[^\\n]+\\[DONE\\]')))\r\nselect ((s.ci_completed)*100)/c.finished_gating_runs as percent_ci_done from c,s",
          "refId": "A",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "column"
              }
            ]
          ],
          "timeColumn": "time",
          "where": [
            {
              "name": "$__timeFilter",
              "params": [],
              "type": "macro"
            }
          ]
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Bodhi Success in %",
      "type": "gauge"
    },
    {
      "datasource": "Datanommer",
      "description": "Percentage of runs, where it reached the notification for the test failing in CI in a given time-limit.",
      "fieldConfig": {
        "defaults": {
          "custom": {},
          "mappings": [],
          "max": 100,
          "min": 0,
          "thresholds": {
            "mode": "percentage",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "green",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 3,
        "x": 12,
        "y": 5
      },
      "id": 7,
      "options": {
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "mean"
          ],
          "fields": "/^percent_ci_done$/",
          "values": false
        },
        "showThresholdLabels": true,
        "showThresholdMarkers": true
      },
      "pluginVersion": "7.2.1",
      "targets": [
        {
          "format": "table",
          "group": [],
          "metricColumn": "none",
          "rawQuery": true,
          "rawSql": "with t as (select * from messages where topic like 'org.fedoraproject.prod.monitor-gating.$test%' and $__timeFilter(\"timestamp\")),\r\nc as (select count(*) as finished_gating_runs from t where topic like '%end%'),\r\ns as (select count(*) as ci_completed from t where (topic like '%succeeded' or ((topic like '%failed' or topic like '%error') and _msg::jsonb#>>'{output_text}' LIKE '%CI (complete) results in datagrepper returned error%')))\r\nselect ((s.ci_completed)*100)/c.finished_gating_runs as percent_ci_done from c,s",
          "refId": "A",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "column"
              }
            ]
          ],
          "timeColumn": "time",
          "where": [
            {
              "name": "$__timeFilter",
              "params": [],
              "type": "macro"
            }
          ]
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "CI Fast & Correct in %",
      "type": "gauge"
    },
    {
      "datasource": "Datanommer",
      "description": "Percentage of runs where resultsdb send out the notification for learning about the result in the given time-limit.",
      "fieldConfig": {
        "defaults": {
          "custom": {},
          "mappings": [],
          "max": 100,
          "min": 0,
          "thresholds": {
            "mode": "percentage",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "green",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 3,
        "x": 15,
        "y": 5
      },
      "id": 14,
      "options": {
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "mean"
          ],
          "fields": "/^percent_ci_done$/",
          "values": false
        },
        "showThresholdLabels": true,
        "showThresholdMarkers": true
      },
      "pluginVersion": "7.2.1",
      "targets": [
        {
          "format": "table",
          "group": [],
          "metricColumn": "none",
          "rawQuery": true,
          "rawSql": "with t as (select * from messages where topic like 'org.fedoraproject.prod.monitor-gating.$test%' and $__timeFilter(\"timestamp\")),\r\nc as (select count(*) as finished_gating_runs from t where topic like '%end%'),\r\ns as (select count(*) as ci_completed from t where (topic like '%succeeded' or ((topic like '%failed' or topic like '%error') and _msg::jsonb#>>'{output_text}' ~ 'resultsdb results in datagrepper returned FAILED[^\\n]+\\[DONE\\]')))\r\nselect ((s.ci_completed)*100)/c.finished_gating_runs as percent_ci_done from c,s",
          "refId": "A",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "column"
              }
            ]
          ],
          "timeColumn": "time",
          "where": [
            {
              "name": "$__timeFilter",
              "params": [],
              "type": "macro"
            }
          ]
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "ResultsDB Fast & Correct in %",
      "type": "gauge"
    },
    {
      "datasource": "Datanommer",
      "description": "Percentage of runs that reached the Greenwave gating a release of a backage based on a failed CI run",
      "fieldConfig": {
        "defaults": {
          "custom": {},
          "mappings": [],
          "max": 100,
          "min": 0,
          "thresholds": {
            "mode": "percentage",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "green",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 3,
        "x": 18,
        "y": 5
      },
      "id": 12,
      "options": {
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "mean"
          ],
          "fields": "/^percent_ci_done$/",
          "values": false
        },
        "showThresholdLabels": true,
        "showThresholdMarkers": true
      },
      "pluginVersion": "7.2.1",
      "targets": [
        {
          "format": "table",
          "group": [],
          "metricColumn": "none",
          "rawQuery": true,
          "rawSql": "with t as (select * from messages where topic like 'org.fedoraproject.prod.monitor-gating.$test%' and $__timeFilter(\"timestamp\")),\r\nc as (select count(*) as finished_gating_runs from t where topic like '%end%'),\r\ns as (select count(*) as ci_completed from t where (topic like '%succeeded' or ((topic like '%failed' or topic like '%error') and _msg::jsonb#>>'{output_text}' ~ 'greenwave results in datagrepper returned False[^\\n]+\\[DONE\\]')))\r\nselect ((s.ci_completed)*100)/c.finished_gating_runs as percent_ci_done from c,s",
          "refId": "A",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "column"
              }
            ]
          ],
          "timeColumn": "time",
          "where": [
            {
              "name": "$__timeFilter",
              "params": [],
              "type": "macro"
            }
          ]
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Greenwave Fast & Correct in %",
      "type": "gauge"
    },
    {
      "datasource": "Datanommer",
      "description": "Percentage of runs successfully waived",
      "fieldConfig": {
        "defaults": {
          "custom": {},
          "mappings": [],
          "max": 100,
          "min": 0,
          "thresholds": {
            "mode": "percentage",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "green",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 3,
        "x": 21,
        "y": 5
      },
      "id": 13,
      "options": {
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "mean"
          ],
          "fields": "/^percent_ci_done$/",
          "values": false
        },
        "showThresholdLabels": true,
        "showThresholdMarkers": true
      },
      "pluginVersion": "7.2.1",
      "targets": [
        {
          "format": "table",
          "group": [],
          "metricColumn": "none",
          "rawQuery": true,
          "rawSql": "with t as (select * from messages where topic like 'org.fedoraproject.prod.monitor-gating.$test%' and $__timeFilter(\"timestamp\")),\r\nc as (select count(*) as finished_gating_runs from t where topic like '%end%'),\r\ns as (select count(*) as ci_completed from t where (topic like '%succeeded' or ((topic like '%failed' or topic like '%error') and _msg::jsonb#>>'{output_text}' ~ 'greenwave results in datagrepper returned True[^\\n]+\\[DONE\\]')))\r\nselect ((s.ci_completed)*100)/c.finished_gating_runs as percent_ci_done from c,s",
          "refId": "A",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "column"
              }
            ]
          ],
          "timeColumn": "time",
          "where": [
            {
              "name": "$__timeFilter",
              "params": [],
              "type": "macro"
            }
          ]
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Waiverdb Fast & Correct in %",
      "type": "gauge"
    },
    {
      "datasource": "Datanommer",
      "fieldConfig": {
        "defaults": {
          "custom": {},
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 20
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 3,
        "x": 0,
        "y": 12
      },
      "id": 17,
      "options": {
        "reduceOptions": {
          "calcs": [
            "mean"
          ],
          "fields": "",
          "values": false
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true
      },
      "pluginVersion": "7.2.1",
      "targets": [
        {
          "format": "time_series",
          "group": [],
          "metricColumn": "none",
          "rawQuery": true,
          "rawSql": "select \"timestamp\" as time, CAST(array_to_string(REGEXP_MATCHES(_msg::jsonb->'output'->>-1, 'Ran for (\\d+).*'),'') as integer)/60  from messages where $__timeFilter(\"timestamp\") and topic like '%gating%$test%.end.%' order by time",
          "refId": "A",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "column"
              }
            ]
          ],
          "timeColumn": "time",
          "where": [
            {
              "name": "$__timeFilter",
              "params": [],
              "type": "macro"
            }
          ]
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Median run in minutes",
      "type": "gauge"
    },
    {
      "datasource": "Datanommer",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": null
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 6,
        "y": 12
      },
      "id": 37,
      "options": {
        "reduceOptions": {
          "calcs": [
            "mean"
          ],
          "fields": "",
          "values": false
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true
      },
      "pluginVersion": "7.2.1",
      "targets": [
        {
          "format": "time_series",
          "group": [],
          "metricColumn": "none",
          "rawQuery": true,
          "rawSql": "select \r\n  \"timestamp\" as time,\r\n  EXTRACT (MINUTES FROM (TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*Retrieving koji tags:.*updates-candidate.*D.*\")')::text,2,8), 'HH24:MI:SS')::TIME \r\n  - \r\n  TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*uilding the package.*\")')::text,2,8), 'HH24:MI:SS')::TIME)::TIME) as interval\r\nfrom messages where $__timeFilter(\"timestamp\") and topic like '%gating%$test%.end.%' order by time",
          "refId": "A",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "column"
              }
            ]
          ],
          "timeColumn": "time",
          "where": [
            {
              "name": "$__timeFilter",
              "params": [],
              "type": "macro"
            }
          ]
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Median build in minutes",
      "type": "gauge"
    },
    {
      "datasource": "Datanommer",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": null
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 12,
        "y": 12
      },
      "id": 26,
      "options": {
        "reduceOptions": {
          "calcs": [
            "mean"
          ],
          "fields": "",
          "values": false
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true
      },
      "pluginVersion": "7.2.1",
      "targets": [
        {
          "format": "time_series",
          "group": [],
          "metricColumn": "none",
          "rawQuery": true,
          "rawSql": "select \n  \"timestamp\" as time,\n  EXTRACT (MINUTES FROM (TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*CI.*running.*D.*\")')::text,2,8), 'HH24:MI:SS')::TIME \n  - \n  TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*bodhi to CI.*DONE.*\")')::text,2,8), 'HH24:MI:SS')::TIME)::TIME) as interval\nfrom messages where $__timeFilter(\"timestamp\") and  topic like '%gating%$test%.end.%' order by time",
          "refId": "A",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "column"
              }
            ]
          ],
          "timeColumn": "time",
          "where": [
            {
              "name": "$__timeFilter",
              "params": [],
              "type": "macro"
            }
          ]
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Median queue in minutes",
      "type": "gauge"
    },
    {
      "datasource": "Datanommer",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": null
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 3,
        "x": 15,
        "y": 12
      },
      "id": 30,
      "options": {
        "reduceOptions": {
          "calcs": [
            "mean"
          ],
          "fields": "",
          "values": false
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true
      },
      "pluginVersion": "7.2.1",
      "targets": [
        {
          "format": "time_series",
          "group": [],
          "metricColumn": "none",
          "rawQuery": true,
          "rawSql": "select \n  \"timestamp\" as time,\n  EXTRACT (MINUTES FROM (TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*resultsdb results in datagrepper returned FAILED[^\\n]+\\[.*D.*\\]\")')::text,2,8), 'HH24:MI:SS')::TIME \n  - \n  TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*CI.*complete.*D.*\")')::text,2,8), 'HH24:MI:SS')::TIME)::TIME) as interval\nfrom messages where ($__timeFilter(\"timestamp\")) AND topic like '%gating%$test%.end.%' order by time",
          "refId": "A",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "column"
              }
            ]
          ],
          "timeColumn": "time",
          "where": [
            {
              "name": "$__timeFilter",
              "params": [],
              "type": "macro"
            }
          ]
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Median to resultsdb",
      "type": "gauge"
    },
    {
      "datasource": "Datanommer",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": null
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 3,
        "x": 18,
        "y": 12
      },
      "id": 29,
      "options": {
        "reduceOptions": {
          "calcs": [
            "mean"
          ],
          "fields": "",
          "values": false
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true
      },
      "pluginVersion": "7.2.1",
      "targets": [
        {
          "format": "time_series",
          "group": [],
          "metricColumn": "none",
          "rawQuery": true,
          "rawSql": "select \n  \"timestamp\" as time,\n  EXTRACT (MINUTES FROM (TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*greenwave results in datagrepper returned False.*DONE.*\")')::text,2,8), 'HH24:MI:SS')::TIME \n  - \n  TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*resultsdb results in datagrepper returned FAILED[^\\n]+\\[.*D.*\\]\")')::text,2,8), 'HH24:MI:SS')::TIME)::TIME) as interval\nfrom messages where ($__timeFilter(\"timestamp\")) AND topic like '%gating%$test%.end.%' order by time",
          "refId": "A",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "column"
              }
            ]
          ],
          "timeColumn": "time",
          "where": [
            {
              "name": "$__timeFilter",
              "params": [],
              "type": "macro"
            }
          ]
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Median from resultsbd to greenwave",
      "type": "gauge"
    },
    {
      "datasource": "Datanommer",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": null
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 3,
        "x": 21,
        "y": 12
      },
      "id": 28,
      "options": {
        "reduceOptions": {
          "calcs": [
            "mean"
          ],
          "fields": "",
          "values": false
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true
      },
      "pluginVersion": "7.2.1",
      "targets": [
        {
          "format": "time_series",
          "group": [],
          "metricColumn": "none",
          "rawQuery": true,
          "rawSql": "select \n  \"timestamp\" as time,\n  EXTRACT (MINUTES FROM (TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*greenwave results in datagrepper returned True.*DONE.*\")')::text,2,8), 'HH24:MI:SS')::TIME \n  - \n  TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*greenwave results in datagrepper returned False.*DONE.*\")')::text,2,8), 'HH24:MI:SS')::TIME)::TIME) as interval\nfrom messages where ($__timeFilter(\"timestamp\")) AND topic like '%gating%$test%.end.%' order by time",
          "refId": "A",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "column"
              }
            ]
          ],
          "timeColumn": "time",
          "where": [
            {
              "name": "$__timeFilter",
              "params": [],
              "type": "macro"
            }
          ]
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Median from unwaived to waived",
      "type": "gauge"
    },
    {
      "datasource": "Datanommer",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": null
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 6,
        "y": 16
      },
      "id": 25,
      "options": {
        "reduceOptions": {
          "calcs": [
            "mean"
          ],
          "fields": "",
          "values": false
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true
      },
      "pluginVersion": "7.2.1",
      "targets": [
        {
          "format": "time_series",
          "group": [],
          "metricColumn": "none",
          "rawQuery": true,
          "rawSql": "select \n  timestamp as time,\n  EXTRACT (MINUTES FROM (TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*Retrieving koji tags:.*testing-.*D.*\")')::text,2,8), 'HH24:MI:SS')::TIME \n  - \n  TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*Retrieving koji tags:.*updates-candidate.*DONE.*\")')::text,2,8), 'HH24:MI:SS')::TIME)::TIME) as interval\nfrom messages where $__timeFilter(\"timestamp\") and topic like '%gating%$test%.end.%' order by time",
          "refId": "A",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "column"
              }
            ]
          ],
          "timeColumn": "time",
          "where": [
            {
              "name": "$__timeFilter",
              "params": [],
              "type": "macro"
            }
          ]
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Median signing in minutes",
      "type": "gauge"
    },
    {
      "datasource": "Datanommer",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": null
          },
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 12,
        "y": 16
      },
      "id": 27,
      "options": {
        "reduceOptions": {
          "calcs": [
            "mean"
          ],
          "fields": "",
          "values": false
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true
      },
      "pluginVersion": "7.2.1",
      "targets": [
        {
          "format": "time_series",
          "group": [],
          "metricColumn": "none",
          "rawQuery": true,
          "rawSql": "select \n  \"timestamp\" as time,\n  EXTRACT (MINUTES FROM (TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*CI.*complete.*D.*\")')::text,2,8), 'HH24:MI:SS')::TIME \n  - \n  TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*CI.*running.*DONE.*\")')::text,2,8), 'HH24:MI:SS')::TIME)::TIME) as interval\nfrom messages where ($__timeFilter(\"timestamp\")) and  topic like '%gating%$test%.end.%' order by time",
          "refId": "A",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "column"
              }
            ]
          ],
          "timeColumn": "time",
          "where": [
            {
              "name": "$__timeFilter",
              "params": [],
              "type": "macro"
            }
          ]
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Median testing in minutes",
      "type": "gauge"
    },
    {
      "datasource": "Datanommer",
      "fieldConfig": {
        "defaults": {
          "custom": {},
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 20
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 3,
        "x": 0,
        "y": 20
      },
      "id": 31,
      "options": {
        "reduceOptions": {
          "calcs": [
            "max"
          ],
          "fields": "",
          "values": false
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true
      },
      "pluginVersion": "7.2.1",
      "targets": [
        {
          "format": "time_series",
          "group": [],
          "metricColumn": "none",
          "rawQuery": true,
          "rawSql": "select \"timestamp\" as time, CAST(array_to_string(REGEXP_MATCHES(_msg::jsonb->'output'->>-1, 'Ran for (\\d+).*'),'') as integer)/60  from messages where $__timeFilter(\"timestamp\") and topic like '%gating%$test%.end.%' order by time",
          "refId": "A",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "column"
              }
            ]
          ],
          "timeColumn": "time",
          "where": [
            {
              "name": "$__timeFilter",
              "params": [],
              "type": "macro"
            }
          ]
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Max run in minutes",
      "type": "gauge"
    },
    {
      "datasource": "Datanommer",
      "description": "",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": null
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 10
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 12,
        "y": 20
      },
      "id": 34,
      "options": {
        "reduceOptions": {
          "calcs": [
            "max"
          ],
          "fields": "",
          "values": false
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true
      },
      "pluginVersion": "7.2.1",
      "targets": [
        {
          "format": "time_series",
          "group": [],
          "metricColumn": "none",
          "rawQuery": true,
          "rawSql": "select \n  \"timestamp\" as time,\n  EXTRACT (MINUTES FROM (TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*CI.*running.*D.*\")')::text,2,8), 'HH24:MI:SS')::TIME \n  - \n  TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*bodhi to CI.*DONE.*\")')::text,2,8), 'HH24:MI:SS')::TIME)::TIME) as interval\nfrom messages where $__timeFilter(\"timestamp\") and  topic like '%gating%$test%.end.%' order by time",
          "refId": "A",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "column"
              }
            ]
          ],
          "timeColumn": "time",
          "where": [
            {
              "name": "$__timeFilter",
              "params": [],
              "type": "macro"
            }
          ]
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Max queue in minutes",
      "type": "gauge"
    },
    {
      "datasource": "Datanommer",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": null
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 10
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 3,
        "x": 15,
        "y": 20
      },
      "id": 36,
      "options": {
        "reduceOptions": {
          "calcs": [
            "max"
          ],
          "fields": "",
          "values": false
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true
      },
      "pluginVersion": "7.2.1",
      "targets": [
        {
          "format": "time_series",
          "group": [],
          "metricColumn": "none",
          "rawQuery": true,
          "rawSql": "select \n  \"timestamp\" as time,\n  EXTRACT (MINUTES FROM (TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*CI results in resultsdb.*D.*\")')::text,2,8), 'HH24:MI:SS')::TIME \n  - \n  TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*CI.*complete.*D.*\")')::text,2,8), 'HH24:MI:SS')::TIME)::TIME) as interval\nfrom messages where ($__timeFilter(\"timestamp\")) AND topic like '%gating%$test%.end.%' order by time",
          "refId": "A",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "column"
              }
            ]
          ],
          "timeColumn": "time",
          "where": [
            {
              "name": "$__timeFilter",
              "params": [],
              "type": "macro"
            }
          ]
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Max to resultsdb",
      "type": "gauge"
    },
    {
      "datasource": "Datanommer",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": null
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 10
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 6,
        "y": 21
      },
      "id": 32,
      "options": {
        "reduceOptions": {
          "calcs": [
            "max"
          ],
          "fields": "",
          "values": false
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true
      },
      "pluginVersion": "7.2.1",
      "targets": [
        {
          "format": "time_series",
          "group": [],
          "metricColumn": "none",
          "rawQuery": true,
          "rawSql": "select \n  \"timestamp\" as time,\n  EXTRACT (MINUTES FROM (TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*Retrieving koji tags:.*updates-candidate.*DONE.*\")')::text,2,8), 'HH24:MI:SS')::TIME \n  - \n  TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*Building the package.*\")')::text,2,8), 'HH24:MI:SS')::TIME)::TIME) as interval\nfrom messages where $__timeFilter(\"timestamp\") and topic like '%gating%$test%.end.%' order by time",
          "refId": "A",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "column"
              }
            ]
          ],
          "timeColumn": "time",
          "where": [
            {
              "name": "$__timeFilter",
              "params": [],
              "type": "macro"
            }
          ]
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Max build in minutes",
      "type": "gauge"
    },
    {
      "datasource": "Datanommer",
      "description": "",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": null
          },
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 10
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 12,
        "y": 24
      },
      "id": 35,
      "options": {
        "reduceOptions": {
          "calcs": [
            "max"
          ],
          "fields": "",
          "values": false
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true
      },
      "pluginVersion": "7.2.1",
      "targets": [
        {
          "format": "time_series",
          "group": [],
          "metricColumn": "none",
          "rawQuery": true,
          "rawSql": "select \n  \"timestamp\" as time,\n  EXTRACT (MINUTES FROM (TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*CI.*complete.*D.*\")')::text,2,8), 'HH24:MI:SS')::TIME \n  - \n  TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*CI.*running.*DONE.*\")')::text,2,8), 'HH24:MI:SS')::TIME)::TIME) as interval\nfrom messages where ($__timeFilter(\"timestamp\")) and  topic like '%gating%$test%.end.%' order by time",
          "refId": "A",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "column"
              }
            ]
          ],
          "timeColumn": "time",
          "where": [
            {
              "name": "$__timeFilter",
              "params": [],
              "type": "macro"
            }
          ]
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Max testing in minutes",
      "type": "gauge"
    },
    {
      "datasource": "Datanommer",
      "description": "",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": null
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 10
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 6,
        "y": 25
      },
      "id": 33,
      "options": {
        "reduceOptions": {
          "calcs": [
            "max"
          ],
          "fields": "",
          "values": false
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true
      },
      "pluginVersion": "7.2.1",
      "targets": [
        {
          "format": "time_series",
          "group": [],
          "metricColumn": "none",
          "rawQuery": true,
          "rawSql": "select \n  timestamp as time,\n  EXTRACT (MINUTES FROM (TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*Retrieving koji tags:.*testing-.*DONE.*\")')::text,2,8), 'HH24:MI:SS')::TIME \n  - \n  TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*Retrieving koji tags:.*updates-candidate.*DONE.*\")')::text,2,8), 'HH24:MI:SS')::TIME)::TIME) as interval\nfrom messages where $__timeFilter(\"timestamp\") and topic like '%gating%$test%.end.%' order by time",
          "refId": "A",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "column"
              }
            ]
          ],
          "timeColumn": "time",
          "where": [
            {
              "name": "$__timeFilter",
              "params": [],
              "type": "macro"
            }
          ]
        }
      ],
      "timeFrom": null,
      "timeShift": null,
      "title": "Max signing in minutes",
      "type": "gauge"
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "Datanommer",
      "fieldConfig": {
        "defaults": {
          "custom": {}
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 8,
        "w": 24,
        "x": 0,
        "y": 29
      },
      "hiddenSeries": false,
      "id": 23,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.2.1",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "format": "time_series",
          "group": [],
          "metricColumn": "none",
          "rawQuery": true,
          "rawSql": "select timestamp as time, CAST(array_to_string(REGEXP_MATCHES(_msg::jsonb->'output'->>-1, 'Ran for (\\d+).*'),'') as integer)/60  from messages where topic like '%gating%$test%.end.%' and $__timeFilter(\"timestamp\") order by time",
          "refId": "A",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "column"
              }
            ]
          ],
          "timeColumn": "time",
          "where": [
            {
              "name": "$__timeFilter",
              "params": [],
              "type": "macro"
            }
          ]
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Time to complete in minutes",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "Datanommer",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": null
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 8,
        "w": 24,
        "x": 0,
        "y": 37
      },
      "hiddenSeries": false,
      "id": 18,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.2.1",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "format": "time_series",
          "group": [],
          "metricColumn": "none",
          "rawQuery": true,
          "rawSql": "select \n  \"timestamp\" as time,\n  EXTRACT (MINUTES FROM (TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*Retrieving koji tags:.*updates-candidate.*DONE.*\")')::text,2,8), 'HH24:MI:SS')::TIME \n  - \n  TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*Building the package.*\")')::text,2,8), 'HH24:MI:SS')::TIME)::TIME) as interval\nfrom messages where $__timeFilter(\"timestamp\") and topic like '%gating%$test%.end.%' order by time",
          "refId": "A",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "column"
              }
            ]
          ],
          "timeColumn": "time",
          "where": [
            {
              "name": "$__timeFilter",
              "params": [],
              "type": "macro"
            }
          ]
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Time for build in minutes",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "Datanommer",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": null
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 8,
        "w": 24,
        "x": 0,
        "y": 45
      },
      "hiddenSeries": false,
      "id": 19,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.2.1",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "format": "time_series",
          "group": [],
          "metricColumn": "none",
          "rawQuery": true,
          "rawSql": "select \n  timestamp as time,\n  EXTRACT (MINUTES FROM (TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*Retrieving koji tags:.*testing-.*DONE.*\")')::text,2,8), 'HH24:MI:SS')::TIME \n  - \n  TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*Retrieving koji tags:.*updates-candidate.*DONE.*\")')::text,2,8), 'HH24:MI:SS')::TIME)::TIME) as interval\nfrom messages where $__timeFilter(\"timestamp\") and topic like '%gating%$test%.end.%' order by time",
          "refId": "A",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "column"
              }
            ]
          ],
          "timeColumn": "time",
          "where": [
            {
              "name": "$__timeFilter",
              "params": [],
              "type": "macro"
            }
          ]
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Time for signing in minutes",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "Datanommer",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": null
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 8,
        "w": 24,
        "x": 0,
        "y": 53
      },
      "hiddenSeries": false,
      "id": 20,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.2.1",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "format": "time_series",
          "group": [],
          "metricColumn": "none",
          "rawQuery": true,
          "rawSql": "select \n  \"timestamp\" as time,\n  EXTRACT (MINUTES FROM (TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*CI.*running.*D.*\")')::text,2,8), 'HH24:MI:SS')::TIME \n  - \n  TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*bodhi to CI.*DONE.*\")')::text,2,8), 'HH24:MI:SS')::TIME)::TIME) as interval\nfrom messages where $__timeFilter(\"timestamp\") and  topic like '%gating%$test%.end.%' order by time",
          "refId": "A",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "column"
              }
            ]
          ],
          "timeColumn": "time",
          "where": [
            {
              "name": "$__timeFilter",
              "params": [],
              "type": "macro"
            }
          ]
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Time in queue in minutes",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "Datanommer",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": null
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 8,
        "w": 24,
        "x": 0,
        "y": 61
      },
      "hiddenSeries": false,
      "id": 21,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.2.1",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "format": "time_series",
          "group": [],
          "metricColumn": "none",
          "rawQuery": true,
          "rawSql": "select \n  \"timestamp\" as time,\n  EXTRACT (MINUTES FROM (TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*CI.*complete.*D.*\")')::text,2,8), 'HH24:MI:SS')::TIME \n  - \n  TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*CI.*running.*DONE.*\")')::text,2,8), 'HH24:MI:SS')::TIME)::TIME) as interval\nfrom messages where ($__timeFilter(\"timestamp\")) and  topic like '%gating%$test%.end.%' order by time",
          "refId": "A",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "column"
              }
            ]
          ],
          "timeColumn": "time",
          "where": [
            {
              "name": "$__timeFilter",
              "params": [],
              "type": "macro"
            }
          ]
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Time in testing",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "Datanommer",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "align": null
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 8,
        "w": 24,
        "x": 0,
        "y": 69
      },
      "hiddenSeries": false,
      "id": 22,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.2.1",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "format": "time_series",
          "group": [],
          "metricColumn": "none",
          "rawQuery": true,
          "rawSql": "select \n  \"timestamp\" as time,\n  EXTRACT (MINUTES FROM (TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*CI results in resultsdb.*D.*\")')::text,2,8), 'HH24:MI:SS')::TIME \n  - \n  TO_TIMESTAMP(substring(jsonb_path_query_first(_msg::jsonb,'$.output[*] ? (@ like_regex \".*CI.*complete.*D.*\")')::text,2,8), 'HH24:MI:SS')::TIME)::TIME) as interval\nfrom messages where ($__timeFilter(\"timestamp\")) AND topic like '%gating%$test%.end.%' order by time",
          "refId": "A",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "column"
              }
            ]
          ],
          "timeColumn": "time",
          "where": [
            {
              "name": "$__timeFilter",
              "params": [],
              "type": "macro"
            }
          ]
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Time from testing to resultsdb",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "Datanommer",
      "fieldConfig": {
        "defaults": {
          "custom": {}
        },
        "overrides": []
      },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 12,
        "w": 24,
        "x": 0,
        "y": 77
      },
      "hiddenSeries": false,
      "id": 15,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
      "pluginVersion": "7.2.1",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "format": "time_series",
          "group": [],
          "metricColumn": "none",
          "rawQuery": true,
          "rawSql": "with t as (select date(\"timestamp\") as time, topic from messages where $__timeFilter(\"timestamp\") and topic like '%gating%'),\nc as (select time, count(*) as finished_gating_runs from t where topic like '%gating%$test%.end.%' group by time),\ns as (select time, count(*) as successful_gating_runs from t where topic like '%gating%$test%.end%succe%' group by time)\nselect s.time as time, (s.successful_gating_runs*100)/c.finished_gating_runs as percent_succesful from c,s where c.time = s.time order by time",
          "refId": "A",
          "select": [
            [
              {
                "params": [
                  "value"
                ],
                "type": "column"
              }
            ]
          ],
          "timeColumn": "time",
          "where": [
            {
              "name": "$__timeFilter",
              "params": [],
              "type": "macro"
            }
          ]
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Monitor Gating Success Percentage",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    }
  ],
  "refresh": false,
  "schemaVersion": 26,
  "style": "dark",
  "tags": [],
  "templating": {
    "list": [
      {
        "allValue": null,
        "current": {
          "selected": false,
          "text": "single",
          "value": "single"
        },
        "hide": 0,
        "includeAll": false,
        "label": null,
        "multi": false,
        "name": "test",
        "options": [
          {
            "selected": true,
            "text": "single",
            "value": "single"
          },
          {
            "selected": false,
            "text": "multi",
            "value": "multi"
          }
        ],
        "query": "single, multi",
        "queryValue": "",
        "skipUrlSync": false,
        "type": "custom"
      }
    ]
  },
  "time": {
    "from": "now-2d",
    "to": "now"
  },
  "timepicker": {
    "refresh_intervals": [
      "5s",
      "10s",
      "30s",
      "1m",
      "5m",
      "15m",
      "30m",
      "1h",
      "2h",
      "1d"
    ]
  },
  "timezone": "",
  "title": "Monitor Gating",
  "uid": "6q2qpcvGz",
  "version": 1
}