Add Discourse support.

master
Simon Quigley 4 years ago
parent 558774de80
commit 6f68733ca1

@ -18,12 +18,13 @@
import argparse import argparse
import logging as log import logging as log
import sqlite3 import sqlite3
from modules.discourse import DiscourseModule
from modules.jenkins import JenkinsModule from modules.jenkins import JenkinsModule
from modules.utilities import * from modules.utilities import *
from os import path from os import path
from shutil import copytree, rmtree from shutil import copytree, rmtree
ENABLED_MODULES = [JenkinsModule] ENABLED_MODULES = [DiscourseModule, JenkinsModule]
def sqlite_run(command, db, return_output=False): def sqlite_run(command, db, return_output=False):
@ -89,8 +90,9 @@ def main(module):
for day in (1, 7, 30, 90, 180): for day in (1, 7, 30, 90, 180):
# Fetch the data and log to debug # Fetch the data and log to debug
run = [module.sqlite_time_range(days=day)] run = [module.sqlite_time_range(days=day)]
log.info(run)
data = sqlite_run(run, db=args.db_location, return_output=True) data = sqlite_run(run, db=args.db_location, return_output=True)
log.debug(data) log.info(data)
# Render the template, which also returns the average values # Render the template, which also returns the average values
_averages[day] = module.render_template(day, data) _averages[day] = module.render_template(day, data)

@ -0,0 +1,220 @@
#!/usr/bin/env python3
# Copyright (C) 2020 Simon Quigley <tsimonq2@lubuntu.me>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import datetime
import requests_cache
import time
from modules.utilities import *
from os import getenv, makedirs, path
from pydiscourse import DiscourseClient
requests_cache.install_cache("discourse", backend="sqlite", expire_after=300)
class DiscourseModule:
"""Discourse module for the Metrics program"""
def __init__(self):
self.name = "Discourse"
def _auth_discourse_server(self):
"""Authenticate to Discourse
This uses the API_SITE, API_USER, and API_KEY env vars.
"""
# Load the config, so we can store secrets outside of env vars
config = load_config()
in_conf = "discourse" in config
# Load the needed secrets either from the config file if it exists
# or the env var if it's defined (which takes precedence)
site = getenv("DISCOURSE_API_SITE") or (in_conf and config["discourse"]["site"])
user = getenv("DISCOURSE_API_USER") or (in_conf and config["discourse"]["user"])
key = getenv("DISCOURSE_API_KEY") or (in_conf and config["discourse"]["key"])
for envvar in [site, user, key]:
if not envvar:
raise ValueError("DISCOURSE_API_SITE, DISCOURSE_API_USER, and",
"DISCOURSE_API_KEY must be defined")
# Authenticate to the server
server = DiscourseClient(site, api_username=user, api_key=key)
return server
def _get_data(self):
"""Get the data from Discourse
This function returns six distinct values as one list:
[open_support, total_support, percent_support, open_all, total_all,
percent_all]
"""
# Authenticate to the server
server = self._auth_discourse_server()
# Initialize the data
data = [0, 0, 0, 0, 0, 0]
for category in server.categories():
# We are limited to 30 topics per page, so we have to loop until
# there are no more topics
page = 0
on_page = 1
c_id = category["id"]
while on_page > 0:
# Get a list of all the topics to then iterate on
page_data = server.category_topics(category_id=c_id, page=page)
topics = page_data["topic_list"]["topics"]
on_page = len(topics)
page += 1
print("Working on " + category["name"])
for topic in topics:
# Increment total_all
data[4] += 1
# If it's open, increment open_all
if not topic["closed"]:
data[3] += 1
# If the topic is in Support, repeat
if category["name"] == "Support":
data[1] += 1
if not topic["closed"]:
data[0] += 1
# Calculate the percentages
data[2] = ((data[0] / data[1]) * 100)
data[5] = ((data[3] / data[4]) * 100)
return data
def sqlite_setup(self):
"""Initially set up the table for usage in SQLite
This returns a str which will then be executed in our SQLite db
Here is the "discourse" table layout:
- date is the primary key, and it is the Unix timestamp as an int
- open_support is the number of open topics in the Support category
- total_support is the number of total topics in the Support category
- percent_support is ((open_support / total_support) * 100)
- open_all is the number of open topics on the Discourse instance
- total_all is the number of total topics on the Discourse instance
- percent_all is ((open_all / total_all) * 100)
"""
command = "CREATE TABLE IF NOT EXISTS discourse (date INTEGER PRIMARY"
command += " KEY, open_support INTEGER, total_support INTEGER, "
command += "percent_support INTEGER, open_all INTEGER, total_all "
command += "INTEGER, percent_all INTEGER);"
return command
def sqlite_add(self):
"""Add data to the SQLite db
This retrieves the current data from the Jenkins server, and returns a
str which will then be executed in our SQLite db
"""
# Match the variable names with the column names in the db
data = tuple(self._get_data())
date = "strftime('%s', 'now')"
# Craft the str
print(data)
command = "INSERT INTO discourse VALUES ("
command += "{}, {}, {}, {}, {}, {}, {});".format(date, *data)
return command
def sqlite_time_range(self, days):
"""Get the rows which have been inserted given days
e.g. if days is 180, it gets all of the values which have been
inserted in the past 180 days.
Note: this just returns the command to be ran, it doesn't actually run
"""
now = datetime.datetime.now()
timedelta = datetime.timedelta(days=days)
unix_time = int(time.mktime((now - timedelta).timetuple()))
command = "SELECT * FROM discourse WHERE date > %s;" % unix_time
return command
def render_template(self, days, data):
"""Render a template with days in the filename, given the data
The above function sqlite_time_range() is ran on the database with
some predetermined date ranges. This function actually interprets that
data, uses Jinja2 to magically render a template, and voila.
"""
# Initialize a (softly) ephemeral dict to store data
discourse = dict()
keys = ["date", "open_support", "total_support", "percent_support",
"open_all", "total_all", "percent_all"]
# Use a lambda to map the data into a dict with the keys
for row in data:
print(row)
_data = {keys[x] : row[x] for x in range(len(row))}
# Add our ephemeral dict to the master dict, and create the key if
# it doesn't already exist
for key in _data.keys():
if key in discourse:
discourse[key].append(_data[key])
else:
discourse[key] = [_data[key]]
data = discourse
# Get human-readable averages and throw it in a dict
average = {}
for datatype in keys:
try:
num = sum(data[datatype]) / len(data[datatype])
num = format(num, ".1f")
except ZeroDivisionError:
num = 0
average[datatype] = num
# Assign data to the dict Jinja2 is actually going to use
discourse = {keys[x] : zip(data["date"], data[keys[x]]) for x in range(len(keys))}
# Make the output dir if it doesn't already exist
if not path.exists("output"):
makedirs("output")
src = path.join("templates", "discourse.html")
dest = path.join("output", "discourse_%sdays.html" % days)
jinja2_template(src, dest, discourse=discourse, average=average,
days=days)
# Return the averages for use in the summary
r_data = []
del keys[0]
for key in keys:
r_data.append(average[key])
return tuple(r_data)

@ -0,0 +1,109 @@
<!DOCTYPE html>
<html lang="en">
<head>
<!-- HTML5 compliance -->
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<!-- Import Chart.js 2.9.3 -->
<script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.3/Chart.bundle.js"></script>
<!-- Import Bootstrap 4.3.1 -->
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css" integrity="sha384-ggOyR0iXCbMQv3Xipma34MD+dH/1fQ784/j6cY/iJTQUOhcWr7x9JvoRxT2MZw1T" crossorigin="anonymous">
<!-- Favicon and page title -->
<link rel="icon" type="image/png" href="/assets/favicon.png"/>
<title>Discourse data for the past {{ days }} day{{ "s" if days > 1 }}</title>
</head>
<body>
<div class="container">
<div class="row justify-content-center">
<div class="col" style="text-align: center;">
<h1>Discourse data for the past {{ days }} day{{ "s" if days > 1 }}</h1>
<canvas id="discoursechart"></canvas>
<h2>Average number of open support topics: {{ average.open_support }}</h2>
<h2>Average number of total support topics: {{ average.total_support }}</h2>
<h2>Average percent of (open / total) support topics: {{ average.percent_support }}%</h2>
<h2>Average number of open topics: {{ average.open_all }}</h2>
<h2>Average number of total topics: {{ average.total_all }}</h2>
<h2>Average percent of (open / total) topics: {{ average.percent_all }}%</h2>
<h2><a href="./">Back to summary page</a></h2>
</div>
</div>
</div>
<!-- This is the actual chart functionality -->
<script>
var ctx = document.getElementById("discoursechart").getContext("2d");
var myChart = new Chart(ctx, {
type: 'line',
data: {
datasets: [
{
label: 'Open Support Topics',
backgroundColor: "#FF0000",
borderColor: "#FF0000",
data: [
{% for timestamp, num in discourse.open_support %}
{
t: new Date({{ timestamp }} * 1000),
y: {{ num }}
},
{% endfor %}
]
},
{
label: 'Open Topics',
backgroundColor: "#FFFF00",
borderColor: "#FFFF00",
data: [
{% for timestamp, num in discourse.open_all %}
{
t: new Date({{ timestamp }} * 1000),
y: {{ num }}
},
{% endfor %}
]
},
{
label: 'Total Support Topics',
backgroundColor: "#32CD32",
borderColor: "#32CD32",
data: [
{% for timestamp, num in discourse.total_support %}
{
t: new Date({{ timestamp }} * 1000),
y: {{ num }}
},
{% endfor %}
]
},
{
label: 'Total Topics',
backgroundColor: "#003A72",
//borderColor: "#003A72",
data: [
{% for timestamp, num in discourse.total_all %}
{
t: new Date({{ timestamp }} * 1000),
y: {{ num }}
},
{% endfor %}
]
},
]
},
options: {
scales: {
xAxes: [{
type: 'time',
time: {
unit: 'day'
}
}]
}
}
});
</script>
</body>
</html>

@ -20,8 +20,9 @@
</div> </div>
</div> </div>
<div class="row justify-content-center"> <div class="row justify-content-center">
<div class="col" style="text-align: center;"> <div class="col-xs-6" style="text-align: center;">
<h2>Jenkins Data</h2> <h2>Jenkins Data</h2>
<div class="table-responsive">
<table class="table table-striped"> <table class="table table-striped">
<thead> <thead>
<tr> <tr>
@ -46,6 +47,37 @@
</table> </table>
</div> </div>
</div> </div>
<div class="col-xs-6" style="text-align: center;">
<h2>Discourse Data</h2>
<div class="table-responsive">
<table class="table table-striped">
<thead>
<tr>
<th>Time (days)</th>
<th>Average Open Support Topics</th>
<th>Average Total Support Topics</th>
<th>Average % (Open / Total) Support Topics</th>
<th>Average Open Topics</th>
<th>Average Total Topics</th>
<th>Average % (Open / Total) Topics</th>
<th>Graph for Data</th>
</tr>
</thead>
<tbody>
{% for day in page.Discourse %}
<tr>
<td>{{ day }}</td>
{% for avg in page.Discourse[day] %}
<td>{{ avg }}</td>
{% endfor %}
<td><a href="discourse_{{ day }}days.html">Here</a></td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
</div> </div>
</body> </body>
</html> </html>

Loading…
Cancel
Save