parent
558774de80
commit
6f68733ca1
@ -0,0 +1,220 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
# Copyright (C) 2020 Simon Quigley <tsimonq2@lubuntu.me>
|
||||||
|
#
|
||||||
|
# This program is free software: you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU General Public License as published by
|
||||||
|
# the Free Software Foundation, either version 3 of the License, or
|
||||||
|
# (at your option) any later version.
|
||||||
|
#
|
||||||
|
# This program is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU General Public License
|
||||||
|
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
import datetime
|
||||||
|
import requests_cache
|
||||||
|
import time
|
||||||
|
from modules.utilities import *
|
||||||
|
from os import getenv, makedirs, path
|
||||||
|
from pydiscourse import DiscourseClient
|
||||||
|
|
||||||
|
requests_cache.install_cache("discourse", backend="sqlite", expire_after=300)
|
||||||
|
|
||||||
|
|
||||||
|
class DiscourseModule:
|
||||||
|
"""Discourse module for the Metrics program"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.name = "Discourse"
|
||||||
|
|
||||||
|
def _auth_discourse_server(self):
|
||||||
|
"""Authenticate to Discourse
|
||||||
|
|
||||||
|
This uses the API_SITE, API_USER, and API_KEY env vars.
|
||||||
|
"""
|
||||||
|
# Load the config, so we can store secrets outside of env vars
|
||||||
|
config = load_config()
|
||||||
|
in_conf = "discourse" in config
|
||||||
|
|
||||||
|
# Load the needed secrets either from the config file if it exists
|
||||||
|
# or the env var if it's defined (which takes precedence)
|
||||||
|
site = getenv("DISCOURSE_API_SITE") or (in_conf and config["discourse"]["site"])
|
||||||
|
user = getenv("DISCOURSE_API_USER") or (in_conf and config["discourse"]["user"])
|
||||||
|
key = getenv("DISCOURSE_API_KEY") or (in_conf and config["discourse"]["key"])
|
||||||
|
for envvar in [site, user, key]:
|
||||||
|
if not envvar:
|
||||||
|
raise ValueError("DISCOURSE_API_SITE, DISCOURSE_API_USER, and",
|
||||||
|
"DISCOURSE_API_KEY must be defined")
|
||||||
|
# Authenticate to the server
|
||||||
|
server = DiscourseClient(site, api_username=user, api_key=key)
|
||||||
|
|
||||||
|
return server
|
||||||
|
|
||||||
|
def _get_data(self):
|
||||||
|
"""Get the data from Discourse
|
||||||
|
|
||||||
|
This function returns six distinct values as one list:
|
||||||
|
|
||||||
|
[open_support, total_support, percent_support, open_all, total_all,
|
||||||
|
percent_all]
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Authenticate to the server
|
||||||
|
server = self._auth_discourse_server()
|
||||||
|
|
||||||
|
# Initialize the data
|
||||||
|
data = [0, 0, 0, 0, 0, 0]
|
||||||
|
|
||||||
|
for category in server.categories():
|
||||||
|
# We are limited to 30 topics per page, so we have to loop until
|
||||||
|
# there are no more topics
|
||||||
|
page = 0
|
||||||
|
on_page = 1
|
||||||
|
c_id = category["id"]
|
||||||
|
|
||||||
|
while on_page > 0:
|
||||||
|
# Get a list of all the topics to then iterate on
|
||||||
|
page_data = server.category_topics(category_id=c_id, page=page)
|
||||||
|
topics = page_data["topic_list"]["topics"]
|
||||||
|
on_page = len(topics)
|
||||||
|
page += 1
|
||||||
|
|
||||||
|
print("Working on " + category["name"])
|
||||||
|
for topic in topics:
|
||||||
|
# Increment total_all
|
||||||
|
data[4] += 1
|
||||||
|
|
||||||
|
# If it's open, increment open_all
|
||||||
|
if not topic["closed"]:
|
||||||
|
data[3] += 1
|
||||||
|
|
||||||
|
# If the topic is in Support, repeat
|
||||||
|
if category["name"] == "Support":
|
||||||
|
data[1] += 1
|
||||||
|
if not topic["closed"]:
|
||||||
|
data[0] += 1
|
||||||
|
|
||||||
|
# Calculate the percentages
|
||||||
|
data[2] = ((data[0] / data[1]) * 100)
|
||||||
|
data[5] = ((data[3] / data[4]) * 100)
|
||||||
|
|
||||||
|
return data
|
||||||
|
|
||||||
|
def sqlite_setup(self):
|
||||||
|
"""Initially set up the table for usage in SQLite
|
||||||
|
|
||||||
|
This returns a str which will then be executed in our SQLite db
|
||||||
|
|
||||||
|
Here is the "discourse" table layout:
|
||||||
|
- date is the primary key, and it is the Unix timestamp as an int
|
||||||
|
- open_support is the number of open topics in the Support category
|
||||||
|
- total_support is the number of total topics in the Support category
|
||||||
|
- percent_support is ((open_support / total_support) * 100)
|
||||||
|
- open_all is the number of open topics on the Discourse instance
|
||||||
|
- total_all is the number of total topics on the Discourse instance
|
||||||
|
- percent_all is ((open_all / total_all) * 100)
|
||||||
|
"""
|
||||||
|
|
||||||
|
command = "CREATE TABLE IF NOT EXISTS discourse (date INTEGER PRIMARY"
|
||||||
|
command += " KEY, open_support INTEGER, total_support INTEGER, "
|
||||||
|
command += "percent_support INTEGER, open_all INTEGER, total_all "
|
||||||
|
command += "INTEGER, percent_all INTEGER);"
|
||||||
|
|
||||||
|
return command
|
||||||
|
|
||||||
|
def sqlite_add(self):
|
||||||
|
"""Add data to the SQLite db
|
||||||
|
|
||||||
|
This retrieves the current data from the Jenkins server, and returns a
|
||||||
|
str which will then be executed in our SQLite db
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Match the variable names with the column names in the db
|
||||||
|
data = tuple(self._get_data())
|
||||||
|
date = "strftime('%s', 'now')"
|
||||||
|
|
||||||
|
# Craft the str
|
||||||
|
print(data)
|
||||||
|
command = "INSERT INTO discourse VALUES ("
|
||||||
|
command += "{}, {}, {}, {}, {}, {}, {});".format(date, *data)
|
||||||
|
|
||||||
|
return command
|
||||||
|
|
||||||
|
def sqlite_time_range(self, days):
|
||||||
|
"""Get the rows which have been inserted given days
|
||||||
|
|
||||||
|
e.g. if days is 180, it gets all of the values which have been
|
||||||
|
inserted in the past 180 days.
|
||||||
|
|
||||||
|
Note: this just returns the command to be ran, it doesn't actually run
|
||||||
|
"""
|
||||||
|
|
||||||
|
now = datetime.datetime.now()
|
||||||
|
timedelta = datetime.timedelta(days=days)
|
||||||
|
unix_time = int(time.mktime((now - timedelta).timetuple()))
|
||||||
|
|
||||||
|
command = "SELECT * FROM discourse WHERE date > %s;" % unix_time
|
||||||
|
|
||||||
|
return command
|
||||||
|
|
||||||
|
def render_template(self, days, data):
|
||||||
|
"""Render a template with days in the filename, given the data
|
||||||
|
|
||||||
|
The above function sqlite_time_range() is ran on the database with
|
||||||
|
some predetermined date ranges. This function actually interprets that
|
||||||
|
data, uses Jinja2 to magically render a template, and voila.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Initialize a (softly) ephemeral dict to store data
|
||||||
|
discourse = dict()
|
||||||
|
keys = ["date", "open_support", "total_support", "percent_support",
|
||||||
|
"open_all", "total_all", "percent_all"]
|
||||||
|
|
||||||
|
# Use a lambda to map the data into a dict with the keys
|
||||||
|
for row in data:
|
||||||
|
print(row)
|
||||||
|
_data = {keys[x] : row[x] for x in range(len(row))}
|
||||||
|
|
||||||
|
# Add our ephemeral dict to the master dict, and create the key if
|
||||||
|
# it doesn't already exist
|
||||||
|
for key in _data.keys():
|
||||||
|
if key in discourse:
|
||||||
|
discourse[key].append(_data[key])
|
||||||
|
else:
|
||||||
|
discourse[key] = [_data[key]]
|
||||||
|
|
||||||
|
data = discourse
|
||||||
|
|
||||||
|
# Get human-readable averages and throw it in a dict
|
||||||
|
average = {}
|
||||||
|
for datatype in keys:
|
||||||
|
try:
|
||||||
|
num = sum(data[datatype]) / len(data[datatype])
|
||||||
|
num = format(num, ".1f")
|
||||||
|
except ZeroDivisionError:
|
||||||
|
num = 0
|
||||||
|
average[datatype] = num
|
||||||
|
|
||||||
|
# Assign data to the dict Jinja2 is actually going to use
|
||||||
|
discourse = {keys[x] : zip(data["date"], data[keys[x]]) for x in range(len(keys))}
|
||||||
|
|
||||||
|
# Make the output dir if it doesn't already exist
|
||||||
|
if not path.exists("output"):
|
||||||
|
makedirs("output")
|
||||||
|
|
||||||
|
src = path.join("templates", "discourse.html")
|
||||||
|
dest = path.join("output", "discourse_%sdays.html" % days)
|
||||||
|
jinja2_template(src, dest, discourse=discourse, average=average,
|
||||||
|
days=days)
|
||||||
|
|
||||||
|
# Return the averages for use in the summary
|
||||||
|
r_data = []
|
||||||
|
del keys[0]
|
||||||
|
for key in keys:
|
||||||
|
r_data.append(average[key])
|
||||||
|
|
||||||
|
return tuple(r_data)
|
||||||
@ -0,0 +1,109 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<!-- HTML5 compliance -->
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
|
||||||
|
|
||||||
|
<!-- Import Chart.js 2.9.3 -->
|
||||||
|
<script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.3/Chart.bundle.js"></script>
|
||||||
|
|
||||||
|
<!-- Import Bootstrap 4.3.1 -->
|
||||||
|
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css" integrity="sha384-ggOyR0iXCbMQv3Xipma34MD+dH/1fQ784/j6cY/iJTQUOhcWr7x9JvoRxT2MZw1T" crossorigin="anonymous">
|
||||||
|
|
||||||
|
<!-- Favicon and page title -->
|
||||||
|
<link rel="icon" type="image/png" href="/assets/favicon.png"/>
|
||||||
|
<title>Discourse data for the past {{ days }} day{{ "s" if days > 1 }}</title>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div class="container">
|
||||||
|
<div class="row justify-content-center">
|
||||||
|
<div class="col" style="text-align: center;">
|
||||||
|
<h1>Discourse data for the past {{ days }} day{{ "s" if days > 1 }}</h1>
|
||||||
|
<canvas id="discoursechart"></canvas>
|
||||||
|
<h2>Average number of open support topics: {{ average.open_support }}</h2>
|
||||||
|
<h2>Average number of total support topics: {{ average.total_support }}</h2>
|
||||||
|
<h2>Average percent of (open / total) support topics: {{ average.percent_support }}%</h2>
|
||||||
|
<h2>Average number of open topics: {{ average.open_all }}</h2>
|
||||||
|
<h2>Average number of total topics: {{ average.total_all }}</h2>
|
||||||
|
<h2>Average percent of (open / total) topics: {{ average.percent_all }}%</h2>
|
||||||
|
<h2><a href="./">Back to summary page</a></h2>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<!-- This is the actual chart functionality -->
|
||||||
|
<script>
|
||||||
|
var ctx = document.getElementById("discoursechart").getContext("2d");
|
||||||
|
|
||||||
|
var myChart = new Chart(ctx, {
|
||||||
|
type: 'line',
|
||||||
|
data: {
|
||||||
|
datasets: [
|
||||||
|
{
|
||||||
|
label: 'Open Support Topics',
|
||||||
|
backgroundColor: "#FF0000",
|
||||||
|
borderColor: "#FF0000",
|
||||||
|
data: [
|
||||||
|
{% for timestamp, num in discourse.open_support %}
|
||||||
|
{
|
||||||
|
t: new Date({{ timestamp }} * 1000),
|
||||||
|
y: {{ num }}
|
||||||
|
},
|
||||||
|
{% endfor %}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: 'Open Topics',
|
||||||
|
backgroundColor: "#FFFF00",
|
||||||
|
borderColor: "#FFFF00",
|
||||||
|
data: [
|
||||||
|
{% for timestamp, num in discourse.open_all %}
|
||||||
|
{
|
||||||
|
t: new Date({{ timestamp }} * 1000),
|
||||||
|
y: {{ num }}
|
||||||
|
},
|
||||||
|
{% endfor %}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: 'Total Support Topics',
|
||||||
|
backgroundColor: "#32CD32",
|
||||||
|
borderColor: "#32CD32",
|
||||||
|
data: [
|
||||||
|
{% for timestamp, num in discourse.total_support %}
|
||||||
|
{
|
||||||
|
t: new Date({{ timestamp }} * 1000),
|
||||||
|
y: {{ num }}
|
||||||
|
},
|
||||||
|
{% endfor %}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: 'Total Topics',
|
||||||
|
backgroundColor: "#003A72",
|
||||||
|
//borderColor: "#003A72",
|
||||||
|
data: [
|
||||||
|
{% for timestamp, num in discourse.total_all %}
|
||||||
|
{
|
||||||
|
t: new Date({{ timestamp }} * 1000),
|
||||||
|
y: {{ num }}
|
||||||
|
},
|
||||||
|
{% endfor %}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
]
|
||||||
|
},
|
||||||
|
options: {
|
||||||
|
scales: {
|
||||||
|
xAxes: [{
|
||||||
|
type: 'time',
|
||||||
|
time: {
|
||||||
|
unit: 'day'
|
||||||
|
}
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
Loading…
Reference in new issue