diff --git a/metrics b/metrics index ba26ed1..212ddc5 100755 --- a/metrics +++ b/metrics @@ -18,12 +18,13 @@ import argparse import logging as log import sqlite3 +from modules.discourse import DiscourseModule from modules.jenkins import JenkinsModule from modules.utilities import * from os import path from shutil import copytree, rmtree -ENABLED_MODULES = [JenkinsModule] +ENABLED_MODULES = [DiscourseModule, JenkinsModule] def sqlite_run(command, db, return_output=False): @@ -89,8 +90,9 @@ def main(module): for day in (1, 7, 30, 90, 180): # Fetch the data and log to debug run = [module.sqlite_time_range(days=day)] + log.info(run) data = sqlite_run(run, db=args.db_location, return_output=True) - log.debug(data) + log.info(data) # Render the template, which also returns the average values _averages[day] = module.render_template(day, data) diff --git a/modules/discourse.py b/modules/discourse.py new file mode 100755 index 0000000..5ae5a9e --- /dev/null +++ b/modules/discourse.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2020 Simon Quigley +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import datetime +import requests_cache +import time +from modules.utilities import * +from os import getenv, makedirs, path +from pydiscourse import DiscourseClient + +requests_cache.install_cache("discourse", backend="sqlite", expire_after=300) + + +class DiscourseModule: + """Discourse module for the Metrics program""" + + def __init__(self): + self.name = "Discourse" + + def _auth_discourse_server(self): + """Authenticate to Discourse + + This uses the API_SITE, API_USER, and API_KEY env vars. + """ + # Load the config, so we can store secrets outside of env vars + config = load_config() + in_conf = "discourse" in config + + # Load the needed secrets either from the config file if it exists + # or the env var if it's defined (which takes precedence) + site = getenv("DISCOURSE_API_SITE") or (in_conf and config["discourse"]["site"]) + user = getenv("DISCOURSE_API_USER") or (in_conf and config["discourse"]["user"]) + key = getenv("DISCOURSE_API_KEY") or (in_conf and config["discourse"]["key"]) + for envvar in [site, user, key]: + if not envvar: + raise ValueError("DISCOURSE_API_SITE, DISCOURSE_API_USER, and", + "DISCOURSE_API_KEY must be defined") + # Authenticate to the server + server = DiscourseClient(site, api_username=user, api_key=key) + + return server + + def _get_data(self): + """Get the data from Discourse + + This function returns six distinct values as one list: + + [open_support, total_support, percent_support, open_all, total_all, + percent_all] + """ + + # Authenticate to the server + server = self._auth_discourse_server() + + # Initialize the data + data = [0, 0, 0, 0, 0, 0] + + for category in server.categories(): + # We are limited to 30 topics per page, so we have to loop until + # there are no more topics + page = 0 + on_page = 1 + c_id = category["id"] + + while on_page > 0: + # Get a list of all the topics to then iterate on + page_data = server.category_topics(category_id=c_id, page=page) + topics = page_data["topic_list"]["topics"] + on_page = len(topics) + page += 1 + + print("Working on " + category["name"]) + for topic in topics: + # Increment total_all + data[4] += 1 + + # If it's open, increment open_all + if not topic["closed"]: + data[3] += 1 + + # If the topic is in Support, repeat + if category["name"] == "Support": + data[1] += 1 + if not topic["closed"]: + data[0] += 1 + + # Calculate the percentages + data[2] = ((data[0] / data[1]) * 100) + data[5] = ((data[3] / data[4]) * 100) + + return data + + def sqlite_setup(self): + """Initially set up the table for usage in SQLite + + This returns a str which will then be executed in our SQLite db + + Here is the "discourse" table layout: + - date is the primary key, and it is the Unix timestamp as an int + - open_support is the number of open topics in the Support category + - total_support is the number of total topics in the Support category + - percent_support is ((open_support / total_support) * 100) + - open_all is the number of open topics on the Discourse instance + - total_all is the number of total topics on the Discourse instance + - percent_all is ((open_all / total_all) * 100) + """ + + command = "CREATE TABLE IF NOT EXISTS discourse (date INTEGER PRIMARY" + command += " KEY, open_support INTEGER, total_support INTEGER, " + command += "percent_support INTEGER, open_all INTEGER, total_all " + command += "INTEGER, percent_all INTEGER);" + + return command + + def sqlite_add(self): + """Add data to the SQLite db + + This retrieves the current data from the Jenkins server, and returns a + str which will then be executed in our SQLite db + """ + + # Match the variable names with the column names in the db + data = tuple(self._get_data()) + date = "strftime('%s', 'now')" + + # Craft the str + print(data) + command = "INSERT INTO discourse VALUES (" + command += "{}, {}, {}, {}, {}, {}, {});".format(date, *data) + + return command + + def sqlite_time_range(self, days): + """Get the rows which have been inserted given days + + e.g. if days is 180, it gets all of the values which have been + inserted in the past 180 days. + + Note: this just returns the command to be ran, it doesn't actually run + """ + + now = datetime.datetime.now() + timedelta = datetime.timedelta(days=days) + unix_time = int(time.mktime((now - timedelta).timetuple())) + + command = "SELECT * FROM discourse WHERE date > %s;" % unix_time + + return command + + def render_template(self, days, data): + """Render a template with days in the filename, given the data + + The above function sqlite_time_range() is ran on the database with + some predetermined date ranges. This function actually interprets that + data, uses Jinja2 to magically render a template, and voila. + """ + + # Initialize a (softly) ephemeral dict to store data + discourse = dict() + keys = ["date", "open_support", "total_support", "percent_support", + "open_all", "total_all", "percent_all"] + + # Use a lambda to map the data into a dict with the keys + for row in data: + print(row) + _data = {keys[x] : row[x] for x in range(len(row))} + + # Add our ephemeral dict to the master dict, and create the key if + # it doesn't already exist + for key in _data.keys(): + if key in discourse: + discourse[key].append(_data[key]) + else: + discourse[key] = [_data[key]] + + data = discourse + + # Get human-readable averages and throw it in a dict + average = {} + for datatype in keys: + try: + num = sum(data[datatype]) / len(data[datatype]) + num = format(num, ".1f") + except ZeroDivisionError: + num = 0 + average[datatype] = num + + # Assign data to the dict Jinja2 is actually going to use + discourse = {keys[x] : zip(data["date"], data[keys[x]]) for x in range(len(keys))} + + # Make the output dir if it doesn't already exist + if not path.exists("output"): + makedirs("output") + + src = path.join("templates", "discourse.html") + dest = path.join("output", "discourse_%sdays.html" % days) + jinja2_template(src, dest, discourse=discourse, average=average, + days=days) + + # Return the averages for use in the summary + r_data = [] + del keys[0] + for key in keys: + r_data.append(average[key]) + + return tuple(r_data) diff --git a/templates/discourse.html b/templates/discourse.html new file mode 100644 index 0000000..2032e06 --- /dev/null +++ b/templates/discourse.html @@ -0,0 +1,109 @@ + + + + + + + + + + + + + + + + Discourse data for the past {{ days }} day{{ "s" if days > 1 }} + + +
+
+
+

Discourse data for the past {{ days }} day{{ "s" if days > 1 }}

+ +

Average number of open support topics: {{ average.open_support }}

+

Average number of total support topics: {{ average.total_support }}

+

Average percent of (open / total) support topics: {{ average.percent_support }}%

+

Average number of open topics: {{ average.open_all }}

+

Average number of total topics: {{ average.total_all }}

+

Average percent of (open / total) topics: {{ average.percent_all }}%

+

Back to summary page

+
+
+
+ + + + diff --git a/templates/index.html b/templates/index.html index db4f22a..0c433e2 100644 --- a/templates/index.html +++ b/templates/index.html @@ -20,30 +20,62 @@
-
+

Jenkins Data

- - - - - - - - - - - - {% for day in page.Jenkins %} - - - {% for avg in page.Jenkins[day] %} - +
+
Time (days)Average Non-passingAverage FailingAverage TotalGraph for Data
{{ day }}{{ avg }}
+ + + + + + + + + + + {% for day in page.Jenkins %} + + + {% for avg in page.Jenkins[day] %} + + {% endfor %} + + + {% endfor %} + +
Time (days)Average Non-passingAverage FailingAverage TotalGraph for Data
{{ day }}{{ avg }}Here
+
+
+
+

Discourse Data

+
+ + + + + + + + + + + + + + + {% for day in page.Discourse %} + + + {% for avg in page.Discourse[day] %} + + {% endfor %} + + {% endfor %} - - - {% endfor %} - -
Time (days)Average Open Support TopicsAverage Total Support TopicsAverage % (Open / Total) Support TopicsAverage Open TopicsAverage Total TopicsAverage % (Open / Total) TopicsGraph for Data
{{ day }}{{ avg }}Here
Here
+ + +