nightly analytics

.gitignore

@@ -2,3 +2,4 @@ download.png
 test.png
 .idea
 secrets.json
+__pycache__

cowmesh_iperf_test.py

@@ -3,6 +3,7 @@ script to do a basic holistic test of the cowmesh network,
 testing internet connection speed using iperf,
 between all nodes in the mesh
 """
+import datetime
 import os
 import re
 import time
@@ -16,10 +17,18 @@ SECRETS_PATH = os.path.join(PROJECT_PATH, "secrets.json")
 with open(SECRETS_PATH, 'r') as f:
     SECRETS = json.loads(f.read())
 
+# nodes = [
+#     "jaaga",
+#     "redcottage",
+#     "new-gazebo2",
+#     "kotemanetp",
+#     "guard"
+# ]
+
 nodes = [
-    "jaaga",
+    # "jaaga",
-    # "new-gazebo",
+    # "redcottage",
-    "redcottage",
+    # "new-gazebo2",
     "kotemanetp",
     "guard"
 ]
@@ -27,6 +36,8 @@ nodes = [
 host_to_ip = {
     "jaaga": "10.56.121.19",
     "redcottage": "10.56.58.194",
+    "redcottage2": "10.56.114.42",
+    "new-gazebo2": "10.56.114.42",
     "new-gazebo": "10.56.113.2",
     "guard": "10.56.121.73",
     "kotemanetp": "10.56.40.113"
@@ -35,73 +46,100 @@ host_to_ip = {
 results = {}
 
 
-def test_between_two_nodes(node_a, node_b):
+class CowmeshIperfTester:
-    print("running test from {} to {}".format(node_a, node_b))
-    u_name = 'root'
-    pswd = SECRETS["ROUTER_PASSWORD"]
 
-    myconn = paramiko.SSHClient()
+    def __init__(self, log=None, debug=False):
-    myconn.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        if log:
+            self.log = log
+        self.debug = debug
 
-    session = myconn.connect(node_a, username =u_name, password=pswd)
+    async def log(self, msg):
+        print(msg)
 
-    ip = host_to_ip[node_b]
+    async def debug_log(self, msg):
+        if self.debug:
+            await self.log(msg)
 
-    remote_cmd = 'iperf -c {ip} -p 5001'.format(ip=ip)
+    async def test_between_two_nodes(self, node_a, node_b):
-    (stdin, stdout, stderr) = myconn.exec_command(remote_cmd)
+        await self.log("++ running test from {} to {}".format(node_a, node_b))
-    output = str(stdout.read())
-    print("output: {}".format(output))
-    print("errors: {}".format(stderr.read()))
-    myconn.close()
-
-    match = re.search("(\S+) Mbits", output)
-    if match:
-        to_return = match.group(1)
-    else:
-        to_return = None
-
-    return to_return
-
-
-async def start_iperf_servers():
-    for node in nodes:
-        print("starting iperf server on {}".format(node))
         u_name = 'root'
         pswd = SECRETS["ROUTER_PASSWORD"]
 
         myconn = paramiko.SSHClient()
         myconn.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 
-        session = myconn.connect(node, username=u_name, password=pswd)
+        myconn.connect(node_a, username =u_name, password=pswd)
 
-        remote_cmd = 'iperf -s &'
+        ip = host_to_ip[node_b]
+
+        remote_cmd = 'iperf -c {ip} -p 5001'.format(ip=ip)
         (stdin, stdout, stderr) = myconn.exec_command(remote_cmd)
-        print("{}".format(stdout.read()))
+        output = str(stdout.read())
-        print("{}".format(type(myconn)))
+        await self.debug_log("output: {}".format(output))
-        print("Options available to deal with the connectios are many like\n{}".format(dir(myconn)))
+        await self.debug_log("errors: {}".format(stderr.read()))
         myconn.close()
 
-async def run_test():
+        match = re.search("(\S+) Mbits", output)
+        if match:
+            to_return = match.group(1)
+        else:
+            to_return = None
 
-    await start_iperf_servers()
+        return to_return
 
-    for node_a in nodes:
+    async def start_iperf_servers(self):
-        for node_b in nodes:
+        for node in nodes:
-            if node_a == node_b:
+            print("++ starting iperf server on {}".format(node))
-                print("skip self")
+            u_name = 'root'
-                continue
+            pswd = SECRETS["ROUTER_PASSWORD"]
 
-            r = test_between_two_nodes(node_a, node_b)
+            myconn = paramiko.SSHClient()
-            result_key = "{} -> {}".format(node_a, node_b)
+            myconn.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-            results[result_key] = r
+
+            myconn.connect(node, username=u_name, password=pswd)
+
+            remote_cmd = 'iperf -s &'
+            (stdin, stdout, stderr) = myconn.exec_command(remote_cmd)
+            await self.debug_log("{}".format(stdout.read()))
+            await self.debug_log("{}".format(type(myconn)))
+            await self.debug_log("Options available to deal with the connections are many like\n{}".format(dir(myconn)))
+            myconn.close()
+
+    async def run_test(self):
+
+        await self.start_iperf_servers()
+
+        for node_a in nodes:
+            for node_b in nodes:
+                if node_a == node_b:
+                    await self.debug_log("skip self")
+                    continue
+
+                r = await self.test_between_two_nodes(node_a, node_b)
+                result_key = "{} -> {}".format(node_a, node_b)
+                results[result_key] = r
+
+    async def output_results(self):
+        results_str = ""
+        now = datetime.datetime.now()
+        date = now.date()
+        time = now.time()
+        results_str += "**** iperf results on {date:%m-%d-%Y} at {time:%H:%M}:\n\n".format(date=date, time=time)
+        for test_name, result in results.items():
+            results_str += "{}: {} mbps\n".format(test_name, result)
+        await self.log(results_str)
 
 
-try:
+if __name__ == "__main__":
-    asyncio.get_event_loop().run_until_complete(run_test())
+    try:
+        tester = CowmeshIperfTester()
 
-    print("** final results **")
+        async def main_fun():
-    for test_name, result in results.items():
+            await tester.run_test()
-        print("{}: {} mbps".format(test_name, result))
+            await tester.output_results()
 
-except (OSError, asyncssh.Error) as exc:
+        asyncio.get_event_loop().run_until_complete(main_fun())
-    sys.exit('SSH connection failed: ' + str(exc))
+
+
+    except (OSError, asyncssh.Error) as exc:
+        sys.exit('SSH connection failed: ' + str(exc))

moonlight_analytics.py

@@ -0,0 +1,121 @@
+import asyncio
+import logging
+from telegram import Update
+from telegram.ext import ApplicationBuilder, ContextTypes, CommandHandler, MessageHandler, filters
+import os
+import json
+
+from cowmesh_iperf_test import CowmeshIperfTester
+
+PROJECT_PATH = os.path.abspath(os.path.dirname(__file__))
+SECRETS_PATH = os.path.join(PROJECT_PATH, "secrets.json")
+with open(SECRETS_PATH, 'r') as f:
+    SECRETS = json.loads(f.read())
+
+logging.basicConfig(
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    level=logging.INFO
+)
+
+
+async def start(update: Update, context: ContextTypes.DEFAULT_TYPE):
+    await context.bot.send_message(chat_id=update.effective_chat.id, text="This is the moonlight analytics bot for mesh network diagnostics.\n\nType /help to see available commands.", message_thread_id=update.message.message_thread_id)
+
+
+async def caps(update: Update, context: ContextTypes.DEFAULT_TYPE):
+    """
+    this is just a function to test that the bot is working as expected
+    """
+    text_caps = ' '.join(context.args).upper()
+    print("chat_id: {}".format(update.effective_chat.id))
+    print("message_thread_id: {}".format(update.message.message_thread_id))
+    await context.bot.send_message(chat_id=update.effective_chat.id, text=text_caps, message_thread_id=update.message.message_thread_id)
+
+
+async def iperf(update: Update, context: ContextTypes.DEFAULT_TYPE):
+    async def log(msg):
+        await context.bot.send_message(chat_id=update.effective_chat.id, text=msg, message_thread_id=update.message.message_thread_id)
+    await log("++ starting iperf test")
+    tester = CowmeshIperfTester(log=log)
+    await tester.run_test()
+    await tester.output_results()
+
+
+async def unknown(update: Update, context: ContextTypes.DEFAULT_TYPE):
+    await context.bot.send_message(chat_id=update.effective_chat.id, text="Sorry, I didn't understand that, please run /help for a list of available commands.", message_thread_id=update.message.message_thread_id)
+
+
+def about_message():
+    msg = "This is a bot designed to help measure the performance of a mesh network through active testing. " \
+          "Every night the bot runs a network test using iperf to measure the connectivity between all the nodes in the mesh, " \
+          "and logs the result to this channel. " \
+          "Members of this channel can also initiate a new network iperf test at any time by sending a message to this channel, " \
+          "with the command /iperf . " \
+          "Telegram users outside of this channel cannot initiate a test, to help keep the network secure from being " \
+          "overrun by malicious users. " \
+          "The network test runs at night because that is when the fewest people are using the network, and so is more likely to " \
+          "give consistent results with less random variability. " \
+          "Please be mindful of initiating too many iperf tests during the day, as it uses a lot of network resources " \
+          "to run the test and could interfere with the internet connections of people using the network. "
+    return msg
+
+
+def help_message():
+    msg = "This bot runs an iperf test every night and logs the results here. You can also initiate a new test using the command /iperf " \
+          "or read a longer message explaining how this bot works by typing the command /readme."
+    return msg
+
+
+async def about(update: Update, context: ContextTypes.DEFAULT_TYPE):
+    text = about_message()
+    await context.bot.send_message(chat_id=update.effective_chat.id, text=text, message_thread_id=update.message.message_thread_id)
+
+
+async def help_fun(update: Update, context: ContextTypes.DEFAULT_TYPE):
+    text = help_message()
+    await context.bot.send_message(chat_id=update.effective_chat.id, text=text, message_thread_id=update.message.message_thread_id)
+
+
+async def nightly_iperf():
+    token = SECRETS["TELEGRAM_TOKEN"]
+    application = ApplicationBuilder().token(token).build()
+    chat_id = SECRETS["TELEGRAM_LOG_CHAT_ID"]
+    message_thread_id = SECRETS.get("TELEGRAM_LOG_MESSAGE_THREAD_ID")
+    bot = application.bot
+
+    async def log(msg):
+        await bot.send_message(chat_id=chat_id, text=msg, message_thread_id=message_thread_id)
+
+    await log("☾☾ starting nightly iperf test")
+    tester = CowmeshIperfTester(log=log)
+    await tester.run_test()
+    await tester.output_results()
+
+
+def init_bot_listener():
+    token = SECRETS["TELEGRAM_TOKEN"]
+    application = ApplicationBuilder().token(token).build()
+
+    start_handler = CommandHandler('start', start)
+    application.add_handler(start_handler)
+
+    caps_handler = CommandHandler('caps', caps)
+    application.add_handler(caps_handler)
+
+    about_handler = CommandHandler('readme', about)
+    application.add_handler(about_handler)
+
+    help_handler = CommandHandler('help', help_fun)
+    application.add_handler(help_handler)
+
+    iperf_handler = CommandHandler('iperf', iperf)
+    application.add_handler(iperf_handler)
+
+    unknown_handler = MessageHandler(filters.COMMAND, unknown)
+    application.add_handler(unknown_handler)
+
+    application.run_polling()
+
+
+if __name__ == '__main__':
+    init_bot_listener()

nightly_test.py

@@ -0,0 +1,7 @@
+import argparse
+from moonlight_analytics import nightly_iperf
+import asyncio
+
+
+if __name__ == '__main__':
+    asyncio.get_event_loop().run_until_complete(nightly_iperf())

server.py

@@ -0,0 +1,19 @@
+import os
+from flask import Flask, request, render_template, jsonify
+
+app = Flask(__name__, static_folder='public', template_folder='templates')
+
+app.config['TEMPLATES_AUTO_RELOAD'] = True
+
+@app.route('/')
+def homepage():
+    return render_template('index.html')
+
+@app.route('/test')
+def testpage():
+    return "hi! its test"
+
+
+# this line start the server with the 'run()' method
+if __name__ == '__main__':
+    app.run()

templates/index.html

@@ -0,0 +1,61 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>🦋</title>
+    <meta name="description" content="moonlight analytics">
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <style>
+      html {
+        box-sizing: border-box;
+      }
+
+      *, *:before, *:after {
+        box-sizing: inherit;
+      }
+
+      body {
+        background-color: lightblue;
+      }
+
+    /* <3<3<3 write all of your loving css ABOVE this line <3<3<3 */
+    </style>
+  </head>
+
+  <body>
+    <!-- <3<3<3 keep all of your html BELOW this line <3<3<3 -->
+
+    <div id="button-container">
+      <button id="button">run network test</button>
+      <div class="output-wrapper"></div>
+    </div>
+
+    <script src="https://code.jquery.com/jquery-2.2.1.min.js"
+            integrity="sha256-gvQgAFzTH6trSrAWoH1iPo9Xc96QxSZ3feW6kem+O00="
+            crossorigin="anonymous">
+              // this part loads in a javascript library called jquery. you can tell when we're using jquery by the $ in front
+  </script>
+    <script>
+        $('#button').bind('click', function() {
+            // below is the code that gets run afer the button get clicked
+            // first we print to the console that the button was clicked
+            console.log("button clicked!");
+
+            // then we use ajax, to make a request to the server at the /generate route
+            $.get('/init-test', function(response) {
+               // below is the code that runs when the python server route returns,
+               // response is a variable which holds the response from the server (a string of text)
+               // and which we can use in whatever way we want
+               // ... first we take this response and print it to the console
+               console.log(response);
+               // and then we insert the response into the .output-wrapper element
+               // (this also replaces whatever was in this element before)
+               $('.output-wrapper').html(response);
+            });
+
+            return false;
+        });
+    </script>
+  </body>
+</html>