cleanup

cowmesh_constants.py

@@ -1,7 +1,7 @@
 NODES = [
     "jaaga",
     "redcottage",
-    "new-gazebo2",
+    "new-gazebo",
     "kotemanetp",
     "guard",
     "ddhills",
@@ -12,11 +12,10 @@ NODES = [
 HOST_TO_IP = {
     "jaaga": "10.56.121.19",
     "redcottage": "10.56.58.194",
-    "redcottage2": "10.56.114.42",
-    "new-gazebo2": "10.56.114.42",
     "new-gazebo": "10.56.113.2",
     "guard": "10.56.121.73",
     "kotemanetp": "10.56.40.113",
     "thimmanayaka": "10.56.39.34",
-    "bambino": "10.56.0.102"
+    "bambino": "10.56.0.102",
+    "ddhills": "10.56.39.66",
 }

cowmesh_helpers.py

@@ -11,7 +11,10 @@ async def cleanup_iperf_server(node, ip, username, password, log):
 
     remote_cmd = 'pgrep iperf | xargs kill'
     (stdin, stdout, stderr) = myconn.exec_command(remote_cmd)
-    await log("{}".format(stdout.read()))
-    await log("{}".format(type(myconn)))
-    await log("Options available to deal with the connections are many like\n{}".format(dir(myconn)))
+    out = stdout.read()
+    err = stderr.read()
+    if out:
+        await log("{}".format(out))
+    if err:
+        await log("err: {}".format(err))
     myconn.close()

cowmesh_pi_speedtest.py

@@ -63,9 +63,16 @@ class CowmeshPiSpeedtestTester:
     async def run_test(self):
 
         for node in nodes:
-            r = await self.speedtest_node(node)
             result_key = "{}".format(node)
+            try:
+                r = await self.speedtest_node(node)
                 self.results[result_key] = r
+            except Exception as e:
+                try:
+                    self.log("error: {}".format(e))
+                except:
+                    continue
+                self.results[result_key] = "error: {}".format(e)
 
     async def output_results(self):
         results_str = ""

cowmesh_router_iperf_test.py

@@ -94,12 +94,19 @@ class CowmeshRouterIperfTester:
                     continue
 
     async def cleanup_iperf_servers(self):
-        await self.log("shutting down iperf servers")
+        await self.log("++ shutting down iperf servers")
         for node in NODES:
+            try:
                 user = "root"
                 password = SECRETS["ROUTER_PASSWORD"]
                 ip = HOST_TO_IP[node]
                 await cleanup_iperf_server(node=node, ip=ip, username=user, password=password, log=self.log)
+            except Exception as e:
+                try:
+                    self.log("++ error stopping {} server: {}".format(node, e))
+                except:
+                    continue
+        await self.log("++ finished shutting down iperf servers")
 
     async def run_test(self):
 

moonlight_analytics.py

@@ -47,14 +47,7 @@ class MoonlightTester:
         log_name = "moonlight-{:%m-%d-%Y}-{:%H-%M}-router-iperf.txt".format(date.today(), datetime.now())
         log_location = os.path.join(LOG_DIR_PATH, log_name)
 
-        async def offline_log(msg):
-            with open(log_location, 'a') as log_file:
-                log_file.write(msg + "\n")
-
-        async def log(msg):
-            await offline_log(msg)
-            await context.bot.send_message(chat_id=update.effective_chat.id, text=msg,
-                                           message_thread_id=update.message.message_thread_id)
+        log = self.get_offline_first_log_fn(bot=context.bot, log_location=log_location)
 
         if update.effective_chat.id != int(SECRETS["TELEGRAM_LOG_CHAT_ID"]):
             await log("++ can only start iperf test from Moonlight Bot group")
@@ -79,12 +72,7 @@ class MoonlightTester:
         log_name = "moonlight-{:%m-%d-%Y}-{:%H-%M}-pi-iperf.txt".format(date.today(), datetime.now())
         log_location = os.path.join(LOG_DIR_PATH, log_name)
 
-        async def offline_log(msg):
-            with open(log_location, 'a') as log_file:
-                log_file.write(msg + "\n")
-        async def log(msg):
-            await offline_log(msg)
-            await context.bot.send_message(chat_id=update.effective_chat.id, text=msg, message_thread_id=update.message.message_thread_id)
+        log = self.get_offline_first_log_fn(bot=context.bot, log_location=log_location)
 
         if update.effective_chat.id != int(SECRETS["TELEGRAM_LOG_CHAT_ID"]):
             await log("++ can only start iperf test from Moonlight Bot group")
@@ -105,14 +93,7 @@ class MoonlightTester:
         log_name = "moonlight-{:%m-%d-%Y}-{:%H-%M}-pi-speedtest.txt".format(date.today(), datetime.now())
         log_location = os.path.join(LOG_DIR_PATH, log_name)
 
-        async def offline_log(msg):
-            with open(log_location, 'a') as log_file:
-                log_file.write(msg + "\n")
-
-        async def log(msg):
-            await offline_log(msg)
-            await context.bot.send_message(chat_id=update.effective_chat.id, text=msg,
-                                           message_thread_id=update.message.message_thread_id)
+        log = self.get_offline_first_log_fn(bot=context.bot, log_location=log_location)
 
         if update.effective_chat.id != int(SECRETS["TELEGRAM_LOG_CHAT_ID"]):
             await log("++ can only start test from Moonlight Bot group")
@@ -162,80 +143,59 @@ class MoonlightTester:
         text = self.help_message()
         await context.bot.send_message(chat_id=update.effective_chat.id, text=text, message_thread_id=update.message.message_thread_id)
 
-    async def send_log(self, bot, chat_id, message_thread_id, log_location):
+    async def send_log(self, bot, log_location):
         # send log file
+        chat_id = SECRETS["TELEGRAM_LOG_CHAT_ID"]
+        message_thread_id = SECRETS.get("TELEGRAM_LOG_MESSAGE_THREAD_ID")
         document = open(log_location, 'rb')
-        bot.send_document(chat_id, document, message_thread_id=message_thread_id)
+        await bot.send_document(chat_id, document, message_thread_id=message_thread_id)
 
     async def nightly_router_iperf(self, time=10):
         token = SECRETS["TELEGRAM_TOKEN"]
         application = ApplicationBuilder().token(token).build()
-        chat_id = SECRETS["TELEGRAM_LOG_CHAT_ID"]
-        message_thread_id = SECRETS.get("TELEGRAM_LOG_MESSAGE_THREAD_ID")
         bot = application.bot
 
-        log_name = "moonlight-{:%M-%d-%Y}-{:%H:%m}-router-iperf.txt".format(datetime.today(), datetime.now())
+        log_name = "moonlight-{:%m-%d-%Y}-{:%H-%M}-router-iperf.txt".format(datetime.today(), datetime.now())
         log_location = os.path.join(LOG_DIR_PATH, log_name)
 
-        async def offline_log(msg):
-            with open(log_location, 'a') as log_file:
-                log_file.write(msg)
-
-        async def log(msg):
-            await offline_log(msg)
-            try:
-                await bot.send_message(chat_id=chat_id, text=msg, message_thread_id=message_thread_id)
-            except:
-                pass
+        log = self.get_offline_first_log_fn(bot=bot, log_location=log_location)
 
         await log("☾☾ starting nightly router-to-router iperf test with {time} seconds per test".format(time=time))
         tester = CowmeshRouterIperfTester(log=log, time=time)
         await tester.run_test()
         await tester.output_results()
-        await self.send_log(bot=bot, log_location=log_location, chat_id=chat_id, message_thread_id=message_thread_id)
+        await self.send_log(bot=bot, log_location=log_location)
 
 
 
     async def nightly_pi_iperf(self, time=10):
         token = SECRETS["TELEGRAM_TOKEN"]
         application = ApplicationBuilder().token(token).build()
-        chat_id = SECRETS["TELEGRAM_LOG_CHAT_ID"]
-        message_thread_id = SECRETS.get("TELEGRAM_LOG_MESSAGE_THREAD_ID")
         bot = application.bot
 
-
-        log_name = "moonlight-{:%M-%d-%Y}-{:%H:%m}-pi-iperf.txt".format(datetime.today(), datetime.now())
+        log_name = "moonlight-{:%m-%d-%Y}-{:%H-%M}-pi-iperf.txt".format(datetime.today(), datetime.now())
         log_location = os.path.join(LOG_DIR_PATH, log_name)
 
-        async def offline_log(msg):
-            with open(log_location, 'a') as log_file:
-                log_file.write(msg)
-
-        async def log(msg):
-            await offline_log(msg)
-            try:
-                await bot.send_message(chat_id=chat_id, text=msg, message_thread_id=message_thread_id)
-            except:
-                pass
+        log = self.get_offline_first_log_fn(bot=bot, log_location=log_location)
 
         await log("☾☾ starting nightly computer-to-computer iperf test with {} seconds per test".format(time))
         tester = CowmeshPiIperfTester(log=log, time=time)
         await tester.run_test()
         await tester.output_results()
-        await self.send_log(bot=bot, log_location=log_location, chat_id=chat_id, message_thread_id=message_thread_id)
+        await self.send_log(bot=bot, log_location=log_location)
 
 
-
-    async def nightly_pi_speedtest(self):
+    def get_bot(self):
         token = SECRETS["TELEGRAM_TOKEN"]
         application = ApplicationBuilder().token(token).build()
+        bot = application.bot
+        return bot
+
+
+    def get_offline_first_log_fn(self, bot, log_location):
+
         chat_id = SECRETS["TELEGRAM_LOG_CHAT_ID"]
         message_thread_id = SECRETS.get("TELEGRAM_LOG_MESSAGE_THREAD_ID")
-        bot = application.bot
-
-        log_name = "moonlight-{:%M-%d-%Y}-{:%H:%m}-pi-iperf.txt".format(datetime.today(), datetime.now())
-        log_location = os.path.join(LOG_DIR_PATH, log_name)
-
         async def offline_log(msg):
             with open(log_location, 'a') as log_file:
                 log_file.write(msg)
@@ -247,13 +207,55 @@ class MoonlightTester:
             except:
                 pass
 
+        return log
+
+    async def nightly_pi_speedtest(self):
+
+        bot = self.get_bot()
+
+        log_name = "moonlight-{:%m-%d-%Y}-{:%H-%M}-pi-speedtest.txt".format(datetime.today(), datetime.now())
+        log_location = os.path.join(LOG_DIR_PATH, log_name)
+
+        log = self.get_offline_first_log_fn(bot=bot, log_location=log_location)
+
         await log("☾☾ starting nightly speedtest-cli speedtest")
         tester = CowmeshPiSpeedtestTester(log=log)
         await tester.run_test()
         await tester.output_results()
-        await self.send_log(bot=bot, log_location=log_location, chat_id=chat_id, message_thread_id=message_thread_id)
+        await self.send_log(bot=bot, log_location=log_location)
 
 
+    async def start_iperf_servers(self, update: Update, context: ContextTypes.DEFAULT_TYPE):
+
+        bot = self.get_bot()
+
+        log_name = "moonlight-{:%m-%d-%Y}-{:%H-%M}-start-iperf-servers.txt".format(datetime.today(), datetime.now())
+        log_location = os.path.join(LOG_DIR_PATH, log_name)
+
+        log = self.get_offline_first_log_fn(bot=bot, log_location=log_location)
+
+        await log("++ ensuring that iperf servers are running")
+
+        tester = CowmeshRouterIperfTester(log=log)
+        for i in range(0, 2):
+            await tester.start_iperf_servers()
+
+        await log("++ iperf servers are now running")
+
+
+    async def cleanup_iperf_servers(self, update: Update, context: ContextTypes.DEFAULT_TYPE):
+
+        bot = self.get_bot()
+
+        log_name = "moonlight-{:%m-%d-%Y}-{:%H-%M}-cleanup-iperf-servers.txt".format(datetime.today(), datetime.now())
+        log_location = os.path.join(LOG_DIR_PATH, log_name)
+
+        log = self.get_offline_first_log_fn(bot=bot, log_location=log_location)
+
+        tester = CowmeshRouterIperfTester(log=log)
+        for i in range(0, 2):
+            await tester.cleanup_iperf_servers()
+
 
     def init_bot_listener(self):
         token = SECRETS["TELEGRAM_TOKEN"]
@@ -277,6 +279,12 @@ class MoonlightTester:
         pi_iperf_handler = CommandHandler('pi_iperf', self.pi_iperf)
         application.add_handler(pi_iperf_handler)
 
+        start_iperf_servers_handler = CommandHandler('start_iperf_servers', self.start_iperf_servers)
+        application.add_handler(start_iperf_servers_handler)
+
+        cleanup_iperf_servers_handler = CommandHandler('cleanup_iperf_servers', self.cleanup_iperf_servers)
+        application.add_handler(cleanup_iperf_servers_handler)
+
         speedtest_handler = CommandHandler('speedtest', self.speedtest)
         application.add_handler(speedtest_handler)
 

nightly_test.py

@@ -13,3 +13,7 @@ if __name__ == '__main__':
         asyncio.get_event_loop().run_until_complete(tester.nightly_pi_iperf(time=time))
     elif sys.argv[1] == "speedtest":
         asyncio.get_event_loop().run_until_complete(tester.nightly_pi_speedtest())
+    elif sys.argv[1] == "start_iperf":
+        asyncio.get_event_loop().run_until_complete(tester.start_iperf_servers())
+    elif sys.argv[1] == "cleanup_iperf":
+        asyncio.get_event_loop().run_until_complete(tester.cleanup_iperf_servers())