try to make script more fault-tolerant against rcon disconnects

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.8.12-alpine3.14
+FROM python:3.10-alpine
 
 COPY requirements.txt minecraft_exporter.py /
 RUN pip install --no-cache-dir -r requirements.txt

minecraft_exporter.py

@@ -1,64 +1,84 @@
-from prometheus_client import start_http_server, REGISTRY, Metric
-import time
-import requests
 import json
-import nbt
-import re
 import os
-import schedule
+import re
-from mcrcon import MCRcon
+import time
 from os import listdir
 from os.path import isfile, join
+
+import nbt
+import requests
+import schedule
+from mcrcon import MCRcon, MCRconException
+from prometheus_client import Metric, REGISTRY, start_http_server
+
+
 class MinecraftCollector(object):
     def __init__(self):
-        self.statsdirectory = "/world/stats"
+        self.stats_directory = "/world/stats"
-        self.playerdirectory = "/world/playerdata"
+        self.player_directory = "/world/playerdata"
-        self.advancementsdirectory = "/world/advancements"
+        self.advancements_directory = "/world/advancements"
-        self.betterquesting = "/world/betterquesting"
+        self.better_questing = "/world/betterquesting"
-        self.map = dict()
+        self.player_map = dict()
-        self.questsEnabled = False
+        self.quests_enabled = False
+
         self.rcon = None
-        if os.path.isdir(self.betterquesting):
+        self.rcon_connected = False
-            self.questsEnabled = True
+        if all(x in os.environ for x in ['RCON_HOST', 'RCON_PASSWORD']):
+            self.rcon = MCRcon(os.environ['RCON_HOST'], os.environ['RCON_PASSWORD'], port=int(os.environ['RCON_PORT']))
+            print("RCON is enabled for " + os.environ['RCON_HOST'])
+
+        if os.path.isdir(self.better_questing):
+            self.quests_enabled = True
+
         schedule.every().day.at("01:00").do(self.flush_playernamecache)
 
     def get_players(self):
-        return [f[:-5] for f in listdir(self.statsdirectory) if isfile(join(self.statsdirectory, f))]
+        return [f[:-5] for f in listdir(self.stats_directory) if isfile(join(self.stats_directory, f))]
 
     def flush_playernamecache(self):
         print("flushing playername cache")
-        self.map = dict()
+        self.player_map = dict()
-        return
 
     def uuid_to_player(self, uuid):
         uuid = uuid.replace('-', '')
-        if uuid in self.map:
+        if uuid in self.player_map:
-            return self.map[uuid]
+            return self.player_map[uuid]
         else:
             try:
                 result = requests.get('https://api.mojang.com/user/profiles/' + uuid + '/names')
-                self.map[uuid] = result.json()[-1]['name']
+                self.player_map[uuid] = result.json()[-1]['name']
                 return (result.json()[-1]['name'])
             except:
                 return
 
-    def rcon_command(self,command):
+    def rcon_connect(self):
-        if self.rcon == None:
+        try:
-            self.rcon = MCRcon(os.environ['RCON_HOST'],os.environ['RCON_PASSWORD'],port=int(os.environ['RCON_PORT']))
             self.rcon.connect()
+            self.rcon_connected = True
+            print("Successfully connected to RCON")
+            return True
+        except Exception as e:
+            print("Failed to connect to RCON")
+            print(e)
+        return False
+
+    def rcon_command(self, command):
         try:
             response = self.rcon.command(command)
-        except BrokenPipeError:
+        except (ConnectionResetError, ConnectionAbortedError, BrokenPipeError, MCRconException):
-            print("Lost RCON Connection, trying to reconnect")
+            print("Lost RCON Connection")
-            self.rcon.connect()
+            self.rcon.disconnect()
-            response = self.rcon.command(command)
+            self.rcon_connected = False
+            response = None
 
         return response
 
     def get_server_stats(self):
-        metrics = []
+        if self.rcon is None or (not self.rcon_connected and not self.rcon_connect()):
-        if not all(x in os.environ for x in ['RCON_HOST','RCON_PASSWORD']):
             return []
+
+        metrics = []
+
         dim_tps = Metric('dim_tps', 'TPS of a dimension', "counter")
         dim_ticktime = Metric('dim_ticktime', "Time a Tick took in a Dimension", "counter")
         overall_tps = Metric('overall_tps', 'overall TPS', "counter")
@@ -69,7 +89,8 @@ class MinecraftCollector(object):
         tps_5m = Metric('paper_tps_5m', '5 Minute TPS', "counter")
         tps_15m = Metric('paper_tps_15m', '15 Minute TPS', "counter")
 
-        metrics.extend([dim_tps,dim_ticktime,overall_tps,overall_ticktime,player_online,entities,tps_1m,tps_5m,tps_15m])
+        metrics.extend(
+            [dim_tps, dim_ticktime, overall_tps, overall_ticktime, player_online, entities, tps_1m, tps_5m, tps_15m])
         if 'PAPER_SERVER' in os.environ and os.environ['PAPER_SERVER'] == "True":
             resp = str(self.rcon_command("tps")).strip().replace("§a", "")
             tpsregex = re.compile("TPS from last 1m, 5m, 15m: (\d*\.\d*), (\d*\.\d*), (\d*\.\d*)")
@@ -83,7 +104,8 @@ class MinecraftCollector(object):
             dimtpsregex = re.compile("Dim\s*(-*\d*)\s\((.*?)\)\s:\sMean tick time:\s(.*?) ms\. Mean TPS: (\d*\.\d*)")
             for dimid, dimname, meanticktime, meantps in dimtpsregex.findall(resp):
                 dim_tps.add_sample('dim_tps', value=meantps, labels={'dimension_id': dimid, 'dimension_name': dimname})
-                dim_ticktime.add_sample('dim_ticktime',value=meanticktime,labels={'dimension_id':dimid,'dimension_name':dimname})
+                dim_ticktime.add_sample('dim_ticktime', value=meanticktime,
+                                        labels={'dimension_id': dimid, 'dimension_name': dimname})
             overallregex = re.compile("Overall\s?: Mean tick time: (.*) ms. Mean TPS: (.*)")
             overall_tps.add_sample('overall_tps', value=overallregex.findall(resp)[0][1], labels={})
             overall_ticktime.add_sample('overall_ticktime', value=overallregex.findall(resp)[0][0], labels={})
@@ -96,23 +118,32 @@ class MinecraftCollector(object):
 
         # dynmap
         if 'DYNMAP_ENABLED' in os.environ and os.environ['DYNMAP_ENABLED'] == "True":
-            dynmap_tile_render_statistics   = Metric('dynmap_tile_render_statistics','Tile Render Statistics reported by Dynmap',"counter")
+            dynmap_tile_render_statistics = Metric('dynmap_tile_render_statistics',
-            dynmap_chunk_loading_statistics_count = Metric('dynmap_chunk_loading_statistics_count','Chunk Loading Statistics reported by Dynmap',"counter")
+                                                   'Tile Render Statistics reported by Dynmap', "counter")
-            dynmap_chunk_loading_statistics_duration = Metric('dynmap_chunk_loading_statistics_duration','Chunk Loading Statistics reported by Dynmap',"counter")
+            dynmap_chunk_loading_statistics_count = Metric('dynmap_chunk_loading_statistics_count',
-            metrics.extend([dynmap_tile_render_statistics,dynmap_chunk_loading_statistics_count,dynmap_chunk_loading_statistics_duration])
+                                                           'Chunk Loading Statistics reported by Dynmap', "counter")
+            dynmap_chunk_loading_statistics_duration = Metric('dynmap_chunk_loading_statistics_duration',
+                                                              'Chunk Loading Statistics reported by Dynmap', "counter")
+            metrics.extend([dynmap_tile_render_statistics, dynmap_chunk_loading_statistics_count,
+                            dynmap_chunk_loading_statistics_duration])
 
             resp = self.rcon_command("dynmap stats")
 
             dynmaptilerenderregex = re.compile("  (.*?): processed=(\d*), rendered=(\d*), updated=(\d*)")
             for dim, processed, rendered, updated in dynmaptilerenderregex.findall(resp):
-                dynmap_tile_render_statistics.add_sample('dynmap_tile_render_statistics',value=processed,labels={'type':'processed','file':dim})
+                dynmap_tile_render_statistics.add_sample('dynmap_tile_render_statistics', value=processed,
-                dynmap_tile_render_statistics.add_sample('dynmap_tile_render_statistics',value=rendered,labels={'type':'rendered','file':dim})
+                                                         labels={'type': 'processed', 'file': dim})
-                dynmap_tile_render_statistics.add_sample('dynmap_tile_render_statistics',value=updated,labels={'type':'updated','file':dim})
+                dynmap_tile_render_statistics.add_sample('dynmap_tile_render_statistics', value=rendered,
+                                                         labels={'type': 'rendered', 'file': dim})
+                dynmap_tile_render_statistics.add_sample('dynmap_tile_render_statistics', value=updated,
+                                                         labels={'type': 'updated', 'file': dim})
 
             dynmapchunkloadingregex = re.compile("Chunks processed: (.*?): count=(\d*), (\d*.\d*)")
             for state, count, duration_per_chunk in dynmapchunkloadingregex.findall(resp):
-                dynmap_chunk_loading_statistics_count.add_sample('dynmap_chunk_loading_statistics',value=count,labels={'type': state})
+                dynmap_chunk_loading_statistics_count.add_sample('dynmap_chunk_loading_statistics', value=count,
-                dynmap_chunk_loading_statistics_duration.add_sample('dynmap_chunk_loading_duration',value=duration_per_chunk,labels={'type': state})
+                                                                 labels={'type': state})
+                dynmap_chunk_loading_statistics_duration.add_sample('dynmap_chunk_loading_duration',
+                                                                    value=duration_per_chunk, labels={'type': state})
 
         # player
         resp = self.rcon_command("list")
@@ -125,7 +156,7 @@ class MinecraftCollector(object):
         return metrics
 
     def get_player_quests_finished(self, uuid):
-        with open(self.betterquesting+"/QuestProgress.json") as json_file:
+        with open(self.better_questing + "/QuestProgress.json") as json_file:
             data = json.load(json_file)
             json_file.close()
         counter = 0
@@ -136,16 +167,16 @@ class MinecraftCollector(object):
         return counter
 
     def get_player_stats(self, uuid):
-        with open(self.statsdirectory+"/"+uuid+".json") as json_file:
+        with open(self.stats_directory + "/" + uuid + ".json") as json_file:
             data = json.load(json_file)
             json_file.close()
-        nbtfile = nbt.nbt.NBTFile(self.playerdirectory+"/"+uuid+".dat",'rb')
+        nbtfile = nbt.nbt.NBTFile(self.player_directory + "/" + uuid + ".dat", 'rb')
         data["stat.XpTotal"] = nbtfile.get("XpTotal").value
         data["stat.XpLevel"] = nbtfile.get("XpLevel").value
         data["stat.Score"] = nbtfile.get("Score").value
         data["stat.Health"] = nbtfile.get("Health").value
         data["stat.foodLevel"] = nbtfile.get("foodLevel").value
-        with open(self.advancementsdirectory+"/"+uuid+".json") as json_file:
+        with open(self.advancements_directory + "/" + uuid + ".json") as json_file:
             count = 0
             advancements = json.load(json_file)
             for key, value in advancements.items():
@@ -154,13 +185,14 @@ class MinecraftCollector(object):
                 if value["done"] == True:
                     count += 1
         data["stat.advancements"] = count
-        if self.questsEnabled:
+        if self.quests_enabled:
             data["stat.questsFinished"] = self.get_player_quests_finished(uuid)
         return data
 
     def update_metrics_for_player(self, uuid):
         name = self.uuid_to_player(uuid)
-        if not name: return
+        if not name:
+            return
 
         data = self.get_player_stats(uuid)
 
@@ -182,21 +214,26 @@ class MinecraftCollector(object):
         player_advancements = Metric('player_advancements', "Number of completed advances of a player", "counter")
         player_slept = Metric('player_slept', "Times a Player slept in a bed", "counter")
         player_quests_finished = Metric('player_quests_finished', 'Number of quests a Player has finished', 'counter')
-        player_used_crafting_table = Metric('player_used_crafting_table',"Times a Player used a Crafting Table","counter")
+        player_used_crafting_table = Metric('player_used_crafting_table', "Times a Player used a Crafting Table",
-        mc_custom           = Metric('mc_custom',"Custom Minectaft stat","counter")
+                                            "counter")
+        mc_custom = Metric('mc_custom', "Custom Minecraft stat", "counter")
         for key, value in data.items():  # pre 1.15
             if key in ("stats", "DataVersion"):
                 continue
             stat = key.split(".")[1]  # entityKilledBy
             if stat == "mineBlock":
-                blocks_mined.add_sample("blocks_mined",value=value,labels={'player':name,'block':'.'.join((key.split(".")[2],key.split(".")[3]))})
+                blocks_mined.add_sample("blocks_mined", value=value, labels={'player': name, 'block': '.'.join(
+                    (key.split(".")[2], key.split(".")[3]))})
             elif stat == "pickup":
-                blocks_picked_up.add_sample("blocks_picked_up",value=value,labels={'player':name,'block':'.'.join((key.split(".")[2],key.split(".")[3]))})
+                blocks_picked_up.add_sample("blocks_picked_up", value=value, labels={'player': name, 'block': '.'.join(
+                    (key.split(".")[2], key.split(".")[3]))})
             elif stat == "entityKilledBy":
                 if len(key.split(".")) == 4:
-                    player_deaths.add_sample('player_deaths',value=value,labels={'player':name,'cause':'.'.join((key.split(".")[2],key.split(".")[3]))})
+                    player_deaths.add_sample('player_deaths', value=value, labels={'player': name, 'cause': '.'.join(
+                        (key.split(".")[2], key.split(".")[3]))})
                 else:
-                    player_deaths.add_sample('player_deaths',value=value,labels={'player':name,'cause':key.split(".")[2]})
+                    player_deaths.add_sample('player_deaths', value=value,
+                                             labels={'player': name, 'cause': key.split(".")[2]})
             elif stat == "jump":
                 player_jumps.add_sample("player_jumps", value=value, labels={'player': name})
             elif stat == "walkOneCm":
@@ -228,13 +265,15 @@ class MinecraftCollector(object):
             elif stat == "Score":
                 player_score.add_sample('player_score', value=value, labels={'player': name})
             elif stat == "killEntity":
-                entities_killed.add_sample('entities_killed',value=value,labels={'player':name,"entity":key.split(".")[2]})
+                entities_killed.add_sample('entities_killed', value=value,
+                                           labels={'player': name, "entity": key.split(".")[2]})
             elif stat == "damageDealt":
                 damage_dealt.add_sample('damage_dealt', value=value, labels={'player': name})
             elif stat == "damageTaken":
                 damage_dealt.add_sample('damage_taken', value=value, labels={'player': name})
             elif stat == "craftItem":
-                blocks_crafted.add_sample('blocks_crafted',value=value,labels={'player':name,'block':'.'.join((key.split(".")[2],key.split(".")[3]))})
+                blocks_crafted.add_sample('blocks_crafted', value=value, labels={'player': name, 'block': '.'.join(
+                    (key.split(".")[2], key.split(".")[3]))})
             elif stat == "playOneMinute":
                 player_playtime.add_sample('player_playtime', value=value, labels={'player': name})
             elif stat == "advancements":
@@ -242,7 +281,8 @@ class MinecraftCollector(object):
             elif stat == "sleepInBed":
                 player_slept.add_sample('player_slept', value=value, labels={'player': name})
             elif stat == "craftingTableInteraction":
-                player_used_crafting_table.add_sample('player_used_crafting_table',value=value,labels={'player':name})
+                player_used_crafting_table.add_sample('player_used_crafting_table', value=value,
+                                                      labels={'player': name})
             elif stat == "questsFinished":
                 player_quests_finished.add_sample('player_quests_finished', value=value, labels={'player': name})
 
@@ -255,10 +295,12 @@ class MinecraftCollector(object):
                     blocks_mined.add_sample("blocks_mined", value=value, labels={'player': name, 'block': block})
             if "minecraft:picked_up" in data["stats"]:
                 for block, value in data["stats"]["minecraft:picked_up"].items():
-                    blocks_picked_up.add_sample("blocks_picked_up",value=value,labels={'player':name,'block':block})
+                    blocks_picked_up.add_sample("blocks_picked_up", value=value,
+                                                labels={'player': name, 'block': block})
             if "minecraft:killed" in data["stats"]:
                 for entity, value in data["stats"]["minecraft:killed"].items():
-                    entities_killed.add_sample('entities_killed',value=value,labels={'player':name,"entity":entity})
+                    entities_killed.add_sample('entities_killed', value=value,
+                                               labels={'player': name, "entity": entity})
             if "minecraft:killed_by" in data["stats"]:
                 for entity, value in data["stats"]["minecraft:killed_by"].items():
                     player_deaths.add_sample('player_deaths', value=value, labels={'player': name, 'cause': entity})
@@ -294,15 +336,20 @@ class MinecraftCollector(object):
                 elif stat == "minecraft:sleep_in_bed":
                     player_slept.add_sample('player_slept', value=value, labels={'player': name})
                 elif stat == "minecraft:interact_with_crafting_table":
-                    player_used_crafting_table.add_sample('player_used_crafting_table',value=value,labels={'player':name})
+                    player_used_crafting_table.add_sample('player_used_crafting_table', value=value,
+                                                          labels={'player': name})
                 else:
                     mc_custom.add_sample('mc_custom', value=value, labels={'stat': stat})
-        return [blocks_mined,blocks_picked_up,player_deaths,player_jumps,cm_traveled,player_xp_total,player_current_level,player_food_level,player_health,player_score,entities_killed,damage_taken,damage_dealt,blocks_crafted,player_playtime,player_advancements,player_slept,player_used_crafting_table,player_quests_finished,mc_custom]
+        return [blocks_mined, blocks_picked_up, player_deaths, player_jumps, cm_traveled, player_xp_total,
+                player_current_level, player_food_level, player_health, player_score, entities_killed, damage_taken,
+                damage_dealt, blocks_crafted, player_playtime, player_advancements, player_slept,
+                player_used_crafting_table, player_quests_finished, mc_custom]
 
     def collect(self):
         for player in self.get_players():
             metrics = self.update_metrics_for_player(player)
-            if not metrics: continue
+            if not metrics:
+                continue
 
             for metric in metrics:
                 yield metric
@@ -310,10 +357,8 @@ class MinecraftCollector(object):
         for metric in self.get_server_stats():
             yield metric
 
-if __name__ == '__main__':
-    if all(x in os.environ for x in ['RCON_HOST','RCON_PASSWORD']):
-        print("RCON is enabled for "+ os.environ['RCON_HOST'])
 
+if __name__ == '__main__':
     start_http_server(8000)
     REGISTRY.register(MinecraftCollector())
     print("Exporter started on Port 8000")

requirements.txt

@@ -1,5 +1,5 @@
-mcrcon==0.5.2
+mcrcon==0.7.0
-NBT==1.5.0
+NBT==1.5.1
-prometheus-client==0.7.1
+prometheus-client==0.12.0
-requests==2.20.0
+requests==2.27.1
-schedule==0.6.0
+schedule==1.1.0