hook for uncaught exceptions (#1703)

* hook for uncaught exceptions

log stdout and stderr

fix

* flake8

packages/helpermodules/command.py

@@ -16,6 +16,7 @@
 from control.chargepoint.chargepoint_template import get_autolock_plan_default, get_chargepoint_template_default
 
 # ToDo: move to module commands if implemented
+from helpermodules.utils.run_command import run_command
 from modules.backup_clouds.onedrive.api import generateMSALAuthCode, retrieveMSALTokens
 
 from helpermodules.broker import InternalBrokerClient
@@ -537,8 +538,8 @@ def sendDebug(self, connection_id: str, payload: dict) -> None:
         pub_user_message(payload, connection_id, "Systembericht wird erstellt...", MessageType.INFO)
         parent_file = Path(__file__).resolve().parents[2]
         previous_log_level = SubData.system_data["system"].data["debug_level"]
-        subprocess.run([str(parent_file / "runs" / "send_debug.sh"),
-                        json.dumps(payload["data"]), parse_send_debug_data()])
+        run_command([str(parent_file / "runs" / "send_debug.sh"),
+                     json.dumps(payload["data"]), parse_send_debug_data()])
         Pub().pub("openWB/set/system/debug_level", previous_log_level)
         pub_user_message(payload, connection_id, "Systembericht wurde versandt.", MessageType.SUCCESS)
 
@@ -559,21 +560,17 @@ def getYearlyLog(self, connection_id: str, payload: dict) -> None:
 
     def initCloud(self, connection_id: str, payload: dict) -> None:
         parent_file = Path(__file__).resolve().parents[2]
-        try:
-            result = subprocess.check_output(
-                ["php", "-f", str(parent_file / "runs" / "cloudRegister.php"), json.dumps(payload["data"])]
-            )
-            # exit status = 0 is success, std_out contains json: {"username", "password"}
-            result_dict = json.loads(result)
-            connect_payload = {
-                "data": result_dict
-            }
-            connect_payload["data"]["partner"] = payload["data"]["partner"]
-            self.connectCloud(connection_id, connect_payload)
-            pub_user_message(payload, connection_id, "Verbindung zur Cloud wurde eingerichtet.", MessageType.SUCCESS)
-        except subprocess.CalledProcessError as error:
-            # exit status = 1 is failure, std_out contains error message
-            pub_user_message(payload, connection_id, error.output.decode("utf-8", MessageType.ERROR))
+        result = run_command(
+            ["php", "-f", str(parent_file / "runs" / "cloudRegister.php"), json.dumps(payload["data"])]
+        )
+        # exit status = 0 is success, std_out contains json: {"username", "password"}
+        result_dict = json.loads(result)
+        connect_payload = {
+            "data": result_dict
+        }
+        connect_payload["data"]["partner"] = payload["data"]["partner"]
+        self.connectCloud(connection_id, connect_payload)
+        pub_user_message(payload, connection_id, "Verbindung zur Cloud wurde eingerichtet.", MessageType.SUCCESS)
 
     def connectCloud(self, connection_id: str, payload: dict) -> None:
         cloud_config = bridge.get_cloud_config()
@@ -610,12 +607,12 @@ def removeMqttBridge(self, connection_id: str, payload: dict) -> None:
     def systemReboot(self, connection_id: str, payload: dict) -> None:
         pub_user_message(payload, connection_id, "Neustart wird ausgeführt.", MessageType.INFO)
         parent_file = Path(__file__).resolve().parents[2]
-        subprocess.run([str(parent_file / "runs" / "reboot.sh")])
+        run_command([str(parent_file / "runs" / "reboot.sh")])
 
     def systemShutdown(self, connection_id: str, payload: dict) -> None:
         pub_user_message(payload, connection_id, "openWB wird heruntergefahren.", MessageType.INFO)
         parent_file = Path(__file__).resolve().parents[2]
-        subprocess.run([str(parent_file / "runs" / "shutdown.sh")])
+        run_command([str(parent_file / "runs" / "shutdown.sh")])
 
     def systemUpdate(self, connection_id: str, payload: dict) -> None:
         log.info("Update requested")
@@ -639,45 +636,34 @@ def systemUpdate(self, connection_id: str, payload: dict) -> None:
                 payload, connection_id,
                 f'Wechsel auf Zweig \'{payload["data"]["branch"]}\' Tag \'{payload["data"]["tag"]}\' gestartet.',
                 MessageType.SUCCESS)
-            subprocess.run([
+            run_command([
                 str(parent_file / "runs" / "update_self.sh"),
                 str(payload["data"]["branch"]),
                 str(payload["data"]["tag"])])
         else:
             pub_user_message(payload, connection_id, "Update gestartet.", MessageType.INFO)
-            subprocess.run([
+            run_command([
                 str(parent_file / "runs" / "update_self.sh"),
                 SubData.system_data["system"].data["current_branch"]])
 
     def systemFetchVersions(self, connection_id: str, payload: dict) -> None:
         log.info("Fetch versions requested")
         pub_user_message(payload, connection_id, "Versionsliste wird aktualisiert...", MessageType.INFO)
         parent_file = Path(__file__).resolve().parents[2]
-        result = subprocess.run([str(parent_file / "runs" / "update_available_versions.sh")])
-        if result.returncode == 0:
-            pub_user_message(payload, connection_id, "Versionsliste erfolgreich aktualisiert.", MessageType.SUCCESS)
-        else:
-            pub_user_message(
-                payload, connection_id,
-                f'Version-Status: {result.returncode}<br />Meldung: {result.stdout.decode("utf-8", MessageType.ERROR)}')
+        run_command([str(parent_file / "runs" / "update_available_versions.sh")])
+        pub_user_message(payload, connection_id, "Versionsliste erfolgreich aktualisiert.", MessageType.SUCCESS)
 
     def createBackup(self, connection_id: str, payload: dict) -> None:
         pub_user_message(payload, connection_id, "Backup wird erstellt...", MessageType.INFO)
         parent_file = Path(__file__).resolve().parents[2]
-        result = subprocess.run(
+        result = run_command(
             [str(parent_file / "runs" / "backup.sh"),
-             "1" if "use_extended_filename" in payload["data"] and payload["data"]["use_extended_filename"] else "0"],
-            stdout=subprocess.PIPE)
-        if result.returncode == 0:
-            file_name = result.stdout.decode("utf-8").rstrip('\n')
-            file_link = "/openWB/data/backup/" + file_name
-            pub_user_message(payload, connection_id,
-                             "Backup erfolgreich erstellt.<br />"
-                             f'Jetzt <a href="{file_link}" target="_blank">herunterladen</a>.', MessageType.SUCCESS)
-        else:
-            pub_user_message(payload, connection_id,
-                             f'Backup-Status: {result.returncode}<br />Meldung: {result.stdout.decode("utf-8")}',
-                             MessageType.ERROR)
+             "1" if "use_extended_filename" in payload["data"] and payload["data"]["use_extended_filename"] else "0"])
+        file_name = result.rstrip('\n')
+        file_link = "/openWB/data/backup/" + file_name
+        pub_user_message(payload, connection_id,
+                         "Backup erfolgreich erstellt.<br />"
+                         f'Jetzt <a href="{file_link}" target="_blank">herunterladen</a>.', MessageType.SUCCESS)
 
     def createCloudBackup(self, connection_id: str, payload: dict) -> None:
         if SubData.system_data["system"].backup_cloud is not None:
@@ -692,18 +678,11 @@ def createCloudBackup(self, connection_id: str, payload: dict) -> None:
 
     def restoreBackup(self, connection_id: str, payload: dict) -> None:
         parent_file = Path(__file__).resolve().parents[2]
-        result = subprocess.run(
-            [str(parent_file / "runs" / "prepare_restore.sh")],
-            stdout=subprocess.PIPE)
-        if result.returncode == 0:
-            pub_user_message(payload, connection_id,
-                             "Wiederherstellung wurde vorbereitet. openWB wird jetzt zum Abschluss neu gestartet.",
-                             MessageType.INFO)
-            self.systemReboot(connection_id, payload)
-        else:
-            pub_user_message(payload, connection_id,
-                             f'Restore-Status: {result.returncode}<br />Meldung: {result.stdout.decode("utf-8")}',
-                             MessageType.ERROR)
+        run_command([str(parent_file / "runs" / "prepare_restore.sh")])
+        pub_user_message(payload, connection_id,
+                         "Wiederherstellung wurde vorbereitet. openWB wird jetzt zum Abschluss neu gestartet.",
+                         MessageType.INFO)
+        self.systemReboot(connection_id, payload)
 
     # ToDo: move to module commands if implemented
     def requestMSALAuthCode(self, connection_id: str, payload: dict) -> None:
@@ -761,6 +740,12 @@ def __exit__(self, exception_type, exception, exception_traceback) -> bool:
                              f'Es ist ein interner Fehler aufgetreten: {exception}', MessageType.ERROR)
             log.error({traceback.format_exc()})
             return True
+        elif isinstance(exception, subprocess.CalledProcessError):
+            log.debug(exception.stdout)
+            pub_user_message(self.payload, self.connection_id,
+                             f'Fehler-Status: {exception.returncode}<br />Meldung: {exception.stderr}',
+                             MessageType.ERROR)
+            return True
         else:
             return False
 

packages/helpermodules/graph.py

@@ -1,13 +1,13 @@
 from dataclasses import dataclass, field
 import json
 from pathlib import Path
-import subprocess
 import time
 import datetime
 import logging
 
 from control import data
 from helpermodules.pub import Pub
+from helpermodules.utils.run_command import run_command
 from modules.common.fault_state import FaultStateLevel
 
 log = logging.getLogger(__name__)
@@ -64,7 +64,7 @@ def _convert_to_kW(value): return round(value/1000, 3)
             Pub().pub("openWB/set/system/lastlivevaluesJson", data_line)
             with open(str(Path(__file__).resolve().parents[2] / "ramdisk"/"graph_live.json"), "a") as f:
                 f.write(f"{json.dumps(data_line, separators=(',', ':'))}\n")
-            subprocess.run([str(Path(__file__).resolve().parents[2] / "runs"/"graphing.sh"),
-                            str(self.data.config.duration*6)])
+            run_command([str(Path(__file__).resolve().parents[2] / "runs"/"graphing.sh"),
+                         str(self.data.config.duration*6)])
         except Exception:
             log.exception("Fehler im Graph-Modul")

packages/helpermodules/logger.py

@@ -2,6 +2,8 @@
 import logging
 from logging.handlers import RotatingFileHandler
 from pathlib import Path
+import sys
+import threading
 import typing_extensions
 
 FORMAT_STR_DETAILED = '%(asctime)s - {%(name)s:%(lineno)s} - {%(levelname)s:%(threadName)s} - %(message)s'
@@ -80,6 +82,15 @@ def mb_to_bytes(megabytes: int) -> int:
     logging.getLogger("pymodbus").setLevel(logging.WARNING)
     logging.getLogger("uModbus").setLevel(logging.WARNING)
 
+    def threading_excepthook(args):
+        logging.getLogger(__name__).error("Uncaught exception in threading.excepthook:", exc_info=(
+            args.exc_type, args.exc_value, args.exc_traceback))
+    threading.excepthook = threading_excepthook
+
+    def handle_unhandled_exception(exc_type, exc_value, exc_traceback):
+        logging.getLogger(__name__).error("Uncaught exception", exc_info=(exc_type, exc_value, exc_traceback))
+    sys.excepthook = handle_unhandled_exception
+
 
 log = logging.getLogger(__name__)
 

packages/helpermodules/subdata.py

@@ -23,6 +23,7 @@
 from helpermodules.abstract_plans import AutolockPlan
 from helpermodules.broker import InternalBrokerClient
 from helpermodules.messaging import MessageType, pub_system_message
+from helpermodules.utils.run_command import run_command
 from helpermodules.utils.topic_parser import decode_payload, get_index, get_second_index
 from control import optional
 from helpermodules.pub import Pub
@@ -602,9 +603,9 @@ def process_general_topic(self, var: general.General, msg: mqtt.MQTTMessage):
                         # 5 Min Handler bis auf Heartbeat, Cleanup, ... beenden
                         self.event_jobs_running.clear()
                     self.set_json_payload_class(var.data, msg)
-                    subprocess.run([
+                    run_command([
                         str(Path(__file__).resolve().parents[2] / "runs" / "setup_network.sh")
-                    ])
+                    ], process_exception=True)
                 elif "openWB/general/modbus_control" == msg.topic:
                     if decode_payload(msg.payload) and self.general_data.data.extern:
                         self.event_modbus_server.set()
@@ -633,9 +634,9 @@ def process_optional_topic(self, var: optional.Optional, msg: mqtt.MQTTMessage):
                     self.set_json_payload_class(var.data.int_display, msg)
                     if re.search("/(standby|active|rotation)$", msg.topic) is not None:
                         # some topics require an update of the display manager or boot settings
-                        subprocess.run([
+                        run_command([
                             str(Path(__file__).resolve().parents[2] / "runs" / "update_local_display.sh")
-                        ])
+                        ], process_exception=True)
                 elif re.search("/optional/et/", msg.topic) is not None:
                     if re.search("/optional/et/get/prices", msg.topic) is not None:
                         var.data.et.get.prices = decode_payload(msg.payload)
@@ -759,12 +760,14 @@ def process_system_topic(self, client: mqtt.Client, var: dict, msg: mqtt.MQTTMes
                 if self.event_subdata_initialized.is_set():
                     index = get_index(msg.topic)
                     parent_file = Path(__file__).resolve().parents[2]
-                    result = subprocess.run(
-                        ["php", "-f", str(parent_file / "runs" / "save_mqtt.php"), index, msg.payload],
-                        stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
-                    if len(result.stdout) > 0:
-                        pub_system_message(msg.payload, result.stdout,
-                                           MessageType.SUCCESS if result.returncode == 0 else MessageType.ERROR)
+                    try:
+                        result = run_command(
+                            ["php", "-f", str(parent_file / "runs" / "save_mqtt.php"), index, msg.payload])
+                        pub_system_message(msg.payload, result, MessageType.SUCCESS)
+                    except subprocess.CalledProcessError as e:
+                        log.debug(e.stdout)
+                        pub_system_message(msg.payload, f'Fehler-Status: {e.returncode}<br />Meldung: {e.stderr}',
+                                           MessageType.ERROR)
                 else:
                     log.debug("skipping mqtt bridge message on startup")
             elif "mqtt" and "valid_partner_ids" in msg.topic:
@@ -779,8 +782,8 @@ def process_system_topic(self, client: mqtt.Client, var: dict, msg: mqtt.MQTTMes
                 token = splitted[0]
                 port = splitted[1] if len(splitted) > 1 else "2223"
                 user = splitted[2] if len(splitted) > 2 else "getsupport"
-                subprocess.run([str(Path(__file__).resolve().parents[2] / "runs" / "start_remote_support.sh"),
-                                token, port, user])
+                run_command([str(Path(__file__).resolve().parents[2] / "runs" / "start_remote_support.sh"),
+                             token, port, user], process_exception=True)
             elif "openWB/system/backup_cloud/config" in msg.topic:
                 config_dict = decode_payload(msg.payload)
                 if config_dict["type"] is None:

packages/helpermodules/system.py

@@ -11,6 +11,7 @@
 
 from helpermodules import pub
 from control import data
+from helpermodules.utils.run_command import run_command
 from modules.common.configurable_backup_cloud import ConfigurableBackupCloud
 
 log = logging.getLogger(__name__)
@@ -40,8 +41,8 @@ def perform_update(self):
             self._trigger_ext_update(train)
             time.sleep(15)
             # aktuell soll kein Update für den Master durchgeführt werden.
-            # subprocess.run([str(Path(__file__).resolve().parents[2]/"runs"/"update_self.sh"), train])
-            subprocess.run(str(Path(__file__).resolve().parents[2]/"runs"/"atreboot.sh"))
+            # run_command([str(Path(__file__).resolve().parents[2]/"runs"/"update_self.sh"), train])
+            run_command(str(Path(__file__).resolve().parents[2]/"runs"/"atreboot.sh"), process_exception=True)
         except Exception:
             log.exception("Fehler im System-Modul")
 
@@ -99,12 +100,13 @@ def create_backup_and_send_to_cloud(self):
             log.debug('Nächtliche Sicherung erstellt und hochgeladen.')
 
     def create_backup(self) -> str:
-        result = subprocess.run([str(self._get_parent_file() / "runs" / "backup.sh"), "1"], stdout=subprocess.PIPE)
-        if result.returncode == 0:
-            file_name = result.stdout.decode("utf-8").rstrip('\n')
+        try:
+            result = run_command([str(self._get_parent_file() / "runs" / "backup.sh"), "1"])
+            file_name = result.rstrip('\n')
             return file_name
-        else:
-            raise Exception(f'Backup-Status: {result.returncode}, Meldung: {result.stdout.decode("utf-8")}')
+        except subprocess.CalledProcessError as e:
+            log.debug(e.stdout)
+            raise Exception(f'Backup-Status: {e.returncode}, Meldung: {e.stderr}')
 
     def _get_parent_file(self) -> Path:
         return Path(__file__).resolve().parents[2]