Index: cloud_sync.py =================================================================== diff -u -r57c12353bf0c2038da88b24dcaddc2a9d19a730b -r3736c29ba097740b4e99684625e78bfaaac6b1a1 --- cloud_sync.py (.../cloud_sync.py) (revision 57c12353bf0c2038da88b24dcaddc2a9d19a730b) +++ cloud_sync.py (.../cloud_sync.py) (revision 3736c29ba097740b4e99684625e78bfaaac6b1a1) @@ -1,65 +1,37 @@ +from cloudsync.handlers.error import Error +from cloudsync.utils.heartbeat import HeartBeatProvider +from cloudsync.busses.file_input_bus import FileInputBus +from cloudsync.handlers.ui_cs_request_handler import UICSMessageHandler +from cloudsync.handlers.cs_mft_dcs_request_handler import NetworkRequestHandler +from cloudsync.handlers.error_handler import ErrorHandler +from cloudsync.busses.file_output_bus import FileOutputBus +from cloudsync.utils.reachability import ReachabilityProvider +from cloudsync.utils.globals import * +from cloudsync.utils.helpers import * +from cloudsync.utils.logging import LoggingConfig +from flask_restplus import Api, Resource, fields, reqparse +from flask import Flask, Response, request import os import sys import logging from logging.handlers import TimedRotatingFileHandler +from cloudsync.handlers.logs_handler import CustomTimedRotatingFileHandlerHandler import werkzeug werkzeug.cached_property = werkzeug.utils.cached_property -from flask import Flask, Response, request -from flask_restplus import Api, Resource, fields, reqparse -from cloudsync.utils.helpers import * -from cloudsync.utils.globals import * -from cloudsync.utils.reachability import ReachabilityProvider -from cloudsync.busses.file_output_bus import FileOutputBus -from cloudsync.handlers.error_handler import ErrorHandler -from cloudsync.handlers.cs_mft_dcs_request_handler import NetworkRequestHandler -from cloudsync.handlers.ui_cs_request_handler import UICSMessageHandler -from cloudsync.busses.file_input_bus import FileInputBus -from cloudsync.utils.heartbeat import HeartBeatProvider -from cloudsync.handlers.error import Error +VERSION = "0.4.10" -VERSION = "0.4.9" - arguments = sys.argv -log_level = int(arguments[1]) -if not os.path.exists(CS_LOG_PATH): - os.makedirs(CS_LOG_PATH) - app = Flask(__name__) api = Api(app=app, version=VERSION, title="CloudSync Registration API", description="Interface with DIA Manufacturing Tool * DCS") -# Remove existing handlers -for handler in app.logger.handlers: - app.logger.removeHandler(handler) +logconf = LoggingConfig() +logconf.initiate(app=app) -# Create a TimedRotatingFileHandler that writes logs to a new file every midnight, in UTC time -handler = TimedRotatingFileHandler(CS_LOG_FILE, when="midnight", interval=1, utc=True) -handler.suffix = "%m-%d-%Y" - -# Set the log level -handler.setLevel(log_level) - -# Add a formatter -default_formatter = logging.Formatter('[%(asctime)s] %(levelname)s in %(module)s: %(message)s | {%(pathname)s:%(lineno)d}') -handler.setFormatter(default_formatter) - -# Add the handler to the logger -app.logger.addHandler(handler) -app.logger.setLevel(log_level) - -# Get the root logger -root_logger = logging.getLogger() - -# Add the handlers to the root logger -root_logger.addHandler(handler) -root_logger.setLevel(log_level) - -g_utils.add_logger(app.logger) - sleep(5) # wait for UI to prepare the configurations partition try: @@ -68,9 +40,19 @@ if g_config[CONFIG_DEVICE][CONFIG_DEVICE_MODE] == 'operation': g_config = helpers_read_config(OPERATION_CONFIG_FILE_PATH) CONFIG_PATH = OPERATION_CONFIG_FILE_PATH +except Exception as e: + g_utils.logger.error("Error reading config file - {0}".format(e)) + sys.exit(0) +try: reachability_provider = ReachabilityProvider(logger=app.logger, url_reachability=g_config[CONFIG_KEBORMED][CONFIG_KEBORMED_REACHABILITY_URL]) +except Exception as e: + g_utils.logger.error( + "Reachability URL missing from config file. Using Default URL - {0}".format(DEFAULT_REACHABILITY_URL)) + reachability_provider = ReachabilityProvider( + logger=app.logger, url_reachability=DEFAULT_REACHABILITY_URL) +try: g_utils.add_reachability_provider(reachability_provider=reachability_provider) output_channel = FileOutputBus(logger=app.logger, max_size=100, file_channels_path=UI2CS_FILE_CHANNELS_PATH) @@ -91,6 +73,11 @@ heartbeat_provider = HeartBeatProvider(logger=app.logger, network_request_handler=network_request_handler, output_channel=output_channel) + logconf.set_log_level(g_config[CONFIG_DEVICE][CONFIG_LOG_LEVEL]) + logconf.set_network_provider(network_request_handler=network_request_handler) + logconf.set_error_provider(error_handler=error_handler) + logconf.set_configuration(g_config=g_config) + if g_config[CONFIG_DEVICE][CONFIG_DEVICE_MODE] == 'operation': heartbeat_provider.send_heartbeat = True else: Index: cloudsync/busses/file_output_bus.py =================================================================== diff -u -rae065dc96f33fc1946785ee833356dae959be2d9 -r3736c29ba097740b4e99684625e78bfaaac6b1a1 --- cloudsync/busses/file_output_bus.py (.../file_output_bus.py) (revision ae065dc96f33fc1946785ee833356dae959be2d9) +++ cloudsync/busses/file_output_bus.py (.../file_output_bus.py) (revision 3736c29ba097740b4e99684625e78bfaaac6b1a1) @@ -73,7 +73,7 @@ message_data = str(round(datetime.datetime.now().timestamp())) + str(self.last_output_message_id) + message_body message_crc8 = helpers.helpers_crc8(message_data.encode('utf-8')) message_body = str(round(datetime.datetime.now().timestamp())) + ',' + str(self.last_output_message_id) + ',' + str(message_crc8) + ',' + message_body - self.logger.debug('Full message: {0}'.format(message_body)) + self.logger.info('CS2UI Message: {0}'.format(message_body)) # print('Full message: {0}'.format(message_body)) f = open(self.file_channels_path + "/" + filename, "a") f.write("{0}\n".format(message_body)) Index: cloudsync/common/enums.py =================================================================== diff -u -rf8d59f1691d5b6850d2fa59d5312c563a42f0d72 -r3736c29ba097740b4e99684625e78bfaaac6b1a1 --- cloudsync/common/enums.py (.../enums.py) (revision f8d59f1691d5b6850d2fa59d5312c563a42f0d72) +++ cloudsync/common/enums.py (.../enums.py) (revision 3736c29ba097740b4e99684625e78bfaaac6b1a1) @@ -73,6 +73,7 @@ UI2CS_SEND_DEVICE_STATE = 1006 UI2CS_SEND_TREATMENT_REPORT = 1007 UI2CS_REQ_DECOMMISSION = 1009 + UI2CS_UPLOAD_DEVICE_LOG = 1010 # DEVICE_LOG # INCOMING ERROR UI2CS_ERROR = 1999 @@ -90,6 +91,7 @@ CS2UI_REQ_DEVICE_STATE = 2006 CS2UI_REQ_TX_CODE_DISPLAY = 2008 CS2UI_REQ_DEVICE_DECOMMISSIONED = 2009 + CS2UI_DEVICE_LOG_UPLOADED = 2010 # OUTGOING ERROR CS2UI_ERROR = 2999 @@ -102,7 +104,10 @@ CS2MFT_REQ_REGISTRATION = 101 CS2DCS_REQ_SET_DEVICE_STATE = 201 CS2DCS_REQ_SEND_TREATMENT_REPORT = 202 + CS2DCS_REQ_SEND_DEVICE_LOG = 203 + CS2DCS_REQ_SEND_CS_LOG = 204 + @unique class ErrorIDs(RootEnum): GENERIC_ERROR = 900 @@ -122,4 +127,5 @@ CS_SAVE_CREDENTIALS_ERROR = 927 CS_UNKNOWN_DEVICE_STATE_ERROR = 928 CS_SAVE_CONFIG_ERROR = 929 - + CS_DEVICE_LOG_ERROR = 930 + CS_LOG_ERROR = 931 Index: cloudsync/config/config_integration.json =================================================================== diff -u -r010bbdec5ebb419130413765886d8da5d1af1d96 -r3736c29ba097740b4e99684625e78bfaaac6b1a1 --- cloudsync/config/config_integration.json (.../config_integration.json) (revision 010bbdec5ebb419130413765886d8da5d1af1d96) +++ cloudsync/config/config_integration.json (.../config_integration.json) (revision 3736c29ba097740b4e99684625e78bfaaac6b1a1) @@ -15,6 +15,8 @@ "dg_serial": "", "sw_version": "", "mode": "registration", - "device_state": "INACTIVE_NOT_OK" + "device_state": "INACTIVE_NOT_OK", + "log_level": "ERROR", + "log_level_duration": "24" } } \ No newline at end of file Index: cloudsync/config/config_production.json =================================================================== diff -u -r010bbdec5ebb419130413765886d8da5d1af1d96 -r3736c29ba097740b4e99684625e78bfaaac6b1a1 --- cloudsync/config/config_production.json (.../config_production.json) (revision 010bbdec5ebb419130413765886d8da5d1af1d96) +++ cloudsync/config/config_production.json (.../config_production.json) (revision 3736c29ba097740b4e99684625e78bfaaac6b1a1) @@ -15,6 +15,8 @@ "dg_serial": "", "sw_version": "", "mode": "registration", - "device_state": "INACTIVE_NOT_OK" + "device_state": "INACTIVE_NOT_OK", + "log_level": "ERROR", + "log_level_duration": "24" } } Index: cloudsync/config/config_quality.json =================================================================== diff -u -r351ef4aabce2ad11d19741f7879530b13c8e946f -r3736c29ba097740b4e99684625e78bfaaac6b1a1 --- cloudsync/config/config_quality.json (.../config_quality.json) (revision 351ef4aabce2ad11d19741f7879530b13c8e946f) +++ cloudsync/config/config_quality.json (.../config_quality.json) (revision 3736c29ba097740b4e99684625e78bfaaac6b1a1) @@ -15,6 +15,8 @@ "dg_serial": "", "sw_version": "", "mode": "registration", - "device_state": "INACTIVE_NOT_OK" + "device_state": "INACTIVE_NOT_OK", + "log_level": "ERROR", + "log_level_duration": "24" } } \ No newline at end of file Index: cloudsync/config/config_staging.json =================================================================== diff -u -r010bbdec5ebb419130413765886d8da5d1af1d96 -r3736c29ba097740b4e99684625e78bfaaac6b1a1 --- cloudsync/config/config_staging.json (.../config_staging.json) (revision 010bbdec5ebb419130413765886d8da5d1af1d96) +++ cloudsync/config/config_staging.json (.../config_staging.json) (revision 3736c29ba097740b4e99684625e78bfaaac6b1a1) @@ -15,6 +15,8 @@ "dg_serial": "", "sw_version": "", "mode": "registration", - "device_state": "INACTIVE_NOT_OK" + "device_state": "INACTIVE_NOT_OK", + "log_level": "ERROR", + "log_level_duration": "24" } } Index: cloudsync/config/device_log_template.json =================================================================== diff -u --- cloudsync/config/device_log_template.json (revision 0) +++ cloudsync/config/device_log_template.json (revision 3736c29ba097740b4e99684625e78bfaaac6b1a1) @@ -0,0 +1,17 @@ +{ + "organizationId": "", + "serialNumber": "", + "checksum": "", + "data": "", + "dataType": "", + "metadata": { + "dataType" : "", + "deviceLogType": "", + "deviceSubType": "", + "deviceFileName": "", + "startTimestamp": "", + "endTimestamp": "" + }, + "completedAt": 0, + "generatedAt": 0 +} \ No newline at end of file Index: cloudsync/config/log_state.json =================================================================== diff -u --- cloudsync/config/log_state.json (revision 0) +++ cloudsync/config/log_state.json (revision 3736c29ba097740b4e99684625e78bfaaac6b1a1) @@ -0,0 +1,7 @@ +{ + "current_log_level": "", + "log_level_duration": 0, + "log_level_start_timestamp": 0, + "log_level_stop_timestamp": 0, + "update_dcs_flag": 0 +} \ No newline at end of file Index: cloudsync/config/treatment_report_template.json =================================================================== diff -u -r373fd8d0c900ff129152e13d896e7de2e222b0e7 -r3736c29ba097740b4e99684625e78bfaaac6b1a1 --- cloudsync/config/treatment_report_template.json (.../treatment_report_template.json) (revision 373fd8d0c900ff129152e13d896e7de2e222b0e7) +++ cloudsync/config/treatment_report_template.json (.../treatment_report_template.json) (revision 3736c29ba097740b4e99684625e78bfaaac6b1a1) @@ -31,11 +31,17 @@ "time": { "start": 0, "end": 0, - "treatmentDuration": 0 + "treatmentDuration": 0, + "targetWeight": "" }, "data": [], "events": [], - "alarms": [] + "alarms": [], + "treatmentPreparation":{ + "totalChlorine":"", + "ph":"", + "conductivity":"" + } }, "deviceTreatmentData": { "dialysateVolumeUsed": 0, @@ -49,7 +55,11 @@ "heparinDelivered": "" }, "diagnostics": { - "waterSampleTestResult": "" + "waterSampleTestResult": "", + "machineWipedDown": "", + "filterLife": "", + "lastChemicalDisinfection": "", + "lastHeatDisinfection": "" } }, "completedAt": 0, Index: cloudsync/handlers/cs_mft_dcs_request_handler.py =================================================================== diff -u -redc3fa3e3e3c36726081dae335254a5f458d8d0a -r3736c29ba097740b4e99684625e78bfaaac6b1a1 --- cloudsync/handlers/cs_mft_dcs_request_handler.py (.../cs_mft_dcs_request_handler.py) (revision edc3fa3e3e3c36726081dae335254a5f458d8d0a) +++ cloudsync/handlers/cs_mft_dcs_request_handler.py (.../cs_mft_dcs_request_handler.py) (revision 3736c29ba097740b4e99684625e78bfaaac6b1a1) @@ -11,17 +11,20 @@ from cloudsync.handlers.error_handler import ErrorHandler from cloudsync.handlers.error import Error from cloudsync.utils.helpers import log_func -from cloudsync.common.enums import * +from cloudsync.utils.logging import LoggingConfig from cloudsync.utils.globals import * +from cloudsync.common.enums import * from cloudsync.handlers.outgoing.handler_cs_to_mft import * from cloudsync.handlers.outgoing.handler_cs_to_dcs import * from cloudsync.handlers.incoming.handler_mft_to_cs import * ERROR_STRING = "{0},2,{1},{2}" + class NetworkRequestHandler: def __init__(self, logger: Logger, max_size, output_channel, reachability_provider, error_handler): self.logger = logger + self.logconf = LoggingConfig() self.reachability_provider = reachability_provider self.logger.info('Created Network Request Handler') self.output_channel = output_channel @@ -96,8 +99,8 @@ error_handler=self.error_handler) except Exception as e: error = Error(ERROR_STRING.format(OutboundMessageIDs.CS2UI_ERROR.value, - ErrorIDs.CS_SAVE_CREDENTIALS_ERROR.value, - e)) + ErrorIDs.CS_SAVE_CREDENTIALS_ERROR.value, + e)) self.error_handler.enqueue_error(error=error) elif req.request_type == NetworkRequestType.MFT2CS_REQ_INIT_CONNECTIVITY_TEST: try: @@ -114,8 +117,8 @@ ) except Exception as e: error = Error(ERROR_STRING.format(OutboundMessageIDs.CS2UI_ERROR.value, - ErrorIDs.CS_VALIDATE_DEVICE_ERROR.value, - e)) + ErrorIDs.CS_VALIDATE_DEVICE_ERROR.value, + e)) self.error_handler.enqueue_error(error=error) elif req.request_type == NetworkRequestType.MFT2CS_REQ_FACTORY_RESET: req.g_config[CONFIG_DEVICE][CONFIG_DEVICE_MODE] = 'operation' @@ -126,24 +129,53 @@ # OPERATION MODE elif req.request_type == NetworkRequestType.CS2DCS_REQ_SET_DEVICE_STATE: try: + device_state_json = req.payload device_state = req.g_config[CONFIG_DEVICE][CONFIG_DEVICE_STATE] - + device_log_json = req.payload base_url = req.g_config[CONFIG_KEBORMED][CONFIG_KEBORMED_DCS_URL] identity_url = req.g_config[CONFIG_KEBORMED][CONFIG_KEBORMED_DEVICE_IDENTITY_URL] client_secret = req.g_config[CONFIG_KEBORMED][CONFIG_KEBORMED_IDP_CLIENT_SECRET] token_verification_url = urllib.parse.urljoin(base_url, DEVICE_TOKEN_VALIDATION) - device_state_url = urllib.parse.urljoin(base_url, "/api/device/state/{0}".format(device_state)) + device_state_url = urllib.parse.urljoin(base_url, "/api/device") access_token = self.get_valid_token(identity_url=identity_url, token_verification_url=token_verification_url, client_secret=client_secret) if access_token is not None: + + # Decide whether to inform DCS for the new log level + if helpers_should_update_dcs_log_level(): + device_state_json['logLevel'] = self.logconf.get_log_level()[1] + self.logconf.set_dcs_flag(0) # Reset the update_dcs_flag + + device_state_json['state'] = DeviceStates.mapped_int_value(device_state).value + + device_state_json = json.dumps(device_state_json) + response = cmd_outgoing_set_device_state(url=device_state_url, access_token=access_token, + device_state_json=device_state_json, error_handler=self.error_handler) - self.logger.debug("DCS Request device state response code: {0} & full response: {1}".format(response.status_code, response.text)) + if response.status_code == 200: + self.logger.debug("CS set Device state: {0}".format(response.json())) + + if helpers_should_update_cs_log_level(response.json()): + new_log_level = response.json()['logLevel'] + # Set the DCS flag + self.logconf.set_dcs_flag(1) + # Calculate the duration of the new log level + log_level_duration = helpers_calculate_log_level_duration(response.json()) + log_level_duration = log_level_duration if log_level_duration else req.g_config[CONFIG_DEVICE][CONFIG_LOG_LEVEL_DURATION] + self.logconf.set_log_duration(log_level_duration) + # Update the log level + self.logconf.set_log_level(new_log_level) + # Start the timer + self.logconf.start_countdown() + + self.logger.debug("DCS Request device state response code: {0} & full response: {1}".format(response.status_code, response.text)) + if response.status_code == UNASSIGNED: error = Error("{0},2,{1}, Invalid device state transition".format(OutboundMessageIDs.CS2UI_ERROR.value, ErrorIDs.CS_SEND_DEVICE_STATE_ERROR.value)) @@ -155,8 +187,8 @@ self.error_handler.enqueue_error(error=error) except Exception as e: error = Error(ERROR_STRING.format(OutboundMessageIDs.CS2UI_ERROR.value, - ErrorIDs.CS_SEND_DEVICE_STATE_ERROR.value, - e)) + ErrorIDs.CS_SEND_DEVICE_STATE_ERROR.value, + e)) self.error_handler.enqueue_error(error=error) elif req.request_type == NetworkRequestType.CS2DCS_REQ_SEND_TREATMENT_REPORT: try: @@ -241,8 +273,15 @@ # Step #4 - Send treatment report association_time = int(round(time() * S_MS_CONVERSION_FACTOR)) - generated_at_time = association_time + 301000 # TEMPORARY DELAY TO COMPENSATE FOR DEVICE INTERNAL CLOCK ERROR - TODO REMOVE AFTER FIX - completed_at_time = association_time + 302000 # TEMPORARY DELAY TO COMPENSATE FOR DEVICE INTERNAL CLOCK ERROR - TODO REMOVE AFTER FIX + # if response.status_code == SUCCESS: + # association_time = int(response.headers.get('X-Timestamp')) + # generated_at_time = association_time + 1000 + # completed_at_time = association_time + 2000 + # else: + # TEMPORARY DELAY TO COMPENSATE FOR DEVICE INTERNAL CLOCK ERROR - TODO REMOVE AFTER FIX + generated_at_time = association_time + 301000 + # TEMPORARY DELAY TO COMPENSATE FOR DEVICE INTERNAL CLOCK ERROR - TODO REMOVE AFTER FIX + completed_at_time = association_time + 302000 treatment_log_json['generatedAt'] = generated_at_time treatment_log_json['completedAt'] = completed_at_time @@ -257,8 +296,18 @@ error_handler=self.error_handler) if response is not None: - self.logger.debug("Treatment upload response: {0}".format(response.json())) + self.logger.debug( + "Treatment upload response: {0}".format(response.json())) + if response.status_code == OK: + # Send TX code to UI app + message_body = str( + OutboundMessageIDs.CS2UI_REQ_TX_CODE_DISPLAY.value) + ',1,' + treatment_id + self.output_channel.enqueue_message(message_body) + else: + g_utils.logger.warning("Treatment {0} upload failed: {1}".format( + treatment_id, response.json())) + # treatment_id = response.json().get("reference", None) # Step #5 - Remove patient/device association @@ -267,15 +316,169 @@ access_token=access_token, associate=False, error_handler=self.error_handler) - # Step #6 - Send TX code to UI app - message_body = str(OutboundMessageIDs.CS2UI_REQ_TX_CODE_DISPLAY.value) + ',1,' + treatment_id - self.output_channel.enqueue_message(message_body) except Exception as e: error = Error(ERROR_STRING.format(OutboundMessageIDs.CS2UI_ERROR.value, - ErrorIDs.CS_SEND_TREATMENT_REPORT_ERROR.value, - e)) + ErrorIDs.CS_SEND_TREATMENT_REPORT_ERROR.value, + e)) self.error_handler.enqueue_error(error=error) + elif req.request_type == NetworkRequestType.CS2DCS_REQ_SEND_DEVICE_LOG: + try: + device_log_json = req.payload + device_log_filename = device_log_json['metadata']['deviceFileName'] + base_url = req.g_config[CONFIG_KEBORMED][CONFIG_KEBORMED_DCS_URL] + identity_url = req.g_config[CONFIG_KEBORMED][CONFIG_KEBORMED_DEVICE_IDENTITY_URL] + client_secret = req.g_config[CONFIG_KEBORMED][CONFIG_KEBORMED_IDP_CLIENT_SECRET] + token_verification_url = urllib.parse.urljoin( + base_url, DEVICE_TOKEN_VALIDATION) + validate_url = urllib.parse.urljoin( + base_url, "api/device/validate") + data_submission_url = urllib.parse.urljoin( + base_url, "/api/device/log") + + # Step #1 - get access token + + access_token = self.get_valid_token(identity_url=identity_url, + token_verification_url=token_verification_url, + client_secret=client_secret) + + if access_token is not None: + + # Step #2 - get organization id for current device + + response = cmd_outgoing_validate_device(access_token=access_token, + hd_serial_number=req.g_config[CONFIG_DEVICE][ + CONFIG_DEVICE_HD_SERIAL], + dg_serial_number=req.g_config[CONFIG_DEVICE][ + CONFIG_DEVICE_DG_SERIAL], + sw_version=req.g_config[CONFIG_DEVICE][CONFIG_DEVICE_SW_VERSION], + url=validate_url, + error_handler=self.error_handler) + + invalid_attributes = response.get("invalidAttributes", None) + + if len(invalid_attributes) > 0: + self.logger.error( + "Device with HD serial number {0}, DG serial number {1}, software version {2} is not a valid " + "device for log upload. Invalid fields: {3}".format( + req.g_config[CONFIG_DEVICE][CONFIG_DEVICE_HD_SERIAL], + req.g_config[CONFIG_DEVICE][CONFIG_DEVICE_DG_SERIAL], + req.g_config[CONFIG_DEVICE][CONFIG_DEVICE_SW_VERSION], + invalid_attributes)) + return response + else: + organization_id = response.get("associatedOrganizationId", None) + device_log_json['organizationId'] = organization_id + + + # Step #3 - upload the device log file + + device_log_json = json.dumps(device_log_json) + + response = cmd_outgoing_send_device_log(url=data_submission_url, + access_token=access_token, + device_log=device_log_json, + error_handler=self.error_handler) + + if response is not None: + self.logger.debug( + "Device log file uploaded: {0}".format(response.json())) + # Send response to the UI + message_body = str( + OutboundMessageIDs.CS2UI_DEVICE_LOG_UPLOADED.value) + ',1,' + device_log_filename + self.output_channel.enqueue_message(message_body) + else: + error = Error( + "{0},2,{1}, Missing access token".format(OutboundMessageIDs.CS2UI_ERROR.value, + ErrorIDs.CS_DEVICE_LOG_ERROR.value)) + self.error_handler.enqueue_error(error=error) + + except Exception as e: + error = Error(ERROR_STRING.format(OutboundMessageIDs.CS2UI_ERROR.value, + ErrorIDs.CS_DEVICE_LOG_ERROR.value, + e)) + self.error_handler.enqueue_error(error=error) + elif req.request_type == NetworkRequestType.CS2DCS_REQ_SEND_CS_LOG: + try: + cs_log_json = req.payload + cs_log_filename = cs_log_json['metadata']['deviceFileName'] + base_url = req.g_config[CONFIG_KEBORMED][CONFIG_KEBORMED_DCS_URL] + identity_url = req.g_config[CONFIG_KEBORMED][CONFIG_KEBORMED_DEVICE_IDENTITY_URL] + client_secret = req.g_config[CONFIG_KEBORMED][CONFIG_KEBORMED_IDP_CLIENT_SECRET] + token_verification_url = urllib.parse.urljoin( + base_url, DEVICE_TOKEN_VALIDATION) + validate_url = urllib.parse.urljoin( + base_url, "api/device/validate") + data_submission_url = urllib.parse.urljoin( + base_url, "/api/device/log") + + # Step #1 - get access token + + access_token = self.get_valid_token(identity_url=identity_url, + token_verification_url=token_verification_url, + client_secret=client_secret) + + if access_token is not None: + + # Step #2 - get organization id for current device + + response = cmd_outgoing_validate_device(access_token=access_token, + hd_serial_number=req.g_config[CONFIG_DEVICE][ + CONFIG_DEVICE_HD_SERIAL], + dg_serial_number=req.g_config[CONFIG_DEVICE][ + CONFIG_DEVICE_DG_SERIAL], + sw_version=req.g_config[CONFIG_DEVICE][CONFIG_DEVICE_SW_VERSION], + url=validate_url, + error_handler=self.error_handler) + + invalid_attributes = response.get("invalidAttributes", None) + + if len(invalid_attributes) > 0: + self.logger.error( + "Device with HD serial number {0}, DG serial number {1}, software version {2} is not a valid " + "device for log upload. Invalid fields: {3}".format( + req.g_config[CONFIG_DEVICE][CONFIG_DEVICE_HD_SERIAL], + req.g_config[CONFIG_DEVICE][CONFIG_DEVICE_DG_SERIAL], + req.g_config[CONFIG_DEVICE][CONFIG_DEVICE_SW_VERSION], + invalid_attributes)) + return response + else: + organization_id = response.get("associatedOrganizationId", None) + cs_log_json['organizationId'] = organization_id + + + # Step #3 - upload the device log file + + cs_log_json = json.dumps(cs_log_json) + + response = cmd_outgoing_send_cs_log(url=data_submission_url, + access_token=access_token, + cs_log=cs_log_json, + error_handler=self.error_handler) + + # Step #4 - remove the uploaded file + + if response.status_code == 200 and response.json().get('sessionId'): + self.logger.debug("CS log file uploaded: {0}".format(response.json())) + file_to_delete = os.path.join(CS_LOG_PATH, cs_log_filename) + if os.path.exists(file_to_delete): + os.remove(file_to_delete) + self.logger.debug(f"CS log file deleted: {file_to_delete}") + else: + self.logger.error("CS log file upload error: {0}".format(response.json())) + + else: + error = Error( + "{0},2,{1}, Missing access token".format(OutboundMessageIDs.CS2UI_ERROR.value, + ErrorIDs.CS_DEVICE_LOG_ERROR.value)) + self.error_handler.enqueue_error(error=error) + + except Exception as e: + error = Error(ERROR_STRING.format(OutboundMessageIDs.CS2UI_ERROR.value, + ErrorIDs.CS_DEVICE_LOG_ERROR.value, + e)) + self.error_handler.enqueue_error(error=error) + else: g_utils.logger.warning("Request type {0} not supported".format(req.request_type)) Index: cloudsync/handlers/logs_handler.py =================================================================== diff -u --- cloudsync/handlers/logs_handler.py (revision 0) +++ cloudsync/handlers/logs_handler.py (revision 3736c29ba097740b4e99684625e78bfaaac6b1a1) @@ -0,0 +1,73 @@ +from cloudsync.handlers.error import Error +from cloudsync.utils.helpers import * +from cloudsync.utils.globals import * + +from logging.handlers import TimedRotatingFileHandler + +class CustomTimedRotatingFileHandlerHandler(TimedRotatingFileHandler): + """ + Handler for logging to a file, rotating the log file and uploading it to cloud at certain timed intervals. + It extends the official TimedRotatingFileHandler to add the upload functionality. + """ + + def __init__(self, filename, prelogger: Logger, *args, **kwargs): + super().__init__(filename, *args, **kwargs) + self.network_request_handler = None + self.error_handler = None + self.g_config = None + prelogger.info("CUSTOM LOG HANDLER INITIATED") + + def doRollover(self): + super().doRollover() + self.__select_files_to_upload() + + def set_network_provider(self, network_request_handler): + self.network_request_handler = network_request_handler + + def set_error_provider(self, error_handler): + self.error_handler = error_handler + + def set_configuration(self, g_config): + self.g_config = g_config + + def __select_files_to_upload(cls): + """ + Checks for existing rotated files in the directory (excluding the newly created one) and uploads them. + """ + # We use the cls.baseFilename because according to code documentation: + # the filename passed in, could be a path object (see Issue #27493), + # but cls.baseFilename will be a string + + existing_files = [] + base_name = os.path.basename(cls.baseFilename) + for filename in os.listdir(os.path.dirname(cls.baseFilename)): + if filename.startswith(base_name) and filename != base_name: + existing_files.append(os.path.join(os.path.dirname(cls.baseFilename), filename)) + + if len(existing_files) > 0: + for existing_file in existing_files: + cls.__upload_cs_log_file(existing_file) + + def __upload_cs_log_file(cls, log_file_path): + + cs_log_json = helpers_construct_cs_log_json(log_file_path) + + if cs_log_json: + cs_log_json['serialNumber'] = cls.g_config[CONFIG_DEVICE][CONFIG_DEVICE_HD_SERIAL] + + g_utils.logger.debug("Device log {0}".format(cs_log_json)) + + try: + + helpers_add_to_network_queue(network_request_handler=cls.network_request_handler, + request_type=NetworkRequestType.CS2DCS_REQ_SEND_CS_LOG, + url='', + payload=cs_log_json, + method='', + g_config=cls.g_config, + success_message='CS2DCS_REQ_SEND_CS_LOG request added to network queue') + except Exception as e: + error = Error("{0},2,{1},{2}".format(OutboundMessageIDs.CS2UI_ERROR.value, + ErrorIDs.CS_LOG_ERROR.value, + e)) + cls.error_handler.enqueue_error(error=error) \ No newline at end of file Index: cloudsync/handlers/outgoing/handler_cs_to_dcs.py =================================================================== diff -u -re2b5bba1ace2613e8cd3ca6d997756b1b61d77a4 -r3736c29ba097740b4e99684625e78bfaaac6b1a1 --- cloudsync/handlers/outgoing/handler_cs_to_dcs.py (.../handler_cs_to_dcs.py) (revision e2b5bba1ace2613e8cd3ca6d997756b1b61d77a4) +++ cloudsync/handlers/outgoing/handler_cs_to_dcs.py (.../handler_cs_to_dcs.py) (revision 3736c29ba097740b4e99684625e78bfaaac6b1a1) @@ -44,7 +44,7 @@ } cert_paths = (path_certificate, path_private_key) - g_utils.logger.debug("Making request: {0}, {1}, {2}, {3}".format(url,headers,payload,cert_paths)) + g_utils.logger.debug("Making request: {0}, {1}, {2}, {3}".format(url, headers, payload, cert_paths)) response = requests.post(url=url, headers=headers, @@ -70,13 +70,13 @@ return None except requests.exceptions.TooManyRedirects: error = Error(TOO_MANY_REDIRECTS_HOLDER.format(OutboundMessageIDs.CS2UI_ERROR.value, - ErrorIDs.CS_GET_NEW_TOKEN_WITH_CERT_ERROR.value)) + ErrorIDs.CS_GET_NEW_TOKEN_WITH_CERT_ERROR.value)) error_handler.enqueue_error(error=error) return None except Exception as e: error = Error(GENERAL_EXCEPTION_HOLDER.format(OutboundMessageIDs.CS2UI_ERROR.value, - ErrorIDs.CS_GET_NEW_TOKEN_WITH_CERT_ERROR.value, - str(e))) + ErrorIDs.CS_GET_NEW_TOKEN_WITH_CERT_ERROR.value, + str(e))) error_handler.enqueue_error(error=error) return None @@ -89,7 +89,7 @@ headers = { 'Authorization': BEARER_HOLDER.format(access_token), 'Content-Type': CONTENT_TYPE, - "X-OrganizationId" : '1', + "X-OrganizationId": '1', 'User-Agent': USER_AGENT } @@ -98,16 +98,16 @@ return resp except requests.exceptions.Timeout: error = Error(REGISTRATION_TIMEOUT_HOLDER.format(OutboundMessageIDs.CS2UI_ERROR.value, - ErrorIDs.CS_VERIFY_TOKEN_ERROR.value)) + ErrorIDs.CS_VERIFY_TOKEN_ERROR.value)) error_handler.enqueue_error(error=error) except requests.exceptions.TooManyRedirects: error = Error(TOO_MANY_REDIRECTS_HOLDER.format(OutboundMessageIDs.CS2UI_ERROR.value, - ErrorIDs.CS_VERIFY_TOKEN_ERROR.value)) + ErrorIDs.CS_VERIFY_TOKEN_ERROR.value)) error_handler.enqueue_error(error=error) except Exception as e: error = Error(GENERAL_EXCEPTION_HOLDER.format(OutboundMessageIDs.CS2UI_ERROR.value, - ErrorIDs.CS_VERIFY_TOKEN_ERROR.value, - str(e))) + ErrorIDs.CS_VERIFY_TOKEN_ERROR.value, + str(e))) error_handler.enqueue_error(error=error) @@ -141,18 +141,18 @@ data=payload) except requests.exceptions.Timeout: error = Error(REGISTRATION_TIMEOUT_HOLDER.format(OutboundMessageIDs.CS2UI_ERROR.value, - ErrorIDs.CS_VALIDATE_DEVICE_ERROR.value)) + ErrorIDs.CS_VALIDATE_DEVICE_ERROR.value)) error_handler.enqueue_error(error=error) return None except requests.exceptions.TooManyRedirects: error = Error(TOO_MANY_REDIRECTS_HOLDER.format(OutboundMessageIDs.CS2UI_ERROR.value, - ErrorIDs.CS_VALIDATE_DEVICE_ERROR.value)) + ErrorIDs.CS_VALIDATE_DEVICE_ERROR.value)) error_handler.enqueue_error(error=error) return None except Exception as e: error = Error(GENERAL_EXCEPTION_HOLDER.format(OutboundMessageIDs.CS2UI_ERROR.value, - ErrorIDs.CS_VALIDATE_DEVICE_ERROR.value, - str(e))) + ErrorIDs.CS_VALIDATE_DEVICE_ERROR.value, + str(e))) error_handler.enqueue_error(error=error) try: @@ -168,8 +168,8 @@ return None except Exception as e: error = Error(GENERAL_EXCEPTION_HOLDER.format(OutboundMessageIDs.CS2UI_ERROR.value, - ErrorIDs.CS_VALIDATE_DEVICE_ERROR.value, - str(e))) + ErrorIDs.CS_VALIDATE_DEVICE_ERROR.value, + str(e))) error_handler.enqueue_error(error=error) @@ -178,10 +178,12 @@ @log_func def cmd_outgoing_set_device_state(url: str, access_token: str, + device_state_json: dict, error_handler: ErrorHandler) -> requests.Response: """ Updates the backend with the current device state :param url: set device state URL + :param device_state_json: device state payload :param access_token: access token :param error_handler: global error handler :return: The response @@ -192,23 +194,23 @@ 'Content-Type': CONTENT_TYPE, 'User-Agent': USER_AGENT } - payload = {} + payload = device_state_json resp = requests.put(url=url, headers=headers, data=payload) return resp except requests.exceptions.Timeout: error = Error(REGISTRATION_TIMEOUT_HOLDER.format(OutboundMessageIDs.CS2UI_ERROR.value, - ErrorIDs.CS_SEND_DEVICE_STATE_ERROR.value)) + ErrorIDs.CS_SEND_DEVICE_STATE_ERROR.value)) error_handler.enqueue_error(error=error) except requests.exceptions.TooManyRedirects: error = Error(TOO_MANY_REDIRECTS_HOLDER.format(OutboundMessageIDs.CS2UI_ERROR.value, - ErrorIDs.CS_SEND_DEVICE_STATE_ERROR.value)) + ErrorIDs.CS_SEND_DEVICE_STATE_ERROR.value)) error_handler.enqueue_error(error=error) except Exception as e: error = Error(GENERAL_EXCEPTION_HOLDER.format(OutboundMessageIDs.CS2UI_ERROR.value, - ErrorIDs.CS_SEND_DEVICE_STATE_ERROR.value, - str(e))) + ErrorIDs.CS_SEND_DEVICE_STATE_ERROR.value, + str(e))) error_handler.enqueue_error(error=error) @@ -227,18 +229,18 @@ headers=headers) except requests.exceptions.Timeout: error = Error(REGISTRATION_TIMEOUT_HOLDER.format(OutboundMessageIDs.CS2UI_ERROR.value, - ErrorIDs.CS_CHECK_IF_PATIENT_WITH_EMR_ID_EXISTS_ERROR.value)) + ErrorIDs.CS_CHECK_IF_PATIENT_WITH_EMR_ID_EXISTS_ERROR.value)) error_handler.enqueue_error(error=error) return None except requests.exceptions.TooManyRedirects: error = Error(TOO_MANY_REDIRECTS_HOLDER.format(OutboundMessageIDs.CS2UI_ERROR.value, - ErrorIDs.CS_CHECK_IF_PATIENT_WITH_EMR_ID_EXISTS_ERROR.value)) + ErrorIDs.CS_CHECK_IF_PATIENT_WITH_EMR_ID_EXISTS_ERROR.value)) error_handler.enqueue_error(error=error) return None except Exception as e: error = Error(GENERAL_EXCEPTION_HOLDER.format(OutboundMessageIDs.CS2UI_ERROR.value, - ErrorIDs.CS_CHECK_IF_PATIENT_WITH_EMR_ID_EXISTS_ERROR.value, - str(e))) + ErrorIDs.CS_CHECK_IF_PATIENT_WITH_EMR_ID_EXISTS_ERROR.value, + str(e))) error_handler.enqueue_error(error=error) try: @@ -254,8 +256,8 @@ return None except Exception as e: error = Error(GENERAL_EXCEPTION_HOLDER.format(OutboundMessageIDs.CS2UI_ERROR.value, - ErrorIDs.CS_CHECK_IF_PATIENT_WITH_EMR_ID_EXISTS_ERROR.value, - str(e))) + ErrorIDs.CS_CHECK_IF_PATIENT_WITH_EMR_ID_EXISTS_ERROR.value, + str(e))) error_handler.enqueue_error(error=error) @@ -277,18 +279,18 @@ data=payload) except requests.exceptions.Timeout: error = Error(REGISTRATION_TIMEOUT_HOLDER.format(OutboundMessageIDs.CS2UI_ERROR.value, - ErrorIDs.CS_CREATE_TEMPORARY_PATIENT_ERROR.value)) + ErrorIDs.CS_CREATE_TEMPORARY_PATIENT_ERROR.value)) error_handler.enqueue_error(error=error) return None except requests.exceptions.TooManyRedirects: error = Error(TOO_MANY_REDIRECTS_HOLDER.format(OutboundMessageIDs.CS2UI_ERROR.value, - ErrorIDs.CS_CREATE_TEMPORARY_PATIENT_ERROR.value)) + ErrorIDs.CS_CREATE_TEMPORARY_PATIENT_ERROR.value)) error_handler.enqueue_error(error=error) return None except Exception as e: error = Error(GENERAL_EXCEPTION_HOLDER.format(OutboundMessageIDs.CS2UI_ERROR.value, - ErrorIDs.CS_CREATE_TEMPORARY_PATIENT_ERROR.value, - str(e))) + ErrorIDs.CS_CREATE_TEMPORARY_PATIENT_ERROR.value, + str(e))) error_handler.enqueue_error(error=error) try: @@ -304,8 +306,8 @@ return None except Exception as e: error = Error(GENERAL_EXCEPTION_HOLDER.format(OutboundMessageIDs.CS2UI_ERROR.value, - ErrorIDs.CS_CREATE_TEMPORARY_PATIENT_ERROR.value, - str(e))) + ErrorIDs.CS_CREATE_TEMPORARY_PATIENT_ERROR.value, + str(e))) error_handler.enqueue_error(error=error) @@ -346,16 +348,16 @@ return resp except requests.exceptions.Timeout: error = Error(REGISTRATION_TIMEOUT_HOLDER.format(OutboundMessageIDs.CS2UI_ERROR.value, - ErrorIDs.CS_SET_PATIENT_DEVICE_ASSOCIATION_ERROR.value)) + ErrorIDs.CS_SET_PATIENT_DEVICE_ASSOCIATION_ERROR.value)) error_handler.enqueue_error(error=error) except requests.exceptions.TooManyRedirects: error = Error(TOO_MANY_REDIRECTS_HOLDER.format(OutboundMessageIDs.CS2UI_ERROR.value, - ErrorIDs.CS_SET_PATIENT_DEVICE_ASSOCIATION_ERROR.value)) + ErrorIDs.CS_SET_PATIENT_DEVICE_ASSOCIATION_ERROR.value)) error_handler.enqueue_error(error=error) except Exception as e: error = Error(GENERAL_EXCEPTION_HOLDER.format(OutboundMessageIDs.CS2UI_ERROR.value, - ErrorIDs.CS_SET_PATIENT_DEVICE_ASSOCIATION_ERROR.value, - str(e))) + ErrorIDs.CS_SET_PATIENT_DEVICE_ASSOCIATION_ERROR.value, + str(e))) error_handler.enqueue_error(error=error) @@ -387,17 +389,120 @@ return resp except requests.exceptions.Timeout: error = Error(REGISTRATION_TIMEOUT_HOLDER.format(OutboundMessageIDs.CS2UI_ERROR.value, - ErrorIDs.CS_SEND_TREATMENT_REPORT_ERROR.value)) + ErrorIDs.CS_SEND_TREATMENT_REPORT_ERROR.value)) error_handler.enqueue_error(error=error) return None except requests.exceptions.TooManyRedirects: error = Error(TOO_MANY_REDIRECTS_HOLDER.format(OutboundMessageIDs.CS2UI_ERROR.value, - ErrorIDs.CS_SEND_TREATMENT_REPORT_ERROR.value)) + ErrorIDs.CS_SEND_TREATMENT_REPORT_ERROR.value)) error_handler.enqueue_error(error=error) return None except Exception as e: error = Error(GENERAL_EXCEPTION_HOLDER.format(OutboundMessageIDs.CS2UI_ERROR.value, - ErrorIDs.CS_SEND_TREATMENT_REPORT_ERROR.value, - str(e))) + ErrorIDs.CS_SEND_TREATMENT_REPORT_ERROR.value, + str(e))) error_handler.enqueue_error(error=error) return None + + +@log_func +def cmd_outgoing_send_device_log(url: str, + access_token: str, + device_log: str, + error_handler: ErrorHandler) -> requests.Response: + """ + Sends a UI logfile to DCS + :param url: set device state URL + :param access_token: access token + :param device_log: Device log payload + :param error_handler: global error handler + :return: The response + """ + try: + + + headers = { + 'Authorization': BEARER_HOLDER.format(access_token), + 'Content-Type': CONTENT_TYPE, + 'User-Agent': USER_AGENT + } + + payload = device_log + + resp = requests.post(url=url, + headers=headers, + data=payload) + return resp + + except FileNotFoundError: + error = Error(FILE_NOT_FOUND.format(OutboundMessageIDs.CS2UI_ERROR.value, + ErrorIDs.CS_DEVICE_LOG_ERROR.value)) + error_handler.enqueue_error(error=error) + return None + except requests.exceptions.Timeout: + error = Error(REGISTRATION_TIMEOUT_HOLDER.format(OutboundMessageIDs.CS2UI_ERROR.value, + ErrorIDs.CS_DEVICE_LOG_ERROR.value)) + error_handler.enqueue_error(error=error) + return None + except requests.exceptions.TooManyRedirects: + error = Error(TOO_MANY_REDIRECTS_HOLDER.format(OutboundMessageIDs.CS2UI_ERROR.value, + ErrorIDs.CS_DEVICE_LOG_ERROR.value)) + error_handler.enqueue_error(error=error) + return None + except Exception as e: + error = Error(GENERAL_EXCEPTION_HOLDER.format(OutboundMessageIDs.CS2UI_ERROR.value, + ErrorIDs.CS_DEVICE_LOG_ERROR.value, + str(e))) + error_handler.enqueue_error(error=error) + return None + +@log_func +def cmd_outgoing_send_cs_log(url: str, + access_token: str, + cs_log: str, + error_handler: ErrorHandler) -> requests.Response: + """ + Sends a CS logfile to DCS + :param url: set device state URL + :param access_token: access token + :param cs_log: Device log payload + :param error_handler: global error handler + :return: The response + """ + try: + + + headers = { + 'Authorization': BEARER_HOLDER.format(access_token), + 'Content-Type': CONTENT_TYPE, + 'User-Agent': USER_AGENT + } + + payload = cs_log + + resp = requests.post(url=url, + headers=headers, + data=payload) + return resp + + except FileNotFoundError: + error = Error(FILE_NOT_FOUND.format(OutboundMessageIDs.CS2UI_ERROR.value, + ErrorIDs.CS_DEVICE_LOG_ERROR.value)) + error_handler.enqueue_error(error=error) + return None + except requests.exceptions.Timeout: + error = Error(REGISTRATION_TIMEOUT_HOLDER.format(OutboundMessageIDs.CS2UI_ERROR.value, + ErrorIDs.CS_DEVICE_LOG_ERROR.value)) + error_handler.enqueue_error(error=error) + return None + except requests.exceptions.TooManyRedirects: + error = Error(TOO_MANY_REDIRECTS_HOLDER.format(OutboundMessageIDs.CS2UI_ERROR.value, + ErrorIDs.CS_DEVICE_LOG_ERROR.value)) + error_handler.enqueue_error(error=error) + return None + except Exception as e: + error = Error(GENERAL_EXCEPTION_HOLDER.format(OutboundMessageIDs.CS2UI_ERROR.value, + ErrorIDs.CS_DEVICE_LOG_ERROR.value, + str(e))) + error_handler.enqueue_error(error=error) + return None Index: cloudsync/handlers/ui_cs_request_handler.py =================================================================== diff -u -rf8d59f1691d5b6850d2fa59d5312c563a42f0d72 -r3736c29ba097740b4e99684625e78bfaaac6b1a1 --- cloudsync/handlers/ui_cs_request_handler.py (.../ui_cs_request_handler.py) (revision f8d59f1691d5b6850d2fa59d5312c563a42f0d72) +++ cloudsync/handlers/ui_cs_request_handler.py (.../ui_cs_request_handler.py) (revision 3736c29ba097740b4e99684625e78bfaaac6b1a1) @@ -57,7 +57,7 @@ else: return False - def handle_message(self, message: UICSMessage): + def handle_message(self, message: UICSMessage) -> None: self.logger.info('UI2CS Message: {0}'.format(message)) message_data = message.timestamp + message.sequence + message.ID + message.size @@ -241,3 +241,47 @@ error_body += ",{0}".format(parameter) error = Error(error_body=error_body) self.error_handler.enqueue_error(error=error) + + # UPLOAD DEVICE LOG TO CLOUD REQUEST + elif InboundMessageIDs.mapped_str_value(message.ID) == InboundMessageIDs.UI2CS_UPLOAD_DEVICE_LOG and \ + (message.g_config[CONFIG_DEVICE][CONFIG_DEVICE_MODE] == 'operation'): + self.logger.info("UI2CS_UPLOAD_DEVICE_LOG request received") + + if (len(message.parameters) != 2) or (message.parameters[0] is None) or (message.parameters[1] is None): + error = Error( + "{0},2,{1},invalid # of parameters for file upload request".format( + OutboundMessageIDs.CS2UI_ERROR.value, + ErrorIDs.CS_DEVICE_LOG_ERROR.value)) + self.error_handler.enqueue_error(error=error) + else: + try: + + local_checksum = helpers_sha256_checksum(message.parameters[0]) + assert (local_checksum == message.parameters[1]), 'No valid sha256 value.' + + hd_serial_number = message.g_config[CONFIG_DEVICE][CONFIG_DEVICE_HD_SERIAL] + dg_serial_number = message.g_config[CONFIG_DEVICE][CONFIG_DEVICE_DG_SERIAL] + sw_version = message.g_config[CONFIG_DEVICE][CONFIG_DEVICE_SW_VERSION] + + self.logger.debug('hd: {0},dg: {1},sw: {2}'.format(hd_serial_number, dg_serial_number, sw_version)) + + device_log_json = helpers_construct_device_log_json(message.parameters[0]) + + if device_log_json: + device_log_json['serialNumber'] = message.g_config[CONFIG_DEVICE][CONFIG_DEVICE_HD_SERIAL] + + g_utils.logger.debug("Device log {0}".format(device_log_json)) + + helpers_add_to_network_queue(network_request_handler=self.network_request_handler, + request_type=NetworkRequestType.CS2DCS_REQ_SEND_DEVICE_LOG, + url='', + payload=device_log_json, + method='', + g_config=message.g_config, + success_message='CS2DCS_REQ_SEND_DEVICE_LOG request added to network queue') + + except Exception as e: + error = Error("{0},2,{1},{2}".format(OutboundMessageIDs.CS2UI_ERROR.value, + ErrorIDs.CS_DEVICE_LOG_ERROR.value, + e)) + self.error_handler.enqueue_error(error=error) Index: cloudsync/utils/globals.py =================================================================== diff -u -r373fd8d0c900ff129152e13d896e7de2e222b0e7 -r3736c29ba097740b4e99684625e78bfaaac6b1a1 --- cloudsync/utils/globals.py (.../globals.py) (revision 373fd8d0c900ff129152e13d896e7de2e222b0e7) +++ cloudsync/utils/globals.py (.../globals.py) (revision 3736c29ba097740b4e99684625e78bfaaac6b1a1) @@ -14,6 +14,8 @@ CONFIG_DEVICE_SW_VERSION = 'sw_version' CONFIG_DEVICE_MODE = 'mode' CONFIG_DEVICE_STATE = 'device_state' +CONFIG_LOG_LEVEL = 'log_level' +CONFIG_LOG_LEVEL_DURATION = 'log_level_duration' CONFIG_KEBORMED = "kebormed_paas" CONFIG_KEBORMED_MFT_URL = "url_mft" @@ -23,6 +25,9 @@ CONFIG_KEBORMED_REACHABILITY_URL = "url_reachability" CONFIG_KEBORMED_DIA_ORG_ID = "dia_org_id" +# DEFAULTS +DEFAULT_REACHABILITY_URL = "https://google.com" + # CONFIG CONFIG_PATH = os.path.join(PATH_HOME, "cloudsync/config/config.json") @@ -105,17 +110,36 @@ SALINE_BOLUS_VOLUME = "Saline Bolus Volume" HEPARIN_DELIVERED_VOLUME = "Heparin Delivered Volume" WATER_SAMPLE_TEST_RESULT = "Water Sample Test Result" +# new fields +TARGET_WEIGHT = " Target Weight" +TOTAL_CHLORINE = "Total Chlorine" +PH = "PH" +CONDUCTIVITY = "Conductivity" +MACHINE_WIPED_DOWN = "Machine Wiped Down" +FILTER_LIFE = "Filter Life" +LAST_CHEMICAL_DISINFECTION = "Last Chemical Disinfection" +LAST_HEAT_DISINFECTION = "Last Heat Disinfection" # TREATMENT TEMPLATE PATH TREATMENT_REPORT_TEMPLATE_PATH = "cloudsync/config/treatment_report_template.json" +# DEVICE LOGS TEMPLATE PATH +DEVICE_LOG_TEMPLATE_PATH = "cloudsync/config/device_log_template.json" + +# CS LOGS TEMPLATE PATH +CS_LOG_TEMPLATE_PATH = "cloudsync/config/device_log_template.json" + +# CS LOG STATE FILE PATH +CS_LOG_STATE_FILE_PATH = "cloudsync/config/log_state.json" + # USER_AGENT USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36" CONTENT_TYPE = "application/json" BEARER_HOLDER = "Bearer {0}" TOO_MANY_REDIRECTS_HOLDER = "{0},2,{1},Too many redirects" GENERAL_EXCEPTION_HOLDER = "{0},2,{1},{2}" REGISTRATION_TIMEOUT_HOLDER = "{0},2,{1},Registration timeout" +FILE_NOT_FOUND = "{0},2,{1},UI logfile not found" # HTTP CODES OK = 200 Index: cloudsync/utils/helpers.py =================================================================== diff -u -r373fd8d0c900ff129152e13d896e7de2e222b0e7 -r3736c29ba097740b4e99684625e78bfaaac6b1a1 --- cloudsync/utils/helpers.py (.../helpers.py) (revision 373fd8d0c900ff129152e13d896e7de2e222b0e7) +++ cloudsync/utils/helpers.py (.../helpers.py) (revision 3736c29ba097740b4e99684625e78bfaaac6b1a1) @@ -3,10 +3,12 @@ import os import shutil import json -import datetime import hashlib import socket +import re +import base64 +from datetime import * from time import time, sleep from typing import Union, Any from logging import Logger @@ -250,6 +252,54 @@ return {} +def helpers_read_device_log_template(path: str) -> dict: + """ + Read the device log template + :param path: the path to the template + :return: the loaded template + """ + if os.path.exists(path): + with open(path, 'r') as f: + template = json.load(f) + return template + else: + g_utils.logger.error( + "Configuration file does not exist: {0}".format(path)) + return {} + + +def helpers_read_cs_log_template(path: str) -> dict: + """ + Read the cs log template + :param path: the path to the template + :return: the loaded template + """ + if os.path.exists(path): + with open(path, 'r') as f: + template = json.load(f) + return template + else: + g_utils.logger.error( + "Configuration file does not exist: {0}".format(path)) + return {} + + +def helpers_read_log_state(path: str) -> dict: + """ + Read the cs log state file + :param path: the path to the file + :return: the contents of the log state file + """ + if os.path.exists(path): + with open(path, 'r') as f: + template = json.load(f) + return template + else: + g_utils.logger.error( + "Configuration file does not exist: {0}".format(path)) + return {} + + def log_func(func): """ Log the function and the parameters passed to it @@ -266,6 +316,124 @@ return _wrapper +def helpers_extract_device_log_metadata(log_file_name:str) -> Union[dict,None]: + local_date_pattern = r"^\d{8}(?=_)" + local_time_pattern = r"^(?:^.*?)_(\d{6})_" + serial_number_pattern = r"^[a-zA-Z0-9]+_[a-zA-Z0-9]+_([a-zA-Z0-9]+)(?=_)" + device_subtype_pattern = r"^[a-zA-Z0-9]+_[a-zA-Z0-9]+_[a-zA-Z0-9]+_([a-zA-Z0-9]+)(?=)" + log_type_pattern = r"[a-zA-Z0-9]+\.u\.(\w+)(?=)" + + local_date_match = re.search(local_date_pattern, log_file_name) + local_time_match = re.search(local_time_pattern, log_file_name) + serial_number_match = re.search(serial_number_pattern, log_file_name) + device_subtype_match = re.search(device_subtype_pattern, log_file_name) + log_type_match = re.search(log_type_pattern, log_file_name) + + return { + "local_date": local_date_match.group(0) if local_date_match else 'unknown', + "local_time": local_time_match.group(1) if local_time_match else 'unknown', + "serial_number": serial_number_match.group(1) if serial_number_match else 'unknown', + "device_sub_type": device_subtype_match.group(1) if device_subtype_match else 'unknown', + "device_log_type": log_type_match.group(1) if log_type_match else 'unknown' + } + + +def helpers_file_to_byte_array(file_path: str) -> bytearray: + try: + with open(file_path, "rb") as f: + # Read the entire file in binary mode + file_bytes = f.read() + return base64.b64encode(file_bytes).decode('utf-8') + except FileNotFoundError as e: + g_utils.logger.error(f"Device log file not found: {e}") + return None + except Exception as e: + g_utils.logger.error(f"Error reading device log file: {e}") + return None + + +def helpers_construct_cs_log_json(path:str): + """ + Constructs the payload for cs log file uploading + :param path: the path to the log file + :returns: the json payload to be uploaded + """ + cs_log_data_template = helpers_read_cs_log_template(CS_LOG_TEMPLATE_PATH) + + # Convert the file into byte array + logs_byte_array = helpers_file_to_byte_array(path) + + # Get the filename + file_name = os.path.basename(path) + + # Completion and generation timestamp from CS (in miliseconds) + local_timestamp = int(datetime.now(timezone.utc).timestamp()*1000) + + # Start and End timestamps calculation + today = datetime.now(timezone.utc) + start_of_day = int(today.replace(hour=0, minute=0, second=0, microsecond=0).timestamp()*1000) + end_of_day = int(today.replace(hour=23, minute=59, second=59, microsecond=999999).timestamp()*1000) + + # Populate JSON object + cs_log_data_template['metadata']['dataType'] = 'cloud-sync-log-category' + cs_log_data_template['metadata']['deviceFileName'] = file_name + cs_log_data_template['metadata']['startTimestamp'] = start_of_day + cs_log_data_template['metadata']['endTimestamp'] = end_of_day + cs_log_data_template['completedAt'] = local_timestamp + cs_log_data_template['generatedAt'] = local_timestamp + cs_log_data_template['data'] = logs_byte_array + cs_log_data_template['checksum'] = helpers_sha256_checksum(logs_byte_array) + + return cs_log_data_template + + +def helpers_construct_device_log_json(path:str): + """ + Constructs the payload for device log file uploading + :param path: the path to the log file + :returns: the json payload to be uploaded + """ + device_log_data = helpers_read_device_log_template(DEVICE_LOG_TEMPLATE_PATH) + + # Convert the file into byte array + logs_byte_array = helpers_file_to_byte_array(path) + + # Extract metadata from the filename + file_name = os.path.basename(path) + extracted_metadata = helpers_extract_device_log_metadata(file_name) + + # Completion and generation timestamp from CS (in miliseconds) + local_timestamp = int(datetime.now(timezone.utc).timestamp()*1000) + + # Calculate the UTC timestamp based on the local date and time the UI provided (in milliseconds) (NEED BETTER SOLUTION HERE) + if extracted_metadata['local_date'] != 'unknown' or extracted_metadata['local_time'] != 'unknown': + datetime_obj = datetime.strptime(f"{extracted_metadata['local_date']}{extracted_metadata['local_time']}", "%Y%m%d%H%M%S") + # Convert to UTC timestamp + ui_utc_timestamp = int(datetime_obj.timestamp()*1000) + else: + ui_utc_timestamp = local_timestamp + + + # Populate JSON object + if extracted_metadata is not None: + device_log_data['metadata']['dataType'] = 'device-log-category' + device_log_data['metadata']['deviceLogType'] = extracted_metadata['device_log_type'] + device_log_data['metadata']['deviceSubType'] = extracted_metadata['device_sub_type'] + device_log_data['metadata']['deviceFileName'] = file_name + device_log_data['metadata']['startTimestamp'] = ui_utc_timestamp + device_log_data['metadata']['endTimestamp'] = ui_utc_timestamp + device_log_data['completedAt'] = local_timestamp + device_log_data['generatedAt'] = local_timestamp + device_log_data['data'] = logs_byte_array + device_log_data['checksum'] = helpers_sha256_checksum(logs_byte_array) + + return device_log_data + + else: + g_utils.logger.error('Device log file name does not match the pattern') + return None + + def helpers_read_treatment_log_file(path: str): treatment_data = helpers_read_treatment_report_template(TREATMENT_REPORT_TEMPLATE_PATH) @@ -313,7 +481,7 @@ data_components = data_line.split(',') parameters = [] if len(data_components) > 2: - parameters = data_components[2:(len(data_components) - 2)] + parameters = data_components[2:len(data_components)] data_record = { "time": int(data_components[0]) * S_MS_CONVERSION_FACTOR, "title": data_components[1], @@ -325,7 +493,7 @@ data_components = data_line.split(',') parameters = [] if len(data_components) > 2: - parameters = data_components[2:(len(data_components) - 2)] + parameters = data_components[2:len(data_components)] data_record = { "time": int(data_components[0]) * S_MS_CONVERSION_FACTOR, "title": data_components[1], @@ -374,12 +542,12 @@ elif line.startswith(HEPARIN_STOP): treatment_data['data']['treatment']['parameters']['heparinStopBeforeTreatmentEnd'] = line.split(',')[1] elif line.startswith(TREATMENT_START_DATE_TIME): - element = datetime.datetime.strptime(line.split(',')[1], "%Y/%m/%d %H:%M") - timestamp = datetime.datetime.timestamp(element) + element = datetime.strptime(line.split(',')[1], "%Y/%m/%d %H:%M") + timestamp = datetime.timestamp(element) treatment_data['data']['treatment']['time']['start'] = int(timestamp) * S_MS_CONVERSION_FACTOR elif line.startswith(TREATMENT_END_DATE_TIME): - element = datetime.datetime.strptime(line.split(',')[1], "%Y/%m/%d %H:%M") - timestamp = datetime.datetime.timestamp(element) + element = datetime.strptime(line.split(',')[1], "%Y/%m/%d %H:%M") + timestamp = datetime.timestamp(element) treatment_data['data']['treatment']['time']['end'] = int(timestamp) * S_MS_CONVERSION_FACTOR elif line.startswith(ACTUAL_TREATMENT_DURATION): treatment_data['data']['treatment']['time']['treatmentDuration'] = int(line.split(',')[1]) @@ -408,7 +576,24 @@ elif line.startswith(HEPARIN_DELIVERED_VOLUME): treatment_data['data']['deviceTreatmentData']['heparinDelivered'] = line.split(',')[1] elif line.startswith(WATER_SAMPLE_TEST_RESULT): - treatment_data['data']['diagnostics']['waterSampleTestResult'] = line.split(',')[1] + treatment_data['data']['treatment']['treatmentPreparation']['totalChlorine'] = line.split(',')[1] + # new treatment fields + elif line.startswith(TARGET_WEIGHT): + treatment_data['data']['treatment']['time']['targetWeight'] = "" + elif line.startswith(TOTAL_CHLORINE): + treatment_data['data']['treatment']['treatmentPreparation']['totalChlorine'] = "" + elif line.startswith(PH): + treatment_data['data']['treatment']['treatmentPreparation']['ph'] = "" + elif line.startswith(CONDUCTIVITY): + treatment_data['data']['treatment']['treatmentPreparation']['conductivity'] = "" + elif line.startswith(MACHINE_WIPED_DOWN): + treatment_data['data']['diagnostics']['machineWipedDown'] = "" + elif line.startswith(FILTER_LIFE): + treatment_data['data']['diagnostics']['filterLife'] = "" + elif line.startswith(LAST_CHEMICAL_DISINFECTION): + treatment_data['data']['diagnostics']['lastChemicalDisinfection'] = "" + elif line.startswith(LAST_HEAT_DISINFECTION): + treatment_data['data']['diagnostics']['lastHeatDisinfection'] = "" counter += 1 return treatment_data @@ -457,14 +642,24 @@ def helpers_sha256_checksum(data: str) -> str: """ Returns the calculated checksum (SHA256) for input data - @param data: input data + @param data: input data. It can be either string or file @return:: checksum """ - checksum = hashlib.sha256(data.encode()).hexdigest() - return checksum + isFile = os.path.isfile(data) + if isFile: + checksum = hashlib.sha256() + with open(data, "rb") as f: + # Read and update hash in blocks of 4K + for byte_block in iter(lambda: f.read(4096), b""): + checksum.update(byte_block) + else: + checksum = hashlib.sha256(data.encode()) + return checksum.hexdigest() + + def helpers_crc8(message_list): """ Returns the calculated crc from a message list @@ -494,3 +689,46 @@ shutil.rmtree(path_to_delete) else: raise FileNotFoundError(f"{path_to_delete} not found or not a directory!") + + +def helpers_should_update_dcs_log_level() -> bool: + """ + Returns True if the state of the log level should be + communicated to the DCS else False. It is controlled by + the update_dcs_flag in the log_state.json. + * 1 --> True + * 0 --> False + """ + log_state = helpers_read_log_state(CS_LOG_STATE_FILE_PATH) + if int(log_state['update_dcs_flag']) == 1: + return True + return False + + +def helpers_should_update_cs_log_level(response:dict) -> bool: + current_log_state = helpers_read_log_state(CS_LOG_STATE_FILE_PATH) + current_log_level:str = current_log_state['current_log_level'] + requested_log_level:str = response['logLevel'] + if requested_log_level.upper() != current_log_level.upper(): + g_utils.logger.debug(f"DCS requested to change the log level from {current_log_level} to {requested_log_level}") + return True + return False + + +def helpers_write_log_state(file_path: str, config: dict) -> None: + with open(file_path, 'w') as f: + json.dump(config, f, indent=4) + +def helpers_calculate_log_level_duration(response): + """ + Returns a tuple with duration_in_seconds, start_timestamp, end_timestamp + if a valid log level duration was given. Otherwise None + """ + duration = int(response['logLevelDuration']) if response['logLevelDuration'] else 0 + start_timestamp = int(datetime.now(timezone.utc).timestamp() * 1000) + end_timestamp = start_timestamp + duration + + if duration > 0: + duration_in_seconds = int(duration/1000) + return duration_in_seconds, start_timestamp, end_timestamp + return None Index: cloudsync/utils/logging.py =================================================================== diff -u --- cloudsync/utils/logging.py (revision 0) +++ cloudsync/utils/logging.py (revision 3736c29ba097740b4e99684625e78bfaaac6b1a1) @@ -0,0 +1,216 @@ +import logging +import os +from logging.handlers import TimedRotatingFileHandler +from cloudsync.handlers.logs_handler import CustomTimedRotatingFileHandlerHandler +from cloudsync.utils.singleton import SingletonMeta +from cloudsync.handlers.error import Error +from cloudsync.utils.globals import * +from cloudsync.utils.helpers import * +from cloudsync.common.enums import * + + + +class LoggingConfig(metaclass=SingletonMeta): + """ + Encapsulates logging configuration and setup. + """ + + DEFAULT_LOG_LEVEL = 'ERROR' + LOG_LEVELS = { + 0: "NOTSET", + 10: "DEBUG", + 20: "INFO", + 30: "WARN", + 40: "ERROR", + 50: "CRITICAL" + } + + def __init__(self): + self.flask_app = None + self.log_level = self.DEFAULT_LOG_LEVEL + self.log_level_name = None + self.log_level_duration = None + self.log_handler = None + + def initiate(self, app): + self.flask_app = app + + # Genereate the Logs Dir + if not os.path.exists(CS_LOG_PATH): + os.makedirs(CS_LOG_PATH) + + # Remove existing handlers form the Flask app + for handler in self.flask_app.logger.handlers: + self.flask_app.logger.removeHandler(handler) + + # Initiate the main configuration of the logger(s) + self.__configure_logging() + + def set_log_level(self, log_level) -> None: + try: + self.log_level = log_level if isinstance(log_level, int) else getattr(logging, log_level.upper()) + self.log_level_name = self.LOG_LEVELS[self.log_level] + + # Set the log level to the custom log handler + self.log_handler.setLevel(self.log_level) + # Set the log level the the flask logger + self.flask_app.logger.setLevel(self.log_level) + # Set the log level ot the root logger + root_logger = logging.getLogger() + root_logger.setLevel(self.log_level) + + # Update the log state file + update_obj = { + 'attribute': 'current_log_level', + 'value': self.log_level_name + } + self.__update_log_state_file(update_object=update_obj) + + g_utils.logger.info(f'Log level set at {self.log_level_name} ({self.log_level})') + + except Exception as e: + g_utils.logger.error(f'Could not update the log level: {e}') + + def set_log_handler(self): + # Add the handler to the Flask app logger + self.flask_app.logger.addHandler(self.log_handler) + # Add the handler to the root logger + root_logger = logging.getLogger() + root_logger.addHandler(self.log_handler) + + def set_dcs_flag(self, flag_state:int): + update_obj = { + 'attribute': 'update_dcs_flag', + 'value': flag_state + } + self.__update_log_state_file(update_object=update_obj) + + def set_log_duration(self, duration: Union[int, tuple]): + + if isinstance(duration, tuple): + self.log_level_duration = int(duration[0]) + # Update the duration at log state file + update_obj = { + 'attribute': 'log_level_duration', + 'value': duration[0] + } + self.__update_log_state_file(update_object=update_obj) + # Update the start timestamp at log state file + update_obj = { + 'attribute': 'log_level_start_timestamp', + 'value': duration[1] + } + self.__update_log_state_file(update_object=update_obj) + # Update the stop timestamp at log state file + update_obj = { + 'attribute': 'log_level_stop_timestamp', + 'value': duration[2] + } + self.__update_log_state_file(update_object=update_obj) + else: + self.log_level_duration = int(duration) + # Update the duration at log state file + update_obj = { + 'attribute': 'log_level_duration', + 'value': duration + } + self.__update_log_state_file(update_object=update_obj) + + g_utils.logger.debug(f"Log level duration set at {self.log_level_duration}") + + def get_log_level(self) -> tuple: + """ + Returns a tuple with both the numeric and text + value of the log level. The first element of the tuple + is the numeric value and the second one is the text. + """ + return self.log_level, self.log_level_name + + def set_network_provider(self, network_request_handler) -> None: + """ + Passes the network provider to the custom log handler so it can + be able to send uploading log requests to the dcs for CS logs. + """ + self.log_handler.set_network_provider(network_request_handler=network_request_handler) + + def set_error_provider(self, error_handler) -> None: + """ + Passes the error provider to the custom log handler so it can + be able to send uploading log requests to the dcs for CS logs. + """ + self.log_handler.set_error_provider(error_handler=error_handler) + + def set_configuration(self, g_config) -> None: + """ + Passes the configuration tothe custom log handler so it can + be able to send uploading log requests to the dcs for CS logs. + """ + self.log_handler.set_configuration(g_config=g_config) + + def clear_prelogger(self) -> None: + """ + Clears the prelogger after the app has been initiated successfully + """ + #TODO: Implement this logic + + def start_countdown(self) -> None: + if self.log_level and self.log_level_duration: + try: + # Initiate the log level reset thread + Thread(target=self.__reset_log_level, args=(self.log_level_duration,)).start() + except Exception as e: + g_utils.logger.error(f"Failed starting the countdown thread: {e}") + + def __configure_logging(cls): + """ + * Sets up the file rotation Handler + * Sets the initial log level to both root and app loggers + * Sets the g_utils logger + """ + + # Create the "prelogger" for handling logs before the main logger is initialized + LOG_FORMAT = ("%(asctime)s [%(levelname)s]: %(message)s in %(pathname)s:%(lineno)d") + prelogger = logging.getLogger("prelogger") + prelogger.setLevel('DEBUG') + prelogger_file_handler = logging.FileHandler(os.path.join(CS_LOG_PATH, 'logs.init')) + prelogger_file_handler.setLevel('DEBUG') + prelogger_file_handler.setFormatter(logging.Formatter(LOG_FORMAT)) + prelogger.addHandler(prelogger_file_handler) + + # Create a custom TimedRotatingFileHandler that writes logs to a new file and uploads them to DCS + # every midnight, in UTC time + cls.log_handler = CustomTimedRotatingFileHandlerHandler( + filename=CS_LOG_FILE, + prelogger=prelogger, + when="midnight", + interval=1, + utc=True + ) + cls.log_handler.suffix = "%m-%d-%Y" + + # Add a formatter + default_formatter = logging.Formatter( + '[%(asctime)s] %(levelname)s in %(module)s: %(message)s | {%(pathname)s:%(lineno)d}') + cls.log_handler.setFormatter(default_formatter) + + # Set the custom handler as global handler + cls.set_log_handler() + # Set the g_utils logger + g_utils.add_logger(cls.flask_app.logger) + # Set the log level globally + cls.set_log_level(cls.log_level) + + def __update_log_state_file(cls, update_object:dict): + log_state_file = helpers_read_log_state(CS_LOG_STATE_FILE_PATH) + log_state_file[update_object['attribute']] = update_object['value'] + helpers_write_log_state(CS_LOG_STATE_FILE_PATH, log_state_file) + + def __reset_log_level(cls, duration:int): + """ + Resets the logger level to the specified default level after a given time interval. + """ + g_utils.logger.info(f"Logger level set to: {cls.log_level_name} for {duration} seconds") + sleep(duration) + cls.set_log_level(cls.DEFAULT_LOG_LEVEL) + cls.set_dcs_flag(1) + g_utils.logger.info(f"Logger level reverted to: {cls.DEFAULT_LOG_LEVEL}")