###########################################################################
#
# Copyright (c) 2021-2024 Diality Inc. - All Rights Reserved.
#
# THIS CODE MAY NOT BE COPIED OR REPRODUCED IN ANY FORM, IN PART OR IN
# WHOLE, WITHOUT THE EXPLICIT PERMISSION OF THE COPYRIGHT OWNER.
#
# @file    nv_ops_utils.py
#
# @author  (last)      Dara Navaei
# @date    (last)      05-Mar-2024
# @author  (original)  Dara Navaei
# @date    (original)  21-Feb-2021
#
############################################################################
import os.path
import struct
import time
from enum import unique
from logging import Logger
from typing import List
from collections import OrderedDict
from .excel_ops import *
from leahi_dialin.utils.base import AbstractObserver, DialinEnum


@unique
class NVRecordsDG(DialinEnum):
    NVDATAMGMT_CALIBRATION_RECORD = 0
    NVDATAMGMT_SYSTEM_RECORD = 1
    NVDATAMGMT_SERVICE_RECORD = 2
    NVDATAMGMT_SCHEDULED_RUNS_RECORD = 3
    NVDATAMGMT_HEATERS_INFO_RECORD = 4
    NVDATAMGMT_USAGE_INFO_RECORD = 5
    NVDATAMGMT_SW_CONFIG_RECORD = 6
    NUM_OF_NVDATMGMT_RECORDS_JOBS = 7

class NVRecordsHD(DialinEnum):
    NVDATAMGMT_CALIBRATION_RECORD = 0
    NVDATAMGMT_SYSTEM_RECORD = 1
    NVDATAMGMT_SERVICE_RECORD = 2
    NVDATAMGMT_INSTITUTIONAL_RECORD = 3
    NVDATAMGMT_USAGE_INFO_RECORD = 4
    NVDATAMGMT_SW_CONFIG_RECORD = 5
    NUM_OF_NVDATMGMT_RECORDS_JOBS = 6


class NVUtilsObserver(AbstractObserver):
    """

    Observation class
    """
    def __init__(self, prop):
        self.received = False
        self.prop = prop

    def update(self, message):
        """
        Publicly accessible function to provide an update of the object that is being observed

        @param message: (str) the message to update its status

        @return none
        """
        self.received = message.get(self.prop, False)


class NVOpsUtils:
    """

    Processes the calibration_record records, service records, system records, and the scheduled runs records.
    The records are prepared to be sent to firmware or to be received from firmware.
    """
    CRC_16_TABLE = (
        0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7,
        0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef,
        0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6,
        0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de,
        0x2462, 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485,
        0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d,
        0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695, 0x46b4,
        0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc,
        0x48c4, 0x58e5, 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823,
        0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b,
        0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12,
        0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a,
        0x6ca6, 0x7c87, 0x4ce4, 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41,
        0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, 0x8d68, 0x9d49,
        0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70,
        0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78,
        0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e, 0xe16f,
        0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067,
        0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e,
        0x02b1, 0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256,
        0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d,
        0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405,
        0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c,
        0x26d3, 0x36f2, 0x0691, 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634,
        0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, 0xb98a, 0xa9ab,
        0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3,
        0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a,
        0x4a75, 0x5a54, 0x6a37, 0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92,
        0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9,
        0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1,
        0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8,
        0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0
    )

    # Public defines
    DEFAULT_CHAR_VALUE = ' '
    CAL_RECORD_TAB_NAME = 'Calibration_Record'
    NON_VOLATILE_RECORD_NAME = 'SW_Config_Report'
    USAGE_INFO_RECORD_TAB_NAME = 'Usage_Info_Record'
    SYSTEM_RECORD_TAB_NAME = 'System_Record'
    SERVICE_RECORD_TAB_NAME = 'Service_Record'
    INSTITUTIONAL_RECORD_TAB_NAME = 'Institutional_Record'
    DEFAULT_EPOCH_VALUE = 0

    _RECORD_START_INDEX = 6
    _RECORD_SPECS_BYTES = 12
    _RECORD_SPECS_BYTE_ARRAY = 3

    _CURRENT_MESSAGE_NUM_INDEX = 0

    _DATA_TYPE_INDEX = 0
    _DATA_VALUE_INDEX = 1
    _CHAR_LENGTH_INDEX = 2
    _DICT_VALUE_LIST_LEN = 2

    _TARGET_BYTES_TO_SEND_TO_FW = 150
    _MIN_PAYLOAD_BYTES_SPACE = 4

    _PAYLOAD_CURRENT_MSG_INDEX = 0
    _PAYLOAD_TOTAL_MSG_INDEX = 1
    _PAYLOAD_TOTAL_BYTES_INDEX = 2

    _SW_CONFIGS_TITLE_COL = 'SW Configurations'
    _SW_CONFIGS_VALUE_COL = 'Status'
    _SW_CONFIGS_REPORT_NAME = 'SW-CONFIGS'
    _CAL_TIME_NAME = 'cal_time'
    _NEW_CAL_TIME_SIGNAL = 'new'
    _CRC_NAME = 'crc'
    _PADDING_GROUP_NAME = 'padding'

    def __init__(self, logger: Logger):
        """
        Constructor for the NVOptsUtils class

        @param logger: (Logger) the logger
        """

        self.logger = logger
        self._workspace_dir = ''
        self._excel_workbook = ''
        self._record_name = ''
        self._firmware_stack = ''
        self._is_writing_to_excel_done = False
        self._is_read_done = False

        # This list contains different data packet lists
        # i.e. [[message 1, payload], [message 2, payload], ...]
        self._record_packets_to_send_to_fw = []

        # Buffer that is used to keep the groups data except the crc to convert them to bytes and calculate the crc
        # of the group.
        self._temp_groups_data_to_calculate_crc = []

    def _create_workspace(self, dir_name):
        """
        Publicly accessible function to get create a workspace for the script that is running.

        @param dir_name: Name of the workspace directory

        @return none
        """
        # Get the root directory of the current script
        scripts_root_dir = os.path.dirname(os.path.dirname(__file__))
        # Get the root directory of the entire scripts folder. The workspace that holds the
        # code review reports and clones other scripts and repositories must be outside of the scripts
        root_dir = os.path.dirname(os.path.dirname(scripts_root_dir))
        # Create the address of the workspace
        self._workspace_dir = os.path.join(root_dir, dir_name)
        # If the path does not exist, make it, otherwise, change to that directory
        if not os.path.isdir(self._workspace_dir):
            # Create the directory and go to it
            os.mkdir(self._workspace_dir)
        os.chdir(self._workspace_dir)

    def prepare_excel_report(self, firmware_stack: str, record_name: str, directory: str, protect_sheet: bool = False):
        """
        Publicly accessible function to prepare the excel report

        @param firmware_stack: (str) firmware stack name (e.g. "HD" or "DG")
        @param record_name: (str) record type to check such as calibration, system, ...
        @param directory: (str) the directory in which to write the excel document.
        @param protect_sheet: (bool) flag to indicate whether to write protect the sheet or not (default False)
        @return none
        """
        path = ''
        is_report_found = False

        # If a directory is provided and there is not a folder in that address, create the
        # directory. Set the workspace directory to the provided directory. If a directory was not
        # provided, create a workspace in the default position
        default_nv_directory = firmware_stack + '_NV_Records'

        if directory is not None:
            directory = os.path.join(directory, default_nv_directory)
            if not os.path.isdir(directory):
                # Create the directory and go to it
                os.mkdir(directory)

            self._workspace_dir = directory
            os.chdir(self._workspace_dir)
        else:
            self._create_workspace(default_nv_directory)

        self._record_name = record_name
        self._firmware_stack = firmware_stack

        # List all the files in the workspace directory
        for file in os.listdir(self._workspace_dir):
            # If the file has an extension of .xlsx
            if file.endswith('.xlsx'):
                # Check if the firmware stack (i.e. DG) is in the file and name of the file
                # does not have lock in it. When the file is open, there is a hidden lock file
                # in there and it is ignored
                file = str(file)
                if self._firmware_stack in file and 'lock' not in file:
                    if str(datetime.datetime.now().date()) in file:
                        if self._SW_CONFIGS_REPORT_NAME in file and self.NON_VOLATILE_RECORD_NAME == record_name:
                            # Create the file path and exit the loop
                            path = os.path.join(self._workspace_dir, file)
                            is_report_found = True
                            break
                        elif self._SW_CONFIGS_REPORT_NAME not in file and self.NON_VOLATILE_RECORD_NAME not in \
                            record_name:
                            # Create the file path and exit the loop
                            path = os.path.join(self._workspace_dir, file)
                            is_report_found = True
                            break

        if is_report_found:
            # Load the excel workbook
            self._excel_workbook = load_excel_report(path)
        else:
            # Get an excel workbook object
            self._excel_workbook = get_an_excel_workbook()

        # Setup worksheet and create the current tab
        setup_excel_worksheet(self._excel_workbook, self._record_name, protection=protect_sheet)

    def write_fw_record_to_excel(self, calibration_record: OrderedDict):
        """
        Writes a calibration record to excel
        @param calibration_record: (dict) the record to write to excel
        @return: None
        """
        try:
            row = 1
            # Let's say the calibration record is:
            # Get the keys of the calibration group {'pressure_sensors': 'ppi', {'fourth_order': ['<f', 0.0], ... }
            for group in calibration_record.keys():
                start_row = row
                start_col = 1
                col = 1
                if isinstance(calibration_record[group], dict):
                    # Write the data to excel
                    write_to_excel(self._excel_workbook, self._record_name, row, col, group, bold=True,
                                   max_col_len=len(group), protect_cell=True)
                    # Get the list of the hardware and write them to excel (i.e. list of pressure sensors)
                    for hardware in calibration_record[group].keys():
                        list_of_keys = list(calibration_record[group].keys())
                        col = 2
                        write_to_excel(self._excel_workbook, self._record_name, row, col, hardware, protect_cell=True)
                        col += 1
                        # If the list of the sensor hardware is a dictionary, loop through the hardware dictionary
                        # and write them to excel
                        if isinstance(calibration_record[group][hardware], dict):
                            for spec in calibration_record[group][hardware]:
                                # Get the spec value from the dictionary
                                spec_value = calibration_record[group][hardware][spec][1]
                                write_to_excel(self._excel_workbook, self._record_name, row, col, spec,
                                               protect_cell=True)
                                col += 1
                                # If the spec is cal_time and the value is not 0, it means it has been calibrated and
                                # the time is in epoch. Convert that to date and time.
                                spec_value = self.get_current_date_time(spec_value) if spec == self._CAL_TIME_NAME and \
                                    spec_value != 0 else spec_value

                                # If the spec value that is read is crc, then the value must be protected too. This is
                                # to make sure the crc is not changed by accident by the user so the calibration fails
                                # in the firmware
                                protect = False if spec is not self._CRC_NAME else True
                                # Write the value to the excel and make the cell protection to false. The actual
                                # calibration value should be edited
                                write_to_excel(self._excel_workbook, self._record_name, row, col, spec_value,
                                               protect_cell=protect)
                                col += 1
                        else:
                            # The hardware value is not a CRC. Write it to excel
                            spec_value = calibration_record[group][hardware][1]
                            write_to_excel(self._excel_workbook, self._record_name, row, col, spec_value,
                                           protect_cell=False)

                        # If all the hardware of a hardware group is written to excel, merge and center the
                        if list_of_keys.index(hardware) == len(list_of_keys) - 1:
                            merge_cells(self._excel_workbook, self._record_name, start_row, start_col, row, start_col)
                        row += 1

                row += 1
            save_report(self._excel_workbook, self._workspace_dir, self._firmware_stack)

        except Exception as e:
            self.logger.error("Failed to write calibration record to excel: {0}".format(e))

    def write_excel_record_to_fw_record(self, record: OrderedDict, report_address: str, record_tab: str):
        """
        Writes the data in the excel report to the firmware

        @param record: (dict) the record to write to firmware from excel
        @param report_address: (str) the address of the excel report to get the data from
        @param record_tab: (str) the name of the excel tab to get the data out (i.e. Calibration Record tab)

        @return: None
        """
        try:
            self._excel_workbook = load_excel_report(report_address)
            row = 1
            col = 1
            # Get the keys of the calibration group {'pressure_sensors': 'ppi', {'fourth_order': ['<f', 0.0], ... }
            for group in record.keys():
                # Get the current value of the excel cell
                cell_value = get_cell_value(self._excel_workbook, record_tab, row, col)
                # Check if the cell value is a string and if it is, lower case it for comparison
                cell_value = cell_value.lower() if isinstance(cell_value, str) else cell_value
                # Check if the current cell value is equal to the group name of the record (i.e. pressure_sensors group
                # are both in the excel and firmware record. Also, if the group name is padding ignore because it is not
                # needed to be read from excel
                if cell_value == group and group is not self._PADDING_GROUP_NAME:
                    # check if the hardware group is a dictionary
                    if isinstance(record[group], dict):
                        # The actual data in the excel starts from column 4
                        col = 4
                        for hardware in record[group].keys():
                            # Loop through the hardware groups and check if it is a dictionary (i.e.
                            # 'ppi', {'fourth_order': ['<f', 0.0], ... }
                            if isinstance(record[group][hardware], dict):
                                for spec in record[group][hardware]:
                                    cell_value = get_cell_value(self._excel_workbook, record_tab, row, col)
                                    # Write the cell value that is read into the firmware record.
                                    record[group][hardware][spec][1] = cell_value
                                    # Check if the cell value is a string and if it is lower case it for comparison
                                    cell_value = cell_value.lower() if isinstance(cell_value, str) else cell_value
                                    # Check if the current spec in the hardware group is cal_time and whether it is
                                    # None or empty or if it has the word 'new' in the value of the cal_time.
                                    # When a new calibration value is entered, the cal_time is set to 'new' which is
                                    # case-insensitive.
                                    if spec == self._CAL_TIME_NAME and cell_value is None or cell_value == '' or \
                                        cell_value == self._NEW_CAL_TIME_SIGNAL:
                                        # So there is a new calibration value. Get the current time in epoch
                                        # The time in epoch is set to the calibration record to be sent to the firmware
                                        epoch = self.get_current_time_in_epoch()
                                        date_time = self.get_current_date_time(epoch)
                                        record[group][hardware][spec][1] = epoch
                                        # Once the epoch time is set to the calibration record, calculate the crc of the
                                        # that hardware group and update the crc to be sent to the firmware
                                        crc = self.get_group_record_crc(record[group][hardware])
                                        record[group][hardware][self._CRC_NAME][1] = crc
                                        # Update the excel report with the newest crc and calibration time
                                        # Calibration time is converted to date and time.
                                        # The date and time is the current column number
                                        write_to_excel(self._excel_workbook, record_tab, row, col, date_time)
                                        # The crc value is 2 columns after the cal_time value. The crc is protected to
                                        # make sure it is not changed by the user. These crcs are checked upon POST in
                                        # the firmware
                                        write_to_excel(self._excel_workbook, record_tab, row, col + 2, crc,
                                                       protect_cell=True)
                                        # Save the excel report
                                        self._excel_workbook.save(filename=report_address)
                                    elif spec == self._CAL_TIME_NAME and isinstance(cell_value, str):
                                        # If the cal_time is in date time and a string, convert that to epoch. This is
                                        # a date that has been there before so it is only converted to epoch and crc is
                                        # not calculated
                                        epoch_time = self.get_date_time_in_epoch(cell_value)
                                        record[group][hardware][spec][1] = epoch_time
                                    # Increment the column by two to get the next value
                                    col += 2
                                # Increment the row to get the next hardware of the same group
                                row += 1
                                # Reset the column to 4 to start reading the next hardware in the same group
                                col = 4
                            else:
                                col = 3
                                cell_value = get_cell_value(self._excel_workbook, record_tab, row, col)
                                record[group][hardware][1] = cell_value
                                if hardware == self._CRC_NAME:
                                    crc = self.get_group_record_crc(record[group])
                                    record[group][self._CRC_NAME][1] = crc
                                row += 1

                        # Go back to the first column to start from the beginning
                        col = 1
                        # Once a hardware group is finished, (i.e. the pressure_sensors group) increment the row
                        # by one. There is an empty row in between each hardware group
                        row += 1

        except Exception as e:
            self.logger.error("Failed to load the excel record as a calibration record: {0}".format(e))

    @staticmethod
    def get_processed_characters(record: list) -> bytearray:
        """
        Gets the list of the characters and makes sure their length is to the define length

        @param record: (list) the list that contains the characters, data type and the target character length
        @return characters (bytearray) that are converted to bytearrays
        """
        data_type = record[0]
        char = record[1]
        char_len = record[2]
        temp = bytearray()

        if len(char) > char_len:
            char = char[:char_len]
        elif len(char) < char_len:
            for i in range(len(char), char_len):
                char += NVOpsUtils.DEFAULT_CHAR_VALUE

        for ch in char:
            temp += struct.pack(data_type, ch.encode('ascii'))

        return temp

    @staticmethod
    def reset_fw_record(record: OrderedDict) -> OrderedDict:
        """
        Gets a record and updated the calibration date and crc

        @param record: (dict) the record to calculate the calibration time and crc
        @return record (OrderedDict) the record with updated calibration time and crc
        """
        for key, value in record.items():
            if isinstance(value, dict):
                for sub_key, sub_value in value.items():
                    if sub_key == 'cal_time':
                        sub_value[sub_key][1] = NVOpsUtils.get_current_time_in_epoch()
                    crc = NVOpsUtils.get_group_record_crc(sub_value)

                    sub_value['crc'][1] = crc
        return record

    @staticmethod
    def reset_fw_system_service_record(record: OrderedDict) -> OrderedDict:
        """
        Gets a record and updated the calibration date and crc

        @param record: (dict) the record to calculate the calibration time and crc
        @return record (OrderedDict) the record with updated calibration time and crc
        """
        for key, value in record.items():
            # Check if there is a CRC in the inner dictionary since some of the structures might not have it.
            # For instance, the software configuration record does not have an inner CRC and it only has a global CRC
            # with the padding
            if isinstance(value, dict):
                crc = NVOpsUtils.get_group_record_crc(value)
                value['crc'][1] = crc
        return record

    @staticmethod
    def get_group_record_crc(group_record: dict) -> int:
        """
        Gets a group record and calculates the crc for the group

        @param group_record: (dict) the record to calculate the crc
        @return crc (int) the calculated crc
        """
        value_in_bytes = b''
        temp = []
        for key, value in group_record.items():
            if key != 'crc':
                data_type = value[0]
                if data_type == '<c':
                    # If the data type is a character, call get processed characters to make sure all the
                    # characters are there or if the characters are not the same length, the extra characters
                    # will be filled with dashes. This way, the crc for the group is calculated correctly
                    temp.append(NVOpsUtils.get_processed_characters(value))
                else:
                    value = value[1]
                    temp.append(struct.pack(data_type, value))

        value_in_bytes += b''.join(temp)
        crc_value = NVOpsUtils.crc_16(value_in_bytes)

        return crc_value

    @staticmethod
    def get_current_date_time(epoch: int):
        """
        Returns the current date and time from an epoch time

        @param: (int) the epoch
        @return: data and time in string
        """
        return time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(epoch))

    @staticmethod
    def get_current_time_in_epoch():
        """
        Returns the current date and time in epoch in integer format. This is a static method.

        @return: (int) data and time in epoch
        """
        return int(datetime.datetime.now().timestamp())

    @staticmethod
    def get_date_time_in_epoch(data_time: str):
        """
        Returns the date time in epoch

        @return: (str) data and time in epoch
        """
        time_struct = time.strptime(data_time, '%Y-%m-%d %H:%M:%S')
        return int(time.mktime(time_struct))

    def process_received_record_from_fw(self, cal_record: dict, fw_raw_records_bytes: List[int]):
        """
        Handles processing the received record from firmware

        @param cal_record: (dict) calibration record from firmware
        @param fw_raw_records_bytes: (List[int]) list of the firmware record from firmware in bytes

        @return: none
        """
        try:
            raw_payload_temp_start_index = 0
            # Convert the concatenated raw data into a byte array since the struct library requires byte arrays.
            fw_raw_records_bytes = bytearray(fw_raw_records_bytes)

            # Loop through the keys for the main calibration_record dictionary
            # DG_Calibration : {pressure_sensors : { ppi : { gain: ['<f', 12.3], offset: ['<f', 5.67]}}, ...}
            # Get the list of keys of the main dictionary (i.e. pressure_sensors, temperature_sensors, ....}
            # These keys are the list of hardware groups
            for group in cal_record.keys():
                # All the hardware groups (i.e. pressure sensors) are nested dictionaries, except the final CRC. It is
                # unpacked here. The CRC is an unsigned 16-bit (short).
                if group == 'crc':
                    # Get the data type ( it must be '<H')
                    data_type = cal_record[group][self._DATA_TYPE_INDEX]
                    # Check the data type of the crc
                    value_bytes = self.get_data_type_bytes(data_type)
                    # Unpack the data
                    data = struct.unpack(data_type, fw_raw_records_bytes[raw_payload_temp_start_index:
                                                                         raw_payload_temp_start_index + value_bytes])[0]
                    # Convert the crc to hex
                    cal_record[group][self._DATA_VALUE_INDEX] = data
                elif group == 'padding':
                    # Get the data type
                    data_type = cal_record[group][self._DATA_TYPE_INDEX]
                    # Check the data type of the padding values (they are unsigned chars)
                    value_bytes = self.get_data_type_bytes(data_type)
                    # Get the length of the padding list
                    padding_length = len(cal_record[group][self._DATA_VALUE_INDEX])
                    # Add the length of the padding bytes to the payload start address
                    raw_payload_temp_start_index += padding_length * value_bytes
                # The rest of the groups are hardware which are layered dictionaries
                else:
                    # If the value is not a CRC, it is nested dictionary of different hardware groups
                    # Loop through the values of each key to get the list of the hardware in a hardware group (i.e
                    # pressure_sensors have ppi, ppo, ...)
                    for key in cal_record[group].keys():
                        # Loop through the keys. Check if the value of the key itself is a dictionary
                        if isinstance(cal_record[group][key], dict):
                            # Check if the values of the key are dictionaries themselves. If yes, loop through them.
                            for inner_key in cal_record[group][key]:
                                values = self._process_fw_record(cal_record, fw_raw_records_bytes, group, key,
                                                                 raw_payload_temp_start_index, inner_key=inner_key)
                                cal_record = values[0]
                                raw_payload_temp_start_index = values[1]
                        else:
                            values = self._process_fw_record(cal_record, fw_raw_records_bytes, group, key,
                                                             raw_payload_temp_start_index)
                            cal_record = values[0]
                            raw_payload_temp_start_index = values[1]

            self._is_read_done = True
        except Exception as e:
            self.logger.error("Failed to process received record from fw: {0}".format(e))

    def _process_fw_record(self, calibration_record: dict, fw_raw_records_bytes: bytearray, group: str, key: str,
                           read_start_index: int, inner_key=None):
        """
        Handles processing the received record from firmware

        @param calibration_record: (dict) that is being updated from firmware
        @param fw_raw_records_bytes: (bytearray) of the firmware record from firmware in bytes
        @param group: (str) record group (i.e CRC, padding)
        @param key: (str) key of the calibration record dictionary
        @param read_start_index: (int) index to read the next set of byte(s) from fw raw records
        @param inner_key: (str) the key to an inner dictionary that is the value of the provided key
        (i.e. pressure: {ppi : [0,1]})

        @return: calibration record dictionary that is updated and read start index as a tuple
        """
        try:
            # The values of the dictionary of each hardware is a list of its data type as well as its value
            # which the first index is data type
            data = ''
            if inner_key is None:
                data_type = calibration_record[group][key][self._DATA_TYPE_INDEX]
                current_value = calibration_record[group][key][self._DATA_VALUE_INDEX]
            else:
                data_type = calibration_record[group][key][inner_key][self._DATA_TYPE_INDEX]
                current_value = calibration_record[group][key][inner_key][self._DATA_VALUE_INDEX]

            value_bytes = self.get_data_type_bytes(data_type)

            if data_type == '<c':
                for _ in current_value:
                    # Unpack the values to be written to the dictionary
                    raw = struct.unpack(data_type, fw_raw_records_bytes[
                                                   read_start_index:read_start_index + value_bytes])[0]
                    # Check if the values is a character first, otherwise it can be received as a byte that was
                    # not a character from firmware
                    try:
                        data += raw.decode("utf-8", errors="ignore")
                    except UnicodeDecodeError:
                        data += 'X'
                    read_start_index += value_bytes
            else:
                # Unpack the values to be written to the dictionary
                data = struct.unpack(data_type, fw_raw_records_bytes[
                                                read_start_index: read_start_index + value_bytes])[0]
                # Update the index of the raw payload for the next read
                read_start_index += value_bytes

            if inner_key is None:
                calibration_record[group][key][self._DATA_VALUE_INDEX] = data
            else:
                calibration_record[group][key][inner_key][self._DATA_VALUE_INDEX] = data

        except Exception as e:
            self.logger.error("Failed to process fw record: {0}".format(e))

        return calibration_record, read_start_index

    def prepare_record_to_send_to_fw(self, new_record):
        """
        Handles preparing the record to be sent to firmware

        @param new_record: the dictionary that is processed to be sent down

        @return: record packets in bytes
        """
        current_payload_length = 0
        current_message_count = 1
        temp_buffer = [0, 0, 0]
        self._record_packets_to_send_to_fw.clear()

        try:
            # Loop through the DG calibration_record dictionary
            for group in new_record.keys():

                # NOTE: crc and padding are checked separately since they do not have layered dictionaries
                # If the group is the final CRC of the dictionary
                if group == 'crc':
                    data_type = new_record[group][self._DATA_TYPE_INDEX]
                    current_payload_length += self.get_data_type_bytes(data_type)
                    temp_buffer[self._CURRENT_MESSAGE_NUM_INDEX] = struct.pack('<i', current_message_count)
                    temp_buffer[self._PAYLOAD_TOTAL_BYTES_INDEX] = struct.pack('<i', current_payload_length)
                    temp_buffer.append(struct.pack(data_type, new_record[group][self._DATA_VALUE_INDEX]))
                    self._record_packets_to_send_to_fw.append(temp_buffer)

                # If the group is padding, and the length of padding is not 0
                elif group == 'padding':
                    if len(new_record[group][self._DATA_VALUE_INDEX]) != 0:
                        # Get the data type and calculate the number of bytes
                        data_type = new_record[group][self._DATA_TYPE_INDEX]
                        data_type_bytes = self.get_data_type_bytes(data_type)

                        # If there are padding zeros, they are in a list. loop through and add them to temp buffer
                        for pad in new_record[group][self._DATA_VALUE_INDEX]:
                            if self._TARGET_BYTES_TO_SEND_TO_FW - current_payload_length > self._MIN_PAYLOAD_BYTES_SPACE:
                                current_payload_length += data_type_bytes
                                temp_buffer[self._PAYLOAD_TOTAL_MSG_INDEX] = struct.pack('<i', 0)
                                temp_buffer.append(struct.pack(data_type, pad))
                            else:
                                temp_buffer[self._PAYLOAD_CURRENT_MSG_INDEX] = struct.pack('<i', current_message_count)
                                temp_buffer[self._PAYLOAD_TOTAL_BYTES_INDEX] = struct.pack('<i', current_payload_length)
                                self._record_packets_to_send_to_fw.append(temp_buffer)

                                current_message_count += 1
                                current_payload_length = data_type_bytes
                                temp_buffer = [struct.pack('<i', current_message_count), struct.pack('<i', 0),
                                               struct.pack('<i', current_payload_length),
                                               struct.pack(data_type, pad)]
                else:
                    for key in new_record[group].keys():
                        # Loop through the keys. Check if the value of the key itself is a dictionary
                        if isinstance(new_record[group][key], dict):
                            # If it is a dictionary, loop through the inner keys and process them
                            for inner_key in new_record[group][key]:
                                processed_values = self._prepare_record_packets_for_fw(new_record,
                                                                                       group, key, temp_buffer,
                                                                                       current_payload_length,
                                                                                       current_message_count,
                                                                                       inner_dict_key=inner_key)
                                # Update the local variables
                                temp_buffer = processed_values[0]
                                current_message_count = processed_values[1]
                                current_payload_length = processed_values[2]
                        else:
                            # The value of the key is not a dictionary itself
                            processed_values = self._prepare_record_packets_for_fw(new_record, group,
                                                                                   key, temp_buffer,
                                                                                   current_payload_length,
                                                                                   current_message_count)
                            # Update the local variables
                            temp_buffer = processed_values[0]
                            current_message_count = processed_values[1]
                            current_payload_length = processed_values[2]

            # Creating a byte array variable and add all the data packets
            # to one to calculate the final CRC
            data = b''
            for svc in self._record_packets_to_send_to_fw:
                # If the data packet is the last one in the list, ignore the last element of the
                # packet since it is the final CRC of the entire dictionary. The final CRC itself
                # is not included in the CRC calculations.
                # The actual calibration_record data is started from the 3rd element of the data packet.
                # The first 3 are current message, total messages, and the payload length of the current
                # packet.
                if self._record_packets_to_send_to_fw.index(svc) == len(self._record_packets_to_send_to_fw) - 1:
                    data += b''.join(svc[self._RECORD_SPECS_BYTE_ARRAY:-1])
                else:
                    # Get all the elements including the last one
                    data += b''.join(svc[self._RECORD_SPECS_BYTE_ARRAY:])

            # Calculate the 16-bit crc.
            crc = self.crc_16(data)
            # Convert the crc into a 2-byte value
            record_crc = struct.pack('<H', crc)
            # Insert the CRC into the last element of the last data packet
            self._record_packets_to_send_to_fw[-1][-1] = record_crc
            # Update the record's crc in hex format
            new_record['crc'][self._DATA_VALUE_INDEX] = crc

            # Update all the data packets with the last message count since is the number of messages that firmware
            # should receive
            for packet in self._record_packets_to_send_to_fw:

                # Add the number of total messages to each data packet. This was delayed
                # until now since it was not known how many message will it take to pack all the data.
                packet[1] = struct.pack('<i', current_message_count)
        except:  # Exception as e:
            raise
            # self.logger.error("Could not prepare record to send to fw: {0}".format(e))

        return self._record_packets_to_send_to_fw

    def _prepare_record_packets_for_fw(self, new_record: OrderedDict, group: str, key: str, temp_buffer: List,
                                       current_payload_length: int, current_message_count: int,
                                       inner_dict_key: str = None):
        """
        Prepares the record packets prior to sending to firmware

        @param new_record: (OrderedDict) the dictionary that is processed to be sent down
        @param group: (str) dictionary group that is being processed (i.e. service_records)
        @param key: dictionary key of the current group
        @param temp_buffer: List temporary buffer that is holding the values to be converted to bytes
        @param current_payload_length: (int) current payload length of the temporary buffer (before it gets to
        maximum allowed)
        @param current_message_count: (int) current message number of all the payloads
        @param inner_dict_key: (str) key of a nested dictionary if exists, None otherwise

        @return: temp buffer, current message count, current payload length
        """
        try:
            # No inner key so no nested dictionary
            if inner_dict_key is None:
                data_type = new_record[group][key][self._DATA_TYPE_INDEX]
                data_type_bytes = self.get_data_type_bytes(data_type)
                current_value = new_record[group][key][self._DATA_VALUE_INDEX]
                # This is used only for the data types of <c which is a character. If the data type is a character,
                # it should come with a length as the third element of the list. If the third element does not
                # exist, assume the len is 1. This value is only used when the data type is a <c.
                char_len = 1 if len(new_record[group][key]) <= self._DICT_VALUE_LIST_LEN else \
                    new_record[group][key][self._CHAR_LENGTH_INDEX]
            else:
                data_type = new_record[group][key][inner_dict_key][self._DATA_TYPE_INDEX]
                data_type_bytes = self.get_data_type_bytes(data_type)
                current_value = new_record[group][key][inner_dict_key][self._DATA_VALUE_INDEX]
                # This is used only for the data types of <c which is a character. If the data type is a character,
                # it should come with a length as the third element of the list. If the third element does not
                # exist, assume the len is 1. This value is only used when the data type is a <c.
                char_len = 1 if len(new_record[group][key][inner_dict_key]) <= self._DICT_VALUE_LIST_LEN else \
                    new_record[group][key][inner_dict_key][self._CHAR_LENGTH_INDEX]

            # If the data type is a <c which is a character, multiply the data type bytes
            # to the character length
            if data_type == '<c':
                data_type_bytes *= char_len

            # If minimum number of bytes space is still available in current temp buffer, continue inserting
            # data into the buffer.
            if self._TARGET_BYTES_TO_SEND_TO_FW - current_payload_length > self._MIN_PAYLOAD_BYTES_SPACE:
                current_payload_length += data_type_bytes
                # Insert a 4-byte 0 to the index of the total messages. This is a place holder and it will
                # be updated with the right value later.
                temp_buffer[self._PAYLOAD_TOTAL_MSG_INDEX] = struct.pack('<i', 0)

                # If the data type is a <c, loop through the current value
                # convert each character to its equivalent ascii number and pack it.
                if data_type == '<c':
                    temp_buffer.append(NVOpsUtils.get_processed_characters(new_record[group][key]))
                else:
                    temp_buffer.append(struct.pack(data_type, current_value))

                temp_buffer[self._PAYLOAD_CURRENT_MSG_INDEX] = struct.pack('<i', current_message_count)
                temp_buffer[self._PAYLOAD_TOTAL_BYTES_INDEX] = struct.pack('<i', current_payload_length)
            else:
                # There is not enough space left in this temp buffer. Update the current message count and
                # the total number of bytes that were packed in this data packet.
                temp_buffer[self._PAYLOAD_CURRENT_MSG_INDEX] = struct.pack('<i', current_message_count)
                temp_buffer[self._PAYLOAD_TOTAL_BYTES_INDEX] = struct.pack('<i', current_payload_length)
                # Append the temp buffer list into the main list to be process later
                self._record_packets_to_send_to_fw.append(temp_buffer)
                # Increment the current message count as a new temp buffer is created for the next data
                # packet. Update the current payload length with the latest addition to the new temp buffer.
                current_message_count += 1
                current_payload_length = data_type_bytes
                # Create a new temp buffer and add the current message count, total length of payload so far
                # and the data that there was not enough space in the previous data packet.
                temp_buffer = [struct.pack('<i', current_message_count), struct.pack('<i', 0),
                               struct.pack('<i', current_payload_length)]
                # If the data type is a <c, loop through the current value
                # convert each character to its equivalent ascii number and pack it
                if data_type == '<c':
                    temp_buffer.append(NVOpsUtils.get_processed_characters(new_record[group][key]))
                else:
                    temp_buffer.append(struct.pack(data_type, current_value))
        except: #Exception as e:
            #raise
            self.logger.error("Failed to prepare record packets for fw")

        # Return the latest calculated values as tuple
        return temp_buffer, current_message_count, current_payload_length

    def calculate_group_byte_size(self, group):
        """
        Handles calculating the byte size of the values of the provided dictionary.

        @param group: the dictionary that is passed to calculate the byte size
        @return: calculated byte size
        """
        byte_size = 0

        for key in group.keys():
            data_type = group[key][self._DATA_TYPE_INDEX]
            current_byte_size = self.get_data_type_bytes(data_type)

            # If data type is a '<c' which is a char, the length must be provided
            # to know how many characters (bytes) should be considered.
            # The characters length should be provided as the third element of the list so check if the length of
            # the list is greater than 2. If yes, use the length value, otherwise ignore the length of characters.
            if data_type == '<c' and len(group[key]) > self._DICT_VALUE_LIST_LEN:
                byte_size += current_byte_size * group[key][self._CHAR_LENGTH_INDEX]
            else:
                byte_size += current_byte_size

        return byte_size

    @classmethod
    def crc_16(cls, data):
        """
        generates crc16 for the provided data
        @param data: byte of data
        @return: (int) the crc code
        """
        crc = 0xFFFF
        length = len(data)
        i = 0

        while length > 0:
            # Make the sure variables are 16-bit integers
            left = (crc << 8) & 0x0000FFFF
            right = (crc >> 8) & 0x0000FFFF
            crc = left ^ cls.CRC_16_TABLE[data[i] ^ (right & 0x00FF)]
            length -= 1
            i += 1

        return crc

    @staticmethod
    def get_data_type_bytes(data):
        """
        Handles converting the string representation of the bytes of the data types in a struct to numbers.
        This is a static method.

        @param data: the data to be converted to bytes in number
        @return: calculated byte size
        """
        number_of_bytes = struct.calcsize(data)

        return number_of_bytes

    @staticmethod
    def calculate_padding_byte_size(total_byte_size, max_buffer_size):
        """
        Handles calculating the padding length based on the provided buffer sizes. This is a static method.

        @param total_byte_size: total byte size of a record dictionary
        @param max_buffer_size: max buffer size that is allowed to be used in the dictionary

        @return: padding size in bytes
        """

        # Calculate the padding size:
        # If bytes in the dictionary % max write bytes to RTC RAM (64) or EEPROM (16) is 0, not padding is needed
        # Else padding = (ceil(dictionary bytes/max write) * max write)-dictionary bytes
        if (total_byte_size % max_buffer_size) == 0:
            padding_size = 0
        else:
            padding_size = (math.ceil(total_byte_size / max_buffer_size) * max_buffer_size) - total_byte_size

        return padding_size

    def write_sw_config_to_excel(self, sw_configs: OrderedDict, stack_name: str):
        """
        Publicly accessible function to write the software configurations into an excel report

        @param sw_configs: (ordered dictionary) the configurations record to write to excel
        @param stack_name: (str) the name of the software stack (i.e. HD)

        @return none
        """
        row = 1
        names_col_number = 2
        write_to_excel(self._excel_workbook, self._record_name, row, names_col_number, self._SW_CONFIGS_TITLE_COL,
                       bold=True, freeze=True, protect_cell=True)

        values_col_number = 3
        write_to_excel(self._excel_workbook, self._record_name, row, values_col_number, self._SW_CONFIGS_VALUE_COL,
                       bold=True, protect_cell=True)

        # Prepare for writing the values
        row += 1

        for key, values in sw_configs['sw_configs'].items():

            write_to_excel(self._excel_workbook, self._record_name, row, names_col_number, key, bold=True,
                           protect_cell=True)

            # Get the configuration value
            config_value = values[1]
            # If the config value is not 0, color the cell as green otherwise, leave it none colored
            color = GREEN if config_value != 0 else NO_COLOR
            write_to_excel(self._excel_workbook, self._record_name, row, values_col_number, config_value, color=color)
            row += 1

        save_report(self._excel_workbook, self._workspace_dir, self._SW_CONFIGS_REPORT_NAME, stack_name=stack_name)

    def get_sw_configs_from_excel(self, sw_configs_dict: OrderedDict, excel_path: str, sw_config_excel_tab: str):
        """
        Publicly accessible function to get the software configurations from excel

        @param sw_configs_dict: (ordered dictionary) the configurations record to write to excel
        @param excel_path: (str): the path to the excel report that its data is read
        @param sw_config_excel_tab: (str): the name of the tab in the excel report that the values are located at

        @return status of the operations
        """
        row = 1
        col = 1
        title_col = None
        value_col = None
        max_col_to_go = 50
        status = False
        # Load the excel report
        self._excel_workbook = load_excel_report(excel_path)
        active_sheet = self._excel_workbook[sw_config_excel_tab]

        while True:
            # Loop through the cells in the title row
            value = active_sheet.cell(row=row, column=col).value
            # If the col number exceeded the maximum column number, break out of the loop
            if col >= max_col_to_go:
                break

            # Check if the value of the read cell is not none
            if value is not None:
                # If the value of the title row is name of the sw configs title column name, update the title col number
                if value.strip() == self._SW_CONFIGS_TITLE_COL:
                    title_col = col
                # If the value of the title row is the name of the values of the sw configs, update the value col number
                if value.strip() == self._SW_CONFIGS_VALUE_COL:
                    value_col = col
            # If the title col and value col numbers are both found and they are not none, then exit the while loop
            # since the values have been found, otherwise, increment the column number and keep looking
            if title_col is not None and value_col is not None:
                break
            else:
                col += 1

        if title_col is not None and value_col is not None:
            # Get the last non-empty row number of the current active sheet
            last_non_empty_row = active_sheet.max_row
            # Get the dictionary of the provided sw configurations dictionary, this can be either HD or DG
            fw_sw_configs = sw_configs_dict['sw_configs']
            # Loop through the excel from row 2 since row 1 is the titles row until the last non-empty row + 1 since the
            # range method does not include that last element so the last non-empty row will not be covered if there is
            # not a +1.
            for row in range(2, last_non_empty_row + 1):
                config = active_sheet.cell(row=row, column=title_col).value

                if config is not None:
                    # Check if the software configuration that has been read from excel exists in dictionary that has
                    # been prepared in Dialin
                    if config.strip() in fw_sw_configs:
                        excel_config_value = active_sheet.cell(row=row, column=value_col).value
                        # Check if the value is an integer and it is a 1 or a 0
                        # The only acceptable values are 1 for enable and 0 for disable
                        if isinstance(excel_config_value, int) and excel_config_value == 1 or excel_config_value == 0:
                            fw_sw_configs[config.strip()][1] = excel_config_value
                        else:
                            # If the value is not acceptable, set the default value that is sent down to firmware to 0
                            # and write incorrect into the report so the user will notice that they had and empty cell
                            # or a cell with a non-acceptable value (i.e. 123). Color the cell as red.
                            fw_sw_configs[config.strip()][1] = 0
                            write_to_excel(self._excel_workbook, sw_config_excel_tab, row, value_col, 'Incorrect Value',
                                           color=RED)
                            # Save back the excel workbook with the changes
                            # This save function is the openpyxl save and not the internal save function that creates
                            # the save path. The save path already exists
                            self._excel_workbook.save(filename=excel_path)
            status = True

        return status