###########################################################################
#
# Copyright (c) 2019-2021 Diality Inc. - All Rights Reserved.
#
# THIS CODE MAY NOT BE COPIED OR REPRODUCED IN ANY FORM, IN PART OR IN
# WHOLE, WITHOUT THE EXPLICIT PERMISSION OF THE COPYRIGHT OWNER.
#
# @file    nv_ops_utils.py
#
# @author  (last)      Quang Nguyen
# @date    (last)      07-Jul-2021
# @author  (original)  Dara Navaei
# @date    (original)  21-Feb-2021
#
############################################################################
import struct
import time
from logging import Logger
from typing import List
from collections import OrderedDict
from .excel_ops import *


class NVOpsUtils:
    """

    Processes the calibration_record records, service records, system records, and the scheduled runs records.
    The records are prepared to be sent to firmware or to be received from firmware.
    """
    CRC_16_TABLE = (
        0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7,
        0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef,
        0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6,
        0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de,
        0x2462, 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485,
        0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d,
        0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695, 0x46b4,
        0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc,
        0x48c4, 0x58e5, 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823,
        0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b,
        0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12,
        0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a,
        0x6ca6, 0x7c87, 0x4ce4, 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41,
        0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, 0x8d68, 0x9d49,
        0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70,
        0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78,
        0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e, 0xe16f,
        0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067,
        0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e,
        0x02b1, 0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256,
        0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d,
        0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405,
        0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c,
        0x26d3, 0x36f2, 0x0691, 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634,
        0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, 0xb98a, 0xa9ab,
        0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3,
        0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a,
        0x4a75, 0x5a54, 0x6a37, 0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92,
        0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9,
        0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1,
        0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8,
        0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0
    )

    DEFAULT_CHAR_VALUE = ' '
    _RECORD_START_INDEX = 6
    _RECORD_SPECS_BYTES = 12
    _RECORD_SPECS_BYTE_ARRAY = 3

    _CURRENT_MESSAGE_NUM_INDEX = 0

    _DATA_TYPE_INDEX = 0
    _DATA_VALUE_INDEX = 1
    _CHAR_LENGTH_INDEX = 2
    _DICT_VALUE_LIST_LEN = 2

    _TARGET_BYTES_TO_SEND_TO_FW = 150
    _MIN_PAYLOAD_BYTES_SPACE = 4

    _PAYLOAD_CURRENT_MSG_INDEX = 0
    _PAYLOAD_TOTAL_MSG_INDEX = 1
    _PAYLOAD_TOTAL_BYTES_INDEX = 2

    def __init__(self, logger: Logger):
        """
        Constructor for the NVOptsUtils class

        @param logger: (Logger) the logger
        """

        self.logger = logger
        self._workspace_dir = ''
        self._excel_workbook = ''
        self._record_name = ''
        self._firmware_stack = ''
        self._is_writing_to_excel_done = False
        self._is_read_done = False

        # This list contains different data packet lists
        # i.e. [[message 1, payload], [message 2, payload], ...]
        self._record_packets_to_send_to_fw = []

        # Buffer that is used to keep the groups data except the crc to convert them to bytes and calculate the crc
        # of the group.
        self._temp_groups_data_to_calculate_crc = []

    def prepare_excel_report(self, firmware_stack: str, record_name: str, directory: str):
        """
        Publicly accessible function to prepare the excel report

        @param firmware_stack: (str) firmware stack name (e.g. "HD" or "DG")
        @param record_name: (str) record type to check such as calibration, system, ...
        @param directory: (str) the directory in which to write the excel doc
        @return none
        """
        path = ''
        is_report_found = False

        if not os.path.isdir(directory):
            # Create the directory and go to it
            os.mkdir(directory)

        self._workspace_dir = directory

        self._record_name = record_name
        self._firmware_stack = firmware_stack

        # List all the files in the workspace directory
        for file in os.listdir(self._workspace_dir):
            # If the file has an extension of .xlsx
            if file.endswith('.xlsx'):
                # Check if the firmware stack (i.e. DG) is in the file and name of the file
                # does not have lock in it. When the file is open, there is a hidden lock file
                # in there and it is ignored
                if self._firmware_stack in str(file) and 'lock' not in str(file):
                    # Create the file path and exit the loop
                    path = os.path.join(self._workspace_dir, file)
                    is_report_found = True
                    break

        if is_report_found:
            # Load the excel workbook
            self._excel_workbook = load_excel_report(path)
        else:
            # Get an excel workbook object
            self._excel_workbook = get_an_excel_workbook()

        # Setup worksheet and create the current tab
        setup_excel_worksheet(self._excel_workbook, self._record_name)

    def write_fw_record_to_excel(self, calibration_record: dict):
        """
        Writes a calibration record to excel
        @param calibration_record: (dict) the record to write to excel
        @return: None
        """

        try:

            row = 1
            for group in calibration_record.keys():

                start_row = row
                start_col = 1
                col = 1
                if isinstance(calibration_record[group], dict):

                    write_to_excel(self._excel_workbook, self._record_name, row, col,
                                   group, bold=True, max_col_len=len(group))

                    for hardware in calibration_record[group].keys():
                        list_of_keys = list(calibration_record[group].keys())

                        col = 2
                        write_to_excel(self._excel_workbook, self._record_name, row, col, hardware)
                        col += 1

                        if isinstance(calibration_record[group][hardware], dict):
                            for spec in calibration_record[group][hardware]:

                                spec_value = calibration_record[group][hardware][spec][1]

                                write_to_excel(self._excel_workbook, self._record_name, row, col, spec)
                                col += 1
                                write_to_excel(self._excel_workbook, self._record_name, row, col, spec_value)
                                col += 1
                        else:
                            spec_value = calibration_record[group][hardware][1]
                            write_to_excel(self._excel_workbook, self._record_name, row, col, spec_value)

                        if list_of_keys.index(hardware) == len(list_of_keys) - 1:
                            merge_cells(self._excel_workbook, self._record_name,
                                        start_row, start_col, row, start_col)
                        row += 1

                row += 1

            save_report(self._excel_workbook, self._workspace_dir, self._firmware_stack)
            # Signal reading is done
            self._is_writing_to_excel_done = True
        except Exception as e:
            self.logger.error("Failed to write calibration record to excel: {0}".format(e))

    def write_excel_record_to_calibration_record(self, calibration_record: dict):
        """
        Writes an excel record to a calibration record

        @param calibration_record: (dict) the calibration record to write
        @return True if reading to firmware records to excel is done otherwise False
        """
        try:
            temp_buffer = []
            is_value_different = False
            row = 1
            for group in calibration_record.keys():

                col = 1
                cell_value = get_cell_value(self._excel_workbook, self._record_name, row, col)

                if cell_value == group:

                    if isinstance(calibration_record[group], dict):
                        for hardware in calibration_record[group].keys():
                            if isinstance(calibration_record[group][hardware], dict):
                                col = 4
                                for spec in calibration_record[group][hardware]:
                                    cell_value = get_cell_value(self._excel_workbook, self._record_name, row, col)
                                    # Check if the cell value is not none. If it is none,
                                    # it cannot be converted to a number like float or integer
                                    is_cell_value_valid = True if cell_value is not None else False
                                    if 'crc' not in spec and 'time' not in spec and is_cell_value_valid:
                                        temp_buffer.append(
                                            struct.pack(calibration_record[group][hardware][spec][0], cell_value))
                                        if calibration_record[group][hardware][spec][1] != cell_value:
                                            is_value_different = True
                                            calibration_record[group][hardware][spec][1] = cell_value

                                    if is_value_different and 'time' in spec:
                                        epoch_time = self.get_current_time_in_epoch()
                                        calibration_record[group][hardware][spec][1] = epoch_time
                                        temp_buffer.append(
                                            struct.pack(calibration_record[group][hardware][spec][0], epoch_time))
                                        date_time = self.get_current_date_time(epoch_time)
                                        write_to_excel(self._excel_workbook, self._record_name, row, col, date_time)
                                    elif 'time' in spec and isinstance(cell_value, str) and is_cell_value_valid:
                                        epoch_time = self.get_date_time_in_epoch(cell_value)
                                        calibration_record[group][hardware][spec][1] = epoch_time

                                    if is_value_different and 'crc' in spec:
                                        # Convert the data in the temp buffer to bytes and calculate its crc.
                                        data = b''.join(temp_buffer)
                                        crc_value = self.crc_16(data)
                                        # Clear the temp buffer for the next round of data
                                        is_value_different = False
                                        calibration_record[group][hardware][spec][1] = crc_value
                                        write_to_excel(self._excel_workbook, self._record_name, row, col, crc_value)
                                    col += 2
                            else:
                                col = 3
                                for _ in calibration_record[group][hardware]:
                                    cell_value = get_cell_value(self._excel_workbook, self._record_name, row, col)
                                    # Check if the cell value is not none. If it is none,
                                    # it cannot be converted to a number like float or integer
                                    is_cell_value_valid = True if cell_value is not None else False
                                    if is_cell_value_valid:
                                        calibration_record[group][hardware][1] = cell_value
                            temp_buffer.clear()
                            row += 1
                row += 1

            save_report(self._excel_workbook, self._workspace_dir, self._firmware_stack)

        except Exception as e:
            self.logger.error("Failed to load the excel record as a calibration record: {0}".format(e))

    def get_writing_to_excel_status(self):
        """
        Publicly accessible function to get the reading status

        @return True if reading to firmware records to excel is done otherwise False
        """

        return self._is_writing_to_excel_done

    def get_reading_record_status(self):
        """
        Gets the reading record status
        @return: True if done, false otherwise
        """
        return self._is_read_done

    @staticmethod
    def get_processed_characters(record: list) -> bytearray:
        """
        Gets the list of the characters and makes sure their length is to the define length

        @param record: (list) the list that contains the characters, data type and the target character length
        @return characters (bytearray) that are converted to bytearrays
        """
        data_type = record[0]
        char = record[1]
        char_len = record[2]
        temp = bytearray()

        if len(char) > char_len:
            char = char[:char_len]
        elif len(char) < char_len:
            for i in range(len(char), char_len):
                char += NVOpsUtils.DEFAULT_CHAR_VALUE

        for ch in char:
            temp += struct.pack(data_type, ch.encode('ascii'))

        return temp

    @staticmethod
    def reset_fw_record(record: OrderedDict) -> OrderedDict:
        """
        Gets a record and updated the calibration date and crc

        @param record: (dict) the record to calculate the calibration time and crc
        @return record (OrderedDict) the record with updated calibration time and crc
        """
        for key, value in record.items():
            if isinstance(value, dict):
                for sub_key, sub_value in value.items():
                    if sub_key == 'cal_time':
                        sub_value[sub_key][1] = NVOpsUtils.get_current_time_in_epoch()
                    crc = NVOpsUtils.get_group_record_crc(sub_value)

                    sub_value['crc'][1] = crc
        return record

    @staticmethod
    def reset_fw_system_service_record(record: OrderedDict) -> OrderedDict:
        """
        Gets a record and updated the calibration date and crc

        @param record: (dict) the record to calculate the calibration time and crc
        @return record (OrderedDict) the record with updated calibration time and crc
        """
        for key, value in record.items():
            if isinstance(value, dict):
                crc = NVOpsUtils.get_group_record_crc(value)
                value['crc'][1] = crc
        return record

    @staticmethod
    def get_group_record_crc(group_record: dict) -> int:
        """
        Gets a group record and calculates the crc for the group

        @param group_record: (dict) the record to calculate the crc
        @return crc (int) the calculated crc
        """
        value_in_bytes = b''
        temp = []
        for key, value in group_record.items():
            if key is not 'crc':
                data_type = value[0]
                if data_type == '<c':
                    # If the data type is a character, call get processed characters to make sure all the
                    # characters are there or if the characters are not the same length, the extra characters
                    # will be filled with dashes. This way, the crc for the group is calculated correctly
                    temp.append(NVOpsUtils.get_processed_characters(value))
                else:
                    value = value[1]
                    temp.append(struct.pack(data_type, value))

        value_in_bytes += b''.join(temp)
        crc_value = NVOpsUtils.crc_16(value_in_bytes)

        return crc_value

    @staticmethod
    def get_current_time_in_epoch():
        """
        Returns the current date and time in epoch in integer format. This is a static method.

        @return: (int) data and time in epoch
        """
        return int(datetime.datetime.now().timestamp())

    @staticmethod
    def get_current_date_time(epoch: int):
        """
        Returns the current date and time from an epoch time

        @param: (int) the epoch
        @return: data and time in string
        """
        return time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(epoch))

    @staticmethod
    def get_date_time_in_epoch(data_time: str):
        """
        Returns the date time in epoch

        @return: (str) data and time in epoch
        """
        time_struct = time.strptime(data_time, '%Y-%m-%d %H:%M:%S')
        return int(time.mktime(time_struct))

    def process_received_record_from_fw(self, cal_record: dict, fw_raw_records_bytes: List[int]):
        """
        Handles processing the received record from firmware

        @param cal_record: (dict) calibration record from firmware
        @param fw_raw_records_bytes: (List[int]) list of the firmware record from firmware in bytes

        @return: none
        """
        try:
            raw_payload_temp_start_index = 0
            # Convert the concatenated raw data into a byte array since the struct library requires byte arrays.
            fw_raw_records_bytes = bytearray(fw_raw_records_bytes)

            # Loop through the keys for the main calibration_record dictionary
            # DG_Calibration : {pressure_sensors : { ppi : { gain: ['<f', 12.3], offset: ['<f', 5.67]}}, ...}
            # Get the list of keys of the main dictionary (i.e. pressure_sensors, temperature_sensors, ....}
            # These keys are the list of hardware groups
            for group in cal_record.keys():
                # All the hardware groups (i.e pressure sensors) are nested dictionaries, except the final CRC. So it is
                # unpacked here. The CRC is an unsigned 16-bit (short).
                if group == 'crc':
                    # Get the data type ( it must be '<H')
                    data_type = cal_record[group][self._DATA_TYPE_INDEX]
                    # Check the data type of the crc
                    value_bytes = self.get_data_type_bytes(data_type)
                    # Unpack the data
                    data = struct.unpack(data_type, fw_raw_records_bytes[raw_payload_temp_start_index:
                                                                         raw_payload_temp_start_index + value_bytes])[0]
                    # Convert the crc to hex
                    cal_record[group][self._DATA_VALUE_INDEX] = data
                elif group == 'padding':
                    # Get the data type
                    data_type = cal_record[group][self._DATA_TYPE_INDEX]
                    # Check the data type of the padding values (they are unsigned chars)
                    value_bytes = self.get_data_type_bytes(data_type)
                    # Get the length of the padding list
                    padding_length = len(cal_record[group][self._DATA_VALUE_INDEX])
                    # Add the length of the padding bytes to the payload start address
                    raw_payload_temp_start_index += padding_length * value_bytes
                # The rest of the groups are hardware which are layered dictionaries
                else:
                    # If the value is not a CRC, it is nested dictionary of different hardware groups
                    # Loop through the values of each key to get the list of the hardware in a hardware group (i.e
                    # pressure_sensors have ppi, ppo, ...)
                    for key in cal_record[group].keys():
                        # Loop through the keys. Check if the value of the key itself is a dictionary
                        if isinstance(cal_record[group][key], dict):
                            # Check if the values of the key are dictionaries themselves. If yes, loop through them.
                            for inner_key in cal_record[group][key]:
                                values = self._process_fw_record(cal_record, fw_raw_records_bytes, group, key,
                                                                 raw_payload_temp_start_index, inner_key=inner_key)
                                cal_record = values[0]
                                raw_payload_temp_start_index = values[1]
                        else:
                            values = self._process_fw_record(cal_record, fw_raw_records_bytes, group, key,
                                                             raw_payload_temp_start_index)
                            cal_record = values[0]
                            raw_payload_temp_start_index = values[1]

            self._is_read_done = True
        except Exception as e:
            self.logger.error("Failed to process received record from fw: {0}".format(e))

    def _process_fw_record(self,
                           calibration_record: dict,
                           fw_raw_records_bytes: bytearray,
                           group: str,
                           key: str,
                           read_start_index: int,
                           inner_key=None):
        """
        Handles processing the received record from firmware

        @param calibration_record: (dict) that is being updated from firmware
        @param fw_raw_records_bytes: (bytearray) of the firmware record from firmware in bytes
        @param group: (str) record group (i.e CRC, padding)
        @param key: (str) key of the calibration record dictionary
        @param read_start_index: (int) index to read the next set of byte(s) from fw raw records
        @param inner_key: (str) the key to an inner dictionary that is the value of the provided key
        (i.e. pressure: {ppi : [0,1]})

        @return: calibration record dictionary that is updated and read start index as a tuple
        """
        try:
            # The values of the dictionary of each hardware is a list of its data type as well as its value
            # which the first index is data type
            data = ''
            if inner_key is None:
                data_type = calibration_record[group][key][self._DATA_TYPE_INDEX]
                current_value = calibration_record[group][key][self._DATA_VALUE_INDEX]
            else:
                data_type = calibration_record[group][key][inner_key][self._DATA_TYPE_INDEX]
                current_value = calibration_record[group][key][inner_key][self._DATA_VALUE_INDEX]

            value_bytes = self.get_data_type_bytes(data_type)

            if data_type == '<c':
                for _ in current_value:
                    # Unpack the values to be written to the dictionary
                    raw = struct.unpack(data_type, fw_raw_records_bytes[
                                                   read_start_index:read_start_index + value_bytes])[0]
                    # Check if the the values is a character first, otherwise it can be received as a byte that was
                    # not a character from firmware
                    try:
                        data += raw.decode("utf-8", errors="ignore")
                    except UnicodeDecodeError:
                        data += 'X'
                    read_start_index += value_bytes
            else:
                # Unpack the values to be written to the dictionary
                data = struct.unpack(data_type, fw_raw_records_bytes[
                                                read_start_index: read_start_index + value_bytes])[0]
                # Update the index of the raw payload for the next read
                read_start_index += value_bytes

            if inner_key is None:
                calibration_record[group][key][self._DATA_VALUE_INDEX] = data
            else:
                calibration_record[group][key][inner_key][self._DATA_VALUE_INDEX] = data

        except Exception as e:
            self.logger.error("Failed to process fw record: {0}".format(e))

        return calibration_record, read_start_index

    def prepare_record_to_send_to_fw(self, new_record):
        """
        Handles preparing the record to be sent to firmware

        @param new_record: the dictionary that is processed to be sent down

        @return: record packets in bytes
        """
        current_payload_length = 0
        current_message_count = 1
        temp_buffer = [0, 0, 0]
        self._record_packets_to_send_to_fw.clear()

        try:
            # Loop through the DG calibration_record dictionary
            for group in new_record.keys():

                # NOTE: crc and padding are checked separately since they do not have layered dictionaries
                # If the group is the final CRC of the dictionary
                if group == 'crc':
                    data_type = new_record[group][self._DATA_TYPE_INDEX]
                    current_payload_length += self.get_data_type_bytes(data_type)
                    temp_buffer[self._CURRENT_MESSAGE_NUM_INDEX] = struct.pack('<i', current_message_count)
                    temp_buffer[self._PAYLOAD_TOTAL_BYTES_INDEX] = struct.pack('<i', current_payload_length)
                    temp_buffer.append(struct.pack(data_type, new_record[group][self._DATA_VALUE_INDEX]))
                    self._record_packets_to_send_to_fw.append(temp_buffer)

                # If the group is padding, and the length of padding is not 0
                elif group == 'padding':
                    if len(new_record[group][self._DATA_VALUE_INDEX]) != 0:
                        # Get the data type and calculate the number of bytes
                        data_type = new_record[group][self._DATA_TYPE_INDEX]
                        data_type_bytes = self.get_data_type_bytes(data_type)

                        # If there are padding zeros, they are in a list. loop through and add them to temp buffer
                        for pad in new_record[group][self._DATA_VALUE_INDEX]:
                            if self._TARGET_BYTES_TO_SEND_TO_FW - current_payload_length > self._MIN_PAYLOAD_BYTES_SPACE:
                                current_payload_length += data_type_bytes
                                temp_buffer[self._PAYLOAD_TOTAL_MSG_INDEX] = struct.pack('<i', 0)
                                temp_buffer.append(struct.pack(data_type, pad))
                            else:
                                temp_buffer[self._PAYLOAD_CURRENT_MSG_INDEX] = struct.pack('<i', current_message_count)
                                temp_buffer[self._PAYLOAD_TOTAL_BYTES_INDEX] = struct.pack('<i', current_payload_length)
                                self._record_packets_to_send_to_fw.append(temp_buffer)

                                current_message_count += 1
                                current_payload_length = data_type_bytes
                                temp_buffer = [struct.pack('<i', current_message_count), struct.pack('<i', 0),
                                               struct.pack('<i', current_payload_length),
                                               struct.pack(data_type, pad)]
                else:
                    for key in new_record[group].keys():
                        # Loop through the keys. Check if the value of the key itself is a dictionary
                        if isinstance(new_record[group][key], dict):
                            # If it is a dictionary, loop through the inner keys and process them
                            for inner_key in new_record[group][key]:
                                processed_values = self._prepare_record_packets_for_fw(new_record,
                                                                                       group, key, temp_buffer,
                                                                                       current_payload_length,
                                                                                       current_message_count,
                                                                                       inner_dict_key=inner_key)
                                # Update the local variables
                                temp_buffer = processed_values[0]
                                current_message_count = processed_values[1]
                                current_payload_length = processed_values[2]
                        else:
                            # The value of the key is not a dictionary itself
                            processed_values = self._prepare_record_packets_for_fw(new_record, group,
                                                                                   key, temp_buffer,
                                                                                   current_payload_length,
                                                                                   current_message_count)
                            # Update the local variables
                            temp_buffer = processed_values[0]
                            current_message_count = processed_values[1]
                            current_payload_length = processed_values[2]

            # Creating a byte array variable and add all the data packets
            # to one to calculate the final CRC
            data = b''
            for svc in self._record_packets_to_send_to_fw:
                # If the data packet is the last one in the list, ignore the last element of the
                # packet since it is the final CRC of the entire dictionary. The final CRC itself
                # is not included in the CRC calculations.
                # The actual calibration_record data is started from the 3rd element of the data packet.
                # The first 3 are current message, total messages, and the payload length of the current
                # packet.
                if self._record_packets_to_send_to_fw.index(svc) == len(self._record_packets_to_send_to_fw) - 1:
                    data += b''.join(svc[self._RECORD_SPECS_BYTE_ARRAY:-1])
                else:
                    # Get all the elements including the last one
                    data += b''.join(svc[self._RECORD_SPECS_BYTE_ARRAY:])

            # Calculate the 16-bit crc.
            crc = self.crc_16(data)
            # Convert the crc into a 2-byte value
            record_crc = struct.pack('<H', crc)
            # Insert the CRC into the last element of the last data packet
            self._record_packets_to_send_to_fw[-1][-1] = record_crc
            # Update the record's crc in hex format
            new_record['crc'][self._DATA_VALUE_INDEX] = crc

            # Update all the data packets with the last message count since is the number of messages that firmware
            # should receive
            for packet in self._record_packets_to_send_to_fw:

                # Add the number of total messages to each data packet. This was delayed
                # until now since it was not known how many message will it take to pack all the data.
                packet[1] = struct.pack('<i', current_message_count)
        except:  # Exception as e:
            raise
            # self.logger.error("Could not prepare record to send to fw: {0}".format(e))

        return self._record_packets_to_send_to_fw

    def _prepare_record_packets_for_fw(self, new_record: OrderedDict,
                                       group: str,
                                       key: str,
                                       temp_buffer: List,
                                       current_payload_length: int,
                                       current_message_count: int,
                                       inner_dict_key: str = None):
        """
        Prepares the record packets prior to sending to firmware

        @param new_record: (OrderedDict) the dictionary that is processed to be sent down
        @param group: (str) dictionary group that is being processed (i.e service_records)
        @param key: dictionary key of the current group
        @param temp_buffer: List temporary buffer that is holding the values to be converted to bytes
        @param current_payload_length: (int) current payload length of the temporary buffer (before it gets to
        maximum allowed)
        @param current_message_count: (int) current message number of all the payloads
        @param inner_dict_key: (str) key of a nested dictionary if exists, None otherwise

        @return: temp buffer, current message count, current payload length
        """
        try:
            # No inner key so no nested dictionary
            if inner_dict_key is None:
                data_type = new_record[group][key][self._DATA_TYPE_INDEX]
                data_type_bytes = self.get_data_type_bytes(data_type)
                current_value = new_record[group][key][self._DATA_VALUE_INDEX]
                # This is used only for the data types of <c which is a character. If the data type is a character,
                # it should come with a length as the third element of the list. If the third element does not
                # exist, assume the len is 1. This value is only used when the data type is a <c.
                char_len = 1 if len(new_record[group][key]) <= self._DICT_VALUE_LIST_LEN else \
                    new_record[group][key][self._CHAR_LENGTH_INDEX]
            else:
                data_type = new_record[group][key][inner_dict_key][self._DATA_TYPE_INDEX]
                data_type_bytes = self.get_data_type_bytes(data_type)
                current_value = new_record[group][key][inner_dict_key][self._DATA_VALUE_INDEX]
                # This is used only for the data types of <c which is a character. If the data type is a character,
                # it should come with a length as the third element of the list. If the third element does not
                # exist, assume the len is 1. This value is only used when the data type is a <c.
                char_len = 1 if len(new_record[group][key][inner_dict_key]) <= self._DICT_VALUE_LIST_LEN else \
                    new_record[group][key][inner_dict_key][self._CHAR_LENGTH_INDEX]

            # If the data type is a <c which is a character, multiply the data type bytes
            # to the character length
            if data_type == '<c':
                data_type_bytes *= char_len

            # If minimum number of bytes space is still available in current temp buffer, continue inserting
            # data into the buffer.
            if self._TARGET_BYTES_TO_SEND_TO_FW - current_payload_length > self._MIN_PAYLOAD_BYTES_SPACE:
                current_payload_length += data_type_bytes
                # Insert a 4-byte 0 to the index of the total messages. This is a place holder and it will
                # be updated with the right value later.
                temp_buffer[self._PAYLOAD_TOTAL_MSG_INDEX] = struct.pack('<i', 0)

                # If the data type is a <c, loop through the current value
                # convert each character to its equivalent ascii number and pack it.
                if data_type == '<c':
                    temp_buffer.append(NVOpsUtils.get_processed_characters(new_record[group][key]))
                else:
                    temp_buffer.append(struct.pack(data_type, current_value))

                temp_buffer[self._PAYLOAD_CURRENT_MSG_INDEX] = struct.pack('<i', current_message_count)
                temp_buffer[self._PAYLOAD_TOTAL_BYTES_INDEX] = struct.pack('<i', current_payload_length)
            else:
                # There is not enough space left in this temp buffer. Update the current message count and
                # the total number of bytes that were packed in this data packet.
                temp_buffer[self._PAYLOAD_CURRENT_MSG_INDEX] = struct.pack('<i', current_message_count)
                temp_buffer[self._PAYLOAD_TOTAL_BYTES_INDEX] = struct.pack('<i', current_payload_length)
                # Append the temp buffer list into the main list to be process later
                self._record_packets_to_send_to_fw.append(temp_buffer)
                # Increment the current message count as a new temp buffer is created for the next data
                # packet. Update the current payload length with the latest addition to the new temp buffer.
                current_message_count += 1
                current_payload_length = data_type_bytes
                # Create a new temp buffer and add the current message count, total length of payload so far
                # and the data that there was not enough space in the previous data packet.
                temp_buffer = [struct.pack('<i', current_message_count), struct.pack('<i', 0),
                               struct.pack('<i', current_payload_length)]
                # If the data type is a <c, loop through the current value
                # convert each character to its equivalent ascii number and pack it
                if data_type == '<c':
                    temp_buffer.append(NVOpsUtils.get_processed_characters(new_record[group][key]))
                else:
                    temp_buffer.append(struct.pack(data_type, current_value))
        except:  # Exception as e:
            raise
        #    self.logger.error("Failed to prepare record packets for fw: {0}".format(e))

        # Return the latest calculated values as tuple
        return temp_buffer, current_message_count, current_payload_length

    def calculate_group_byte_size(self, group):
        """
        Handles calculating the byte size of the values of the provided dictionary.

        @param group: the dictionary that is passed to calculate the byte size
        @return: calculated byte size
        """
        byte_size = 0

        for key in group.keys():
            data_type = group[key][self._DATA_TYPE_INDEX]
            current_byte_size = self.get_data_type_bytes(data_type)

            # If data type is a '<c' which is a char, the length must be provided
            # to know how many characters (bytes) should be considered.
            # The characters length should be provided as the third element of the list so check if the length of
            # the list is greater than 2. If yes, use the length value, otherwise ignore the length of characters.
            if data_type == '<c' and len(group[key]) > self._DICT_VALUE_LIST_LEN:
                byte_size += current_byte_size * group[key][self._CHAR_LENGTH_INDEX]
            else:
                byte_size += current_byte_size

        return byte_size

    @classmethod
    def crc_16(cls, data):
        """
        generates crc16 for the provided data
        @param data: byte of data
        @return: (int) the crc code
        """
        crc = 0xFFFF
        length = len(data)
        i = 0

        while length > 0:
            # Make the sure variables are 16-bit integers
            left = (crc << 8) & 0x0000FFFF
            right = (crc >> 8) & 0x0000FFFF
            crc = left ^ cls.CRC_16_TABLE[data[i] ^ (right & 0x00FF)]
            length -= 1
            i += 1

        return crc

    @staticmethod
    def get_data_type_bytes(data):
        """
        Handles converting the string representation of the bytes of the data types in a struct to numbers.
        This is a static method.

        @param data: the data to be converted to bytes in number
        @return: calculated byte size
        """
        number_of_bytes = struct.calcsize(data)

        return number_of_bytes

    @staticmethod
    def calculate_padding_byte_size(total_byte_size, max_buffer_size):
        """
        Handles calculating the padding length based on the provided buffer sizes. This is a static method.

        @param total_byte_size: total byte size of a record dictionary
        @param max_buffer_size: max buffer size that is allowed to be used in the dictionary

        @return: padding size in bytes
        """

        # Calculate the padding size:
        # If bytes in the dictionary % max write bytes to RTC RAM (64) or EEPROM (16) is 0, not padding is needed
        # Else padding = (ceil(dictionary bytes/max write) * max write)-dictionary bytes
        if (total_byte_size % max_buffer_size) == 0:
            padding_size = 0
        else:
            padding_size = (math.ceil(total_byte_size / max_buffer_size) * max_buffer_size) - total_byte_size

        return padding_size