Index: dialin/dg/calibration.py
===================================================================
diff -u -r6d387cadbda3027d23510126942eaa9f6317d357 -r6154b064dd181c32b09982c04ca9ddbca41bf509
--- dialin/dg/calibration.py	(.../calibration.py)	(revision 6d387cadbda3027d23510126942eaa9f6317d357)
+++ dialin/dg/calibration.py	(.../calibration.py)	(revision 6154b064dd181c32b09982c04ca9ddbca41bf509)
@@ -1,5 +1,6 @@
 
 import struct
+from collections import OrderedDict
 from ..utils.conversions import integer_to_bytearray
 from ..common.msg_defs import MsgIds, MsgFieldPositions
 from ..protocols.CAN import (DenaliMessage, DenaliChannels)
@@ -14,7 +15,22 @@
     """
 
     CALIBRATION_RECORD_START_INDEX = 6
+    CALIBRATION_RECORD_SPECS_BYTES = 12
+    DEFAULT_GAIN_VALUE = 1
+    DEFAULT_OFFSET_VALUE = 0
+    DEFAULT_RATIO_VALUE = 1
+    DEFAULT_CONSTANT_VALUE = 0
+    DEFAULT_CAL_TIME_VALUE = 0
+    DEFAULT_CAL_CRC_VALUE = 0
 
+    CURRENT_MESSAGE_NUM_INDEX = 0
+    TOTAL_MESSAGES_NUM_INDEX = 4
+    PAYLOAD_LENGTH_INDEX = 8
+    PAYLOAD_START_INDEX = 12
+
+    # DG calibration main record
+    DG_CALIBRATION_RECORD = OrderedDict()
+
     def __init__(self, can_interface, logger: Logger):
         """
 
@@ -33,34 +49,25 @@
 
         if self.can_interface is not None:
 
-            channel_id = DenaliChannels.dg_sync_broadcast_ch_id
+            channel_id = DenaliChannels.dg_to_dialin_ch_id
             msg_id = MsgIds.MSG_ID_DG_SEND_CALIBRATION_DATA.value
             self.can_interface.register_receiving_publication_function(channel_id, msg_id,
                                                                        self._handler_dg_calibration_sync)
+        # Prepare the calibration record by putting sub-dictionaries together
+        self._prepare_dg_calibration_record()
 
-    @_publish(["current_message", "total_messages", "length", "cal_data"])
-    def _handler_dg_calibration_sync(self, message):
+    def get_dg_calibration_data(self):
         """
-        Handles published ro pump data messages.  RO pump data are captured
-        for reference.
+        Handles getting dg calibration data from firmware.
 
-        @param message:  published RO pump data message
-
         @return: None
         """
-        curr = struct.unpack('i', bytearray(
-            message['message'][MsgFieldPositions.START_POS_FIELD_1:MsgFieldPositions.END_POS_FIELD_1]))[0]
-        total = struct.unpack('i', bytearray(
-            message['message'][MsgFieldPositions.START_POS_FIELD_2:MsgFieldPositions.END_POS_FIELD_2]))[0]
-        length = struct.unpack('i', bytearray(
-            message['message'][MsgFieldPositions.START_POS_FIELD_3:MsgFieldPositions.END_POS_FIELD_3]))[0]
+        # Clear the list for the next call
+        self.raw_cal_record.clear()
+        # Run the firmware commands to get the calibration record
+        self._request_dg_fw_calibration_data()
 
-        self.current_message = curr
-        self.total_messages = total
-        self.length = length
-        self.cal_data = message['message'][self.CALIBRATION_RECORD_START_INDEX:]
-
-    def get_dg_calibration_data(self):
+    def _request_dg_fw_calibration_data(self):
         """
         Handles getting DG calibration record from firmware.
 
@@ -81,6 +88,221 @@
             self.logger.debug("Timeout!!!!")
             return False
 
+    @_publish(["current_message", "total_messages", "length", "cal_data"]) #TODO do we need to publish these?
+    def _handler_dg_calibration_sync(self, message):
+        """
+        Handles published DG calibration record messages. DG calibration records are captured for processing and
+        updating the DG calibration record.
+
+        @param message:  published DG calibration record data message
+
+        @return: None
+        """
+        curr = struct.unpack('i', bytearray(
+            message['message'][MsgFieldPositions.START_POS_FIELD_1:MsgFieldPositions.END_POS_FIELD_1]))[0]
+        total = struct.unpack('i', bytearray(
+            message['message'][MsgFieldPositions.START_POS_FIELD_2:MsgFieldPositions.END_POS_FIELD_2]))[0]
+        length = struct.unpack('i', bytearray(
+            message['message'][MsgFieldPositions.START_POS_FIELD_3:MsgFieldPositions.END_POS_FIELD_3]))[0]
+
+        self.current_message = curr
+        self.total_messages = total
+        self.length = length
+        # The end of calibration record payload is from the start index + 12 bytes for the current message + total
+        # messages + the length of calibration. The rest is the CAN messaging CRC that is not needed to be kept
+        end_of_data_index = self.CALIBRATION_RECORD_START_INDEX + self.CALIBRATION_RECORD_SPECS_BYTES + self.length
+
+        # Get the calibration data only
+        self.cal_data = message['message'][self.CALIBRATION_RECORD_START_INDEX:end_of_data_index]
+
+        # Continue getting calibration records until the all the calibration messages are received. Concatenate the
+        # calibration records to each other
+        if self.current_message <= self.total_messages:
+            self.raw_cal_record += (message['message'][self.CALIBRATION_RECORD_START_INDEX +
+                                                       self.CALIBRATION_RECORD_SPECS_BYTES:end_of_data_index])
+            # If all the messages have been received, call another function to process the raw data
+            if self.current_message == self.total_messages:
+
+                self._update_dg_calibration_record_from_fw()
+
+    def _update_dg_calibration_record_from_fw(self):
+        """
+        Handles parsing the calibration messages that were received from DG firmware.
+
+        @return: None
+        """
+        value_bytes = 0
+        raw_payload_temp_start_index = 0
+        data = ''
+        # Convert the concatenated raw data into a byte array since the struct library requires byte arrays.
+        self.raw_cal_record = bytearray(self.raw_cal_record)
+
+        # Loop though the keys for the main calibration dictionary
+        # DG_Calibration : {pressure_sensors : { ppi : { gain: ['<f', 12.3], offset: ['<f', 5.67]}}, ...}
+        # Get the list of keys of the main dictionary (i.e. pressure_sensors, temperature_sensors, ....}
+        # These keys are the list of hardware groups
+        for group in self.DG_CALIBRATION_RECORD.keys():
+            # All the hardware groups (i.e pressure sensors) are nested dictionaries, except the final CRC. So it is
+            # unpacked exceptionally here. The CRC is an unsigned 16-bit (short).
+            if group == 'crc':
+
+                # Get the data type ( it must be 'h')
+                data_type = self.DG_CALIBRATION_RECORD[group][0]
+                # Check the data type of the crc
+                value_bytes = 2 if data_type == '<h' else 4
+                # Unpack the data
+                data = struct.unpack(data_type, self.raw_cal_record[raw_payload_temp_start_index:
+                                                                    raw_payload_temp_start_index + value_bytes])[0]
+            else:
+                # If the value is not a CRC, it is nested dictionary of different hardware groups
+                # Loop through the values of each key to get the list of the hardware in a hardware group (i.e
+                # pressure_sensors have ppi, ppo, ...)
+                for hardware_group in self.DG_CALIBRATION_RECORD[group].keys():
+                    # Each hardware has a set of calibration values (i.e ppi: {gain: ['<f',  12.3]}. Loop though those
+                    for hardware in self.DG_CALIBRATION_RECORD[group][hardware_group]:
+                        # The values of the dictionary of each hardware is a list of its data type as well as its value
+                        # which the first index is data type
+                        data_type = self.DG_CALIBRATION_RECORD[group][hardware_group][hardware][0]
+
+                        # The characters are needed for the unpack method the struct class and the integers are needed
+                        # to offset the index of the byte array for the next value to be parsed
+                        if data_type == '<f' or data_type == '<i':
+                            value_bytes = 4
+                        elif data_type == '<h':
+                            value_bytes = 2
+
+                        data = struct.unpack(data_type, self.raw_cal_record[raw_payload_temp_start_index:
+                                                                            raw_payload_temp_start_index +
+                                                                            value_bytes])[0]
+                        # Update the index of the raw payload for the next read
+                        raw_payload_temp_start_index += value_bytes
+
+                        # For testing only remove TODO Remove
+                        print(raw_payload_temp_start_index, group, hardware_group, hardware, data)
+
+    def _prepare_dg_calibration_record(self):
+        """
+        Handles assembling the sub dictionaries of each hardware group to make the main DG calibration record.
+
+        @return: None
+        """
+        # Call the other functions to get the dictionaries of each hardware group. All the dictionaries are
+        # ordered dictionaries to maintain the order in which they are inserted.
+        pressure = self._prepare_pressure_sensors_cal_record()
+        flow = self._prepare_flow_sensors_cal_record()
+        load_cells = self._prepare_load_cells_record()
+        temperature = self._prepare_temperature_sensors_record()
+        conductivity = self._prepare_conductivity_sensors_record()
+        # Build the CRC of the main calibration record
+        cal_record_crc = OrderedDict({'crc': ['<h', self.DEFAULT_CAL_CRC_VALUE]})
+        # Insert the sub dictionaries into the main calibration record
+        self.DG_CALIBRATION_RECORD = {**pressure, **flow, **load_cells, **temperature, **conductivity, **cal_record_crc}
+
+    def _prepare_pressure_sensors_cal_record(self):
+        """
+        Handles creating the calibration dictionary of the pressure sensors hardware group.
+
+        @return: pressure sensors hardware group dictionary
+        """
+        hardware_names = ['ppi', 'ppo', 'prd', 'pdr', 'reserved_1', 'reserved_2', 'reserved_3', 'reserved_4',
+                          'reserved_5', 'reserved_6']
+        group_name = 'pressure_sensors'
+        hardware_group = OrderedDict({group_name: OrderedDict()})
+        hardware = OrderedDict()
+
+        for i in hardware_names:
+
+            hardware[i] = {'gain': ['<f', self.DEFAULT_GAIN_VALUE], 'offset': ['<f', self.DEFAULT_OFFSET_VALUE],
+                           'cal_time': ['<i', self.DEFAULT_CAL_TIME_VALUE], 'crc': ['<h', self.DEFAULT_CAL_CRC_VALUE]}
+
+        hardware_group[group_name] = hardware
+
+        return hardware_group
+
+    def _prepare_flow_sensors_cal_record(self):
+        """
+        Handles creating the calibration dictionary of the flow sensors hardware group.
+
+        @return: flow sensors hardware group dictionary
+        """
+        hardware_names = ['flow_sensor', 'reserved_1', 'reserved_2', 'reserved_3']
+        group_name = 'flow_sensors'
+        hardware_group = OrderedDict({group_name: OrderedDict()})
+        hardware = OrderedDict()
+
+        for i in hardware_names:
+
+            hardware[i] = {'gain': ['<f', self.DEFAULT_GAIN_VALUE], 'offset': ['<f', self.DEFAULT_OFFSET_VALUE],
+                           'cal_time': ['<i', self.DEFAULT_CAL_TIME_VALUE], 'crc': ['<h', self.DEFAULT_CAL_CRC_VALUE]}
+
+        hardware_group[group_name] = hardware
+
+        return hardware_group
+
+    def _prepare_load_cells_record(self):
+        """
+        Handles creating the calibration dictionary of the load cells hardware group.
+
+        @return: load cells hardware group dictionary
+        """
+        hardware_names = ['load_cell_a1', 'load_cell_a2', 'load_cell_b1', 'load_cell_b2']
+        group_name = 'load_cells'
+        hardware_group = OrderedDict({group_name: OrderedDict()})
+        hardware = OrderedDict()
+
+        for i in hardware_names:
+
+            hardware[i] = {'gain': ['<f', self.DEFAULT_GAIN_VALUE], 'offset': ['<f', self.DEFAULT_OFFSET_VALUE],
+                           'cal_time': ['<i', self.DEFAULT_CAL_TIME_VALUE], 'crc': ['<h', self.DEFAULT_CAL_CRC_VALUE]}
+
+        hardware_group[group_name] = hardware
+
+        return hardware_group
+
+    def _prepare_temperature_sensors_record(self):
+        """
+        Handles creating the calibration dictionary of the temperature sensors hardware group.
+
+        @return: temperature sensors hardware group dictionary
+        """
+        hardware_names = ['fpga_board', 'load_cell_a1_b1', 'load_cell_a2_b2', 'thdo_rtd', 'tdi_rtd', 'internal_cond',
+                          'onboard_ntc', 'power_supply_1', 'power_supply_2', 'outlet_redundant', 'inlet_dialysate',
+                          'inlet_primary_heater', 'outlet_primary_heater', 'cond_sensor_1', 'cond_sensor_2',
+                          'reserved_1', 'reserved_2', 'reserved_3', 'reserved_4', 'reserved_5']
+        group_name = 'temperature_sensors'
+        hardware_group = OrderedDict({group_name: OrderedDict()})
+        hardware = OrderedDict()
+
+        for i in hardware_names:
+
+            hardware[i] = {'gain': ['<f', self.DEFAULT_GAIN_VALUE], 'offset': ['<f', self.DEFAULT_OFFSET_VALUE],
+                           'cal_time': ['<i', self.DEFAULT_CAL_TIME_VALUE], 'crc': ['<h', self.DEFAULT_CAL_CRC_VALUE]}
+
+        hardware_group[group_name] = hardware
+
+        return hardware_group
+
+    def _prepare_conductivity_sensors_record(self):
+        """
+        Handles creating the calibration dictionary of the conductivity sensors hardware group.
+
+        @return: conductivity sensors hardware group dictionary
+        """
+        hardware_names = ['cpi', 'cpo', 'cd1', 'cd2', 'reserved_1', 'reserved_2']
+        group_name = 'conductivity_sensors'
+        hardware_group = OrderedDict({group_name: OrderedDict()})
+        hardware = OrderedDict()
+
+        for i in hardware_names:
+
+            hardware[i] = {'gain': ['<f', self.DEFAULT_GAIN_VALUE], 'offset': ['<f', self.DEFAULT_OFFSET_VALUE],
+                           'cal_time': ['<i', self.DEFAULT_CAL_TIME_VALUE], 'crc': ['<h', self.DEFAULT_CAL_CRC_VALUE]}
+
+        hardware_group[group_name] = hardware
+
+        return hardware_group
+
+    #TODO update this function
     def set_dg_calibration_data(self, data):
 
         #test = {'sensors': {'ppi': {'gain': ['<f', 12.1], 'offset': ['<f', 2.2], 'cal_time': ['<i', 12345]}}}
Index: tests/hd_valves_test.py
===================================================================
diff -u -r91dc90bd009bdbf5621dcaa1bc12ab3d691673f7 -r6154b064dd181c32b09982c04ca9ddbca41bf509
--- tests/hd_valves_test.py	(.../hd_valves_test.py)	(revision 91dc90bd009bdbf5621dcaa1bc12ab3d691673f7)
+++ tests/hd_valves_test.py	(.../hd_valves_test.py)	(revision 6154b064dd181c32b09982c04ca9ddbca41bf509)
@@ -35,25 +35,67 @@
     #sleep(2)
 
     overall_counter = 0
-    sleep_time = 0.01
+    sleep_time = 0.05
     overall_time_seconds = 4
+    loops = 5
+    counter = 0
+    valve = ValvesEnum.VBV
 
-    valves.cmd_home_hd_valve(ValvesEnum.VBA.value)
+    valves.cmd_hd_valves_broadcast_interval_override(100)
     sleep(3)
 
-    valves.cmd_hd_valves_broadcast_interval_override(10)
-    sleep(3)
+    valves.cmd_home_hd_valve(valve.value)
+    sleep(5)
 
-    valves.cmd_set_hd_valve_position(ValvesEnum.VBA.value, ValvesPositions.VALVE_POSITION_C_CLOSE.value)
+    f = open("Valves_Test_200.txt", "w")
 
-    while overall_counter < (overall_time_seconds/sleep_time):
+    while counter < loops:
 
-        print(valves.valves_status[ValvesEnum.VBA.name])
+        print('Loop: {}'.format(counter))
+        overall_counter = 0
+        valves.cmd_set_hd_valve_position(valve.value, ValvesPositions.VALVE_POSITION_B_OPEN.value)
 
-        overall_counter = overall_counter + 1
+        while overall_counter < (overall_time_seconds/sleep_time):
+            sleep(sleep_time)
+            status = ('ID, {}, Pos, {}, PosCnt, {}, Cmd, {}, State, {}, Current, {}, PosA, {}, PosB, {}, PosC, {}, PWM, {} \r'
+                      .format(valves.valves_status[valve.name]['Valve'],
+                              valves.valves_status[valve.name]['PosID'],
+                              valves.valves_status[valve.name]['PosCnt'],
+                              valves.valves_status[valve.name]['Cmd'],
+                              valves.valves_status[valve.name]['State'],
+                              valves.valves_status[valve.name]['Current'],
+                              valves.valves_status[valve.name]['PosA'],
+                              valves.valves_status[valve.name]['PosB'],
+                              valves.valves_status[valve.name]['PosC'],
+                              valves.valves_status[valve.name]['PWM']))
+            print(status)
+            f.write(status)
+            overall_counter = overall_counter + 1
 
-        sleep(sleep_time)
+        overall_counter = 0
+        valves.cmd_set_hd_valve_position(valve.value, ValvesPositions.VALVE_POSITION_C_CLOSE.value)
 
+        while overall_counter < (overall_time_seconds / sleep_time):
+            sleep(sleep_time)
+            status = ('ID, {}, Pos, {}, PosCnt, {}, Cmd, {}, State, {}, Current, {}, PosA, {}, PosB, {}, PosC, {}, PWM, {} \r'
+                      .format(valves.valves_status[valve.name]['Valve'],
+                              valves.valves_status[valve.name]['PosID'],
+                              valves.valves_status[valve.name]['PosCnt'],
+                              valves.valves_status[valve.name]['Cmd'],
+                              valves.valves_status[valve.name]['State'],
+                              valves.valves_status[valve.name]['Current'],
+                              valves.valves_status[valve.name]['PosA'],
+                              valves.valves_status[valve.name]['PosB'],
+                              valves.valves_status[valve.name]['PosC'],
+                              valves.valves_status[valve.name]['PWM']))
+            print(status)
+            f.write(status)
+            overall_counter = overall_counter + 1
+
+        counter += 1
+
+    f.close()
+
     """
     print("Reset")
     #valves.cmd_set_hd_valve_pwm(2, 50, 0)
Index: tests/test_dg_calibration.py
===================================================================
diff -u -r6d387cadbda3027d23510126942eaa9f6317d357 -r6154b064dd181c32b09982c04ca9ddbca41bf509
--- tests/test_dg_calibration.py	(.../test_dg_calibration.py)	(revision 6d387cadbda3027d23510126942eaa9f6317d357)
+++ tests/test_dg_calibration.py	(.../test_dg_calibration.py)	(revision 6154b064dd181c32b09982c04ca9ddbca41bf509)
@@ -43,8 +43,8 @@
 
     try:
         while True:
-            sleep(0.05)
-            print(cal.temp, cal.cal_data)
+            sleep(0.2)
+            #print(cal.current_message, cal.total_messages, cal.cal_data)
 
             if cal.cal_data != 0:
                 f.write(str(cal.cal_data) + '\r')
@@ -53,10 +53,17 @@
                 if cal.current_message == cal.total_messages:
                     f.close()
                     break
-
         #[print(i) for i in dara]
-        value = struct.unpack('f', bytearray(dara[0][18:(18+4)]))[0]
-        print(value)
+        #value = struct.unpack('f', bytearray(dara[0][18:(18+4)]))[0]
+        #print(cal.DG_CALIBRATION_RECORD.keys())
 
+        #load_cells = OrderedDict({'group_name': OrderedDict()})
+        #individual_sensor = OrderedDict()
+        #list1 = ['load_1', 'load_2', 'load_3']
+        #for i in list1:
+        #    individual_sensor[i] = {'gain': ['<f', 12.5], 'offset': ['<f', 13.2], 'cal_time': ['<i', 7]}
+        #load_cells['group_name'] = individual_sensor
+        #print(load_cells['group_name']['load_2']['cal_time'])
+
     except KeyboardInterrupt:
         f.close()