Index: lib/MsgUtils/CMakeLists.txt =================================================================== diff -u -rcfc0df719cb5033078d0cac45ce0f6243810f2e7 -rfa691c15183dfe8596de699ef8c3a7fcdb90edab --- lib/MsgUtils/CMakeLists.txt (.../CMakeLists.txt) (revision cfc0df719cb5033078d0cac45ce0f6243810f2e7) +++ lib/MsgUtils/CMakeLists.txt (.../CMakeLists.txt) (revision fa691c15183dfe8596de699ef8c3a7fcdb90edab) @@ -21,6 +21,7 @@ # set(DENALI_MSG_CSV ${CMAKE_CURRENT_SOURCE_DIR}/../../data/FW_Messages_List.csv) set(LEAHI_MSG_CSV ${CMAKE_CURRENT_SOURCE_DIR}/../../data/Leahi_Staging_FW_Messages_List.csv) +set(LEAHI_MSG_CONF ${CMAKE_CURRENT_SOURCE_DIR}/../../data/Unhandled.conf) set(INCLUDES include/CanMessage.h @@ -68,8 +69,10 @@ # generate_msg_defs_cpp(DENALI_MSG_CSV Denali ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/src denali) # generate_protobuf(DENALI_MSG_CSV DenaliMsgDefs.proto ${CMAKE_CURRENT_BINARY_DIR}/data denali) # generate_protobuf_cpp(DenaliMsgDefs.proto ${CMAKE_CURRENT_BINARY_DIR}/data ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/src) -generate_msg_defs_cpp(LEAHI_MSG_CSV Leahi ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/src leahi) -generate_protobuf(LEAHI_MSG_CSV LeahiMsgDefs.proto ${CMAKE_CURRENT_BINARY_DIR}/data leahi) +# generate_msg_defs_cpp_csvs(LEAHI_MSG_CSV Leahi ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/src leahi) +# generate_protobuf_csvs(LEAHI_MSG_CSV LeahiMsgDefs.proto ${CMAKE_CURRENT_BINARY_DIR}/data leahi) +generate_msg_defs_cpp(LEAHI_MSG_CONF Leahi ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/src leahi) +generate_protobuf(LEAHI_MSG_CONF LeahiMsgDefs.proto ${CMAKE_CURRENT_BINARY_DIR}/data leahi) generate_protobuf_cpp(LeahiMsgDefs.proto ${CMAKE_CURRENT_BINARY_DIR}/data ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/src) add_library(${PROJECT_NAME} SHARED) Index: lib/MsgUtils/cmake/MsgUtils.cmake =================================================================== diff -u -rcfc0df719cb5033078d0cac45ce0f6243810f2e7 -rfa691c15183dfe8596de699ef8c3a7fcdb90edab --- lib/MsgUtils/cmake/MsgUtils.cmake (.../MsgUtils.cmake) (revision cfc0df719cb5033078d0cac45ce0f6243810f2e7) +++ lib/MsgUtils/cmake/MsgUtils.cmake (.../MsgUtils.cmake) (revision fa691c15183dfe8596de699ef8c3a7fcdb90edab) @@ -1,10 +1,101 @@ +# \brief This function adds a custom command to generate message defintions from inputted message conf files +# \param[in] _input_confs list of message conf files to use to generate the header +# \param[in] _device_name name of device these files are being generated for +# \param[in] _header_dir directory where the header file will be outputted +# \param[in] _source_dir directory where the source file will be outputted +# \param[in] _namespace C++ namespace for the outputted message definitions +function(generate_msg_defs_cpp _input_confs _device_name _header_dir _source_dir _namespace) + find_package(Python3 COMPONENTS Interpreter Development REQUIRED) + + # create a variable with the full path output header file + get_filename_component(_header_dir ${_header_dir} ABSOLUTE) # cmake < 3.20 + # cmake >= 3.20: cmake_path(ABSOLUTE_PATH _header_path) + string(JOIN "/" _header_path ${_header_dir} ${_device_name}MsgDefs.h) + + # create a variable with the full path output header file + get_filename_component(_source_dir ${_source_dir} ABSOLUTE) # cmake < 3.20 + # cmake >= 3.20: cmake_path(ABSOLUTE_PATH _source_path) + string(JOIN "/" _source_path ${_source_dir} ${_device_name}MsgDefs.cpp) + + # generate a pretty string containing a list of the relative paths of all csv files + foreach(_path ${${_input_confs}}) + # cmake >= 3.20: cmake_path(ABSOLUTE_PATH _path) + get_filename_component(_path ${_path} ABSOLUTE) + list(APPEND _files ${_path}) + endforeach() + list(JOIN _files ", " _file_list) + + add_custom_command( + DEPENDS + ${MSGUTILS_SCRIPTS_DIR}/MsgData.py + ${MSGUTILS_SCRIPTS_DIR}/MsgCpp.py + ${MSGUTILS_SCRIPTS_DIR}/GenerateMsgDefsCpp.py + ${MSGUTILS_SCRIPTS_DIR}/templates/MsgDefs_h.jinja + ${MSGUTILS_SCRIPTS_DIR}/templates/MsgDefs_cpp.jinja + ${${_input_confs}} + OUTPUT + ${_header_path} + ${_source_path} + COMMAND + ${Python3_EXECUTABLE} ${MSGUTILS_SCRIPTS_DIR}/GenerateMsgDefsCpp.py + --namespace ${_namespace} + --header_dir ${_header_dir} + --source_dir ${_source_dir} + ${${_input_confs}} + ${_device_name} + COMMENT "Generating MsgDefs C++ header ${_header_path} and source ${_source_path} from input ${_file_list}" + VERBATIM + ) +endfunction() + +# \brief This function adds a custom command to generate a Protobuf file from inputted message conf files +# \param[in] _input_confs list of message conf files to use to generate the header +# \param[in] _protobuf_filename name of the generated protobuf file +# \param[in] _output_path directory where the header file will be outputted +# \param[in] _namespace C++ namespace for the outputted message definitions +function(generate_protobuf _input_confs _protobuf_filename _output_path _namespace) + find_package(Python3 COMPONENTS Interpreter Development REQUIRED) + + # create a variable with the full path output header file + string(JOIN "/" _protobuf_abs_filename ${_output_path} ${_protobuf_filename}) + # cmake >= 3.20: cmake_path(ABSOLUTE_PATH _protobuf_abs_filename) + get_filename_component(_protobuf_abs_filename ${_protobuf_abs_filename} ABSOLUTE) # cmake < 3.20 + + # generate a pretty string containing a list of the relative paths of all csv files + foreach(_path ${${_input_confs}}) + # cmake >= 3.20: cmake_path(ABSOLUTE_PATH _path) + get_filename_component(_path ${_path} ABSOLUTE) # cmake < 3.20 + list(APPEND _files ${_path}) + endforeach() + list(JOIN _files ", " _file_list) + + add_custom_command( + DEPENDS + ${MSGUTILS_SCRIPTS_DIR}/MsgData.py + ${MSGUTILS_SCRIPTS_DIR}/MsgProtobuf.py + ${MSGUTILS_SCRIPTS_DIR}/GenerateProtobuf.py + ${MSGUTILS_SCRIPTS_DIR}/templates/MsgDefs_proto.jinja + ${${_input_confs}} + OUTPUT + ${_protobuf_abs_filename} + COMMAND + ${Python3_EXECUTABLE} ${MSGUTILS_SCRIPTS_DIR}/GenerateProtobuf.py + --namespace ${_namespace} + --output_dir ${_output_path} + ${${_input_confs}} + ${_protobuf_filename} + COMMENT "Generating Protobuf definitions ${_protobuf_abs_filename} from input ${_file_list}" + VERBATIM + ) +endfunction() + # \brief This function adds a custom command to generate message defintions from inputted CSV files # \param[in] _input_csvs list of CSV files to use to generate the header # \param[in] _device_name name of device these files are being generated for # \param[in] _header_dir directory where the header file will be outputted # \param[in] _source_dir directory where the source file will be outputted # \param[in] _namespace C++ namespace for the outputted message definitions -function(generate_msg_defs_cpp _input_csvs _device_name _header_dir _source_dir _namespace) +function(generate_msg_defs_cpp_csvs _input_csvs _device_name _header_dir _source_dir _namespace) find_package(Python3 COMPONENTS Interpreter Development REQUIRED) # create a variable with the full path output header file @@ -53,7 +144,7 @@ # \param[in] _protobuf_filename name of the generated protobuf file # \param[in] _output_path directory where the header file will be outputted # \param[in] _namespace C++ namespace for the outputted message definitions -function(generate_protobuf _input_csvs _protobuf_filename _output_path _namespace) +function(generate_protobuf_csvs _input_csvs _protobuf_filename _output_path _namespace) find_package(Python3 COMPONENTS Interpreter Development REQUIRED) # create a variable with the full path output header file Index: lib/MsgUtils/scripts/GenerateMsgDefsCpp.py =================================================================== diff -u -rcfc0df719cb5033078d0cac45ce0f6243810f2e7 -rfa691c15183dfe8596de699ef8c3a7fcdb90edab --- lib/MsgUtils/scripts/GenerateMsgDefsCpp.py (.../GenerateMsgDefsCpp.py) (revision cfc0df719cb5033078d0cac45ce0f6243810f2e7) +++ lib/MsgUtils/scripts/GenerateMsgDefsCpp.py (.../GenerateMsgDefsCpp.py) (revision fa691c15183dfe8596de699ef8c3a7fcdb90edab) @@ -8,9 +8,9 @@ def main(): parser = argparse.ArgumentParser( - description='Tool for generating C++ header and source files containing messages definitions for the inputted message csv file' + description='Tool for generating C++ header and source files containing messages definitions for the inputted message conf file' ) - parser.add_argument('csv', nargs='+') + parser.add_argument('conf', nargs='+') parser.add_argument('device_name', help='name of the device; used for naming the output C++ message definitions header and source files') parser.add_argument('--header_dir', help='output directory for the generated .h header file', default='.') parser.add_argument('--source_dir', help='output directory for the generated .cpp source file', default='.') @@ -23,8 +23,8 @@ else: msg_cpp = MsgCpp() try: - for csv in args.csv: - msg_cpp.load(csv) + for conf in args.conf: + msg_cpp.loadConf(conf) if args.header_dir is not None: os.makedirs(args.header_dir, exist_ok=True) if args.source_dir is not None: Index: lib/MsgUtils/scripts/GenerateProtobuf.py =================================================================== diff -u -rcfc0df719cb5033078d0cac45ce0f6243810f2e7 -rfa691c15183dfe8596de699ef8c3a7fcdb90edab --- lib/MsgUtils/scripts/GenerateProtobuf.py (.../GenerateProtobuf.py) (revision cfc0df719cb5033078d0cac45ce0f6243810f2e7) +++ lib/MsgUtils/scripts/GenerateProtobuf.py (.../GenerateProtobuf.py) (revision fa691c15183dfe8596de699ef8c3a7fcdb90edab) @@ -8,11 +8,11 @@ def main(): parser = argparse.ArgumentParser( - description='Tool for generating the protobuf input file from the inputted message csv file' + description='Tool for generating the protobuf input file from the inputted message conf file' ) parser.add_argument('--output_dir', help='output directory for the generated Protobuf file', default='.') parser.add_argument('--namespace', help='namespace for protobuf package definition') - parser.add_argument('csv', nargs='+', help='csv input file') + parser.add_argument('conf', nargs='+', help='conf input file') parser.add_argument('protobuf', help='filename of outputted Protobuf file') args = parser.parse_args() @@ -21,8 +21,8 @@ else: msg_protobuf = MsgProtobuf() try: - for csv in args.csv: - msg_protobuf.load(csv) + for conf in args.conf: + msg_protobuf.loadConf(conf) if args.output_dir is not None: os.makedirs(args.output_dir, exist_ok=True) msg_protobuf.write_proto(args.protobuf, args.namespace, args.output_dir) Index: lib/MsgUtils/scripts/MsgCpp.py =================================================================== diff -u -rcfc0df719cb5033078d0cac45ce0f6243810f2e7 -rfa691c15183dfe8596de699ef8c3a7fcdb90edab --- lib/MsgUtils/scripts/MsgCpp.py (.../MsgCpp.py) (revision cfc0df719cb5033078d0cac45ce0f6243810f2e7) +++ lib/MsgUtils/scripts/MsgCpp.py (.../MsgCpp.py) (revision fa691c15183dfe8596de699ef8c3a7fcdb90edab) @@ -17,12 +17,24 @@ # \param[in] clear If true, clear out any already loaded data before processing new data, # otherwise new data will be added to the existing data # \return none - def load(self, filename, clear=True): - super().load(filename, clear) + def loadCSV(self, filename, clear=True): + super().loadCSV(filename, clear) for (msg_id_value, msg) in self.data.items(): msg['cpp_struct'] = self.__struct_data(msg) + # \brief Load a .conf file (Unhandled.conf format) and cache the data. + # \note This will clear any previously cached data. + # \param[in] filename Filename of the .conf to load. + # \param[in] clear If true, clear out any already loaded data before processing new data, + # otherwise new data will be added to the existing data + # \return none + def loadConf(self, filename, clear=True): + super().loadConf(filename, clear) + for (msg_id_value, msg) in self.data.items(): + msg['cpp_struct'] = self.__struct_data(msg) + + # \brief Write the loaded MsgData to a C++ header file # \param[in] device_name name of the device used to name the output message definitions header file # \param[in] output_dir directory where the output header file will be written Index: lib/MsgUtils/scripts/MsgData.py =================================================================== diff -u -rcfc0df719cb5033078d0cac45ce0f6243810f2e7 -rfa691c15183dfe8596de699ef8c3a7fcdb90edab --- lib/MsgUtils/scripts/MsgData.py (.../MsgData.py) (revision cfc0df719cb5033078d0cac45ce0f6243810f2e7) +++ lib/MsgUtils/scripts/MsgData.py (.../MsgData.py) (revision fa691c15183dfe8596de699ef8c3a7fcdb90edab) @@ -5,6 +5,7 @@ import csv import os from pathlib import Path +import re import sys # \brief Class that reads and stores the message data. @@ -65,7 +66,7 @@ # \param[in] clear If true, clear out any already loaded data before processing new data, # otherwise new data will be added to the existing data # \return none - def load(self, filename, clear=True): + def loadCSV(self, filename, clear=True): filename = Path(filename).resolve() def get_field_from_entry(entry, column): if column in self._msg_col and len(entry) > self._msg_col.index(column): @@ -127,6 +128,91 @@ print(f'Loaded {filename} with {entry_count} entries') + # \brief Load a .conf file (Unhandled.conf format) and cache the data. + # \note This will clear any previously cached data. + # \param[in] filename Filename of the .conf to load. + # \param[in] clear If true, clear out any already loaded data before processing new data, + # otherwise new data will be added to the existing data + # \return none + def loadConf(self, filename, clear=True): + name_regex = re.compile(r'[a-zA-Z][a-zA-Z0-9_]*') + filename = Path(filename).resolve() + if clear: + self.data = { } + entry_count = 0 + row_data = None + with open(filename, mode='r', encoding='utf-8') as in_file: + for line in in_file: + line = line.strip() + # blank line and message has been collected, end of message section + if not line and row_data is not None: + # check to see if data has already been collected for this msg_id_value + if row_data['msg_id_value'] in self.data: + print(f"WARNING: found MesgIDs with same value, {self.data[row_data['msg_id_value']]['msg_id']} will be replaced by {row_data['msg_id']} for value {row_data['msg_id_hex_string']}") + for item in self.data.values(): + if item['msg_name'] == row_data['msg_name']: + print(f"WARNING: found MesgIDs with same name, {row_data['msg_name']}, skipping message") + row_data = None + break + if row_data is not None: + if row_data['msg_id'] == '': + print(f"WARNING: message with ID {row_data['msg_id_hex_string']} contains blank msg_id, skipping message") + elif row_data['msg_name'] == '': + print(f"WARNING: message with ID {row_data['msg_id_hex_string']} contains blank msg_name, skipping message") + else: + self.data[row_data['msg_id_value']] = row_data + entry_count += 1 + row_data = None + elif line.startswith('#'): + if row_data is not None: + row_data['raw'] += line + '\n' + continue + # new message section start + elif line.startswith('[') and line.endswith(']') and row_data is None: + id_value = int(line[1:-1], 16) + row_data = { + 'msg_id': '', + 'msg_id_value': id_value, + 'msg_id_hex_string': MsgData.value_to_hex_string(id_value), + 'can_channel': '', + 'can_channel_value': 0, + 'ack_status': False, + 'msg_name': '', + 'payload': [], + 'raw': line + '\n', + } + # message name (first line of message section) + elif row_data is not None and row_data['msg_id'] == '' and re.fullmatch(name_regex, line): + row_data['raw'] += line + '\n' + row_data['msg_id'] = 'MSG_ID_' + '_'.join(word.upper() for word in line.split('_')) + row_data['msg_name'] = self.__message_name(row_data['msg_id']) + # payload parameter + elif row_data is not None and '=' in line: + row_data['raw'] += line + '\n' + result = line.strip().split('=', 1) + field_type = result[0] + field_name = result[1][:1].lower() + result[1][1:] if result[1] else '' + if not re.fullmatch(name_regex, field_name): + print(f"WARNING: {row_data['msg_id']} ({row_data['msg_id_hex_string']}) contains invalid field name \"{result[1]}\", skipping message") + row_data = None + continue + count = 0 + base_name = field_name + while any(field['name'] == field_name for field in row_data['payload']): + if count == 0: + print(f"WARNING: {row_data['msg_id']} ({row_data['msg_id_hex_string']}) contains duplicate field \"{field_name}\"") + count += 1 + field_name = f"{base_name}{count}" + row_data['payload'].append({ + 'raw': line.strip(), + 'name': field_name, + 'type': field_type, + }) + self.data = OrderedDict(sorted(self.data.items(), + key=lambda x: (int(x[1]['msg_id_value'] if x[1]['msg_id_value'] is not None else 99999), x[0]))) + print(f'Loaded {filename} with {entry_count} entries') + + # \brief Sort the cached csv data. # \return none def sort(self): Index: lib/MsgUtils/scripts/templates/MsgDefs_cpp.jinja =================================================================== diff -u -rcfc0df719cb5033078d0cac45ce0f6243810f2e7 -rfa691c15183dfe8596de699ef8c3a7fcdb90edab --- lib/MsgUtils/scripts/templates/MsgDefs_cpp.jinja (.../MsgDefs_cpp.jinja) (revision cfc0df719cb5033078d0cac45ce0f6243810f2e7) +++ lib/MsgUtils/scripts/templates/MsgDefs_cpp.jinja (.../MsgDefs_cpp.jinja) (revision fa691c15183dfe8596de699ef8c3a7fcdb90edab) @@ -130,7 +130,7 @@ auto proto = payload.toProtobuf(); updateHeader(proto.mutable_header()); std::ostringstream out; - proto.SerializeToOstream(&out); + (void)proto.SerializeToOstream(&out); return out.str().c_str(); break; } Index: lib/MsgUtils/src/format.cpp =================================================================== diff -u -rcfc0df719cb5033078d0cac45ce0f6243810f2e7 -rfa691c15183dfe8596de699ef8c3a7fcdb90edab --- lib/MsgUtils/src/format.cpp (.../format.cpp) (revision cfc0df719cb5033078d0cac45ce0f6243810f2e7) +++ lib/MsgUtils/src/format.cpp (.../format.cpp) (revision fa691c15183dfe8596de699ef8c3a7fcdb90edab) @@ -78,7 +78,13 @@ { QByteArray mData; - switch (static_cast(vData.type())) { + switch (static_cast( +#if QT_VERSION >= QT_VERSION_CHECK(6, 0, 0) + vData.typeId() +#else + vData.type() +#endif + )) { case QMetaType::QString: // string { mData += vData.toByteArray();