Index: lib/MsgUtils/CMakeLists.txt =================================================================== diff -u -rac303b902c681a25ff0d910dd56ab309669e381f -rbde1243eb2ff6af2868f6c6ad0cb4f5760aaf68b --- lib/MsgUtils/CMakeLists.txt (.../CMakeLists.txt) (revision ac303b902c681a25ff0d910dd56ab309669e381f) +++ lib/MsgUtils/CMakeLists.txt (.../CMakeLists.txt) (revision bde1243eb2ff6af2868f6c6ad0cb4f5760aaf68b) @@ -15,7 +15,7 @@ find_package(QT NAMES Qt6 Qt5 REQUIRED COMPONENTS Core) find_package(Qt${QT_VERSION_MAJOR} REQUIRED COMPONENTS Core Network SerialBus) -set(MSGUTILS_SCRIPTS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/scripts) +set(MSGUTILS_SCRIPTS_DIR ${CMAKE_SOURCE_DIR}/scripts/MsgUtils) # Create a build-local venv so pip install works on distros that enforce PEP 668 # (Debian/Ubuntu with Python 3.11+, etc.) without polluting system or user site-packages. Index: lib/MsgUtils/MsgUtilsConfig.cmake.in =================================================================== diff -u -rcfc0df719cb5033078d0cac45ce0f6243810f2e7 -rbde1243eb2ff6af2868f6c6ad0cb4f5760aaf68b --- lib/MsgUtils/MsgUtilsConfig.cmake.in (.../MsgUtilsConfig.cmake.in) (revision cfc0df719cb5033078d0cac45ce0f6243810f2e7) +++ lib/MsgUtils/MsgUtilsConfig.cmake.in (.../MsgUtilsConfig.cmake.in) (revision bde1243eb2ff6af2868f6c6ad0cb4f5760aaf68b) @@ -7,7 +7,7 @@ # Compute paths get_filename_component(MSGUTILS_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) set(MSGUTILS_INCLUDE_DIRS "@CONF_INCLUDE_DIRS@") -set(MSGUTILS_SCRIPTS_DIR "${MSGUTILS_SOURCE_DIR}/scripts") +set(MSGUTILS_SCRIPTS_DIR "${CMAKE_SOURCE_DIR}/scripts/MsgUtils") # Our library dependencies (contains definitions for IMPORTED targets) if(NOT TARGET MsgUtils AND NOT MsgUtils_BINARY_DIR) Fisheye: Tag bde1243eb2ff6af2868f6c6ad0cb4f5760aaf68b refers to a dead (removed) revision in file `lib/MsgUtils/scripts/GenerateMsgDefsCpp.py'. Fisheye: No comparison available. Pass `N' to diff? Fisheye: Tag bde1243eb2ff6af2868f6c6ad0cb4f5760aaf68b refers to a dead (removed) revision in file `lib/MsgUtils/scripts/GenerateProtobuf.py'. Fisheye: No comparison available. Pass `N' to diff? Fisheye: Tag bde1243eb2ff6af2868f6c6ad0cb4f5760aaf68b refers to a dead (removed) revision in file `lib/MsgUtils/scripts/msgutils/MsgCpp.py'. Fisheye: No comparison available. Pass `N' to diff? Fisheye: Tag bde1243eb2ff6af2868f6c6ad0cb4f5760aaf68b refers to a dead (removed) revision in file `lib/MsgUtils/scripts/msgutils/MsgData.py'. Fisheye: No comparison available. Pass `N' to diff? Fisheye: Tag bde1243eb2ff6af2868f6c6ad0cb4f5760aaf68b refers to a dead (removed) revision in file `lib/MsgUtils/scripts/msgutils/MsgProtobuf.py'. Fisheye: No comparison available. Pass `N' to diff? Fisheye: Tag bde1243eb2ff6af2868f6c6ad0cb4f5760aaf68b refers to a dead (removed) revision in file `lib/MsgUtils/scripts/msgutils/__init__.py'. Fisheye: No comparison available. Pass `N' to diff? Fisheye: Tag bde1243eb2ff6af2868f6c6ad0cb4f5760aaf68b refers to a dead (removed) revision in file `lib/MsgUtils/scripts/msgutils/templates/MsgDefs_cpp.jinja'. Fisheye: No comparison available. Pass `N' to diff? Fisheye: Tag bde1243eb2ff6af2868f6c6ad0cb4f5760aaf68b refers to a dead (removed) revision in file `lib/MsgUtils/scripts/msgutils/templates/MsgDefs_h.jinja'. Fisheye: No comparison available. Pass `N' to diff? Fisheye: Tag bde1243eb2ff6af2868f6c6ad0cb4f5760aaf68b refers to a dead (removed) revision in file `lib/MsgUtils/scripts/msgutils/templates/MsgDefs_proto.jinja'. Fisheye: No comparison available. Pass `N' to diff? Fisheye: Tag bde1243eb2ff6af2868f6c6ad0cb4f5760aaf68b refers to a dead (removed) revision in file `lib/MsgUtils/scripts/pyproject.toml'. Fisheye: No comparison available. Pass `N' to diff? Index: scripts/MsgUtils/GenerateMsgDefsCpp.py =================================================================== diff -u --- scripts/MsgUtils/GenerateMsgDefsCpp.py (revision 0) +++ scripts/MsgUtils/GenerateMsgDefsCpp.py (revision bde1243eb2ff6af2868f6c6ad0cb4f5760aaf68b) @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import argparse +import os +import sys +from msgutils import MsgCpp + +def main(): + parser = argparse.ArgumentParser( + description='Tool for generating C++ header and source files containing messages definitions for the inputted message conf file' + ) + parser.add_argument('conf', nargs='+') + parser.add_argument('device_name', help='name of the device; used for naming the output C++ message definitions header and source files') + parser.add_argument('--header_dir', help='output directory for the generated .h header file', default='.') + parser.add_argument('--source_dir', help='output directory for the generated .cpp source file', default='.') + parser.add_argument('--namespace', dest='namespace', help='C++ namespace scope for generated C++ files', required=False) + parser.add_argument('--proto', action="store_true", help='add protobuf utility functions to generated C++ files', required=False) + + args = parser.parse_args() + if len(sys.argv) < 3: + parser.print_help() + else: + msg_cpp = MsgCpp() + try: + for conf in args.conf: + msg_cpp.loadConf(conf) + if args.header_dir is not None: + os.makedirs(args.header_dir, exist_ok=True) + if args.source_dir is not None: + os.makedirs(args.source_dir, exist_ok=True) + if args.device_name is not None: + msg_cpp.write_msg_defs_header(f"{args.device_name}", args.header_dir, args.namespace, args.proto) + msg_cpp.write_msg_defs_source(f"{args.device_name}", args.source_dir, args.namespace, args.proto) + except Exception as e: + print('Error: %s' % e) + sys.exit(1) + +if __name__ == "__main__": # calling main function + main() Index: scripts/MsgUtils/GenerateProtobuf.py =================================================================== diff -u --- scripts/MsgUtils/GenerateProtobuf.py (revision 0) +++ scripts/MsgUtils/GenerateProtobuf.py (revision bde1243eb2ff6af2868f6c6ad0cb4f5760aaf68b) @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import argparse +import os +import sys +from msgutils import MsgProtobuf + +def main(): + parser = argparse.ArgumentParser( + description='Tool for generating the protobuf input file from the inputted message conf file' + ) + parser.add_argument('--output_dir', help='output directory for the generated Protobuf file', default='.') + parser.add_argument('--namespace', help='namespace for protobuf package definition') + parser.add_argument('conf', nargs='+', help='conf input file') + parser.add_argument('protobuf', help='filename of outputted Protobuf file') + + args = parser.parse_args() + if len(sys.argv) < 2: + parser.print_help() + else: + msg_protobuf = MsgProtobuf() + try: + for conf in args.conf: + msg_protobuf.loadConf(conf) + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + msg_protobuf.write_proto(args.protobuf, args.namespace, args.output_dir) + except Exception as e: + print('Error: %s' % e) + sys.exit(1) + # msg_data.dump() + +if __name__ == "__main__": # calling main function + main() Index: scripts/MsgUtils/msgutils.egg-info/PKG-INFO =================================================================== diff -u --- scripts/MsgUtils/msgutils.egg-info/PKG-INFO (revision 0) +++ scripts/MsgUtils/msgutils.egg-info/PKG-INFO (revision bde1243eb2ff6af2868f6c6ad0cb4f5760aaf68b) @@ -0,0 +1,6 @@ +Metadata-Version: 2.4 +Name: msgutils +Version: 1.0.0 +Summary: Diality Message Utilities +Requires-Python: >=3.9 +Requires-Dist: jinja2 Index: scripts/MsgUtils/msgutils.egg-info/SOURCES.txt =================================================================== diff -u --- scripts/MsgUtils/msgutils.egg-info/SOURCES.txt (revision 0) +++ scripts/MsgUtils/msgutils.egg-info/SOURCES.txt (revision bde1243eb2ff6af2868f6c6ad0cb4f5760aaf68b) @@ -0,0 +1,16 @@ +GenerateMsgDefsCpp.py +GenerateProtobuf.py +pyproject.toml +msgutils/MsgCpp.py +msgutils/MsgData.py +msgutils/MsgProtobuf.py +msgutils/__init__.py +msgutils.egg-info/PKG-INFO +msgutils.egg-info/SOURCES.txt +msgutils.egg-info/dependency_links.txt +msgutils.egg-info/entry_points.txt +msgutils.egg-info/requires.txt +msgutils.egg-info/top_level.txt +msgutils/templates/MsgDefs_cpp.jinja +msgutils/templates/MsgDefs_h.jinja +msgutils/templates/MsgDefs_proto.jinja \ No newline at end of file Index: scripts/MsgUtils/msgutils.egg-info/dependency_links.txt =================================================================== diff -u --- scripts/MsgUtils/msgutils.egg-info/dependency_links.txt (revision 0) +++ scripts/MsgUtils/msgutils.egg-info/dependency_links.txt (revision bde1243eb2ff6af2868f6c6ad0cb4f5760aaf68b) @@ -0,0 +1 @@ + Index: scripts/MsgUtils/msgutils.egg-info/entry_points.txt =================================================================== diff -u --- scripts/MsgUtils/msgutils.egg-info/entry_points.txt (revision 0) +++ scripts/MsgUtils/msgutils.egg-info/entry_points.txt (revision bde1243eb2ff6af2868f6c6ad0cb4f5760aaf68b) @@ -0,0 +1,3 @@ +[console_scripts] +GenerateMsgDefsCpp = GenerateMsgDefsCpp:main +GenerateProtobuf = GenerateProtobuf:main Index: scripts/MsgUtils/msgutils.egg-info/requires.txt =================================================================== diff -u --- scripts/MsgUtils/msgutils.egg-info/requires.txt (revision 0) +++ scripts/MsgUtils/msgutils.egg-info/requires.txt (revision bde1243eb2ff6af2868f6c6ad0cb4f5760aaf68b) @@ -0,0 +1 @@ +jinja2 Index: scripts/MsgUtils/msgutils.egg-info/top_level.txt =================================================================== diff -u --- scripts/MsgUtils/msgutils.egg-info/top_level.txt (revision 0) +++ scripts/MsgUtils/msgutils.egg-info/top_level.txt (revision bde1243eb2ff6af2868f6c6ad0cb4f5760aaf68b) @@ -0,0 +1 @@ +msgutils Index: scripts/MsgUtils/msgutils/MsgCpp.py =================================================================== diff -u --- scripts/MsgUtils/msgutils/MsgCpp.py (revision 0) +++ scripts/MsgUtils/msgutils/MsgCpp.py (revision bde1243eb2ff6af2868f6c6ad0cb4f5760aaf68b) @@ -0,0 +1,176 @@ +from jinja2 import Environment, FileSystemLoader +from .MsgData import MsgData +from pathlib import Path + +# \brief Class that usees the loaded MsgData and outputs converts to Protobuf +class MsgCpp(MsgData): + # \brief Initializer + def __init__(self): + super().__init__() + + + # \brief Load the csv files and cache the data. + # \note This will clear any previously cached csv data. + # \param[in] filename Filename of the csv to load. + # \param[in] clear If true, clear out any already loaded data before processing new data, + # otherwise new data will be added to the existing data + # \return none + def loadCSV(self, filename, clear=True): + super().loadCSV(filename, clear) + for (msg_id_value, msg) in self.data.items(): + msg['cpp_struct'] = self.__struct_data(msg) + + + # \brief Load a .conf file (Unhandled.conf format) and cache the data. + # \note This will clear any previously cached data. + # \param[in] filename Filename of the .conf to load. + # \param[in] clear If true, clear out any already loaded data before processing new data, + # otherwise new data will be added to the existing data + # \return none + def loadConf(self, filename, clear=True): + super().loadConf(filename, clear) + for (msg_id_value, msg) in self.data.items(): + msg['cpp_struct'] = self.__struct_data(msg) + + + # \brief Write the loaded MsgData to a C++ header file + # \param[in] device_name name of the device used to name the output message definitions header file + # \param[in] output_dir directory where the output header file will be written + # \param[in] namespace namespace to use in the header file, if namespace is blank then no namespace will be added + # \param[in] proto flag to add protobuf utility function to generated header file + def write_msg_defs_header(self, device_name, output_dir, namespace=None, proto=False): + env = Environment(loader = FileSystemLoader(f'{Path(__file__).parent.absolute()}/templates')) + template = env.get_template('MsgDefs_h.jinja') + render = template.render({ + 'msg_cpp': self, + 'cpp_namespace': namespace, + 'proto': proto, + 'device_name': device_name + }) + header_path = Path(output_dir).joinpath(f"{device_name}MsgDefs.h") + with open(header_path, mode='w', encoding='utf-8', newline='\n') as out_file: + out_file.write(render) + out_file.close() + print(f"Wrote C++ message definitions header file {header_path}") + + + # \brief Write the loaded MsgData to a source file + # \param[in] device_name name of the device used to name the output message definitions source file + # \param[in] output_dir directory where the output header file will be written + # \param[in] namespace namespace to use in the header file, if namespace is blank then no namespace will be added + # \param[in] proto flag to add protobuf utility function to generated header file + def write_msg_defs_source(self, device_name, output_dir, namespace=None, proto=False): + env = Environment(loader = FileSystemLoader(f'{Path(__file__).parent.absolute()}/templates')) + template = env.get_template('MsgDefs_cpp.jinja') + render = template.render({ + 'msg_cpp': self, + 'msg_defs_header': f"{device_name}MsgDefs.h", + 'cpp_namespace': namespace, + 'proto': proto, + }) + srcPath = Path(output_dir).joinpath(f"{device_name}MsgDefs.cpp") + with open(srcPath, mode='w', encoding='utf-8', newline='\n') as out_file: + out_file.write(render) + out_file.close() + print(f"Wrote C++ message definitions source file {srcPath}") + + + def write_log_parser(self, basename, header_dir, source_dir, namespace, msg_defs_header): + with open((header_dir.rstrip('/') + "/" if header_dir else "") + basename + '.h', mode='w', encoding='utf-8', newline='\n') as out_file: + includes = set() + includes.add(f"\"{msg_defs_header}\"") + if len(includes): + for key, value in self.__includeGroups(includes): + out_file.write("\n".join(f"#include {include}" for include in value) + "\n\n") + if namespace: + out_file.write(f"namespace {namespace}\n{{\n\n") + if namespace: + out_file.write(f"\n}} // namespace {namespace}\n") + out_file.close() + print(f'Wrote message source file {(output_dir.rstrip("/") + "/" if output_dir else "") + basename}.cpp') + + + def write_source(self, basename, output_dir, namespace): + with open((output_dir.rstrip('/') + "/" if output_dir else "") + basename + '.cpp', mode='w', encoding='utf-8', newline='\n') as out_file: + includes = set() + includes.add(f"\"{Path(basename + '.h').name}\"") + if len(includes): + for key, value in self.__includeGroups(includes): + out_file.write("\n".join(f"#include {include}" for include in value) + "\n\n") + if namespace: + out_file.write(f"namespace {namespace}\n{{\n\n") + if namespace: + out_file.write(f"\n}} // namespace {namespace}\n") + out_file.close() + print(f'Wrote message source file {(output_dir.rstrip("/") + "/" if output_dir else "") + basename}.cpp') + + + def __includeGroups(self, includes): + # sort the includes + groups = { } + # create groups for the includes based on their search order (i.e. , "include") + for include in includes: + if include[0] not in groups: + groups[include[0]] = [ ] + groups[include[0]].append(include) + # sort the groups, system includes () first and local includes ("include") second + groups = sorted(groups.items(), key=lambda x: x[0], reverse=True) + # sort each group of includes based on alphabetic order of the filename + for key, value in groups: + value.sort() + return groups + + + def __struct_data(self, msg_def): + def field_type_to_cpp_type(field_type): + if field_type == 'BOOL': + return "Types::U32" + elif field_type == 'U08': + return "Types::U08" + elif field_type == 'U16': + return "Types::U16" + elif field_type == 'U32': + return "Types::U32" + elif field_type == 'S08': + return "Types::S08" + elif field_type == 'S16': + return "Types::S16" + elif field_type == 'S32': + return "Types::S32" + elif field_type == 'F32': + return "Types::F32" + elif field_type == 'union': + return "ParamUnion" + else: + return None + def field_type_size(field_type): + if field_type == 'BOOL': + return 4 + elif field_type == 'U08': + return 1 + elif field_type == 'U16': + return 2 + elif field_type == 'U32': + return 4 + elif field_type == 'S08': + return 1 + elif field_type == 'S16': + return 2 + elif field_type == 'S32': + return 4 + elif field_type == 'F32': + return 4 + elif field_type == 'union': + return 4 + else: + return 0 + struct_info = { + 'payload': [ ], + 'payload_size': 0, + } + for field in msg_def['payload']: + struct_info['payload_size'] += field_type_size(field['type']) + struct_info['payload'].append({ 'name': field['name'], 'type': field['type'], 'cpp_type': field_type_to_cpp_type(field['type']) }) + if struct_info['payload'][-1]['type'] is None: + print(f"WARNING: unhandled type \"{field['type']}\" for field {field['name']} in message {msg_def['msg_id']} ({msg_def['msg_id_hex_string']})") + return struct_info Index: scripts/MsgUtils/msgutils/MsgData.py =================================================================== diff -u --- scripts/MsgUtils/msgutils/MsgData.py (revision 0) +++ scripts/MsgUtils/msgutils/MsgData.py (revision bde1243eb2ff6af2868f6c6ad0cb4f5760aaf68b) @@ -0,0 +1,234 @@ + +from collections import OrderedDict +import csv +import os +from pathlib import Path +import re +import sys + +# \brief Class that reads and stores the message data. +class MsgData(object): + # \brief Initializer + def __init__(self): + self.data = { } + self._msg_col = [ 'hex_msg_id', 'dec_msg_id', 'msg_id', 'source', 'module', 'type', 'can_channel', 'hex_can_channel', 'ack_status', 'payload' ] + + + @property + def field_types(self): + types = set() + for (msg_id_value, msg) in self.data.items(): + types.update(field['type'].upper() for field in msg['payload']) + return list(sorted(types)) + + + def field_list(self, msg_id_value, empty_string=True): + fields = [ ] + for field in self.data[msg_id_value]['payload']: + fields.append(f"{field['type']}-{field['name']}") + if len(fields) == 0: + fields.append("") + return fields + + + # \brief Converts a value to a hex string + # \return hex string representation of the inputted value + @staticmethod + def value_to_hex_string(value): + return f"0x{f'{value:x}'.upper()}" + + + # \brief Print all loaded data to the console + # \return none + def dump(self): + for row in self.data: + print(f"{row}:") + print(f" msg_name: {self.data[row]['msg_name']}") + print(f" msg_id: {self.data[row]['msg_id_hex_string']} ({self.data[row]['msg_id_value']})") + print(f" can_channel: {self.data[row]['can_channel']} ({MsgData.value_to_hex_string(self.data[row]['can_channel_value'])})") + print(f" ack_status: {self.data[row]['ack_status']}") + if (len(self.data[row]['payload'])): + print(f" payload: [") + for field in self.data[row]['payload']: + print(f" {field['name']} : {field['type']}") + print(f" ]") + else: + print(f" payload: [ ]") + print(f" raw: {self.data[row]['raw']}") + print(f"") + + + # \brief Load the csv files and cache the data. + # \note This will clear any previously cached csv data. + # \param[in] filename Filename of the csv to load. + # \param[in] clear If true, clear out any already loaded data before processing new data, + # otherwise new data will be added to the existing data + # \return none + def loadCSV(self, filename, clear=True): + filename = Path(filename).resolve() + def get_field_from_entry(entry, column): + if column in self._msg_col and len(entry) > self._msg_col.index(column): + return entry[self._msg_col.index(column)].strip() + return '' + def get_int_field_from_entry(entry, column): + value = get_field_from_entry(entry, column) + return int(value) if value != '' else 0 + def get_hex_field_from_entry(entry, column): + value = get_field_from_entry(entry, column) + return int(value, 16) if value != '' else 0 + # clear the previously cached csv data, if requested + if clear: + self.data = { } + with open(filename, mode='r', encoding='utf-8') as in_file: + # skip the first row, it is the header + header = in_file.readline() + entry_count = 0 + # loop thru each entry/row in the file and read in the data + for entry in list(csv.reader(in_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)): + entry_count += 1 + msg_id_value = get_int_field_from_entry(entry, 'dec_msg_id') + row_data = { + 'msg_id': entry[self._msg_col.index('msg_id')].strip(), + 'msg_id_value': get_int_field_from_entry(entry, 'dec_msg_id'), + 'msg_id_hex_string': MsgData.value_to_hex_string(get_int_field_from_entry(entry, 'dec_msg_id')), + 'can_channel': get_field_from_entry(entry, 'can_channel'), + 'can_channel_value': get_hex_field_from_entry(entry, 'hex_can_channel'), + 'ack_status': bool(get_field_from_entry(entry, 'ack_status') == 'ACK_REQUIRED'), + 'msg_name': self.__message_name(entry[self._msg_col.index('msg_id')].strip()), + 'payload': [ ], + 'raw': entry, + } + for i in range(len(self._msg_col), len(entry)): + if '-' in entry[i]: + result = entry[i].split('-', 1) + if '.' in result[1]: + print(f"WARNING: {row_data['msg_id']} ({row_data['msg_id_hex_string']}) field \"{entry[i]}\" contains illegal character \'.\'") + result[1] = result[1].replace(' ', '').split('.')[-1] + field_name = result[1] + count = 0 + while any(field['name'] == field_name for field in row_data['payload']): + if count == 0: + print(f"WARNING: {row_data['msg_id']} ({row_data['msg_id_hex_string']}) contains duplicate field \"{field_name}\"") + count += 1 + field_name = f"{result[1]}{count}" + row_data['payload'].append({ + 'raw': entry[i], + 'name': field_name, + 'type': result[0], + }) + if msg_id_value in self.data: + print(f"WARNING: found MesgIDs with same value, {self.data[msg_id_value]['msg_id']} will be replaced by {row_data['msg_id']} for value {row_data['msg_id_hex_string']}") + self.data[msg_id_value] = row_data + in_file.close() + # sort the collected entries + self.data = OrderedDict(sorted(self.data.items(), + key=lambda x: (int(x[1]['msg_id_value'] if x[1]['msg_id_value'] is not None else 99999), x[0]))) + print(f'Loaded {filename} with {entry_count} entries') + + + # \brief Load a .conf file (Unhandled.conf format) and cache the data. + # \note This will clear any previously cached data. + # \param[in] filename Filename of the .conf to load. + # \param[in] clear If true, clear out any already loaded data before processing new data, + # otherwise new data will be added to the existing data + # \return none + def loadConf(self, filename, clear=True): + name_regex = re.compile(r'[a-zA-Z][a-zA-Z0-9_]*') + filename = Path(filename).resolve() + if clear: + self.data = { } + entry_count = 0 + row_data = None + with open(filename, mode='r', encoding='utf-8') as in_file: + for line in in_file: + line = line.strip() + # blank line and message has been collected, end of message section + if not line and row_data is not None: + # check to see if data has already been collected for this msg_id_value + if row_data['msg_id_value'] in self.data: + print(f"WARNING: found MesgIDs with same value, {self.data[row_data['msg_id_value']]['msg_id']} will be replaced by {row_data['msg_id']} for value {row_data['msg_id_hex_string']}") + for item in self.data.values(): + if item['msg_name'] == row_data['msg_name']: + print(f"WARNING: found MesgIDs with same name, {row_data['msg_name']}, skipping message") + row_data = None + break + if row_data is not None: + if row_data['msg_id'] == '': + print(f"WARNING: message with ID {row_data['msg_id_hex_string']} contains blank msg_id, skipping message") + elif row_data['msg_name'] == '': + print(f"WARNING: message with ID {row_data['msg_id_hex_string']} contains blank msg_name, skipping message") + else: + self.data[row_data['msg_id_value']] = row_data + entry_count += 1 + row_data = None + elif line.startswith('#'): + if row_data is not None: + row_data['raw'] += line + '\n' + continue + # new message section start + elif line.startswith('[') and line.endswith(']') and row_data is None: + id_value = int(line[1:-1], 16) + row_data = { + 'msg_id': '', + 'msg_id_value': id_value, + 'msg_id_hex_string': MsgData.value_to_hex_string(id_value), + 'can_channel': '', + 'can_channel_value': 0, + 'ack_status': False, + 'msg_name': '', + 'payload': [], + 'raw': line + '\n', + } + # message name (first line of message section) + elif row_data is not None and row_data['msg_id'] == '' and re.fullmatch(name_regex, line): + row_data['raw'] += line + '\n' + row_data['msg_id'] = 'MSG_ID_' + '_'.join(word.upper() for word in line.split('_')) + row_data['msg_name'] = self.__message_name(row_data['msg_id']) + # payload parameter + elif row_data is not None and '=' in line: + row_data['raw'] += line + '\n' + result = line.strip().split('=', 1) + field_type = result[0] + field_name = result[1][:1].lower() + result[1][1:] if result[1] else '' + if not re.fullmatch(name_regex, field_name): + print(f"WARNING: {row_data['msg_id']} ({row_data['msg_id_hex_string']}) contains invalid field name \"{result[1]}\", skipping message") + row_data = None + continue + count = 0 + base_name = field_name + while any(field['name'] == field_name for field in row_data['payload']): + if count == 0: + print(f"WARNING: {row_data['msg_id']} ({row_data['msg_id_hex_string']}) contains duplicate field \"{field_name}\"") + count += 1 + field_name = f"{base_name}{count}" + row_data['payload'].append({ + 'raw': line.strip(), + 'name': field_name, + 'type': field_type, + }) + self.data = OrderedDict(sorted(self.data.items(), + key=lambda x: (int(x[1]['msg_id_value'] if x[1]['msg_id_value'] is not None else 99999), x[0]))) + print(f'Loaded {filename} with {entry_count} entries') + + + # \brief Sort the cached csv data. + # \return none + def sort(self): + try: + self.data = OrderedDict(sorted(self.data.items(), + key=lambda x: (int(x[1]['msg_id_value'] if x[1]['msg_id_value'] is not None else 99999), x[0]))) + except Exception as e: + pass + + + def __message_name(self, msg_id): + # if python version >= 3.9.x, just use str.removeprefix built-in string function + if sys.version_info >= (3,9,0): + return ''.join(word.capitalize() for word in msg_id.removeprefix('MSG_ID_').split('_')) + # otherwise need to define and use remove_prefix custom function + else: + def remove_prefix(text, prefix): + if text.startswith(prefix): + return text[len(prefix):] + return text + return ''.join(word.capitalize() for word in remove_prefix(msg_id, 'MSG_ID_').split('_')) Index: scripts/MsgUtils/msgutils/MsgProtobuf.py =================================================================== diff -u --- scripts/MsgUtils/msgutils/MsgProtobuf.py (revision 0) +++ scripts/MsgUtils/msgutils/MsgProtobuf.py (revision bde1243eb2ff6af2868f6c6ad0cb4f5760aaf68b) @@ -0,0 +1,42 @@ +from jinja2 import Environment, FileSystemLoader +from .MsgData import MsgData +import os +from pathlib import Path + +# \brief Class that usees the loaded MsgData and outputs converts to Protobuf +class MsgProtobuf(MsgData): + # \brief Initializer + def __init__(self): + super().__init__() + + + # \brief Write the loaded MsgData to a C++ header message to protobuf utility class + # \param[in] device_name name of device (e.g. Denali, Leahi) + # \param[in] output_dir directory where the output header file will be written + # \param[in] namespace namespace to use in the header file, if namespace is blank then no namespace will be added + def write_msg_proto_header(self, device_name, output_dir=None, namespace=None): + env = Environment(loader = FileSystemLoader(f'{Path(__file__).parent.absolute()}/templates')) + template = env.get_template('MsgDefs_h.jinja') + render = template.render(msg_data = self.data, device_name = device_name, cpp_namespace = namespace) + header_path = Path(output_dir).joinpath(f"{device_name}MsgProto.h") + with open({header_path}, mode='w', encoding='utf-8', newline='\n') as out_file: + out_file.write(render) + out_file.close() + print(f"Wrote C++ Protobuf utiltity header file {header_path}") + + + # \brief Write the loaded MsgData to a protobuf file + # \param[in] filename name of the outputted protobuf file + # \param[in] output_dir directory where the outputted file will be written + def write_proto(self, filename, namespace=None, output_dir=None): + env = Environment(loader = FileSystemLoader(f'{Path(__file__).parent.absolute()}/templates')) + template = env.get_template('MsgDefs_proto.jinja') + render = template.render({ + 'msg_proto': self, + 'proto_namespace': namespace + }) + proto_path = Path(output_dir).joinpath(filename) + with open(proto_path, mode='w', encoding='utf-8', newline='\n') as out_file: + out_file.write(render) + out_file.close() + print(f"Wrote Protobuf message defintion file {proto_path}") Index: scripts/MsgUtils/msgutils/__init__.py =================================================================== diff -u --- scripts/MsgUtils/msgutils/__init__.py (revision 0) +++ scripts/MsgUtils/msgutils/__init__.py (revision bde1243eb2ff6af2868f6c6ad0cb4f5760aaf68b) @@ -0,0 +1,3 @@ +from .MsgData import MsgData +from .MsgCpp import MsgCpp +from .MsgProtobuf import MsgProtobuf Index: scripts/MsgUtils/msgutils/templates/MsgDefs_cpp.jinja =================================================================== diff -u --- scripts/MsgUtils/msgutils/templates/MsgDefs_cpp.jinja (revision 0) +++ scripts/MsgUtils/msgutils/templates/MsgDefs_cpp.jinja (revision bde1243eb2ff6af2868f6c6ad0cb4f5760aaf68b) @@ -0,0 +1,147 @@ +{%- if msg_defs_header | length -%} +#include +#include "{{ msg_defs_header }}" +{%- endif %} +{%- if cpp_namespace is defined and cpp_namespace is not none %} + +namespace {{ cpp_namespace }} +{ +{%- endif %} +{%- for (msg_id_value, msg) in msg_cpp.data.items() %} +{%- if proto is defined %} + +// {{ msg['msg_id'] }} ({{ msg['msg_id_hex_string'] }}) +// payload: {{ msg_cpp.field_list(msg_id_value) | join(", ") }} +void {{ msg['msg_name'] }}Payload::fromProtobuf([[maybe_unused]] const messages::{{ msg['msg_name'] }} &src) +{ +{%- for field in msg['payload'] %} +{%- if field['type'] != "union" %} + // {{ field['type'] ~ "-" ~ field['name'] }} + {{ field['name'] }}.value = src.{{ field['name'].lower() }}(); +{%- else %} + // TODO: {{ field['type'] ~ "-" ~ field['name'] }} +{%- endif %} +{%- endfor %} +} +{%- endif %} + +// {{ msg['msg_id'] }} ({{ msg['msg_id_hex_string'] }}) +// payload: {{ msg_cpp.field_list(msg_id_value) | join(", ") }} +bool {{ msg['msg_name'] }}Payload::fromQByteArray([[maybe_unused]] const QByteArray &src) +{ +{%- if msg['payload'] | length %} + int offset = 0; + return ( +{%- set fields = namespace(count = 0) %} +{%- for field in msg['payload'] %} +{%- if field['type'] == "union" %} + // TODO: {{ field['type'] ~ "-" ~ field['name'] }} +{%- if fields.count < msg['payload'] | length - 1 %} + true && +{%- else %} + true +{%- endif %} +{%- else %} + // {{ field['type'] ~ "-" ~ field['name'] }} +{%- if fields.count < msg['payload'] | length - 1 %} + Types::getValue<>(src, offset, {{ field['name'] }}, QT_STRINGIFY({{ field['name'] }})) && +{%- else %} + Types::getValue<>(src, offset, {{ field['name'] }}, QT_STRINGIFY({{ field['name'] }})) +{%- endif %} +{%- endif %} +{%- set fields.count = fields.count + 1 %} +{%- endfor %} + ); +{%- else %} + return true; +{%- endif %} +} +{%- if proto is defined %} + +// {{ msg['msg_id'] }} ({{ msg['msg_id_hex_string'] }}) +// payload: {{ msg_cpp.field_list(msg_id_value) | join(", ") }} +messages::{{ msg['msg_name'] }} {{ msg['msg_name'] }}Payload::toProtobuf() const +{ + messages::{{ msg['msg_name'] }} protoMsg; +{%- for field in msg['payload'] %} +{%- if field['type'] != "union" %} + // {{ field['type'] ~ "-" ~ field['name'] }} + protoMsg.set_{{ field['name'].lower() }}({{ field['name'] }}.value); +{%- else %} + // TODO: {{ field['type'] ~ "-" ~ field['name'] }} +{%- endif %} +{%- endfor %} + return protoMsg; +} +{%- endif %} + +// {{ msg['msg_id'] }} ({{ msg['msg_id_hex_string'] }}) +// payload: {{ msg_cpp.field_list(msg_id_value) | join(", ") }} +void {{ msg['msg_name'] }}Payload::toQByteArray([[maybe_unused]] QByteArray &dst) const +{ +{%- if msg['payload'] | length %} +{%- for field in msg['payload'] %} +{%- if field['type'] == "union" %} + // TODO: {{ field['type'] ~ "-" ~ field['name'] }} +{%- else %} + // {{ field['type'] ~ "-" ~ field['name'] }} + Types::setValue<>({{ field['name'] }}, dst); +{%- endif %} +{%- endfor %} +{%- endif %} +} + +void {{ msg['msg_name'] }}Payload::dump() const +{ + QStringList params; +{%- for field in msg['payload'] %} +{%- if field['type'] != "union" %} + params << QString("{{ field['name'] }}=%1").arg({{ field['name'] }}.value); +{%- endif %} +{%- endfor %} + qDebug().noquote() << QString("{{ msg['msg_name'] }}Payload: %1").arg(params.count() ? params.join(", ") : ""); +} +{%- endfor %} + +QByteArray canMessageToProtobufByteArray(const QDateTime ×tamp, const QString &deviceSerialNum, const Can::Message &msg) +{ + static const auto updateHeader = [&](messages::Header *header) { + if (header) { + const auto msecs = timestamp.toMSecsSinceEpoch(); + header->set_deviceserialnum(deviceSerialNum.toStdString()); + auto proto_timestamp = header->mutable_timestamp(); + if (proto_timestamp) { + proto_timestamp->set_seconds(msecs / 1000); + proto_timestamp->set_nanos((msecs % 1000) * 1000000); + } + header->set_msgid(msg.msgId); + header->set_sequence(msg.sequence); + } + }; + + switch (msg.msgId) { +{%- for (msg_id_value, msg) in msg_cpp.data.items() %} + case {{ msg['msg_id'] }}: { + {{ msg['msg_name'] }}Payload payload; + if (payload.fromQByteArray(msg.data) == false) { + qDebug().noquote() << "ERROR: could not convert CAN message with MsgId={{ msg['msg_name'] }} to struct"; + } + payload.dump(); + auto proto = payload.toProtobuf(); + updateHeader(proto.mutable_header()); + std::ostringstream out; + (void)proto.SerializeToOstream(&out); + return out.str().c_str(); + break; + } +{%- endfor %} + default: + qDebug().noquote() << QString("WARNING: MsgId=0x%1 not handled").arg(msg.msgId, 4, 16, QChar('0')); + break; + } + return QByteArray(); +} +{%- if cpp_namespace is defined and cpp_namespace is not none %} + +} // namespace {{ cpp_namespace }} +{%- endif %} Index: scripts/MsgUtils/msgutils/templates/MsgDefs_h.jinja =================================================================== diff -u --- scripts/MsgUtils/msgutils/templates/MsgDefs_h.jinja (revision 0) +++ scripts/MsgUtils/msgutils/templates/MsgDefs_h.jinja (revision bde1243eb2ff6af2868f6c6ad0cb4f5760aaf68b) @@ -0,0 +1,96 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "CanMessage.h" +#include "types.h" +{%- if proto is defined %} +#include "{{ device_name }}MsgDefs.pb.h" +{%- endif %} + +{%- if cpp_namespace is defined and cpp_namespace is not none %} + +namespace {{ cpp_namespace }} +{ +{%- endif %} + +enum MsgId : quint16 { +{%- for (msg_id_value, msg) in msg_cpp.data.items() %} + {{ msg['msg_id'] }} = {{ msg['msg_id_hex_string'] }}, +{%- endfor %} +}; + +enum ParamType { +{%- for type in msg_cpp.paramTypes %} + {{ type }}, +{%- endfor %} +}; + +// typedef union { +// Types::U08 uint8Value; +// Types::U16 uint16Value; +// Types::U32 uint32Value; +// Types::S08 int8Value; +// Types::S16 int16Value; +// Types::S32 int32Value; +// Types::F32 floatValue; +// } ParamUnion; +union ParamUnion { + uint8_t uint8Value; + uint16_t uint16Value; + uint32_t uint32Value; + int8_t int8Value; + int16_t int16Value; + int32_t int32Value; + float floatValue; + uint8_t bytes[std::max(sizeof(float), sizeof(uint32_t))]; +}; +{%- if msg_cpp.data.items() | length %} + +const QMap payloadSize = { +{%- for (msg_id_value, msg) in msg_cpp.data.items() %} + { {{ msg['msg_id'] }}, {{ msg['cpp_struct']['payload_size'] }} }, +{%- endfor %} +}; +{%- endif %} +{%- for (msg_id_value, msg) in msg_cpp.data.items() %} + +// {{ msg['msg_id'] }} ({{ msg['msg_id_hex_string'] }}) +// payload: {{ msg_cpp.field_list(msg_id_value) | join(", ") }} +struct {{ msg['msg_name'] }}Payload { +{%- for field in msg['cpp_struct']['payload'] %} + {{ field['cpp_type'] }} {{ field['name'] }}; +{%- endfor %} +{%- if msg['cpp_struct']['payload'] | length %} +{% endif %} +{%- if proto is defined %} +{%- if cpp_namespace is defined and cpp_namespace is not none %} + void fromProtobuf(const {{ cpp_namespace }}::messages::{{ msg['msg_name'] }} &src); +{%- else %} + void fromProtobuf(const messages::{{ msg['msg_name'] }} &src); +{%- endif %} +{%- endif %} + bool fromQByteArray(const QByteArray &src); +{%- if proto is defined %} +{%- if cpp_namespace is defined and cpp_namespace is not none %} + {{ cpp_namespace }}::messages::{{ msg['msg_name'] }} toProtobuf() const; +{%- else %} + messages::{{ msg['msg_name'] }} toProtobuf() const; +{%- endif %} +{%- endif %} + void toQByteArray(QByteArray &dst) const; + void dump() const; +}; +{%- endfor %} +{%- if cpp_namespace is defined and cpp_namespace is not none %} + +QByteArray canMessageToProtobufByteArray(const QDateTime ×tamp, const QString &deviceSerialNumber, const Can::Message &msg); + +} // namespace {{ cpp_namespace }} +{%- endif %} Index: scripts/MsgUtils/msgutils/templates/MsgDefs_proto.jinja =================================================================== diff -u --- scripts/MsgUtils/msgutils/templates/MsgDefs_proto.jinja (revision 0) +++ scripts/MsgUtils/msgutils/templates/MsgDefs_proto.jinja (revision bde1243eb2ff6af2868f6c6ad0cb4f5760aaf68b) @@ -0,0 +1,60 @@ +syntax = "proto3"; +{% if proto_namespace is defined and proto_namespace is not none %} +package {{ proto_namespace }}.messages; +{%- else %} +package messages; +{%- endif %} + +import "google/protobuf/timestamp.proto"; + +message Header { + string deviceSerialNum = 1; + google.protobuf.Timestamp timestamp = 2; + uint32 msgId = 3; + uint32 sequence = 4; +} +{%- for (msg_id, msg) in msg_proto.data.items() %} + +// {{ msg['msg_id'] }} ({{ msg['msg_id_hex_string'] }}) +{%- if msg['payload'] | length %} +{%- set fields = namespace(payloadList = []) %} +{%- for field in msg['payload'] %} +{%- set fields.payloadList = fields.payloadList + [ field['type'] ~ "-" ~ field['name'] ] %} +{%- endfor %} +{%- else %} +{%- set fields = namespace(payloadList = [""]) %} +{%- endif %} +// payload: {{ fields.payloadList | join(", ") }} +message {{ msg['msg_name'] }} { + Header header = 1; +{%- set fields = namespace(id = 2) %} +{%- for field in msg['payload'] %} +{%- if field['type'] == 'BOOL' %} + bool {{ field['name'] }} = {{ fields.id }}; +{%- set fields.id = fields.id + 1 %} +{%- elif field['type'] == 'U08' or field['type'] == 'U16' or field['type'] == 'U32' %} + uint32 {{ field['name'] }} = {{ fields.id }}; +{%- set fields.id = fields.id + 1 %} +{%- elif field['type'] == 'S16' or field['type'] == 'S32' %} + int32 {{field['name']}} = {{fields.id}}; +{%- set fields.id = fields.id + 1 %} +{%- elif field['type'] == 'F32' %} + float {{field['name']}} = {{fields.id}}; +{%- set fields.id = fields.id + 1 %} +{%- elif field['type'] == 'union' %} + oneof {{field['name']}} { + bool {{field['name']}}_bool = {{fields.id}}; +{%- set fields.id = fields.id + 1 %} + uint32 {{field['name']}}_uint32 = {{fields.id}}; +{%- set fields.id = fields.id + 1 %} + int32 {{field['name']}}_int32 = {{fields.id}}; +{%- set fields.id = fields.id + 1 %} + float {{field['name']}}_float = {{fields.id}}; +{%- set fields.id = fields.id + 1 %} + } +{%- else %} + // {{field['type']}} {{field['name']}} +{%- endif %} +{%- endfor %} +} +{%- endfor %} Index: scripts/MsgUtils/pyproject.toml =================================================================== diff -u --- scripts/MsgUtils/pyproject.toml (revision 0) +++ scripts/MsgUtils/pyproject.toml (revision bde1243eb2ff6af2868f6c6ad0cb4f5760aaf68b) @@ -0,0 +1,20 @@ +[build-system] +requires = ["setuptools>=42"] +build-backend = "setuptools.build_meta" + +[project] +name = "msgutils" +version = "1.0.0" +description = "Diality Message Utilities" +requires-python = ">=3.9" +dependencies = ["jinja2"] + +[project.scripts] +GenerateMsgDefsCpp = "GenerateMsgDefsCpp:main" +GenerateProtobuf = "GenerateProtobuf:main" + +[tool.setuptools.packages.find] +exclude = ["test*"] + +[tool.setuptools.package-data] +msgutils = ["templates/*"] Index: tools/CMakeLists.txt =================================================================== diff -u -r70e672866cdc9e334ed725b82bde6ba6574420bf -rbde1243eb2ff6af2868f6c6ad0cb4f5760aaf68b --- tools/CMakeLists.txt (.../CMakeLists.txt) (revision 70e672866cdc9e334ed725b82bde6ba6574420bf) +++ tools/CMakeLists.txt (.../CMakeLists.txt) (revision bde1243eb2ff6af2868f6c6ad0cb4f5760aaf68b) @@ -1,4 +1,3 @@ -add_subdirectory(AgentSim) if(UNIX AND NOT APPLE) add_subdirectory(CANDumpPlayer) endif()