Index: cloudsync/handlers/logs_handler.py =================================================================== diff -u -ra19403bb936dd50d786e22fd52105f586a93f252 -r7ef7642362334bb325692b852cb10387d25b9b5c --- cloudsync/handlers/logs_handler.py (.../logs_handler.py) (revision a19403bb936dd50d786e22fd52105f586a93f252) +++ cloudsync/handlers/logs_handler.py (.../logs_handler.py) (revision 7ef7642362334bb325692b852cb10387d25b9b5c) @@ -3,6 +3,7 @@ from cloudsync.utils.globals import * from logging.handlers import TimedRotatingFileHandler +from time import time class CustomTimedRotatingFileHandlerHandler(TimedRotatingFileHandler): """ @@ -59,13 +60,19 @@ g_utils.logger.debug(f"CS log data: {cs_log_data}") try: + # Generate synthetic correlation ID for CS log uploads. + # CS logs are triggered by timer (log rotation), not bus messages, + # so there is no originating message to derive a correlation from. + serial = cls.g_config[CONFIG_DEVICE][CONFIG_DEVICE_HD_SERIAL] + correlation_id = f"cs-{serial}-cslog-0-{str(round(time() * 1000))}" helpers_add_to_network_queue(network_request_handler=cls.network_request_handler, request_type=NetworkRequestType.CS2DCS_REQ_SEND_CS_LOG, url='', payload=cs_log_data, method='', g_config=cls.g_config, + correlation_id=correlation_id, success_message='CS2DCS_REQ_SEND_CS_LOG request added to network queue') except Exception as e: error = Error.general(OutboundMessageIDs.CS2UI_ERROR.value, Index: cloudsync/utils/helpers.py =================================================================== diff -u -ra19403bb936dd50d786e22fd52105f586a93f252 -r7ef7642362334bb325692b852cb10387d25b9b5c --- cloudsync/utils/helpers.py (.../helpers.py) (revision a19403bb936dd50d786e22fd52105f586a93f252) +++ cloudsync/utils/helpers.py (.../helpers.py) (revision 7ef7642362334bb325692b852cb10387d25b9b5c) @@ -609,36 +609,76 @@ def helpers_extract_device_log_metadata(log_file_name:str) -> Union[dict,None]: - # Format 1: YYYYMMDD_HHMMSS_SERIAL_SUBTYPE.u.LOGTYPE.gz (original expected) - local_date_pattern = r"^\d{8}(?=_)" - local_time_pattern = r"^(?:^.*?)_(\d{6})_" - serial_number_pattern = r"^[a-zA-Z0-9]+_[a-zA-Z0-9]+_([a-zA-Z0-9]+)(?=_)" - device_subtype_pattern = r"^[a-zA-Z0-9]+_[a-zA-Z0-9]+_[a-zA-Z0-9]+_([a-zA-Z0-9]+)(?=)" + """ + Extract metadata from device log filenames by parsing right-to-left. - # Format 2: SERIAL_YYYYMMDD.u.LOGTYPE.gz (actual device format, e.g. DVT-BN_20260320.u.log.gz) - alt_date_pattern = r"_(\d{8})\.u\." + Device log naming convention: + {serial}[_{LABEL}_]{YYYYMMDD}.{xxx}.{yyy}.{zzz} - log_type_pattern = r"[a-zA-Z0-9]+\.u\.(\w+)(?=)" + Where: + - {serial}: variable format (NOT extracted — use CS-known value) + - [_{LABEL}_]: optional label like BOOTPOST + - {YYYYMMDD}: date (8 digits) + - {xxx}: arbitrary identifier (u, u1, u2, anything) + - {yyy}: file type (log, err, etc.) + - {zzz}: compression (gz) - local_date_match = re.search(local_date_pattern, log_file_name) - local_time_match = re.search(local_time_pattern, log_file_name) - serial_number_match = re.search(serial_number_pattern, log_file_name) - device_subtype_match = re.search(device_subtype_pattern, log_file_name) - log_type_match = re.search(log_type_pattern, log_file_name) + Also supports Format 1: YYYYMMDD_HHMMSS_SERIAL_SUBTYPE.{xxx}.{yyy}.{zzz} - # Fallback to Format 2 date if Format 1 didn't match - if not local_date_match: - alt_date_match = re.search(alt_date_pattern, log_file_name) - if alt_date_match: - local_date_match = alt_date_match + Parsing strategy: right-to-left from the triple-dot extension, avoiding + ambiguity from _ appearing in both serials and labels. + """ + # Step 1: Strip extension from the right + # Try triple extension .{xxx}.{yyy}.{zzz} first (e.g., .u1.log.gz) + # Fall back to double extension .{xxx}.{yyy} (e.g., .u.DeviceLog) + triple_ext_match = re.search(r'\.([^.]+)\.([^.]+)\.([^.]+)$', log_file_name) + double_ext_match = re.search(r'\.([^.]+)\.([^.]+)$', log_file_name) + if triple_ext_match: + device_log_type = triple_ext_match.group(2) # {yyy} = log, err, etc. + stem = log_file_name[:triple_ext_match.start()] + elif double_ext_match: + device_log_type = double_ext_match.group(2) # {yyy} = DeviceLog, etc. + stem = log_file_name[:double_ext_match.start()] + else: + return { + "local_date": "unknown", + "local_time": "unknown", + "serial_number": "unknown", + "device_sub_type": "unknown", + "device_log_type": "unknown" + } + + # Step 2: Extract date from the stem + # Format 1: starts with YYYYMMDD_HHMMSS (e.g., 20260320_120000_...) + format1_match = re.match(r'^(\d{8})_(\d{6})_', stem) + if format1_match: + local_date = format1_match.group(1) + local_time = format1_match.group(2) + # Format 1 also has serial and subtype after the time + remainder = stem[format1_match.end():] + parts = remainder.split('_', 1) + serial_number = parts[0] if parts else 'unknown' + device_sub_type = parts[1] if len(parts) > 1 else 'unknown' + return { + "local_date": local_date, + "local_time": local_time, + "serial_number": serial_number, + "device_sub_type": device_sub_type, + "device_log_type": device_log_type + } + + # Format 2: date is the last 8 digits before the triple extension + # e.g., DVT-BN_20260320 or DVT-BN_BOOTPOST_20260320 + date_match = re.search(r'(\d{8})$', stem) + local_date = date_match.group(1) if date_match else 'unknown' + return { - "local_date": local_date_match.group(1) if local_date_match and local_date_match.lastindex else - (local_date_match.group(0) if local_date_match else 'unknown'), - "local_time": local_time_match.group(1) if local_time_match else 'unknown', - "serial_number": serial_number_match.group(1) if serial_number_match else 'unknown', - "device_sub_type": device_subtype_match.group(1) if device_subtype_match else 'unknown', - "device_log_type": log_type_match.group(1) if log_type_match else 'unknown' + "local_date": local_date, + "local_time": "unknown", + "serial_number": "unknown", + "device_sub_type": "unknown", + "device_log_type": device_log_type } @@ -687,6 +727,8 @@ cs_log_json['start_session']['macAddress'] = 'device-data' cs_log_json['start_session']['organizationId'] = cs_log_data['organizationId'] cs_log_json['start_session']['metadata']['dataType'] = 'cloud-sync-log-category' + cs_log_json['start_session']['metadata']['deviceLogType'] = 'cloudsync' + cs_log_json['start_session']['metadata']['deviceSubType'] = 'log' # Prepend serial to CS log filename for DCS dedup uniqueness per device. # Without this, two devices uploading cloudsync.log.03-27-2026 on the same day # would collide in the filename-only dedup check (DataLogExistsByFileName). @@ -726,17 +768,20 @@ local_timestamp = int(datetime.now(timezone.utc).timestamp()*1000) # Derive deterministic timestamps from filename date (required for DCS dedup composite key). - # When date+time are both available, use the exact datetime. - # When only date is available (Format 2: SERIAL_YYYYMMDD.u.log.gz), use start-of-day. + # When date+time are both available, use the exact datetime for both start and end. + # When only date is available, use 00:00:00 for start and 23:59:59 for end. # Fallback to upload time only if no date can be extracted at all. if extracted_metadata['local_date'] != 'unknown' and extracted_metadata['local_time'] != 'unknown': datetime_obj = datetime.strptime(f"{extracted_metadata['local_date']}{extracted_metadata['local_time']}", "%Y%m%d%H%M%S") - ui_utc_timestamp = int(datetime_obj.replace(tzinfo=timezone.utc).timestamp() * 1000) + start_timestamp = int(datetime_obj.replace(tzinfo=timezone.utc).timestamp() * 1000) + end_timestamp = start_timestamp elif extracted_metadata['local_date'] != 'unknown': - date_obj = datetime.strptime(extracted_metadata['local_date'], "%Y%m%d") - ui_utc_timestamp = int(date_obj.replace(tzinfo=timezone.utc).timestamp() * 1000) + date_obj = datetime.strptime(extracted_metadata['local_date'], "%Y%m%d").replace(tzinfo=timezone.utc) + start_timestamp = int(date_obj.replace(hour=0, minute=0, second=0, microsecond=0).timestamp() * 1000) + end_timestamp = int(date_obj.replace(hour=23, minute=59, second=59, microsecond=0).timestamp() * 1000) else: - ui_utc_timestamp = local_timestamp + start_timestamp = local_timestamp + end_timestamp = local_timestamp # Populate JSON object device_log_json['start_session']['reference'] = str(uuid.uuid4()) @@ -754,8 +799,8 @@ if not file_name.startswith(serial): file_name = f"{serial}_{file_name}" device_log_json['start_session']['metadata']['deviceFileName'] = file_name - device_log_json['start_session']['metadata']['startTimestamp'] = ui_utc_timestamp - device_log_json['start_session']['metadata']['endTimestamp'] = ui_utc_timestamp + device_log_json['start_session']['metadata']['startTimestamp'] = start_timestamp + device_log_json['start_session']['metadata']['endTimestamp'] = end_timestamp device_log_json['end_session']['checksum'] = checksum device_log_json['general']['file_size'] = file_size device_log_json['general']['file_path'] = device_log_data['path']