ASCII Reader/Writer enhancements #820
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -6,7 +6,7 @@ | |
| - under `test/data/logfile.asc` | ||
| """ | ||
|
|
||
| from typing import cast, Any, Generator, IO, List, Optional, Union, Dict | ||
| from can import typechecking | ||
|
|
||
| from datetime import datetime | ||
|
|
@@ -53,122 +53,165 @@ def __init__( | |
| if not self.file: | ||
| raise ValueError("The given file cannot be None") | ||
| self.base = base | ||
| self._converted_base = self._check_base(base) | ||
| self.date = None | ||
| self.timestamps_format = None | ||
| self.internal_events_logged = None | ||
|
|
||
| def _extract_header(self): | ||
| for line in self.file: | ||
| line = line.strip() | ||
| lower_case = line.lower() | ||
| if lower_case.startswith("date"): | ||
| self.date = line[5:] | ||
| elif lower_case.startswith("base"): | ||
| try: | ||
| _, base, _, timestamp_format = line.split() | ||
| except ValueError: | ||
| raise Exception("Unsupported header string format: {}".format(line)) | ||
| self.base = base | ||
|
Collaborator
There was a problem hiding this comment. The If you were intending to use this value from the
Contributor
Author
There was a problem hiding this comment. @karlding In both the current and previous implementations, we were ignoring the base passed on to the constructor and using the base defined in the ASCII logs file to parse all the messages. It seems accurate to set base to the value actually used during parsing so I think overwriting it is fine.
Collaborator
There was a problem hiding this comment. Hmm, if the idea is for the I don't have access to Vector's tools so I can't check, but are there cases where an ASC file is considered valid but it doesn't specify a base?
Contributor
Author
There was a problem hiding this comment. Perhaps the only use case I can think of is when we have a logs file split across two ASCII files. Not completely sure if the header is present in files other than the first file. If the intention is perhaps to read only the second file, then having a manually specified base can be useful. Your call.
Collaborator
There was a problem hiding this comment. Let's just leave this for now then. If someone has a use case for this, we can revisit it at that point. |
||
| self._converted_base = self._check_base(self.base) | ||
| self.timestamps_format = timestamp_format | ||
| elif lower_case.endswith("internal events logged"): | ||
| self.internal_events_logged = not lower_case.startswith("no") | ||
| # Currently the last line in the header which is parsed | ||
| break | ||
| else: | ||
| break | ||
|
|
||
| def _extract_can_id(self, str_can_id: str, msg_kwargs: Dict[str, Any]) -> None: | ||
| if str_can_id[-1:].lower() == "x": | ||
| msg_kwargs["is_extended_id"] = True | ||
| can_id = int(str_can_id[0:-1], self._converted_base) | ||
| else: | ||
| msg_kwargs["is_extended_id"] = False | ||
| can_id = int(str_can_id, self._converted_base) | ||
| msg_kwargs["arbitration_id"] = can_id | ||
|
|
||
| @staticmethod | ||
| def _check_base(base: str) -> int: | ||
| if base not in ["hex", "dec"]: | ||
| raise ValueError('base should be either "hex" or "dec"') | ||
| return BASE_DEC if base == "dec" else BASE_HEX | ||
|
|
||
| def _process_data_string( | ||
| self, data_str: str, data_length: int, msg_kwargs: Dict[str, Any] | ||
| ) -> None: | ||
| frame = bytearray() | ||
| data = data_str.split() | ||
| for byte in data[:data_length]: | ||
| frame.append(int(byte, self._converted_base)) | ||
| msg_kwargs["data"] = frame | ||
|
|
||
| def _process_classic_can_frame( | ||
| self, line: str, msg_kwargs: Dict[str, Any] | ||
| ) -> Message: | ||
|
|
||
| # CAN error frame | ||
| if line.strip()[0:10].lower() == "errorframe": | ||
| # Error Frame | ||
| msg_kwargs["is_error_frame"] = True | ||
| else: | ||
| abr_id_str, dir, rest_of_message = line.split(None, 2) | ||
| msg_kwargs["is_rx"] = dir == "Rx" | ||
| self._extract_can_id(abr_id_str, msg_kwargs) | ||
|
|
||
| if rest_of_message[0].lower() == "r": | ||
| # CAN Remote Frame | ||
| msg_kwargs["is_remote_frame"] = True | ||
| remote_data = rest_of_message.split() | ||
| if len(remote_data) > 1: | ||
| dlc_str = remote_data[1] | ||
| if dlc_str.isdigit(): | ||
| msg_kwargs["dlc"] = int(dlc_str, self._converted_base) | ||
| else: | ||
| # Classic CAN Message | ||
| try: | ||
| # There is data after DLC | ||
| _, dlc_str, data = rest_of_message.split(None, 2) | ||
| except ValueError: | ||
| # No data after DLC | ||
| _, dlc_str = rest_of_message.split(None, 1) | ||
| data = "" | ||
|
|
||
| dlc = int(dlc_str, self._converted_base) | ||
| msg_kwargs["dlc"] = dlc | ||
| self._process_data_string(data, dlc, msg_kwargs) | ||
|
|
||
| return Message(**msg_kwargs) | ||
|
|
||
| def _process_fd_can_frame(self, line: str, msg_kwargs: Dict[str, Any]) -> Message: | ||
| channel, dir, rest_of_message = line.split(None, 2) | ||
| # See ASCWriter | ||
| msg_kwargs["channel"] = int(channel) - 1 | ||
| msg_kwargs["is_rx"] = dir == "Rx" | ||
|
|
||
| # CAN FD error frame | ||
| if rest_of_message.strip()[:10].lower() == "errorframe": | ||
| # Error Frame | ||
| # TODO: maybe use regex to parse BRS, ESI, etc? | ||
| msg_kwargs["is_error_frame"] = True | ||
| else: | ||
| can_id_str, frame_name_or_brs, rest_of_message = rest_of_message.split( | ||
| None, 2 | ||
| ) | ||
|
|
||
| if frame_name_or_brs.isdigit(): | ||
| brs = frame_name_or_brs | ||
| esi, dlc_str, data_length_str, data = rest_of_message.split(None, 3) | ||
| else: | ||
| brs, esi, dlc_str, data_length_str, data = rest_of_message.split( | ||
| None, 4 | ||
| ) | ||
|
|
||
| self._extract_can_id(can_id_str, msg_kwargs) | ||
| msg_kwargs["bitrate_switch"] = brs == "1" | ||
| msg_kwargs["error_state_indicator"] = esi == "1" | ||
| dlc = int(dlc_str, self._converted_base) | ||
| msg_kwargs["dlc"] = dlc | ||
| data_length = int(data_length_str) | ||
|
|
||
| # CAN remote Frame | ||
| msg_kwargs["is_remote_frame"] = data_length == 0 | ||
|
|
||
| self._process_data_string(data, data_length, msg_kwargs) | ||
|
|
||
| return Message(**msg_kwargs) | ||
|
|
||
| def __iter__(self) -> Generator[Message, None, None]: | ||
| # This is guaranteed to not be None since we raise ValueError in __init__ | ||
| self.file = cast(IO[Any], self.file) | ||
| self._extract_header() | ||
|
|
||
| for line in self.file: | ||
| temp = line.strip() | ||
| if not temp or not temp[0].isdigit(): | ||
| # Could be a comment | ||
| continue | ||
| msg_kwargs = {} | ||
| try: | ||
| timestamp, channel, rest_of_message = temp.split(None, 2) | ||
| timestamp = float(timestamp) | ||
| msg_kwargs["timestamp"] = timestamp | ||
| if channel == "CANFD": | ||
| msg_kwargs["is_fd"] = True | ||
| elif channel.isdigit(): | ||
| # See ASCWriter | ||
| msg_kwargs["channel"] = int(channel) - 1 | ||
| else: | ||
| # Not a CAN message. Possible values include "statistic", J1939TP | ||
| continue | ||
| except ValueError: | ||
| # Some other unprocessed or unknown format | ||
| continue | ||
|
|
||
| if "is_fd" not in msg_kwargs: | ||
| msg = self._process_classic_can_frame(rest_of_message, msg_kwargs) | ||
| else: | ||
| msg = self._process_fd_can_frame(rest_of_message, msg_kwargs) | ||
| if msg is not None: | ||
| yield msg | ||
|
|
||
| self.stop() | ||
|
|
||
|
|
||
|
|
@@ -190,7 +233,7 @@ class ASCWriter(BaseIOHandler, Listener): | |
| "{id:>8} {symbolic_name:>32}", | ||
| "{brs}", | ||
| "{esi}", | ||
| "{dlc:x}", | ||
| "{data_length:>2}", | ||
| "{data}", | ||
| "{message_duration:>8}", | ||
|
|
@@ -281,10 +324,10 @@ def on_message_received(self, msg: Message) -> None: | |
| self.log_event("{} ErrorFrame".format(self.channel), msg.timestamp) | ||
| return | ||
| if msg.is_remote_frame: | ||
| dtype = "r {:x}".format(msg.dlc) # New after v8.5 | ||
| data: List[str] = [] | ||
| else: | ||
| dtype = "d {:x}".format(msg.dlc) | ||
| data = ["{:02X}".format(byte) for byte in msg.data] | ||
| arb_id = "{:X}".format(msg.arbitration_id) | ||
| if msg.is_extended_id: | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| date Sam Sep 30 15:06:13.191 2017 | ||
| base hex timestamps absolute | ||
| internal events logged | ||
| // version 9.0.0 | ||
| Begin Triggerblock Sam Sep 30 15:06:13.191 2017 | ||
| 0.000000 Start of measurement | ||
| 2.501000 1 ErrorFrame | ||
| 3.501000 1 ErrorFrame ECC: 10100010 | ||
| 4.501000 2 ErrorFrame Flags = 0xe CodeExt = 0x20a2 Code = 0x82 ID = 0 DLC = 0 Position = 5 Length = 11300 | ||
| 30.806898 CANFD 5 Tx ErrorFrame Not Acknowledge error, dominant error flag fffe c7 31ca Arb. 556 44 0 0 f 64 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1331984 11 0 46500250 460a0250 20011736 20010205 | ||
| End TriggerBlock |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| date Sam Sep 30 15:06:13.191 2017 | ||
| base hex timestamps absolute | ||
| internal events logged | ||
| // version 9.0.0 | ||
| Begin Triggerblock Sam Sep 30 15:06:13.191 2017 | ||
| 0.000000 Start of measurement | ||
| 30.005021 CANFD 1 Rx 300 1 0 8 8 11 c2 03 04 05 06 07 08 102203 133 303000 e0006659 46500250 4b140250 20011736 2001040d | ||
| 30.005041 CANFD 2 Tx 1C4D80A7x 0 1 8 8 12 c2 03 04 05 06 07 08 102203 133 303000 e0006659 46500250 4b140250 20011736 2001040d | ||
| 30.005071 CANFD 3 Rx 30a Generic_Name_12 1 1 8 8 01 02 03 04 05 06 07 08 102203 133 303000 e0006659 46500250 4b140250 20011736 2001040d | ||
| End TriggerBlock |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| date Sam Sep 30 15:06:13.191 2017 | ||
| base hex timestamps absolute | ||
| internal events logged | ||
| // version 9.0.0 | ||
| Begin Triggerblock Sam Sep 30 15:06:13.191 2017 | ||
| 0.000000 Start of measurement | ||
| 30.506898 CANFD 4 Rx 4EE 0 1 f 64 A1 02 03 04 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 64 1331984 11 0 46500250 460a0250 20011736 20010205 | ||
| 31.506898 CANFD 4 Rx 1C4D80A7x AlphaNumericName_2 1 0 f 64 b1 02 03 04 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 64 1331984 11 0 46500250 460a0250 20011736 20010205 | ||
| End TriggerBlock |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| date Sam Sep 30 15:06:13.191 2017 | ||
| base hex timestamps absolute | ||
| internal events logged | ||
| // version 9.0.0 | ||
| Begin Triggerblock Sam Sep 30 15:06:13.191 2017 | ||
| 0.000000 Start of measurement | ||
| 30.300981 CANFD 3 Tx 50005x 0 1 5 0 140000 73 200050 7a60 46500250 460a0250 20011736 20010205 | ||
| End TriggerBlock |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| date Sam Sep 30 15:06:13.191 2017 | ||
| base hex timestamps absolute | ||
| internal events logged | ||
| // version 9.0.0 | ||
| Begin Triggerblock Sam Sep 30 15:06:13.191 2017 | ||
| 0.000000 Start of measurement | ||
| 2.5010 2 C8 Tx d 8 09 08 07 06 05 04 03 02 | ||
| 17.876708 1 6F9 Rx d 8 05 0C 00 00 00 00 00 00 Length = 240015 BitCount = 124 ID = 1785 | ||
| End TriggerBlock |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| date Sam Sep 30 15:06:13.191 2017 | ||
| base hex timestamps absolute | ||
| internal events logged | ||
| // version 9.0.0 | ||
| Begin Triggerblock Sam Sep 30 15:06:13.191 2017 | ||
| 0.000000 Start of measurement | ||
| 2.510001 2 100 Rx r | ||
| 2.520002 3 200 Tx r Length = 1704000 BitCount = 145 ID = 88888888x | ||
| 2.584921 4 300 Rx r 8 Length = 1704000 BitCount = 145 ID = 88888888x | ||
| End TriggerBlock |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This doesn't seem to be used anywhere. It seems to just indicate the presence of the following line:
Can you help me understand why this needs to be exposed?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@karlding The line can be either "internal events logged" or "no internal events logged". It may be useful in the future as we add support for more kinds of frames/events in the AscReader.