From bee6d4dcd1cc7dec1d9b4d30c3044bbfc9198621 Mon Sep 17 00:00:00 2001 From: danielhrisca Date: Mon, 18 Feb 2019 14:12:31 +0200 Subject: [PATCH 01/28] add extra dependency --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index c4127baa0..a1dfed9b3 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,7 @@ extras_require = { 'serial': ['pyserial~=3.0'], 'neovi': ['python-ics>=2.12'] + 'mf4': ['asammdf>=5.0.0'] } tests_require = [ From 1fc9bf239da83791af705b7de7bc58498a78f60f Mon Sep 17 00:00:00 2001 From: danielhrisca Date: Thu, 18 Apr 2019 16:29:19 +0300 Subject: [PATCH 02/28] add mf4 io writer --- can/__init__.py | 3 + can/io/__init__.py | 3 + can/io/mf4.py | 160 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 166 insertions(+) create mode 100644 can/io/mf4.py diff --git a/can/__init__.py b/can/__init__.py index fb48c6dd4..374f18d9f 100644 --- a/can/__init__.py +++ b/can/__init__.py @@ -7,6 +7,7 @@ from __future__ import absolute_import import logging +import sys __version__ = "3.1.1" @@ -34,6 +35,8 @@ class CanError(IOError): from .io import CanutilsLogReader, CanutilsLogWriter from .io import CSVWriter, CSVReader from .io import SqliteWriter, SqliteReader +if sys.hexversion >= 0x03060000: + from .io import MF4Writer from .util import set_logging_level diff --git a/can/io/__init__.py b/can/io/__init__.py index 967b9e555..adbbc30cb 100644 --- a/can/io/__init__.py +++ b/can/io/__init__.py @@ -6,6 +6,7 @@ """ from __future__ import absolute_import +import sys # Generic from .logger import Logger @@ -18,3 +19,5 @@ from .csv import CSVWriter, CSVReader from .sqlite import SqliteReader, SqliteWriter from .printer import Printer +if sys.hexversion >= 0x03060000: + from .mf4 import MF4Writer diff --git a/can/io/mf4.py b/can/io/mf4.py new file mode 100644 index 000000000..2c72e74b0 --- /dev/null +++ b/can/io/mf4.py @@ -0,0 +1,160 @@ +# coding: utf-8 + +""" +Contains handling of MF4 logging files. + +""" + +from __future__ import absolute_import + +from datetime import datetime +from pathlib import Path +import logging + +from ..listener import Listener +from ..util import channel2int +from .generic import BaseIOHandler + +from asammdf import MDF, Signal +import numpy as np + +CAN_MSG_EXT = 0x80000000 +CAN_ID_MASK = 0x1FFFFFFF + +DTYPE = np.dtype( + [ + ('BusChannel', ' Date: Thu, 18 Apr 2019 16:31:58 +0300 Subject: [PATCH 03/28] update asammdf requirement --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0404aea31..e2e965482 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ extras_require = { 'serial': ['pyserial~=3.0'], 'neovi': ['python-ics>=2.12'] - 'mf4': ['asammdf>=5.0.0'] + 'mf4': ['asammdf>=5.5.0'] } tests_require = [ From b6f73af1ba3b5f9fe0551d12bad9d26c76528b1c Mon Sep 17 00:00:00 2001 From: danielhrisca Date: Mon, 22 Apr 2019 09:57:38 +0300 Subject: [PATCH 04/28] add missing comma in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e2e965482..f739e115c 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ # Dependencies extras_require = { 'serial': ['pyserial~=3.0'], - 'neovi': ['python-ics>=2.12'] + 'neovi': ['python-ics>=2.12'], 'mf4': ['asammdf>=5.5.0'] } From 26cd2556dee5314f585ac0e256a876d21cffc8b5 Mon Sep 17 00:00:00 2001 From: danielhrisca Date: Mon, 22 Apr 2019 11:44:49 +0300 Subject: [PATCH 05/28] changes after initial review --- can/__init__.py | 4 +- can/io/__init__.py | 4 +- can/io/mf4.py | 291 ++++++++++++++++++++++++++------------------- setup.py | 2 +- 4 files changed, 174 insertions(+), 127 deletions(-) diff --git a/can/__init__.py b/can/__init__.py index 374f18d9f..96abb2418 100644 --- a/can/__init__.py +++ b/can/__init__.py @@ -35,8 +35,10 @@ class CanError(IOError): from .io import CanutilsLogReader, CanutilsLogWriter from .io import CSVWriter, CSVReader from .io import SqliteWriter, SqliteReader -if sys.hexversion >= 0x03060000: +try: from .io import MF4Writer +except ImportError: + pass from .util import set_logging_level diff --git a/can/io/__init__.py b/can/io/__init__.py index 70ab2261f..fd0f82b23 100644 --- a/can/io/__init__.py +++ b/can/io/__init__.py @@ -19,5 +19,7 @@ from .csv import CSVWriter, CSVReader from .sqlite import SqliteReader, SqliteWriter from .printer import Printer -if sys.hexversion >= 0x03060000: +try: from .mf4 import MF4Writer +except ImportError: + pass diff --git a/can/io/mf4.py b/can/io/mf4.py index 2c72e74b0..59f28fb89 100644 --- a/can/io/mf4.py +++ b/can/io/mf4.py @@ -15,130 +15,160 @@ from ..util import channel2int from .generic import BaseIOHandler -from asammdf import MDF, Signal import numpy as np -CAN_MSG_EXT = 0x80000000 -CAN_ID_MASK = 0x1FFFFFFF - -DTYPE = np.dtype( - [ - ('BusChannel', '=2.12'], - 'mf4': ['asammdf>=5.5.0'] + 'mf4': ['asammdf>=5.5.0', 'numpy>=1.16.0'] } tests_require = [ From 74e84722b34da6c065f82dd912554e1dac03906a Mon Sep 17 00:00:00 2001 From: danielhrisca Date: Mon, 22 Apr 2019 11:46:46 +0300 Subject: [PATCH 06/28] simplify append call --- can/io/mf4.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/can/io/mf4.py b/can/io/mf4.py index 59f28fb89..020646f37 100644 --- a/can/io/mf4.py +++ b/can/io/mf4.py @@ -94,37 +94,34 @@ def __init__(self, file, database=None): attachment = None # standard frames group - sigs = [ + self._mdf.append( Signal( name="CAN_DataFrame", samples=np.array([], dtype=DTYPE), timestamps=np.array([], dtype=' Date: Mon, 22 Apr 2019 13:46:17 +0300 Subject: [PATCH 07/28] add MF4Reader class --- can/io/mf4.py | 209 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 207 insertions(+), 2 deletions(-) diff --git a/can/io/mf4.py b/can/io/mf4.py index 020646f37..bea3902cc 100644 --- a/can/io/mf4.py +++ b/can/io/mf4.py @@ -3,6 +3,9 @@ """ Contains handling of MF4 logging files. +MF4 files represent Measurement Data Format (MDF) version 4 as specified by +the ASAM MDF standard (see https://www.asam.net/standards/detail/mdf/) + """ from __future__ import absolute_import @@ -11,12 +14,13 @@ from pathlib import Path import logging +import numpy as np + +from ..message import Message from ..listener import Listener from ..util import channel2int from .generic import BaseIOHandler -import numpy as np - try: from asammdf import MDF, Signal @@ -58,6 +62,11 @@ 64: 15, } + FD_DLC2LEN = { + value: key + for key, value in FD_LEN2DLC.items() + } + logger = logging.getLogger('can.io.mf4') @@ -194,6 +203,202 @@ def on_message_received(self, msg): else: self._mdf.extend(0, sigs) + + class MF4Reader(BaseIOHandler): + """ + Iterator of CAN messages from a MF4 logging file. + + """ + + def __init__(self, file): + """ + :param file: a path-like object or as file-like object to read from + If this is a file-like object, is has to opened in + binary read mode, not text read mode. + """ + self.start_timestamp = datetime.now().timestamp() + super(MF4Reader, self).__init__(file, mode='rb') + + self._mdf = MDF(file) + + masters = [ + self._mdf.get_master(i, copy_master=False) + for i in range(3) + ] + + masters = [ + np.core.records.fromarrays( + (master, np.ones(len(master))) + ) + for master in masters + ] + + self.masters = np.sort(masters) + + def __iter__(self): + standard_counter = 0 + error_counter = 0 + rtr_counter = 0 + + for timestamp, group_index in self.masters: + + # standard frames + if group_index == 0: + sample = self._mdf.get( + 'CAN_DataFrame', + group=group_index, + raw=True, + record_offset=standard_counter, + record_count=1, + ) + + if sample['CAN_DataFrame.EDL'] == 0: + + is_extended_id = bool(sample['CAN_DataFrame.IDE']) + channel = sample['CAN_DataFrame.ID'] + arbitration_id = int(sample['CAN_DataFrame.ID']) + size = int(sample['CAN_DataFrame.DataLength']) + dlc = int(sample['CAN_DataFrame.DLC']) + data = sample['CAN_DataFrame.DataBytes'][:size].tobytes() + + msg = Message( + timestamp=timestamp + self.start_timestamp, + is_error_frame=False, + is_remote_frame=False, + if_fd=False, + is_extended_id=is_extended_id, + channel=channel, + arbitration_id=arbitration_id, + data=data, + dlc=dlc, + ) + + else: + is_extended_id = bool(sample['CAN_DataFrame.IDE']) + channel = sample['CAN_DataFrame.ID'] + arbitration_id = int(sample['CAN_DataFrame.ID']) + size = int(sample['CAN_DataFrame.DataLength']) + dlc = FD_DLC2LEN(sample['CAN_DataFrame.DLC']) + data = sample['CAN_DataFrame.DataBytes'][:size].tobytes() + error_state_indicator = int(sample['CAN_DataFrame.ESI']) + bitrate_switch = int(sample['CAN_DataFrame.BRS']) + + msg = Message( + timestamp=timestamp + self.start_timestamp, + is_error_frame=False, + is_remote_frame=False, + if_fd=True, + is_extended_id=is_extended_id, + channel=channel, + arbitration_id=arbitration_id, + data=data, + dlc=dlc, + bitrate_switch=bitrate_switch, + error_state_indicator=error_state_indicator, + ) + + yield msg + standard_counter += 1 + + # error frames + elif group_index == 1: + sample = self._mdf.get( + 'CAN_ErrorFrame', + group=group_index, + raw=True, + record_offset=error_counter, + record_count=1, + ) + + if sample['CAN_ErrorFrame.EDL'] == 0: + + is_extended_id = bool(sample['CAN_ErrorFrame.IDE']) + channel = sample['CAN_ErrorFrame.ID'] + arbitration_id = int(sample['CAN_ErrorFrame.ID']) + size = int(sample['CAN_ErrorFrame.DataLength']) + dlc = int(sample['CAN_ErrorFrame.DLC']) + data = sample['CAN_ErrorFrame.DataBytes'][:size].tobytes() + + msg = Message( + timestamp=timestamp + self.start_timestamp, + is_error_frame=True, + is_remote_frame=False, + if_fd=False, + is_extended_id=is_extended_id, + channel=channel, + arbitration_id=arbitration_id, + data=data, + dlc=dlc, + ) + + else: + is_extended_id = bool(sample['CAN_ErrorFrame.IDE']) + channel = sample['CAN_ErrorFrame.ID'] + arbitration_id = int(sample['CAN_ErrorFrame.ID']) + size = int(sample['CAN_ErrorFrame.DataLength']) + dlc = FD_DLC2LEN(sample['CAN_ErrorFrame.DLC']) + data = sample['CAN_ErrorFrame.DataBytes'][:size].tobytes() + error_state_indicator = int(sample['CAN_ErrorFrame.ESI']) + bitrate_switch = int(sample['CAN_ErrorFrame.BRS']) + + msg = Message( + timestamp=timestamp + self.start_timestamp, + is_error_frame=True, + is_remote_frame=False, + if_fd=True, + is_extended_id=is_extended_id, + channel=channel, + arbitration_id=arbitration_id, + data=data, + dlc=dlc, + bitrate_switch=bitrate_switch, + error_state_indicator=error_state_indicator, + ) + + yield msg + error_counter += 1 + + # remote frames + else: + sample = self._mdf.get( + 'CAN_DataFrame', + group=group_index, + raw=True, + record_offset=rtr_counter, + record_count=1, + ) + + if sample['CAN_DataFrame.EDL'] == 0: + + is_extended_id = bool(sample['CAN_DataFrame.IDE']) + channel = sample['CAN_DataFrame.ID'] + arbitration_id = int(sample['CAN_DataFrame.ID']) + dlc = int(sample['CAN_DataFrame.DLC']) + + msg = Message( + timestamp=timestamp + self.start_timestamp, + is_error_frame=False, + is_remote_frame=True, + if_fd=False, + is_extended_id=is_extended_id, + channel=channel, + arbitration_id=arbitration_id, + dlc=dlc, + ) + + yield msg + + else: + logger.warning("CAN FD does not have remote frames") + + rtr_counter += 1 + + self.stop() + + def stop(self): + self._mdf.close() + super(MF4Writer, self).stop() + ASAMMDF_AVAILABLE = True except ImportError: From efdfdc9c890992e32142ff56ace3402093393658 Mon Sep 17 00:00:00 2001 From: danielhrisca Date: Mon, 22 Apr 2019 16:51:57 +0300 Subject: [PATCH 08/28] start testing --- can/__init__.py | 2 +- can/io/__init__.py | 2 +- can/io/mf4.py | 156 +++++++++++++++++++++++----------------- test/logformats_test.py | 23 +++++- 4 files changed, 111 insertions(+), 72 deletions(-) diff --git a/can/__init__.py b/can/__init__.py index 96abb2418..782357224 100644 --- a/can/__init__.py +++ b/can/__init__.py @@ -36,7 +36,7 @@ class CanError(IOError): from .io import CSVWriter, CSVReader from .io import SqliteWriter, SqliteReader try: - from .io import MF4Writer + from .io import MF4Writer, MF4Reader except ImportError: pass diff --git a/can/io/__init__.py b/can/io/__init__.py index fd0f82b23..66d2d4bca 100644 --- a/can/io/__init__.py +++ b/can/io/__init__.py @@ -20,6 +20,6 @@ from .sqlite import SqliteReader, SqliteWriter from .printer import Printer try: - from .mf4 import MF4Writer + from .mf4 import MF4Writer, MF4Reader except ImportError: pass diff --git a/can/io/mf4.py b/can/io/mf4.py index bea3902cc..29f27c3ae 100644 --- a/can/io/mf4.py +++ b/can/io/mf4.py @@ -27,28 +27,43 @@ CAN_MSG_EXT = 0x80000000 CAN_ID_MASK = 0x1FFFFFFF - DTYPE = np.dtype( + STD_DTYPE = np.dtype( [ - ('BusChannel', ' Date: Tue, 23 Apr 2019 00:45:20 +0300 Subject: [PATCH 09/28] passes tests --- can/io/mf4.py | 33 ++++++++++++++------------------- test/logformats_test.py | 29 +++++++++++++++++------------ 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/can/io/mf4.py b/can/io/mf4.py index 29f27c3ae..11ad9f52b 100644 --- a/can/io/mf4.py +++ b/can/io/mf4.py @@ -182,12 +182,12 @@ def on_message_received(self, msg): buffer['CAN_DataFrame.DataLength'] = size buffer['CAN_DataFrame.DataBytes'][0, :size] = data if msg.is_fd: - buffer['CAN_DataFrame.DLC'] = FD_LEN2DLC[size] + buffer['CAN_DataFrame.DLC'] = FD_LEN2DLC[msg.dlc] buffer['CAN_DataFrame.ESI'] = int(msg.error_state_indicator) buffer['CAN_DataFrame.BRS'] = int(msg.bitrate_switch) buffer['CAN_DataFrame.EDL'] = 1 else: - buffer['CAN_DataFrame.DLC'] = size + buffer['CAN_DataFrame.DLC'] = msg.dlc buffer['CAN_DataFrame.ESI'] = 0 buffer['CAN_DataFrame.BRS'] = 0 buffer['CAN_DataFrame.EDL'] = 0 @@ -201,14 +201,16 @@ def on_message_received(self, msg): timestamp, ) + timestamp -= self._start_time + if msg.is_remote_frame: sigs = [ - (np.array([self.last_timestamp]), None), + (np.array([timestamp]), None), (rtr_buffer, None) ] else: sigs = [ - (np.array([self.last_timestamp]), None), + (np.array([timestamp]), None), (buffer, None) ] @@ -232,18 +234,17 @@ def __init__(self, file): If this is a file-like object, is has to opened in binary read mode, not text read mode. """ - self.start_timestamp = datetime.now().timestamp() super(MF4Reader, self).__init__(file, mode='rb') self._mdf = MDF(file) + self.start_timestamp = self._mdf.header.start_time.timestamp() + masters = [ self._mdf.get_master(i, copy_master=False) for i in range(3) ] - print('++++++++++ ', [len(s) for s in masters]) - masters = [ np.core.records.fromarrays( (master, np.ones(len(master)) * i) @@ -251,8 +252,6 @@ def __init__(self, file): for i, master in enumerate(masters) ] - print - self.masters = np.sort(np.concatenate(masters)) def __iter__(self): @@ -260,6 +259,8 @@ def __iter__(self): error_counter = 0 rtr_counter = 0 + + for timestamp, group_index in self.masters: # standard frames @@ -279,7 +280,7 @@ def __iter__(self): arbitration_id = int(sample['CAN_DataFrame.ID']) size = int(sample['CAN_DataFrame.DataLength']) dlc = int(sample['CAN_DataFrame.DLC']) - data = sample['CAN_DataFrame.DataBytes'][:size].tobytes() + data = sample['CAN_DataFrame.DataBytes'][0, :size].tobytes() msg = Message( timestamp=timestamp + self.start_timestamp, @@ -299,7 +300,7 @@ def __iter__(self): arbitration_id = int(sample['CAN_DataFrame.ID']) size = int(sample['CAN_DataFrame.DataLength']) dlc = FD_DLC2LEN(sample['CAN_DataFrame.DLC']) - data = sample['CAN_DataFrame.DataBytes'][:size].tobytes() + data = sample['CAN_DataFrame.DataBytes'][0, :size].tobytes() error_state_indicator = int(sample['CAN_DataFrame.ESI']) bitrate_switch = int(sample['CAN_DataFrame.BRS']) @@ -338,7 +339,7 @@ def __iter__(self): arbitration_id = int(sample['CAN_ErrorFrame.ID']) size = int(sample['CAN_ErrorFrame.DataLength']) dlc = int(sample['CAN_ErrorFrame.DLC']) - data = sample['CAN_ErrorFrame.DataBytes'][:size].tobytes() + data = sample['CAN_ErrorFrame.DataBytes'][0, :size].tobytes() msg = Message( timestamp=timestamp + self.start_timestamp, @@ -358,7 +359,7 @@ def __iter__(self): arbitration_id = int(sample['CAN_ErrorFrame.ID']) size = int(sample['CAN_ErrorFrame.DataLength']) dlc = FD_DLC2LEN(sample['CAN_ErrorFrame.DLC']) - data = sample['CAN_ErrorFrame.DataBytes'][:size].tobytes() + data = sample['CAN_ErrorFrame.DataBytes'][0, :size].tobytes() error_state_indicator = int(sample['CAN_ErrorFrame.ESI']) bitrate_switch = int(sample['CAN_ErrorFrame.BRS']) @@ -409,12 +410,6 @@ def __iter__(self): rtr_counter += 1 - - print('!!!!\n!!!!!!!\n!!!!!!! ', standard_counter, - error_counter, - rtr_counter, - [gp.channel_group.cycles_nr for gp in self._mdf.groups] - ) self.stop() def stop(self): diff --git a/test/logformats_test.py b/test/logformats_test.py index 34018cc7b..4cae66bfc 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -132,6 +132,7 @@ def test_path_like_explicit_stop(self): writer = self.writer_constructor(self.test_file_name) self._write_all(writer) self._ensure_fsync(writer) + writer.stop() if hasattr(writer.file, 'closed'): self.assertTrue(writer.file.closed) @@ -235,8 +236,6 @@ def test_file_like_context_manager(self): self.assertEqual(len(read_messages), len(self.original_messages), "the number of written messages does not match the number of read messages") - print('!!!!!! ', len(read_messages)) - self.assertMessagesEqual(self.original_messages, read_messages) self.assertIncludesComments(self.test_file_name) @@ -395,19 +394,25 @@ def _setup_instance(self): ) -class TestMF4FileFormat(ReaderWriterTest): - """Tests can.MF4Writer and can.MF4Reader""" +try: + from can import MF4Writer, MF4Reader - __test__ = True + class TestMF4FileFormat(ReaderWriterTest): + """Tests can.MF4Writer and can.MF4Reader""" - def _setup_instance(self): - super(TestMF4FileFormat, self)._setup_instance_helper( - can.MF4Writer, can.MF4Reader, - binary_file=True, - check_comments=False, - preserves_channel=False, adds_default_channel=0 - ) + __test__ = True + def _setup_instance(self): + super(TestMF4FileFormat, self)._setup_instance_helper( + MF4Writer, MF4Reader, + binary_file=True, + check_comments=False, + preserves_channel=False, + adds_default_channel=0, + ) + +except ImportError: + pass class TestSqliteDatabaseFormat(ReaderWriterTest): From a11440b912a15725bd9dd745d8cb0c31008882a7 Mon Sep 17 00:00:00 2001 From: danielhrisca Date: Tue, 23 Apr 2019 01:05:44 +0300 Subject: [PATCH 10/28] update documentation --- README.rst | 2 +- can/io/mf4.py | 6 ++++-- doc/listeners.rst | 26 ++++++++++++++++++++++++++ 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 12b917355..7a99f3311 100644 --- a/README.rst +++ b/README.rst @@ -48,7 +48,7 @@ Features - receiving, sending, and periodically sending messages - normal and extended arbitration IDs - limited `CAN FD `__ support -- many different loggers and readers supporting playback: ASC (CANalyzer format), BLF (Binary Logging Format by Vector), CSV, SQLite and Canutils log +- many different loggers and readers supporting playback: ASC (CANalyzer format), BLF (Binary Logging Format by Vector), CSV, SQLite, Canutils log and MF4 (Measurement Data Format v4 by ASAM) - efficient in-kernel or in-hardware filtering of messages on supported interfaces - bus configuration reading from file or environment variables - CLI tools for working with CAN busses (see the `docs `__) diff --git a/can/io/mf4.py b/can/io/mf4.py index 11ad9f52b..750449941 100644 --- a/can/io/mf4.py +++ b/can/io/mf4.py @@ -88,6 +88,8 @@ class MF4Writer(BaseIOHandler, Listener): """Logs CAN data to an ASAM Measurement Data File v4 (.mf4). + MF4Writer does not support append mode. + If a message has a timestamp smaller than the previous one or None, it gets assigned the timestamp that was written for the last message. It the first message does not have a timestamp, it is set to zero. @@ -96,7 +98,7 @@ class MF4Writer(BaseIOHandler, Listener): def __init__(self, file, database=None): """ :param file: a path-like object or as file-like object to write to - If this is a file-like object, is has to opened in + If this is a file-like object, is has to be opened in binary write mode, not text write mode. :param database: optional path to a DBC or ARXML file that contains message description. @@ -231,7 +233,7 @@ class MF4Reader(BaseIOHandler): def __init__(self, file): """ :param file: a path-like object or as file-like object to read from - If this is a file-like object, is has to opened in + If this is a file-like object, is has to be opened in binary read mode, not text read mode. """ super(MF4Reader, self).__init__(file, mode='rb') diff --git a/doc/listeners.rst b/doc/listeners.rst index 975de6fd1..6f2bb3d48 100644 --- a/doc/listeners.rst +++ b/doc/listeners.rst @@ -160,3 +160,29 @@ The following class can be used to read messages from BLF file: .. autoclass:: can.BLFReader :members: + + +MF4 (Measurement Data Format v4) +-------------------------------- + +Implements support for MF4 (Measurement Data Format v4) which is a proprietary +format from ASAM, widely used in many automotive software (Vector CANape, ETAS INCA, dSPACE COntrolDesk, etc.). + +MF4 support requires Python >= 3.6 + +The data is stored in a compressed format which makes it compact. + +.. note:: Channels will be converted to integers. + +.. note:: MF4Writer does not suppport append mode + +`MF4Writer` has the following init arguments + +* **file** : a path-like object or as file-like object to write to. + If this is a file-like object, is has to be opened in binary write mode, not text write mode. +* **database** : optional path to a DBC or ARXML file that contains message description. + +`MF4Reader` has the following init arguments + +* **file** : a path-like object or as file-like object to write to. + If this is a file-like object, is has to be opened in binary write mode, not text write mode. From b736542bd65af43ab4aeb4c394d3e3c26983afb7 Mon Sep 17 00:00:00 2001 From: danielhrisca Date: Tue, 23 Apr 2019 09:25:17 +0300 Subject: [PATCH 11/28] update docs and CI scripts --- .appveyor.yml | 2 +- .travis.yml | 2 +- doc/listeners.rst | 16 +++++++--------- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 3b52143e7..a625b8266 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -19,7 +19,7 @@ install: - set PATH=%PYTHON_INSTALL%;%PYTHON_INSTALL%\\Scripts;%PATH% # We need to install the python-can library itself including the dependencies - - "python -m pip install .[test,neovi]" + - "python -m pip install .[test,neovi,mf4]" build: off diff --git a/.travis.yml b/.travis.yml index 0a063684f..7a746a71c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,7 +49,7 @@ matrix: install: - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo bash test/open_vcan.sh ; fi - if [[ "$TRAVIS_PYTHON_VERSION" == "3.6" ]]; then travis_retry pip install -r doc/doc-requirements.txt; fi - - travis_retry pip install .[test] + - travis_retry pip install .[test,mf4] script: - pytest diff --git a/doc/listeners.rst b/doc/listeners.rst index 6f2bb3d48..71d9af504 100644 --- a/doc/listeners.rst +++ b/doc/listeners.rst @@ -168,21 +168,19 @@ MF4 (Measurement Data Format v4) Implements support for MF4 (Measurement Data Format v4) which is a proprietary format from ASAM, widely used in many automotive software (Vector CANape, ETAS INCA, dSPACE COntrolDesk, etc.). -MF4 support requires Python >= 3.6 - The data is stored in a compressed format which makes it compact. +.. note:: MF4 support requires Python >= 3.6 and has to be installed as an extra with for example ``pip install python-can[mf4]``. + .. note:: Channels will be converted to integers. .. note:: MF4Writer does not suppport append mode -`MF4Writer` has the following init arguments -* **file** : a path-like object or as file-like object to write to. - If this is a file-like object, is has to be opened in binary write mode, not text write mode. -* **database** : optional path to a DBC or ARXML file that contains message description. +.. autoclass:: can.MF4Writer + :members: -`MF4Reader` has the following init arguments +MF4Reader can only replay files created with MF4Writer. The following class can be used to read messages from MF4 file: -* **file** : a path-like object or as file-like object to write to. - If this is a file-like object, is has to be opened in binary write mode, not text write mode. +.. autoclass:: can.MF4Reader + :members: From c2db9f6c7173c8f8454e33133b4b7069543e36e6 Mon Sep 17 00:00:00 2001 From: danielhrisca Date: Thu, 25 Apr 2019 16:26:25 +0300 Subject: [PATCH 12/28] retrigger build --- doc/listeners.rst | 2 +- examples/receive_all.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/listeners.rst b/doc/listeners.rst index 71d9af504..42e90cdf0 100644 --- a/doc/listeners.rst +++ b/doc/listeners.rst @@ -174,7 +174,7 @@ The data is stored in a compressed format which makes it compact. .. note:: Channels will be converted to integers. -.. note:: MF4Writer does not suppport append mode +.. note:: MF4Writer does not suppport append mode. .. autoclass:: can.MF4Writer diff --git a/examples/receive_all.py b/examples/receive_all.py index 44a495de7..69901f593 100755 --- a/examples/receive_all.py +++ b/examples/receive_all.py @@ -8,9 +8,9 @@ def receive_all(): - bus = can.interface.Bus(bustype='pcan', channel='PCAN_USBBUS1', bitrate=250000) +# bus = can.interface.Bus(bustype='pcan', channel='PCAN_USBBUS1', bitrate=250000) #bus = can.interface.Bus(bustype='ixxat', channel=0, bitrate=250000) - #bus = can.interface.Bus(bustype='vector', app_name='CANalyzer', channel=0, bitrate=250000) + bus = can.interface.Bus(bustype='vector', app_name='daxil', channel=0, bitrate=250000) bus.state = BusState.ACTIVE # or BusState.PASSIVE From 2a2e77e32b8b55feb867b6c0471b44c2a05ca42c Mon Sep 17 00:00:00 2001 From: danielhrisca Date: Mon, 6 May 2019 09:51:19 +0300 Subject: [PATCH 13/28] update setup.py according to review --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ea4787368..d668de03e 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ extras_require = { 'serial': ['pyserial~=3.0'], 'neovi': ['python-ics>=2.12'], - 'mf4': ['asammdf>=5.5.0', 'numpy>=1.16.0'] + 'mf4': ['asammdf>=5.5.0;python_version>="3.6"', 'numpy>=1.16.0;python_version>="3.6"'] } tests_require = [ From be0d168bf0b2a55f91d82608fdf493574e29fb84 Mon Sep 17 00:00:00 2001 From: danielhrisca Date: Mon, 6 May 2019 10:04:12 +0300 Subject: [PATCH 14/28] remove debug save file --- can/io/mf4.py | 1 - 1 file changed, 1 deletion(-) diff --git a/can/io/mf4.py b/can/io/mf4.py index 750449941..a819dce7b 100644 --- a/can/io/mf4.py +++ b/can/io/mf4.py @@ -154,7 +154,6 @@ def __init__(self, file, database=None): def stop(self): self._mdf.save(self.file, compression=2) - self._mdf.save(r'D:\TMP\test.mf4', overwrite=True) self._mdf.close() super(MF4Writer, self).stop() From 9f7e6b4291e3eaff9f8285844a832f83a860b271 Mon Sep 17 00:00:00 2001 From: danielhrisca Date: Mon, 6 May 2019 10:13:06 +0300 Subject: [PATCH 15/28] install and test mf4 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 70a5c0df1..442845b5d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -59,7 +59,7 @@ matrix: install: - if [[ "$TEST_SOCKETCAN" ]]; then sudo bash test/open_vcan.sh ; fi - if [[ "$BUILD_ONLY_DOCS" ]]; then travis_retry pip install -r doc/doc-requirements.txt; fi - - travis_retry pip install .[test] + - travis_retry pip install .[test,mf4] script: From 368c5055ea27c9df45fe25f7088552c1d0cfcdff Mon Sep 17 00:00:00 2001 From: danielhrisca Date: Sun, 12 May 2019 07:40:58 +0300 Subject: [PATCH 16/28] changes after review --- can/__init__.py | 1 - can/io/mf4.py | 3 +-- doc/listeners.rst | 9 +++++++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/can/__init__.py b/can/__init__.py index a71df6a8b..74635efa3 100644 --- a/can/__init__.py +++ b/can/__init__.py @@ -7,7 +7,6 @@ from __future__ import absolute_import import logging -import sys __version__ = "3.2.0-a0" diff --git a/can/io/mf4.py b/can/io/mf4.py index a819dce7b..c51b3495c 100644 --- a/can/io/mf4.py +++ b/can/io/mf4.py @@ -14,8 +14,6 @@ from pathlib import Path import logging -import numpy as np - from ..message import Message from ..listener import Listener from ..util import channel2int @@ -23,6 +21,7 @@ try: from asammdf import MDF, Signal + import numpy as np CAN_MSG_EXT = 0x80000000 CAN_ID_MASK = 0x1FFFFFFF diff --git a/doc/listeners.rst b/doc/listeners.rst index 42e90cdf0..cd06ef72f 100644 --- a/doc/listeners.rst +++ b/doc/listeners.rst @@ -166,7 +166,7 @@ MF4 (Measurement Data Format v4) -------------------------------- Implements support for MF4 (Measurement Data Format v4) which is a proprietary -format from ASAM, widely used in many automotive software (Vector CANape, ETAS INCA, dSPACE COntrolDesk, etc.). +format from ASAM, widely used in many automotive software (Vector CANape, ETAS INCA, dSPACE ControlDesk, etc.). The data is stored in a compressed format which makes it compact. @@ -180,7 +180,12 @@ The data is stored in a compressed format which makes it compact. .. autoclass:: can.MF4Writer :members: -MF4Reader can only replay files created with MF4Writer. The following class can be used to read messages from MF4 file: +The MDF format is very flexible regarding the internal structure and it is used to handle data from multiple sources, not just CAN bus logging. +MDF4Writer will always create a fixed internal file structure where there will be three channel groups (for standard, error and remote frames). +Using this fixed file structure allows for a simple implementation of MDF4Writer and MF4Reader classes. +Therefor MF4Reader can only replay files created with MF4Writer. + +The following class can be used to read messages from MF4 file: .. autoclass:: can.MF4Reader :members: From a8abbf0aabefeda79e771d862525bf200fbc8261 Mon Sep 17 00:00:00 2001 From: danielhrisca Date: Sat, 18 May 2019 13:35:52 +0300 Subject: [PATCH 17/28] updates after review --- can/io/__init__.py | 1 - can/io/mf4.py | 2 -- setup.py | 2 +- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/can/io/__init__.py b/can/io/__init__.py index 66d2d4bca..c8edc5744 100644 --- a/can/io/__init__.py +++ b/can/io/__init__.py @@ -6,7 +6,6 @@ """ from __future__ import absolute_import -import sys # Generic from .logger import Logger diff --git a/can/io/mf4.py b/can/io/mf4.py index c51b3495c..28d9bba1e 100644 --- a/can/io/mf4.py +++ b/can/io/mf4.py @@ -259,8 +259,6 @@ def __iter__(self): error_counter = 0 rtr_counter = 0 - - for timestamp, group_index in self.masters: # standard frames diff --git a/setup.py b/setup.py index d668de03e..ea4787368 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ extras_require = { 'serial': ['pyserial~=3.0'], 'neovi': ['python-ics>=2.12'], - 'mf4': ['asammdf>=5.5.0;python_version>="3.6"', 'numpy>=1.16.0;python_version>="3.6"'] + 'mf4': ['asammdf>=5.5.0', 'numpy>=1.16.0'] } tests_require = [ From c9a4a5793b7efad517b3a79805fe3dd96a228caa Mon Sep 17 00:00:00 2001 From: danielhrisca Date: Wed, 10 Jul 2019 16:59:07 +0300 Subject: [PATCH 18/28] add cython requirement --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c4c36a110..0afd4682e 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ # Dependencies extras_require = { "seeedstudio": ["pyserial>=3.0"], - "mf4": ["asammdf>=5.5.0", "numpy>=1.16.0"], + "mf4": ["asammdf>=5.5.0", "numpy>=1.16.0", "Cython"], "serial": ["pyserial~=3.0"], "neovi": ["python-ics>=2.12", "filelock"], } From 9901b774742162fa22039fc27edeff7d1823398e Mon Sep 17 00:00:00 2001 From: danielhrisca Date: Mon, 30 Sep 2019 09:08:47 +0300 Subject: [PATCH 19/28] fixes after review: * fix documentation * fix item access on FD_DLC2LEN dict * add compression argument to MF4Writer stop method --- can/io/mf4.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/can/io/mf4.py b/can/io/mf4.py index 28d9bba1e..48ee63c7f 100644 --- a/can/io/mf4.py +++ b/can/io/mf4.py @@ -91,12 +91,12 @@ class MF4Writer(BaseIOHandler, Listener): If a message has a timestamp smaller than the previous one or None, it gets assigned the timestamp that was written for the last message. - It the first message does not have a timestamp, it is set to zero. + If the first message does not have a timestamp, it is set to zero. """ def __init__(self, file, database=None): """ - :param file: a path-like object or as file-like object to write to + :param file: a path-like object or a file-like object to write to If this is a file-like object, is has to be opened in binary write mode, not text write mode. :param database: optional path to a DBC or ARXML file that contains @@ -151,8 +151,15 @@ def __init__(self, file, database=None): self._buffer = np.zeros(1, dtype=STD_DTYPE) self._rtr_buffer = np.zeros(1, dtype=RTR_DTYPE) - def stop(self): - self._mdf.save(self.file, compression=2) + def stop(self, compression=2): + """ + :param file: compression option as integer (default 2) + * 0 - no compression + * 1 - deflate (slower, but produces smaller files) + * 2 - transposition + deflate (slowest, but produces + the smallest files) + """ + self._mdf.save(self.file, compression=compression) self._mdf.close() super(MF4Writer, self).stop() @@ -297,7 +304,7 @@ def __iter__(self): channel = sample['CAN_DataFrame.ID'] arbitration_id = int(sample['CAN_DataFrame.ID']) size = int(sample['CAN_DataFrame.DataLength']) - dlc = FD_DLC2LEN(sample['CAN_DataFrame.DLC']) + dlc = FD_DLC2LEN[sample['CAN_DataFrame.DLC']] data = sample['CAN_DataFrame.DataBytes'][0, :size].tobytes() error_state_indicator = int(sample['CAN_DataFrame.ESI']) bitrate_switch = int(sample['CAN_DataFrame.BRS']) @@ -356,7 +363,7 @@ def __iter__(self): channel = sample['CAN_ErrorFrame.ID'] arbitration_id = int(sample['CAN_ErrorFrame.ID']) size = int(sample['CAN_ErrorFrame.DataLength']) - dlc = FD_DLC2LEN(sample['CAN_ErrorFrame.DLC']) + dlc = FD_DLC2LEN[sample['CAN_ErrorFrame.DLC']] data = sample['CAN_ErrorFrame.DataBytes'][0, :size].tobytes() error_state_indicator = int(sample['CAN_ErrorFrame.ESI']) bitrate_switch = int(sample['CAN_ErrorFrame.BRS']) From ceb33051567419d6a772a72c3f38af457d3ec96d Mon Sep 17 00:00:00 2001 From: danielhrisca Date: Mon, 30 Sep 2019 09:25:07 +0300 Subject: [PATCH 20/28] add MF4Writer and MF4Reader to logger and player modules --- can/io/logger.py | 72 +++++++++++++++++++++++++++--------------------- can/io/mf4.py | 2 +- can/io/player.py | 8 ++++++ 3 files changed, 49 insertions(+), 33 deletions(-) diff --git a/can/io/logger.py b/can/io/logger.py index 3c9cf5e46..99e0c3b40 100644 --- a/can/io/logger.py +++ b/can/io/logger.py @@ -1,11 +1,12 @@ +# coding: utf-8 + """ See the :class:`Logger` class. """ -import pathlib -import typing +from __future__ import absolute_import -import can.typechecking +import logging from ..listener import Listener from .generic import BaseIOHandler @@ -16,8 +17,15 @@ from .sqlite import SqliteWriter from .printer import Printer +try: + from .mf4 import MF4Writer +except ImportError: + MF4Writer = None + +log = logging.getLogger("can.io.logger") + -class Logger(BaseIOHandler, Listener): # pylint: disable=abstract-method +class Logger(BaseIOHandler, Listener): """ Logs CAN messages to a file. @@ -27,42 +35,42 @@ class Logger(BaseIOHandler, Listener): # pylint: disable=abstract-method * .csv: :class:`can.CSVWriter` * .db: :class:`can.SqliteWriter` * .log :class:`can.CanutilsLogWriter` - * .txt :class:`can.Printer` - - The **filename** may also be *None*, to fall back to :class:`can.Printer`. + * .mf4 :class:`can.MF4Writer` + * other: :class:`can.Printer` The log files may be incomplete until `stop()` is called due to buffering. .. note:: - This class itself is just a dispatcher, and any positional and keyword + This class itself is just a dispatcher, and any positional an keyword arguments are passed on to the returned instance. """ @staticmethod - def __new__( - cls, filename: typing.Optional[can.typechecking.StringPathLike], *args, **kwargs - ): + def __new__(cls, filename, *args, **kwargs): """ - :param filename: the filename/path of the file to write to, - may be a path-like object or None to - instantiate a :class:`~can.Printer` - :raises ValueError: if the filename's suffix is of an unknown file type + :type filename: str or None or path-like + :param filename: the filename/path the file to write to, + may be a path-like object if the target logger supports + it, and may be None to instantiate a :class:`~can.Printer` + """ - if filename is None: - return Printer(*args, **kwargs) + if filename: + if filename.endswith(".asc"): + return ASCWriter(filename, *args, **kwargs) + elif filename.endswith(".blf"): + return BLFWriter(filename, *args, **kwargs) + elif filename.endswith(".csv"): + return CSVWriter(filename, *args, **kwargs) + elif filename.endswith(".db"): + return SqliteWriter(filename, *args, **kwargs) + elif filename.endswith(".log"): + return CanutilsLogWriter(filename, *args, **kwargs) + elif filename.endswith(".mf4"): + if MF4Writer is not None: + return MF4Writer(filename, *args, **kwargs) + else: + log.info('Could not import MF4 logger, falling pack to can.Printer') - lookup = { - ".asc": ASCWriter, - ".blf": BLFWriter, - ".csv": CSVWriter, - ".db": SqliteWriter, - ".log": CanutilsLogWriter, - ".txt": Printer, - } - suffix = pathlib.PurePath(filename).suffix - try: - return lookup[suffix](filename, *args, **kwargs) - except KeyError: - raise ValueError( - f'No write support for this unknown log format "{suffix}"' - ) from None + # else: + log.info('unknown file type "%s", falling pack to can.Printer', filename) + return Printer(filename, *args, **kwargs) diff --git a/can/io/mf4.py b/can/io/mf4.py index 48ee63c7f..368939257 100644 --- a/can/io/mf4.py +++ b/can/io/mf4.py @@ -237,7 +237,7 @@ class MF4Reader(BaseIOHandler): def __init__(self, file): """ - :param file: a path-like object or as file-like object to read from + :param file: a path-like object or a file-like object to read from If this is a file-like object, is has to be opened in binary read mode, not text read mode. """ diff --git a/can/io/player.py b/can/io/player.py index bd206061c..78168f743 100644 --- a/can/io/player.py +++ b/can/io/player.py @@ -18,6 +18,11 @@ from .csv import CSVReader from .sqlite import SqliteReader +try: + from .mf4 import MF4Reader +except ImportError: + MF4Reader = None + class LogReader(BaseIOHandler): """ @@ -59,6 +64,9 @@ def __new__(cls, filename: "can.typechecking.PathLike", *args, **kwargs): ".db": SqliteReader, ".log": CanutilsLogReader, } + if MF4Reader is not None: + lookup[".mf4"] = MF4Reader + suffix = pathlib.PurePath(filename).suffix try: return lookup[suffix](filename, *args, **kwargs) From 355783f4ac2bd82f11512dc5909b0ad83474ad53 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Wed, 4 Sep 2019 09:24:53 +0200 Subject: [PATCH 21/28] reformat and change setup.py accordingly --- can/__init__.py | 1 + can/io/__init__.py | 3 +- can/io/mf4.py | 211 ++++++++++++++++++---------------------- setup.py | 13 ++- test/logformats_test.py | 4 +- 5 files changed, 107 insertions(+), 125 deletions(-) diff --git a/can/__init__.py b/can/__init__.py index 88ba76f88..6ce56c9d9 100644 --- a/can/__init__.py +++ b/can/__init__.py @@ -27,6 +27,7 @@ class CanError(IOError): from .io import CanutilsLogReader, CanutilsLogWriter from .io import CSVWriter, CSVReader from .io import SqliteWriter, SqliteReader + try: from .io import MF4Writer, MF4Reader except ImportError: diff --git a/can/io/__init__.py b/can/io/__init__.py index 819abac57..e09f969ad 100644 --- a/can/io/__init__.py +++ b/can/io/__init__.py @@ -14,7 +14,8 @@ from .csv import CSVWriter, CSVReader from .sqlite import SqliteReader, SqliteWriter from .printer import Printer + try: - from .mf4 import MF4Writer, MF4Reader + from .mf4 import MF4Writer, MF4Reader except ImportError: pass diff --git a/can/io/mf4.py b/can/io/mf4.py index 368939257..a42c0e31f 100644 --- a/can/io/mf4.py +++ b/can/io/mf4.py @@ -28,61 +28,49 @@ STD_DTYPE = np.dtype( [ - ('CAN_DataFrame.BusChannel', '=2.12", "filelock"], } -tests_require = [ +extras_require["test"] = [ "pytest~=4.3", "pytest-timeout~=1.3", "pytest-cov~=2.6", @@ -41,7 +44,9 @@ "hypothesis", ] + extras_require["serial"] -extras_require["test"] = tests_require +# see GitHub issue #696: MF4 does not run on PyPy +if IS_CPYTHON: + extras_require["test"] += extras_require["mf4"] # Check for 'pytest-runner' only if setup.py was invoked with 'test'. # This optimizes setup.py for cases when pytest-runner is not needed, @@ -105,10 +110,10 @@ "aenum", 'windows-curses;platform_system=="Windows"', "filelock", - "mypy_extensions >= 0.4.0, < 0.5.0", + "mypy_extensions~=0.4.0", 'pywin32;platform_system=="Windows"', ], setup_requires=pytest_runner, extras_require=extras_require, - tests_require=tests_require, + tests_require=extras_require["test"], ) diff --git a/test/logformats_test.py b/test/logformats_test.py index 23ca62e21..7e406f2d3 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -423,13 +423,15 @@ class TestMF4FileFormat(ReaderWriterTest): def _setup_instance(self): super(TestMF4FileFormat, self)._setup_instance_helper( - MF4Writer, MF4Reader, + MF4Writer, + MF4Reader, binary_file=True, check_comments=False, preserves_channel=False, adds_default_channel=0, ) + except ImportError: pass From 90f9ecf6c1da9422fe980dc0ca7e8ff6e37a2ead Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Wed, 4 Sep 2019 09:27:55 +0200 Subject: [PATCH 22/28] cleanup test file --- test/logformats_test.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/test/logformats_test.py b/test/logformats_test.py index 7e406f2d3..46dbcde67 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -415,14 +415,17 @@ def _setup_instance(self): try: from can import MF4Writer, MF4Reader +except ImportError: + # seems to unsupported on this platform + # see GitHub issue #696: MF4 does not run on PyPy + pass +else: class TestMF4FileFormat(ReaderWriterTest): """Tests can.MF4Writer and can.MF4Reader""" - __test__ = True - def _setup_instance(self): - super(TestMF4FileFormat, self)._setup_instance_helper( + super()._setup_instance_helper( MF4Writer, MF4Reader, binary_file=True, @@ -432,10 +435,6 @@ def _setup_instance(self): ) -except ImportError: - pass - - class TestSqliteDatabaseFormat(ReaderWriterTest): """Tests can.SqliteWriter and can.SqliteReader""" From d06cc520ffe65f637f53c2623a6bca33b3c5deb7 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Wed, 4 Sep 2019 09:29:33 +0200 Subject: [PATCH 23/28] cleanup docs --- doc/listeners.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/listeners.rst b/doc/listeners.rst index 0160eb80d..1830eaeed 100644 --- a/doc/listeners.rst +++ b/doc/listeners.rst @@ -177,11 +177,11 @@ format from ASAM, widely used in many automotive software (Vector CANape, ETAS I The data is stored in a compressed format which makes it compact. -.. note:: MF4 support requires Python >= 3.6 and has to be installed as an extra with for example ``pip install python-can[mf4]``. +.. note:: MF4 support has to be installed as an extra with for example ``pip install python-can[mf4]``. .. note:: Channels will be converted to integers. -.. note:: MF4Writer does not suppport append mode. +.. note:: MF4Writer does not suppport the append mode. .. autoclass:: can.MF4Writer From d9d43c112ff1332d134ca69d9ead8f835b6913f0 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Wed, 4 Sep 2019 11:36:13 +0200 Subject: [PATCH 24/28] remove orphaned pip extention --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 30b353283..a83d461af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ env: install: - if [[ "$TEST_SOCKETCAN" ]]; then sudo bash test/open_vcan.sh ; fi - - travis_retry pip install .[test,mf4] + - travis_retry pip install .[test] script: - | From e42126caa83b60306384ebf8a2a5fbf2a7c69154 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Wed, 4 Sep 2019 12:41:26 +0200 Subject: [PATCH 25/28] cleanups and fix linter problems --- can/io/mf4.py | 659 +++++++++++++++++++++++++------------------------- 1 file changed, 330 insertions(+), 329 deletions(-) diff --git a/can/io/mf4.py b/can/io/mf4.py index a42c0e31f..4ad85ce8a 100644 --- a/can/io/mf4.py +++ b/can/io/mf4.py @@ -1,15 +1,10 @@ -# coding: utf-8 - """ Contains handling of MF4 logging files. MF4 files represent Measurement Data Format (MDF) version 4 as specified by the ASAM MDF standard (see https://www.asam.net/standards/detail/mdf/) - """ -from __future__ import absolute_import - from datetime import datetime from pathlib import Path import logging @@ -20,381 +15,387 @@ from .generic import BaseIOHandler try: - from asammdf import MDF, Signal + from asammdf import MDF4, Signal import numpy as np - CAN_MSG_EXT = 0x80000000 - CAN_ID_MASK = 0x1FFFFFFF - - STD_DTYPE = np.dtype( - [ - ("CAN_DataFrame.BusChannel", " Date: Thu, 3 Oct 2019 17:14:06 +0200 Subject: [PATCH 26/28] re-add change to can.io.Logger; it somehow went lost while rebasing --- can/io/logger.py | 66 ++++++++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 30 deletions(-) diff --git a/can/io/logger.py b/can/io/logger.py index 99e0c3b40..e31458ac0 100644 --- a/can/io/logger.py +++ b/can/io/logger.py @@ -7,6 +7,10 @@ from __future__ import absolute_import import logging +import pathlib +import typing + +import can.typechecking from ..listener import Listener from .generic import BaseIOHandler @@ -22,10 +26,8 @@ except ImportError: MF4Writer = None -log = logging.getLogger("can.io.logger") - -class Logger(BaseIOHandler, Listener): +class Logger(BaseIOHandler, Listener): # pylint: disable=abstract-method """ Logs CAN messages to a file. @@ -36,41 +38,45 @@ class Logger(BaseIOHandler, Listener): * .db: :class:`can.SqliteWriter` * .log :class:`can.CanutilsLogWriter` * .mf4 :class:`can.MF4Writer` - * other: :class:`can.Printer` + * .txt :class:`can.Printer` + + The **filename** may also be *None*, to fall back to :class:`can.Printer`. The log files may be incomplete until `stop()` is called due to buffering. .. note:: - This class itself is just a dispatcher, and any positional an keyword + This class itself is just a dispatcher, and any positional and keyword arguments are passed on to the returned instance. """ @staticmethod - def __new__(cls, filename, *args, **kwargs): + def __new__( + cls, filename: typing.Optional[can.typechecking.StringPathLike], *args, **kwargs + ): """ - :type filename: str or None or path-like - :param filename: the filename/path the file to write to, - may be a path-like object if the target logger supports - it, and may be None to instantiate a :class:`~can.Printer` - + :param filename: the filename/path of the file to write to, + may be a path-like object or None to + instantiate a :class:`~can.Printer` + :raises ValueError: if the filename's suffix is of an unknown file type """ - if filename: - if filename.endswith(".asc"): - return ASCWriter(filename, *args, **kwargs) - elif filename.endswith(".blf"): - return BLFWriter(filename, *args, **kwargs) - elif filename.endswith(".csv"): - return CSVWriter(filename, *args, **kwargs) - elif filename.endswith(".db"): - return SqliteWriter(filename, *args, **kwargs) - elif filename.endswith(".log"): - return CanutilsLogWriter(filename, *args, **kwargs) - elif filename.endswith(".mf4"): - if MF4Writer is not None: - return MF4Writer(filename, *args, **kwargs) - else: - log.info('Could not import MF4 logger, falling pack to can.Printer') + if filename is None: + return Printer(*args, **kwargs) + + lookup = { + ".asc": ASCWriter, + ".blf": BLFWriter, + ".csv": CSVWriter, + ".db": SqliteWriter, + ".log": CanutilsLogWriter, + ".txt": Printer, + } + if MF4Writer is not None: + lookup[".mf4"] = MF4Writer - # else: - log.info('unknown file type "%s", falling pack to can.Printer', filename) - return Printer(filename, *args, **kwargs) + suffix = pathlib.PurePath(filename).suffix + try: + return lookup[suffix](filename, *args, **kwargs) + except KeyError: + raise ValueError( + f'No write support for this unknown log format "{suffix}"' + ) from None From 5cb4b6398ac18143ee7587e7803a4b109ee77e5a Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Thu, 3 Oct 2019 17:17:19 +0200 Subject: [PATCH 27/28] remove leftover __future__ import --- can/io/logger.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/can/io/logger.py b/can/io/logger.py index e31458ac0..05ef5d20f 100644 --- a/can/io/logger.py +++ b/can/io/logger.py @@ -4,8 +4,6 @@ See the :class:`Logger` class. """ -from __future__ import absolute_import - import logging import pathlib import typing From c311af4d2351df28f05b0ebd8d88af3baa17508d Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Sun, 6 Oct 2019 16:25:05 +0200 Subject: [PATCH 28/28] fix typing error in can.io's player.py and logger.py --- can/io/logger.py | 7 +++++-- can/io/player.py | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/can/io/logger.py b/can/io/logger.py index 05ef5d20f..9582ee5be 100644 --- a/can/io/logger.py +++ b/can/io/logger.py @@ -22,7 +22,8 @@ try: from .mf4 import MF4Writer except ImportError: - MF4Writer = None + # be careful when using MF4Writer, it might NameError + pass class Logger(BaseIOHandler, Listener): # pylint: disable=abstract-method @@ -68,8 +69,10 @@ def __new__( ".log": CanutilsLogWriter, ".txt": Printer, } - if MF4Writer is not None: + try: lookup[".mf4"] = MF4Writer + except NameError: + pass suffix = pathlib.PurePath(filename).suffix try: diff --git a/can/io/player.py b/can/io/player.py index 78168f743..85eb00df8 100644 --- a/can/io/player.py +++ b/can/io/player.py @@ -21,7 +21,8 @@ try: from .mf4 import MF4Reader except ImportError: - MF4Reader = None + # be careful when using MF4Writer, it might NameError + pass class LogReader(BaseIOHandler): @@ -64,8 +65,10 @@ def __new__(cls, filename: "can.typechecking.PathLike", *args, **kwargs): ".db": SqliteReader, ".log": CanutilsLogReader, } - if MF4Reader is not None: + try: lookup[".mf4"] = MF4Reader + except NameError: + pass suffix = pathlib.PurePath(filename).suffix try: