From 9259ec09c4b3ae78c05521551e6a2f1cf6aba0b3 Mon Sep 17 00:00:00 2001 From: j-c-cook Date: Wed, 17 Aug 2022 14:59:13 -0500 Subject: [PATCH 1/8] Quick fix to make blf.gz writer functional --- can/io/blf.py | 15 ++++++++------- can/io/logger.py | 4 +++- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/can/io/blf.py b/can/io/blf.py index 911177d20..39d5e9b35 100644 --- a/can/io/blf.py +++ b/can/io/blf.py @@ -578,11 +578,12 @@ def file_size(self) -> int: def stop(self): """Stops logging and closes the file.""" self._flush() - if self.file.seekable(): - filesize = self.file.tell() - # Write header in the beginning of the file - self.file.seek(0) - self._write_header(filesize) - else: - LOG.error("Could not write BLF header since file is not seekable") + # Note: This does not appear to be necessary if the append option is not being used. + # if self.file.seekable(): + # filesize = self.file.tell() + # # Write header in the beginning of the file + # self.file.seek(0) # TODO: fix the OSError that occurs here + # self._write_header(filesize) + # else: + # LOG.error("Could not write BLF header since file is not seekable") super().stop() diff --git a/can/io/logger.py b/can/io/logger.py index 071eeb300..689f5bb2c 100644 --- a/can/io/logger.py +++ b/can/io/logger.py @@ -104,7 +104,9 @@ def compress(filename: StringPathLike) -> Tuple[str, FileLike]: File will automatically recompress upon close. """ real_suffix = pathlib.Path(filename).suffixes[-2].lower() - mode = "ab" if real_suffix == ".blf" else "at" + # Why is this setup to append by default? + # mode = "ab" if real_suffix == ".blf" else "at" + mode = "wb" if real_suffix == ".blf" else "wt" return real_suffix, gzip.open(filename, mode) From 5b9b49a0a24c140fee8e30831edbfebcb0eec2d6 Mon Sep 17 00:00:00 2001 From: j-c-cook Date: Wed, 17 Aug 2022 16:03:11 -0500 Subject: [PATCH 2/8] Fix rollover name creation --- can/io/logger.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/can/io/logger.py b/can/io/logger.py index 3f4d54e1f..44bdf72b3 100644 --- a/can/io/logger.py +++ b/can/io/logger.py @@ -346,12 +346,20 @@ def do_rollover(self) -> None: def _default_name(self) -> StringPathLike: """Generate the default rotation filename.""" path = pathlib.Path(self.base_filename) + if len(path.suffixes) == 1: + _stem = path.stem + _suffix = path.suffix + elif len(path.suffixes) == 2: + _stem = path.parts[-1].split('.')[0] + _suffix = ''.join(path.suffixes) + else: + raise ValueError("More than two suffixes are seen.") new_name = ( - path.stem + _stem + "_" + datetime.now().strftime("%Y-%m-%dT%H%M%S") + "_" + f"#{self.rollover_count:03}" - + path.suffix + + _suffix ) return str(path.parent / new_name) From 81298052adfe48cd79e615883407ab503774d4fa Mon Sep 17 00:00:00 2001 From: j-c-cook Date: Mon, 22 Aug 2022 17:24:58 -0500 Subject: [PATCH 3/8] Fix formatting with black --- can/io/logger.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/can/io/logger.py b/can/io/logger.py index 44bdf72b3..1202a4b66 100644 --- a/can/io/logger.py +++ b/can/io/logger.py @@ -350,8 +350,8 @@ def _default_name(self) -> StringPathLike: _stem = path.stem _suffix = path.suffix elif len(path.suffixes) == 2: - _stem = path.parts[-1].split('.')[0] - _suffix = ''.join(path.suffixes) + _stem = path.parts[-1].split(".")[0] + _suffix = "".join(path.suffixes) else: raise ValueError("More than two suffixes are seen.") new_name = ( From 4ed0a17e0ca8a52fdcd3df5f768d21190e33058d Mon Sep 17 00:00:00 2001 From: j-c-cook Date: Mon, 22 Aug 2022 17:33:03 -0500 Subject: [PATCH 4/8] Clean up _default_name changes --- can/io/logger.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/can/io/logger.py b/can/io/logger.py index 1202a4b66..597673533 100644 --- a/can/io/logger.py +++ b/can/io/logger.py @@ -346,20 +346,14 @@ def do_rollover(self) -> None: def _default_name(self) -> StringPathLike: """Generate the default rotation filename.""" path = pathlib.Path(self.base_filename) - if len(path.suffixes) == 1: - _stem = path.stem - _suffix = path.suffix - elif len(path.suffixes) == 2: - _stem = path.parts[-1].split(".")[0] - _suffix = "".join(path.suffixes) - else: - raise ValueError("More than two suffixes are seen.") + stem = path.parts[-1].split(".")[0] + suffix = "".join(pathlib.Path(self.base_filename).suffixes[-2:]) new_name = ( - _stem + stem + "_" + datetime.now().strftime("%Y-%m-%dT%H%M%S") + "_" + f"#{self.rollover_count:03}" - + _suffix + + suffix ) return str(path.parent / new_name) From 28e540573fda266a45cc04d2d79b79086819e4f5 Mon Sep 17 00:00:00 2001 From: j-c-cook Date: Tue, 23 Aug 2022 09:00:49 -0500 Subject: [PATCH 5/8] Compressed logger now acknowledges append kwarg --- can/io/logger.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/can/io/logger.py b/can/io/logger.py index 597673533..6a3a4b985 100644 --- a/can/io/logger.py +++ b/can/io/logger.py @@ -88,7 +88,7 @@ def __new__( # type: ignore file_or_filename: AcceptedIOType = filename if suffix == ".gz": - suffix, file_or_filename = Logger.compress(filename) + suffix, file_or_filename = Logger.compress(filename, *args, **kwargs) try: return Logger.message_writers[suffix](file_or_filename, *args, **kwargs) @@ -98,15 +98,16 @@ def __new__( # type: ignore ) from None @staticmethod - def compress(filename: StringPathLike) -> Tuple[str, FileLike]: + def compress(filename: StringPathLike, *args: Any, **kwargs: Any) -> Tuple[str, FileLike]: """ Return the suffix and io object of the decompressed file. File will automatically recompress upon close. """ real_suffix = pathlib.Path(filename).suffixes[-2].lower() - # Why is this setup to append by default? - # mode = "ab" if real_suffix == ".blf" else "at" - mode = "wb" if real_suffix == ".blf" else "wt" + if kwargs.get('append', False): + mode = "ab" if real_suffix == ".blf" else "at" + else: + mode = "wb" if real_suffix == ".blf" else "wt" return real_suffix, gzip.open(filename, mode) From ac664ca43813d8cec1d62535b748f4cb1c156855 Mon Sep 17 00:00:00 2001 From: j-c-cook Date: Tue, 23 Aug 2022 09:01:18 -0500 Subject: [PATCH 6/8] For gzip.GzipFile, don't call seek twice --- can/io/blf.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/can/io/blf.py b/can/io/blf.py index 381615b89..a4f0ff0ca 100644 --- a/can/io/blf.py +++ b/can/io/blf.py @@ -11,7 +11,7 @@ of uncompressed data each. This data contains the actual CAN messages and other objects types. """ - +import gzip import struct import zlib import datetime @@ -583,12 +583,12 @@ def file_size(self) -> int: def stop(self): """Stops logging and closes the file.""" self._flush() - # Note: This does not appear to be necessary if the append option is not being used. - # if self.file.seekable(): - # filesize = self.file.tell() - # # Write header in the beginning of the file - # self.file.seek(0) # TODO: fix the OSError that occurs here - # self._write_header(filesize) - # else: - # LOG.error("Could not write BLF header since file is not seekable") + if self.file.seekable(): + filesize = self.file.tell() # == self.file.seek(0, 1) if type == gzip.GzipFile + if type(self.file) != gzip.GzipFile: + # Write header in the beginning of the file + self.file.seek(0) + self._write_header(filesize) + else: + LOG.error("Could not write BLF header since file is not seekable") super().stop() From 876dacb8926d495a90279b1880385a5a18664a0f Mon Sep 17 00:00:00 2001 From: j-c-cook Date: Tue, 23 Aug 2022 09:02:11 -0500 Subject: [PATCH 7/8] Format with black --- can/io/blf.py | 4 +++- can/io/logger.py | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/can/io/blf.py b/can/io/blf.py index a4f0ff0ca..a6c505437 100644 --- a/can/io/blf.py +++ b/can/io/blf.py @@ -584,7 +584,9 @@ def stop(self): """Stops logging and closes the file.""" self._flush() if self.file.seekable(): - filesize = self.file.tell() # == self.file.seek(0, 1) if type == gzip.GzipFile + filesize = ( + self.file.tell() + ) # == self.file.seek(0, 1) if type == gzip.GzipFile if type(self.file) != gzip.GzipFile: # Write header in the beginning of the file self.file.seek(0) diff --git a/can/io/logger.py b/can/io/logger.py index 6a3a4b985..d8decfc1b 100644 --- a/can/io/logger.py +++ b/can/io/logger.py @@ -98,13 +98,15 @@ def __new__( # type: ignore ) from None @staticmethod - def compress(filename: StringPathLike, *args: Any, **kwargs: Any) -> Tuple[str, FileLike]: + def compress( + filename: StringPathLike, *args: Any, **kwargs: Any + ) -> Tuple[str, FileLike]: """ Return the suffix and io object of the decompressed file. File will automatically recompress upon close. """ real_suffix = pathlib.Path(filename).suffixes[-2].lower() - if kwargs.get('append', False): + if kwargs.get("append", False): mode = "ab" if real_suffix == ".blf" else "at" else: mode = "wb" if real_suffix == ".blf" else "wt" From f64b6e79d9174be7ca9fbf076bc635a887ab5e90 Mon Sep 17 00:00:00 2001 From: j-c-cook Date: Wed, 24 Aug 2022 08:45:10 -0500 Subject: [PATCH 8/8] Hotfix blf.gz log by temporarily removing dflt header --- can/io/blf.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/can/io/blf.py b/can/io/blf.py index a6c505437..ff6c3d57c 100644 --- a/can/io/blf.py +++ b/can/io/blf.py @@ -12,6 +12,7 @@ objects types. """ import gzip +import os import struct import zlib import datetime @@ -432,7 +433,8 @@ def __init__( self.start_timestamp = None self.stop_timestamp = None # Write a default header which will be updated when stopped - self._write_header(FILE_HEADER_SIZE) + # Temporarily removed for the sake of GzipFiles + # self._write_header(FILE_HEADER_SIZE) def _write_header(self, filesize): header = [b"LOGG", FILE_HEADER_SIZE, self.application_id, 0, 0, 0, 2, 6, 8, 1] @@ -540,8 +542,9 @@ def _add_object(self, obj_type, data, timestamp=None): self._buffer_size += obj_size + padding_size self.object_count += 1 - if self._buffer_size >= self.max_container_size: - self._flush() + # Don't accidentally write to blf file prior to writing header + # if self._buffer_size >= self.max_container_size: + # self._flush() def _flush(self): """Compresses and writes data in the buffer to file.""" @@ -582,15 +585,13 @@ def file_size(self) -> int: def stop(self): """Stops logging and closes the file.""" - self._flush() if self.file.seekable(): - filesize = ( - self.file.tell() - ) # == self.file.seek(0, 1) if type == gzip.GzipFile - if type(self.file) != gzip.GzipFile: - # Write header in the beginning of the file - self.file.seek(0) + # For a GzipFile, the file currently must be empty prior to writing + # this final header, else an OSError will occur + filesize = self.file.tell() + self.file.seek(0) self._write_header(filesize) else: LOG.error("Could not write BLF header since file is not seekable") + self._flush() super().stop()