From 84db6f503d2dd971476fd224627eeed43fc188eb Mon Sep 17 00:00:00 2001 From: glopesdev Date: Mon, 30 Oct 2023 12:45:09 +0000 Subject: [PATCH 1/9] Remove reflex-generator from black formatter --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 9b1cbdd..b5d4169 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,5 +62,6 @@ extend-exclude = ''' ^/LICENSE ^/README.md | harp/model.py + | reflex-generator ) ''' From 6daba190d1ddd85c054d1f670add5a1e262852c1 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Mon, 30 Oct 2023 12:45:46 +0000 Subject: [PATCH 2/9] Add base implementation of register reader --- harp/reader.py | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 harp/reader.py diff --git a/harp/reader.py b/harp/reader.py new file mode 100644 index 0000000..7c1a122 --- /dev/null +++ b/harp/reader.py @@ -0,0 +1,71 @@ +import re +from functools import partial, reduce +from pandas import DataFrame +from typing import Iterable, Callable +from harp.model import Model, Register +from harp.io import read + +_camel_to_snake_regex = re.compile(r"(? None: + self.register = register + self.read = read + + +def compose(f, g): + return lambda *a, **kw: f(g(*a, **kw)) + + +def id_camel_to_snake(id: str): + return _camel_to_snake_regex.sub("_", id).lower() + + +def keys_camel_to_snake(keys: Iterable[str]): + return [id_camel_to_snake(k) for k in keys] + + +def create_bitreader(mask): + return lambda xs: ((xs & mask) != 0) + + +def create_reader(device: Model, name: str): + register = device.registers[name] + reader = read + + if register.maskType is not None: + key = register.maskType.root + bitMask = device.bitMasks.get(key) + if bitMask is not None: + lookup = [ + (id_camel_to_snake(k), create_bitreader(v.root)) + for k, v in bitMask.bits.items() + ] + + def unpack(df): + return DataFrame({n: f(df[0]) for n, f in lookup}, index=df.index) + + reader = compose(unpack, reader) + return Reader(register, reader) + + groupMask = device.groupMasks.get(key) + if groupMask is not None: + name = id_camel_to_snake(name) + lookup = {value.root: name for name, value in groupMask.values.items()} + reader = partial(reader, columns=[name]) + reader = compose(lambda df: df.map(lambda x: lookup[x]), reader) + return Reader(register, reader) + + if register.payloadSpec is not None: + columns = register.payloadSpec.keys() + columns = keys_camel_to_snake(columns) + reader = partial(reader, columns=columns) + return Reader(register, reader) + + columns = [id_camel_to_snake(name)] + reader = partial(reader, columns=columns) + return Reader(register, reader) From 3ece4c1cb4b83218755ef89498a704cd710a387d Mon Sep 17 00:00:00 2001 From: glopesdev Date: Mon, 30 Oct 2023 12:56:27 +0000 Subject: [PATCH 3/9] Add support for creating full device reader --- harp/reader.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/harp/reader.py b/harp/reader.py index 7c1a122..d121e62 100644 --- a/harp/reader.py +++ b/harp/reader.py @@ -8,7 +8,7 @@ _camel_to_snake_regex = re.compile(r"(? None self.read = read +class DeviceReader: + model: Model + registers: dict[str, RegisterReader] + + def __init__(self, model: Model, registers: dict[str, RegisterReader]) -> None: + self.model = model + self.registers = registers + + def compose(f, g): return lambda *a, **kw: f(g(*a, **kw)) @@ -33,7 +42,14 @@ def create_bitreader(mask): return lambda xs: ((xs & mask) != 0) -def create_reader(device: Model, name: str): +def create_reader(device: Model): + reg_readers = { + create_register_reader(device, name) for name in device.registers.keys() + } + return DeviceReader(device, reg_readers) + + +def create_register_reader(device: Model, name: str): register = device.registers[name] reader = read @@ -50,7 +66,7 @@ def unpack(df): return DataFrame({n: f(df[0]) for n, f in lookup}, index=df.index) reader = compose(unpack, reader) - return Reader(register, reader) + return RegisterReader(register, reader) groupMask = device.groupMasks.get(key) if groupMask is not None: @@ -58,14 +74,14 @@ def unpack(df): lookup = {value.root: name for name, value in groupMask.values.items()} reader = partial(reader, columns=[name]) reader = compose(lambda df: df.map(lambda x: lookup[x]), reader) - return Reader(register, reader) + return RegisterReader(register, reader) if register.payloadSpec is not None: columns = register.payloadSpec.keys() columns = keys_camel_to_snake(columns) reader = partial(reader, columns=columns) - return Reader(register, reader) + return RegisterReader(register, reader) columns = [id_camel_to_snake(name)] reader = partial(reader, columns=columns) - return Reader(register, reader) + return RegisterReader(register, reader) From c36a70d4e22928aa58cfd7bb75f0b138954c5a86 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Mon, 30 Oct 2023 13:41:39 +0000 Subject: [PATCH 4/9] Ensure compatibility of IO type annotations --- harp/io.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/harp/io.py b/harp/io.py index d358fb1..d5340c7 100644 --- a/harp/io.py +++ b/harp/io.py @@ -1,5 +1,5 @@ from os import PathLike -from typing import Any, Optional, Union +from typing import Any, BinaryIO, Optional, Union from pandas._typing import Axes import numpy as np import pandas as pd @@ -19,7 +19,7 @@ def read( - file: Union[str, bytes, PathLike[Any], np._IOProtocol], + file: Union[str, bytes, PathLike[Any], BinaryIO], columns: Optional[Axes] = None, ): """ From 62b562c5c0635ad620897f29359d801a3ef84312 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Mon, 30 Oct 2023 14:12:24 +0000 Subject: [PATCH 5/9] Ensure setuptools compatibility --- pyproject.toml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b5d4169..78e2c27 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,15 +31,17 @@ classifiers = [ [project.optional-dependencies] dev = [ "datamodel-code-generator", - "setuptools_scm", "black" ] +jupyter = [ + "ipykernel" +] [build-system] requires = [ - "setuptools>=45", "wheel", - "setuptools_scm[toml]>=6.2", + "setuptools", + "setuptools_scm[toml]", ] build-backend = "setuptools.build_meta" From 593baa940008c488905d7a17809f65ef8e3ab46e Mon Sep 17 00:00:00 2001 From: glopesdev Date: Mon, 30 Oct 2023 14:34:46 +0000 Subject: [PATCH 6/9] Fix enumeration of device register attributes --- harp/reader.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/harp/reader.py b/harp/reader.py index d121e62..9424a09 100644 --- a/harp/reader.py +++ b/harp/reader.py @@ -25,6 +25,12 @@ def __init__(self, model: Model, registers: dict[str, RegisterReader]) -> None: self.model = model self.registers = registers + def __dir__(self) -> Iterable[str]: + return self.registers.keys() + + def __getattr__(self, __name: str) -> RegisterReader: + return self.registers[__name] + def compose(f, g): return lambda *a, **kw: f(g(*a, **kw)) @@ -44,7 +50,7 @@ def create_bitreader(mask): def create_reader(device: Model): reg_readers = { - create_register_reader(device, name) for name in device.registers.keys() + name: create_register_reader(device, name) for name in device.registers.keys() } return DeviceReader(device, reg_readers) From 2879ea6dab8363f5a8ce561f999bf7af027f37b3 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Mon, 30 Oct 2023 20:55:15 +0000 Subject: [PATCH 7/9] Refactor device reader attributes --- harp/reader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/harp/reader.py b/harp/reader.py index 9424a09..b645164 100644 --- a/harp/reader.py +++ b/harp/reader.py @@ -18,11 +18,11 @@ def __init__(self, register: Register, read: Callable[[str], DataFrame]) -> None class DeviceReader: - model: Model + device: Model registers: dict[str, RegisterReader] - def __init__(self, model: Model, registers: dict[str, RegisterReader]) -> None: - self.model = model + def __init__(self, device: Model, registers: dict[str, RegisterReader]) -> None: + self.device = device self.registers = registers def __dir__(self) -> Iterable[str]: From f8ef6fa80556ed9b64844ee670c50ba97eea94be Mon Sep 17 00:00:00 2001 From: glopesdev Date: Wed, 1 Nov 2023 21:10:09 +0000 Subject: [PATCH 8/9] Refactor internal helpers with type hinting --- harp/reader.py | 78 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 31 deletions(-) diff --git a/harp/reader.py b/harp/reader.py index b645164..aabc1ff 100644 --- a/harp/reader.py +++ b/harp/reader.py @@ -1,8 +1,8 @@ import re -from functools import partial, reduce -from pandas import DataFrame -from typing import Iterable, Callable -from harp.model import Model, Register +from functools import partial +from pandas import DataFrame, Series +from typing import Iterable, Callable, Union +from harp.model import BitMask, GroupMask, MaskValueItem, Model, Register from harp.io import read _camel_to_snake_regex = re.compile(r"(? RegisterReader: return self.registers[__name] -def compose(f, g): +def _compose(f, g): return lambda *a, **kw: f(g(*a, **kw)) -def id_camel_to_snake(id: str): +def _id_camel_to_snake(id: str): return _camel_to_snake_regex.sub("_", id).lower() -def keys_camel_to_snake(keys: Iterable[str]): - return [id_camel_to_snake(k) for k in keys] +def _keys_camel_to_snake(keys: Iterable[str]): + return [_id_camel_to_snake(k) for k in keys] -def create_bitreader(mask): - return lambda xs: ((xs & mask) != 0) +def _create_bit_parser(mask: Union[int, MaskValueItem]): + def parser(xs: Series) -> Series: + return (xs & mask) != 0 + return parser -def create_reader(device: Model): - reg_readers = { - name: create_register_reader(device, name) for name in device.registers.keys() - } - return DeviceReader(device, reg_readers) +def _create_bitmask_parser(bitMask: BitMask): + lookup = [ + (_id_camel_to_snake(k), _create_bit_parser(v.root)) + for k, v in bitMask.bits.items() + ] + + def parser(df: DataFrame): + return DataFrame({n: f(df[0]) for n, f in lookup}, index=df.index) + + return parser + + +def _create_groupmask_parser(name: str, groupMask: GroupMask): + name = _id_camel_to_snake(name) + lookup = {v.root: n for n, v in groupMask.values.items()} + + def parser(df: DataFrame): + return DataFrame({name: df.map(lambda x: lookup[x])}) + + return parser -def create_register_reader(device: Model, name: str): + +def _create_register_reader(device: Model, name: str): register = device.registers[name] reader = read @@ -63,31 +81,29 @@ def create_register_reader(device: Model, name: str): key = register.maskType.root bitMask = device.bitMasks.get(key) if bitMask is not None: - lookup = [ - (id_camel_to_snake(k), create_bitreader(v.root)) - for k, v in bitMask.bits.items() - ] - - def unpack(df): - return DataFrame({n: f(df[0]) for n, f in lookup}, index=df.index) - - reader = compose(unpack, reader) + parser = _create_bitmask_parser(bitMask) + reader = _compose(parser, reader) return RegisterReader(register, reader) groupMask = device.groupMasks.get(key) if groupMask is not None: - name = id_camel_to_snake(name) - lookup = {value.root: name for name, value in groupMask.values.items()} - reader = partial(reader, columns=[name]) - reader = compose(lambda df: df.map(lambda x: lookup[x]), reader) + parser = _create_groupmask_parser(name, groupMask) + reader = _compose(parser, reader) return RegisterReader(register, reader) if register.payloadSpec is not None: columns = register.payloadSpec.keys() - columns = keys_camel_to_snake(columns) + columns = _keys_camel_to_snake(columns) reader = partial(reader, columns=columns) return RegisterReader(register, reader) - columns = [id_camel_to_snake(name)] + columns = [_id_camel_to_snake(name)] reader = partial(reader, columns=columns) return RegisterReader(register, reader) + + +def create_reader(device: Model): + reg_readers = { + name: _create_register_reader(device, name) for name in device.registers.keys() + } + return DeviceReader(device, reg_readers) From 0b9a13293407cfd2c36b8333dcf755438b5c2e35 Mon Sep 17 00:00:00 2001 From: glopesdev Date: Wed, 1 Nov 2023 23:58:16 +0000 Subject: [PATCH 9/9] Fix constant definition --- harp/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/harp/io.py b/harp/io.py index d5340c7..2ff6efc 100644 --- a/harp/io.py +++ b/harp/io.py @@ -4,7 +4,7 @@ import numpy as np import pandas as pd -_SECONDS_PER_TICK = 32e6 +_SECONDS_PER_TICK = 32e-6 payloadtypes = { 1: np.dtype(np.uint8), 2: np.dtype(np.uint16),