From 2cc6a5f73d6947610c673c4900036c3e62e6e3e2 Mon Sep 17 00:00:00 2001 From: "MIDE\\cflanigan" Date: Tue, 15 Jun 2021 17:03:56 -0400 Subject: [PATCH 01/41] Modified how chaches are being made, unit tests currently failing. --- idelib/dataset.py | 179 +++++++++++++++++++++++++++++++++++++--- idelib/importer.py | 4 + testing/test_dataset.py | 5 +- 3 files changed, 173 insertions(+), 15 deletions(-) diff --git a/idelib/dataset.py b/idelib/dataset.py index 6e26294f..dec44485 100644 --- a/idelib/dataset.py +++ b/idelib/dataset.py @@ -50,6 +50,7 @@ from bisect import bisect_right from collections.abc import Iterable, Sequence from datetime import datetime +from threading import Lock from functools import partial import os.path @@ -62,7 +63,7 @@ import numpy as np from .transforms import Transform, CombinedPoly, PolyPoly -from .parsers import getParserTypes, getParserRanges +from .parsers import getParserTypes, getParserRanges, ChannelDataBlock SCHEMA_FILE = 'mide_ide.xml' @@ -255,6 +256,8 @@ def __init__(self, stream, name=None, exitCondition=None, quiet=True): self.loadCancelled = False self.loading = True self.filename = getattr(stream, "name", None) + + self._channelDataLock = Lock() # Subsets: used when importing multiple files into the same dataset. self.subsets = [] @@ -490,6 +493,17 @@ def updateTransforms(self): for ch in self.channels.values(): ch.updateTransforms() + def allocateCaches(self): + for channel in self.channels.values(): + for ea in channel.sessions.values(): + with self._channelDataLock: + ea.allocateCache() + + def fillCaches(self): + for channel in self.channels.values(): + for ea in channel.sessions.values(): + ea.fillCache() + #=============================================================================== # @@ -1195,6 +1209,29 @@ def __init__(self, parentChannel, session=None, parentList=None): self._mean = None + format = self.parent.parser.format + if len(format) == 0: + self._npType = np.uint8 + else: + print(format) + + if format[0] in ['<', '>', '=']: + endian = format[0] + dtypes = [endian + ChannelDataBlock.TO_NP_TYPESTR[x] for x in format[1:]] + else: + dtypes = [ChannelDataBlock.TO_NP_TYPESTR[x] for x in format] + + self._npType = np.dtype([(str(i), dtype) for i, dtype in enumerate(dtypes)]) + + self._channelDataLock = parentChannel.dataset._channelDataLock + self._cacheArray = None + self._cacheBytes = None + self._fullyCached = False + self._cacheStart = None + self._cacheEnd = None + self._cacheBlockStart = None + self._cacheBlockEnd = None + def updateTransforms(self, recurse=True): """ (Re-)Build and (re-)apply the transformation functions. @@ -1797,12 +1834,32 @@ def arrayValues(self, start=None, end=None, step=1, subchannels=True, """ # TODO: Optimize; times don't need to be computed since they aren't used # -> take directly from _blockSlice - arrayEvents = self.arraySlice(start, end, step, display) - if self.hasSubchannels and subchannels is not True: - return arrayEvents[np.asarray(subchannels)+1] + if not isinstance(start, slice): + start = slice(start, end, step) + start, end, step = start.indices(len(self)) + + if self.useAllTransforms: + xform = self._fullXform + if display: + xform = self._displayXform or xform + else: + xform = self._comboXform + + rawData = self._accessCache(start, end, step) + + if isinstance(self.parent, SubChannel): + out = np.zeros((1, len(rawData))) + else: + out = np.zeros((len(rawData.dtype), len(rawData))) + + if isinstance(self.parent, SubChannel): + xform.polys[self.subchannelId].inplace(rawData, out=out) else: - return arrayEvents[1:] + for i, (k, _) in enumerate(rawData.dtype.descr): + xform.polys[i].inplace(rawData[k], out=out[i]) + + return out def _blockSlice(self, start=None, end=None, step=1, display=False): @@ -1883,15 +1940,33 @@ def arraySlice(self, start=None, end=None, step=1, display=False): 'display' transform) will be applied to the data. :return: a structured array of events in the specified index range. """ - raw_slice = [ - [times[np.newaxis].T, values.T] - for times, values in self._blockSlice(start, end, step, display) - ] - if not raw_slice: - no_of_chs = (len(self.parent.types) if self.hasSubchannels else 1) - return np.empty((no_of_chs+1, 0), dtype=np.float) + if not isinstance(start, slice): + start = slice(start, end, step) + start, end, step = start.indices(len(self)) - return np.block(raw_slice).T + if self.useAllTransforms: + xform = self._fullXform + if display: + xform = self._displayXform or xform + else: + xform = self._comboXform + + rawData = self._accessCache(start, end, step) + + if isinstance(self.parent, SubChannel): + out = np.zeros((2, len(rawData))) + else: + out = np.zeros((len(rawData.dtype) + 1, len(rawData))) + + self._inplaceTime(start, end, step, out=out[0]) + + if isinstance(self.parent, SubChannel): + xform.polys[self.subchannelId].inplace(rawData, out=out[1]) + else: + for i, (k, _) in enumerate(rawData.dtype.descr): + xform.polys[i].inplace(rawData[k], out=out[i + 1]) + + return out def _blockJitterySlice(self, start=None, end=None, step=1, jitter=0.5, @@ -2708,6 +2783,7 @@ def exportCsv(self, stream, start=None, stop=None, step=1, subchannels=True, try: for num, evt in enumerate(_self.iterSlice(start, stop, step, display=display)): stream.write("%s\n" % formatter(evt)) + # print(evt - np.array([float(x) for x in formatter(evt).split(', ')])) if callback is not None: if getattr(callback, 'cancelled', False): callback(done=True) @@ -2724,6 +2800,83 @@ def exportCsv(self, stream, start=None, stop=None, step=1, subchannels=True, return num+1, datetime.now() - t0 + def allocateCache(self): + + payloadLen = sum((d.payloadSize for d in self._data)) + self._cacheBytes = np.zeros(payloadLen, dtype=np.uint8) + + self._cacheArray = self._cacheBytes.view(self._npType) + + def fillCache(self): + idx = 0 + for d in self._data: + with self.dataset._channelDataLock: + payloadLen = len(d._payloadEl.value) + self._cacheBytes[idx:idx + payloadLen] = np.frombuffer(d._payloadEl.value, dtype=np.uint8) + d._payloadEl._value = self._cacheBytes[idx:idx + payloadLen] + d._payload = self._cacheBytes[idx:idx + payloadLen].view(self._npType) + idx += payloadLen + + def _accessCache(self, start, end, step): + """ Access cached data in a thread-safe way. If data has not fully + loaded, instead allocate an appropriately large array, fill it with + data, then return that instead. + """ + + if isinstance(self.parent, SubChannel): + schId = self.subchannelId + ch = self.parent.parent + rawData = ch.getSession(self.session.sessionId)._accessCache(start, end, step) + schKey = rawData.dtype.names[schId] + return rawData[schKey] + + if self.dataset.loading: + nSamples = int(np.ceil((end - start)/step)) + newBytes = np.zeros(nSamples*self._npType.itemsize, dtype=np.uint8) + newArray = newBytes.view(self._npType) + + nextStart = start + idx = 0 + for d in self._data: + with self.dataset._channelDataLock: + blockStart, blockEnd = d.indexRange + if blockStart > end: + return newArray + elif blockEnd < nextStart: + continue + + relativeStart = nextStart - blockStart + if end > blockEnd: + relativeEnd = blockEnd - blockStart + else: + relativeEnd = end - blockStart + + payloadView = d.payload.view(self._npType)[relativeStart:relativeEnd:step] + newArray[idx:idx + len(payloadView)] = payloadView + idx += len(payloadView) + nextStart = idx*step + + return newArray + else: + with self.dataset._channelDataLock: + return self._cacheArray[start:end:step] + + def _inplaceTime(self, start, end, step, out=None): + if out is None: + out = np.zeros((int(np.ceil((end - start)/step)),)) + + arrayStart = float(self._data[0].startTime) + arrayEnd = float(self._data[-1].endTime) + nSamples = len(self) + samplePeriod = (arrayEnd - arrayStart)/(nSamples - 1) + + out[:] = np.linspace( + arrayStart + start*samplePeriod, + arrayStart + end*samplePeriod, + len(out), + ) + return out + #=============================================================================== # diff --git a/idelib/importer.py b/idelib/importer.py index 6aac7b1f..bdd39026 100644 --- a/idelib/importer.py +++ b/idelib/importer.py @@ -504,6 +504,10 @@ def readData(doc, source=None, updater=nullUpdater, numUpdates=500, updateInterv # (typically the last) doc.fileDamaged = True + doc.allocateCaches() + + doc.fillCaches() + doc.loading = False updater(done=True) return eventsRead diff --git a/testing/test_dataset.py b/testing/test_dataset.py index 08009a2e..727dcbf3 100644 --- a/testing/test_dataset.py +++ b/testing/test_dataset.py @@ -2039,10 +2039,11 @@ def testCalibratedExport(self): accel.exportCsv(out) out.seek(0) - new = np.genfromtxt(out, delimiter=', ') + new = np.genfromtxt(out, delimiter=', ').T old = accel.__getitem__(slice(None), display=True) + old = np.round(1e6*old)/1e6 - np.testing.assert_allclose(new.T, old, rtol=1e-4) + np.testing.assert_allclose(new, old, rtol=1e-4) def testUncalibratedExport(self): From 62b3f85da5297a86749008443e8a3aaf26d84912 Mon Sep 17 00:00:00 2001 From: "MIDE\\cflanigan" Date: Wed, 16 Jun 2021 13:44:12 -0400 Subject: [PATCH 02/41] fix copy --- idelib/dataset.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/idelib/dataset.py b/idelib/dataset.py index dec44485..3c0417c1 100644 --- a/idelib/dataset.py +++ b/idelib/dataset.py @@ -1299,6 +1299,14 @@ def copy(self, newParent=None): newList.noBivariates = self.noBivariates newList._blockIndices = self._blockIndices newList._blockTimes = self._blockTimes + newList._channelDataLock = self._channelDataLock + newList._cacheArray = self._cacheArray + newList._cacheBytes = self._cacheBytes + newList._fullyCached = self._fullyCached + newList._cacheStart = self._cacheStart + newList._cacheEnd = self._cacheEnd + newList._cacheBlockStart = self._cacheBlockStart + newList._cacheBlockEnd = self._cacheBlockEnd return newList @@ -2068,7 +2076,7 @@ def arrayJitterySlice(self, start=None, end=None, step=1, jitter=0.5, :return: a structured array of events in the specified index range. """ self._computeMinMeanMax() - + raw_slice = [ [times[np.newaxis].T, values.T] for times, values in self._blockJitterySlice( @@ -2778,6 +2786,13 @@ def exportCsv(self, stream, start=None, stop=None, step=1, subchannels=True, if headers: stream.write('"Time"%s%s\n' % (delimiter, delimiter.join(['"%s"' % n for n in names]))) + + data = _self.arraySlice(start, stop, step) + if useUtcTime and _self.session.utcStartTime: + if useIsoFormat: + times = data[0] + data = data.astype([('time', ' Date: Mon, 21 Jun 2021 15:50:19 -0400 Subject: [PATCH 03/41] updating `arrayJitterySlice` --- idelib/dataset.py | 77 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 64 insertions(+), 13 deletions(-) diff --git a/idelib/dataset.py b/idelib/dataset.py index 694c37f5..ea113049 100644 --- a/idelib/dataset.py +++ b/idelib/dataset.py @@ -1212,13 +1212,15 @@ def __init__(self, parentChannel, session=None, parentList=None): if len(format) == 0: self._npType = np.uint8 else: + if isinstance(format, bytes): + format = format.decode() print(format) if format[0] in ['<', '>', '=']: endian = format[0] dtypes = [endian + ChannelDataBlock.TO_NP_TYPESTR[x] for x in format[1:]] else: - dtypes = [ChannelDataBlock.TO_NP_TYPESTR[x] for x in format] + dtypes = [ChannelDataBlock.TO_NP_TYPESTR[x] for x in str(format)] self._npType = np.dtype([(str(i), dtype) for i, dtype in enumerate(dtypes)]) @@ -1968,10 +1970,10 @@ def arraySlice(self, start=None, end=None, step=1, display=False): self._inplaceTime(start, end, step, out=out[0]) if isinstance(self.parent, SubChannel): - xform.polys[self.subchannelId].inplace(rawData, out=out[1]) + xform.polys[self.subchannelId].inplace(rawData, out=out[1], timestamp=out[0]) else: for i, (k, _) in enumerate(rawData.dtype.descr): - xform.polys[i].inplace(rawData[k], out=out[i + 1]) + xform.polys[i].inplace(rawData[k], out=out[i + 1], timestamp=out[0]) return out @@ -1999,6 +2001,12 @@ def _blockJitterySlice(self, start=None, end=None, step=1, jitter=0.5, jitter = 0.5 scaledJitter = jitter * abs(step) + indices = np.arange(start, end, step) + if scaledJitter > 0.5: + indices[1:-1] += np.rint( + scaledJitter*np.random.uniform(-1, 1, max(0, len(indices) - 2)) + ).astype(indices.dtype) + startBlockIdx = self._getBlockIndexWithIndex(start) if start > 0 else 0 endBlockIdx = self._getBlockIndexWithIndex(end-1, start=startBlockIdx) @@ -2074,19 +2082,62 @@ def arrayJitterySlice(self, start=None, end=None, step=1, jitter=0.5, 'display' transform) will be applied to the data. :return: a structured array of events in the specified index range. """ + + # Check for non-jittered cases + if jitter is True: + jitter = 0.5 + scaledJitter = jitter * abs(step) + + if scaledJitter <= 0.5: + return self.arraySlice(start=start, end=end, step=step, display=display) + + # begin as normal self._computeMinMeanMax() - raw_slice = [ - [times[np.newaxis].T, values.T] - for times, values in self._blockJitterySlice( - start, end, step, jitter, display - ) - ] - if not raw_slice: - no_of_chs = (len(self.parent.types) if self.hasSubchannels else 1) - return np.empty((no_of_chs+1, 0), dtype=np.float) + if not isinstance(start, slice): + start = slice(start, end, step) + start, end, step = start.indices(len(self)) + + if self.useAllTransforms: + xform = self._fullXform + if display: + xform = self._displayXform or xform + else: + xform = self._comboXform + + # grab all raw data + rawData = self._accessCache(start, end, 1) + + # slightly janky way of enforcing output length + if isinstance(self.parent, SubChannel): + out = np.zeros((2, len(self._accessCache(start, end, step)))) + else: + out = np.zeros((len(rawData.dtype) + 1, len(self._accessCache(start, end, step)))) + + # save on space by being really clever and storing indices in timestamps + indices = out[0].view(np.int64) + indices[:] = np.arange(start, end, step, dtype=np.int64) + if len(indices) > 2: + indices[1:-1] += np.rint(np.random.uniform( + -scaledJitter, scaledJitter, (len(indices) - 2,) + )).astype(np.int64) - return np.block(raw_slice).T + # now index raw data + rawData = rawData[indices] + + # now times + times = self._inplaceTime(start, end, 1) + times = times[indices] + out[0] = times[:] + del times + + if isinstance(self.parent, SubChannel): + xform.polys[self.subchannelId].inplace(rawData, out=out[1], timestamp=out[0]) + else: + for i, (k, _) in enumerate(rawData.dtype.descr): + xform.polys[i].inplace(rawData[k], out=out[i + 1], timestamp=out[0]) + + return out def getEventIndexBefore(self, t): From 81899cf949277351d765b366fb169f4329ecf324 Mon Sep 17 00:00:00 2001 From: "MIDE\\cflanigan" Date: Thu, 24 Jun 2021 16:07:58 -0400 Subject: [PATCH 04/41] fixing eventArray tests, partly by making them conform to pytest standards Also fixing broken methods in eventarray --- idelib/dataset.py | 27 +-- testing/test_dataset.py | 384 ++++++++++++++++++++++------------------ 2 files changed, 225 insertions(+), 186 deletions(-) diff --git a/idelib/dataset.py b/idelib/dataset.py index ea113049..2c25e530 100644 --- a/idelib/dataset.py +++ b/idelib/dataset.py @@ -1214,7 +1214,6 @@ def __init__(self, parentChannel, session=None, parentList=None): else: if isinstance(format, bytes): format = format.decode() - print(format) if format[0] in ['<', '>', '=']: endian = format[0] @@ -2148,14 +2147,17 @@ def getEventIndexBefore(self, t): :return: The index of the event preceding the given time, -1 if the time occurs before the first event. """ - if t <= self._data[0].startTime: + if t < self._data[0].startTime: return -1 + + if t >= self._data[-1].endTime: + return self._data[-1].indexRange[1] + blockIdx = self._getBlockIndexWithTime(t) try: block = self._data[blockIdx] except IndexError: - blockIdx = len(self._data)-1 - block = self._data[blockIdx] + block = self._data[-1] return int(block.indexRange[0] + \ ((t - block.startTime) / self._getBlockSampleTime(blockIdx))) @@ -2168,11 +2170,16 @@ def getEventIndexNear(self, t): """ if t <= self._data[0].startTime: return 0 + + if t >= self._data[-1].endTime: + return self._data[-1].indexRange[1] + idx = self.getEventIndexBefore(t) - events = self[idx:idx+2] - if events[0][0] == t or len(events) == 1: + events = self[idx:idx+2][0] + if len(events) == 1: return idx - return min((t - events[0][0], idx), (events[1][0] - t, idx+1))[1] + + return idx + abs(events - t).argmin() def getRangeIndices(self, startTime, endTime): @@ -2935,11 +2942,7 @@ def _inplaceTime(self, start, end, step, out=None): nSamples = len(self) samplePeriod = (arrayEnd - arrayStart)/(nSamples - 1) - out[:] = np.linspace( - arrayStart + start*samplePeriod, - arrayStart + end*samplePeriod, - len(out), - ) + out[:] = np.arange(start, end, step)*samplePeriod + arrayStart return out diff --git a/testing/test_dataset.py b/testing/test_dataset.py index 727dcbf3..fce28641 100644 --- a/testing/test_dataset.py +++ b/testing/test_dataset.py @@ -13,7 +13,11 @@ import sys import unittest import mock +import os +import pytest + +import idelib from idelib.dataset import (Cascading, Channel, Dataset, @@ -35,6 +39,37 @@ from .file_streams import makeStreamLike + +# ============================================================================== +# Fixtures +# ============================================================================== + +_fileStrings = {} + + +def _load_file(filePath): + if filePath not in _fileStrings: + with open(filePath, 'rb') as f: + _fileStrings[filePath] = f.read() + out = BytesIO(_fileStrings[filePath]) + out.name = filePath + return out + + +@pytest.fixture +def testIDE(): + doc = idelib.importer.openFile(_load_file('./test.IDE')) + idelib.importer.readData(doc) + return doc + + +@pytest.fixture +def SSX70065IDE(): + doc = idelib.importer.openFile(_load_file('./testing/SSX70065.IDE')) + idelib.importer.readData(doc) + return doc + + #=============================================================================== # #=============================================================================== @@ -79,7 +114,7 @@ def parseByIndexWith(self, parser, indices, subchannel): # #=============================================================================== -class CascadingTestCase(unittest.TestCase): +class TestCascading(unittest.TestCase): """ Test case for methods in the Cascading class. """ def setUp(self): @@ -117,7 +152,7 @@ def testRepr(self): # #=============================================================================== -class TransformableTestCase(unittest.TestCase): +class TestTransformable(unittest.TestCase): """ Test case for methods in the Transformable class. """ def setUp(self): @@ -190,7 +225,7 @@ def testGetTransforms(self): # #=============================================================================== -class DatasetTestCase(unittest.TestCase): +class TestDataset(unittest.TestCase): """ Test case for methods in the Dataset class. """ def setUp(self): @@ -429,7 +464,7 @@ def mockUpdateTransforms(): # #=============================================================================== -class SessionTestCase(unittest.TestCase): +class TestSession(unittest.TestCase): """ Test case for methods in the Session class. """ def testInitAndEQ(self): @@ -462,7 +497,7 @@ def testRepr(self): # #=============================================================================== -class SensorTestCase(unittest.TestCase): +class TestSensor(unittest.TestCase): """ Test case for methods in the Sensor class. """ def setUp(self): @@ -531,7 +566,7 @@ def testBandwidthRolloff(self): # #=============================================================================== -class ChannelTestCase(unittest.TestCase): +class TestChannel(unittest.TestCase): """ Test case for methods in the Channel class. """ def setUp(self): @@ -701,7 +736,7 @@ def testUpdateTransforms(self): # #=============================================================================== -class SubChannelTestCase(unittest.TestCase): +class TestSubChannel(unittest.TestCase): """ Test case for methods in the SubChannel class. """ def setUp(self): @@ -849,39 +884,47 @@ def testGetSubchannel(self): # #=============================================================================== -class EventArrayTestCase(unittest.TestCase): +class TestEventArray: """ Test case for methods in the EventArray class. """ - def assertArrayEqual(self, array1, array2): - self.assertTrue(np.all(array1 == array2)) + @pytest.fixture + def dataset(self, SSX70065IDE): + SSX70065IDE.addSession(0, 1, 2) + SSX70065IDE.addSensor(0) - def setUp(self): - self.dataset = importer.importFile('./testing/SSX70065.IDE') - self.dataset.addSession(0, 1, 2) - self.dataset.addSensor(0) + return SSX70065IDE - self.fakeParser = GenericObject() - self.fakeParser.types = [0] - self.fakeParser.format = [] + @pytest.fixture + def fakeParser(self): + parser = GenericObject() + parser.types = [0] + parser.format = [] - self.channel1 = Channel( - self.dataset, channelId=0, name="channel1", parser=self.fakeParser, + return parser + + @pytest.fixture + def _channel1(self, dataset, fakeParser): + return Channel( + dataset, channelId=0, name="channel1", parser=fakeParser, displayRange=[0] ) - self.eventArray1 = EventArray(self.channel1, - session=self.dataset.sessions[0]) - self.channel1.addSubChannel(subchannelId=0) + @pytest.fixture + def eventArray1(self, _channel1, dataset): + return EventArray(_channel1, session=dataset.sessions[0]) - self.subChannel1 = SubChannel(self.channel1, 0) + @pytest.fixture + def channel1(self, _channel1): + _channel1.addSubChannel(subchannelId=0) - def tearDown(self): - self.dataset = None - self.fakeParser = None - self.channel1 = None - self.eventArray1 = None + return _channel1 - def mockData(self): + @pytest.fixture + def subchannel1(self, channel1): + return SubChannel(channel1, 0) + + @pytest.fixture + def mockData(self, eventArray1): """ mock up a bit of fake data so I don't have to worry that external classes are working during testing. """ @@ -890,7 +933,7 @@ def mockData(self): fakeData.indexRange = [0, 4] fakeData.sampleTime = 1 fakeData.numSamples = 1 - self.eventArray1._data = [fakeData] + eventArray1._data = [fakeData] def mockIterSlice(self, *args, **kwargs): """ Mock up iterslice so it doesn't get called while testing @@ -919,110 +962,103 @@ def mockXform(self, times, session=None, noBivariates=None): # Base Method Tests # -------------------------------------------------------------------------- - def testConstructor(self): + def testConstructor(self, eventArray1, channel1, dataset): """ Test the __init__ method. """ - self.assertEqual(self.eventArray1._blockIndices, []) - self.assertEqual(self.eventArray1._blockTimes, []) - self.assertEqual(self.eventArray1._childLists, []) - self.assertEqual(self.eventArray1._data, []) - self.assertEqual(self.eventArray1._firstTime, None) - self.assertFalse(self.eventArray1._hasSubsamples) - self.assertEqual(self.eventArray1._lastTime, None) - self.assertEqual(self.eventArray1._length, 0) - self.assertEqual(self.eventArray1._parentList, None) - self.assertEqual(self.eventArray1._singleSample, - self.channel1.singleSample) - - self.assertEqual(self.eventArray1.channelId, self.channel1.id) - self.assertEqual(self.eventArray1.dataset, self.channel1.dataset) - self.assertEqual(self.eventArray1.displayRange, - self.channel1.displayRange) - self.assertTrue(self.eventArray1.hasMinMeanMax) - self.assertEqual(self.eventArray1.hasDisplayRange, - self.channel1.hasDisplayRange) - self.assertTrue(self.eventArray1.hasSubchannels) - self.assertFalse(self.eventArray1.noBivariates) - self.assertEqual(self.eventArray1.parent, self.channel1) - self.assertFalse(self.eventArray1.removeMean) - self.assertEqual(self.eventArray1.rollingMeanSpan, - EventArray.DEFAULT_MEAN_SPAN) - self.assertEqual(self.eventArray1.session, self.dataset.sessions[0]) - self.assertEqual(self.eventArray1.subchannelId, None) - - self.assertEqual(self.eventArray1._blockIndicesArray.size, 0) - self.assertEqual(self.eventArray1._blockTimesArray.size, 0) - - # TODO add test - #def test_JoinTimesValues(self): - - def testUpdateTransforms(self): + assert eventArray1._blockIndices == [] + assert eventArray1._blockTimes == [] + assert eventArray1._childLists == [] + assert eventArray1._data == [] + assert eventArray1._firstTime is None + assert eventArray1._hasSubsamples is False + assert eventArray1._lastTime is None + assert eventArray1._length == 0 + assert eventArray1._parentList is None + assert eventArray1._singleSample == channel1.singleSample + + assert eventArray1.channelId == channel1.id + assert eventArray1.dataset == channel1.dataset + assert eventArray1.displayRange == channel1.displayRange + assert eventArray1.hasMinMeanMax is True + assert eventArray1.hasDisplayRange == channel1.hasDisplayRange + assert eventArray1.hasSubchannels is True + assert eventArray1.noBivariates is False + assert eventArray1.parent == channel1 + assert eventArray1.removeMean is False + assert eventArray1.rollingMeanSpan == EventArray.DEFAULT_MEAN_SPAN + assert eventArray1.session == dataset.sessions[0] + assert eventArray1.subchannelId is None + + assert eventArray1._blockIndicesArray.size == 0 + assert eventArray1._blockTimesArray.size == 0 + + @pytest.mark.skip('not yet implemented') + def testJoinTimesValues(self): + pass + + def testUpdateTransformsNoRecursion(self, eventArray1, channel1, dataset): """ Test the updateTransforms method. """ # update transforms without recursion - self.eventArray1.updateTransforms(False) - self.assertEqual( - self.eventArray1._comboXform, - PolyPoly([self.channel1.transform]*len(self.channel1.types)) - ) + eventArray1.updateTransforms(False) + assert eventArray1._comboXform == PolyPoly([channel1.transform]*len(channel1.types)) + xs = [c.transform if c is not None else None - for c in self.channel1.subchannels] - xs = [CombinedPoly(t, x=self.channel1.transform, dataset=self.dataset) + for c in channel1.subchannels] + xs = [CombinedPoly(t, x=channel1.transform, dataset=dataset) for t in xs] - self.assertEqual(self.eventArray1._fullXform, - PolyPoly(xs, dataset=self.dataset)) - self.tearDown() + assert eventArray1._fullXform == PolyPoly(xs, dataset=dataset) + + def testUpdateTransformsYesRecursion(self, eventArray1, channel1, dataset): # update transforms with recursion - self.setUp() - self.eventArray1.updateTransforms(True) - self.assertEqual(self.eventArray1._displayXform, - PolyPoly(xs, dataset=self.dataset)) - self.tearDown() + xs = [c.transform if c is not None else None + for c in channel1.subchannels] + xs = [CombinedPoly(t, x=channel1.transform, dataset=dataset) + for t in xs] + eventArray1.updateTransforms(True) + assert eventArray1._displayXform == PolyPoly(xs, dataset=self.dataset) + + def testUpdateTransformsOther(self, eventArray1, channel1, dataset): # test for when there's a subchannel with a corresponding session - self.setUp() - self.eventArray1.session.sessionId = 'session0' - self.eventArray1.parent.subchannels[0]._sessions = {'session0': self.eventArray1} - self.eventArray1.updateTransforms() + eventArray1.session.sessionId = 'session0' + eventArray1.parent.subchannels[0]._sessions = {'session0': eventArray1} + eventArray1.updateTransforms() xs = [c.transform if c is not None else None - for c in self.eventArray1.parent.subchannels] - self.assertEqual( - self.eventArray1._displayXform, - PolyPoly( - [CombinedPoly(self.eventArray1.transform, - x=xs[0], dataset=self.dataset)], - dataset=self.dataset, - ) - ) + for c in eventArray1.parent.subchannels] + assert eventArray1._displayXform == PolyPoly( + [CombinedPoly(eventArray1.transform, x=xs[0], dataset=dataset)], + dataset=dataset, + ) - def testUnits(self): + def testUnits(self, eventArray1): """ Test the units property. """ - self.assertEqual(self.eventArray1.units, ('', '')) + assert eventArray1.units == ('', '') - def testPath(self): + def testPath(self, eventArray1): """ Test the path method. """ - self.assertEqual(self.eventArray1.path(), "channel1, 0") + assert eventArray1.path() == "channel1, 0" - def testCopy(self): + def testCopy(self, eventArray1): """ Test the copy method. Since this is a shallow copy, don't use the build in equality check. """ - eventArrayCopy = self.eventArray1.copy() - self.assertEqual(self.eventArray1.parent, eventArrayCopy.parent) - self.assertEqual(self.eventArray1.session, eventArrayCopy.session) - self.assertEqual(self.eventArray1.dataset, eventArrayCopy.dataset) - self.assertEqual(self.eventArray1.hasSubchannels, eventArrayCopy.hasSubchannels) - self.assertEqual(self.eventArray1.noBivariates, eventArrayCopy.noBivariates) - self.assertEqual(self.eventArray1.channelId, eventArrayCopy.channelId) - self.assertEqual(self.eventArray1.subchannelId, eventArrayCopy.subchannelId) - self.assertEqual(self.eventArray1.channelId, eventArrayCopy.channelId) - self.assertEqual(self.eventArray1.hasDisplayRange, eventArrayCopy.hasDisplayRange) - self.assertEqual(self.eventArray1.displayRange, eventArrayCopy.displayRange) - self.assertEqual(self.eventArray1.removeMean, eventArrayCopy.removeMean) - self.assertEqual(self.eventArray1.hasMinMeanMax, eventArrayCopy.hasMinMeanMax) - self.assertEqual(self.eventArray1.rollingMeanSpan, eventArrayCopy.rollingMeanSpan) - self.assertEqual(self.eventArray1.transform, eventArrayCopy.transform) - self.assertEqual(self.eventArray1.useAllTransforms, eventArrayCopy.useAllTransforms) - self.assertEqual(self.eventArray1.allowMeanRemoval, eventArrayCopy.allowMeanRemoval) + eventArrayCopy = eventArray1.copy() + assert eventArray1.parent == eventArrayCopy.parent + assert eventArray1.session == eventArrayCopy.session + assert eventArray1.dataset == eventArrayCopy.dataset + assert eventArray1.hasSubchannels == eventArrayCopy.hasSubchannels + assert eventArray1.noBivariates == eventArrayCopy.noBivariates + assert eventArray1.channelId == eventArrayCopy.channelId + assert eventArray1.subchannelId == eventArrayCopy.subchannelId + assert eventArray1.channelId == eventArrayCopy.channelId + assert eventArray1.hasDisplayRange == eventArrayCopy.hasDisplayRange + assert eventArray1.displayRange == eventArrayCopy.displayRange + assert eventArray1.removeMean == eventArrayCopy.removeMean + assert eventArray1.hasMinMeanMax == eventArrayCopy.hasMinMeanMax + assert eventArray1.rollingMeanSpan == eventArrayCopy.rollingMeanSpan + assert eventArray1.transform == eventArrayCopy.transform + assert eventArray1.useAllTransforms == eventArrayCopy.useAllTransforms + assert eventArray1.allowMeanRemoval == eventArrayCopy.allowMeanRemoval @unittest.skip('failing, poorly formed') @@ -1037,70 +1073,70 @@ def testAppend(self): fakeData.parseMinMeanMax = lambda x: x # append boring basic fakeData - self.eventArray1.append(fakeData) - - self.assertEqual(fakeData.blockIndex, 0) - self.assertFalse(fakeData.cache) - self.assertEqual(fakeData.indexRange, (0, 1)) - self.assertEqual(self.eventArray1._blockIndices, [0]) - self.assertEqual(self.eventArray1._blockTimes, [2]) - self.assertEqual(self.eventArray1._firstTime, 2) - self.assertEqual(self.eventArray1._lastTime, 4) - self.assertEqual(self.eventArray1._length, 1) - self.assertTrue(self.eventArray1._singleSample) + eventArray1.append(fakeData) + + assert fakeData.blockIndex == 0 + assert fakeData.cache is False + assert fakeData.indexRange == (0, 1) + assert eventArray1._blockIndices == [0] + assert eventArray1._blockTimes == [2] + assert eventArray1._firstTime == 2 + assert eventArray1._lastTime == 4 + assert eventArray1._length == 1 + assert eventArray1._singleSample is True # append single sample fakeData - self.eventArray1._singleSample = True + eventArray1._singleSample = True - self.eventArray1.append(fakeData) + eventArray1.append(fakeData) - self.assertEqual(fakeData.blockIndex, 1) - self.assertFalse(fakeData.cache) - self.assertEqual(fakeData.indexRange, (1, 2)) - self.assertEqual(self.eventArray1._blockIndices, [0, 1]) - self.assertEqual(self.eventArray1._blockTimes, [2, 2]) - self.assertEqual(self.eventArray1._firstTime, 2) - self.assertEqual(self.eventArray1._lastTime, 4) - self.assertEqual(self.eventArray1._length, 2) - self.assertTrue(self.eventArray1._singleSample) + assert fakeData.blockIndex == 1 + assert fakeData.cache is False + assert fakeData.indexRange == (1, 2) + assert eventArray1._blockIndices == [0, 1] + assert eventArray1._blockTimes == [2, 2] + assert eventArray1._firstTime == 2 + assert eventArray1._lastTime == 4 + assert eventArray1._length == 2 + assert eventArray1._singleSample is True # append with times stripped out - self.eventArray1.session.firstTime = None - self.eventArray1.session.lastTime = None - self.eventArray1._firstTime = None - - self.eventArray1.append(fakeData) - - self.assertEqual(fakeData.blockIndex, 2) - self.assertFalse(fakeData.cache) - self.assertEqual(fakeData.indexRange, (2, 3)) - self.assertEqual(self.eventArray1._blockIndices, [0, 1, 2]) - self.assertEqual(self.eventArray1._blockTimes, [2, 2, 2]) - self.assertEqual(self.eventArray1._firstTime, 2) - self.assertEqual(self.eventArray1._lastTime, 4) - self.assertEqual(self.eventArray1._length, 3) - self.assertTrue(self.eventArray1._singleSample) - - def testGetInterval(self): + eventArray1.session.firstTime = None + eventArray1.session.lastTime = None + eventArray1._firstTime = None + + eventArray1.append(fakeData) + + assert fakeData.blockIndex == 2 + assert fakeData.cache is False + assert fakeData.indexRange == (2, 3) + assert eventArray1._blockIndices == [0, 1, 2] + assert eventArray1._blockTimes == [2, 2, 2] + assert eventArray1._firstTime == 2 + assert eventArray1._lastTime == 4 + assert eventArray1._length == 3 + assert eventArray1._singleSample is True + + def testGetInterval(self, dataset): """ Test the getInterval method. """ fakeObject = GenericObject() fakeObject.startTime = 3 fakeObject.endTime = 1 - accel = self.dataset.channels[32].getSession() + accel = dataset.channels[32].getSession() # without _data, return None - self.assertEqual(accel.getInterval(), None) - self.assertEqual(accel._lastTime, None) + assert accel.getInterval() == None + assert accel._lastTime is None # with mocked data accel._data = [fakeObject] - self.assertEqual(accel.getInterval(), (3, 1)) - self.assertEqual(accel._lastTime, 1) + assert accel.getInterval() == (3, 1) + assert accel._lastTime == 1 # with mocked data and a mocked dataset accel.dataset = GenericObject() accel.dataset.loading = True - self.assertEqual(accel.getInterval(), (3, 1)) + assert accel.getInterval() == (3, 1) def mockForGetItem(self, section): """ Mock different things for testGetItem. """ @@ -1124,15 +1160,15 @@ def mockXform(time, val, session=None, noBivariates=False): def mockParseBlock(block, start=None, end=None, step=None): return [(block,)] - self.eventArray1._getBlockIndexRange = mockGetBlockIndexRange - self.eventArray1._getBlockIndexWithIndex = mockGetBlockIndexWithIndex - self.eventArray1.parent.parseBlock = mockParseBlock - self.eventArray1._displayXform = self.eventArray1._comboXform = \ - self.eventArray1._fullXform = mockXform + eventArray1._getBlockIndexRange = mockGetBlockIndexRange + eventArray1._getBlockIndexWithIndex = mockGetBlockIndexWithIndex + eventArray1.parent.parseBlock = mockParseBlock + eventArray1._displayXform = eventArray1._comboXform = \ + eventArray1._fullXform = mockXform - self.eventArray1._data = [GenericObject() for _ in range(4)] + eventArray1._data = [GenericObject() for _ in range(4)] - for i, datum in enumerate(self.eventArray1._data): + for i, datum in enumerate(eventArray1._data): datum.id = i # mock without xforms @@ -1141,23 +1177,23 @@ def mockParseBlock(block, start=None, end=None, step=None): def mockXform(time, val, session=None, noBivariates=False): return None - self.eventArray1._displayXform = self.eventArray1._comboXform = \ - self.eventArray1._fullXform = mockXform + eventArray1._displayXform = eventArray1._comboXform = \ + eventArray1._fullXform = mockXform # with blockRollingMean elif section == 2: self.mockForGetItem(0) - self.eventArray1._getBlockRollingMean = lambda x: [1] + eventArray1._getBlockRollingMean = lambda x: [1] def mockXform(time, val, session=None, noBivariates=False): if type(val) is tuple: return time, val return time, [val.id] - self.eventArray1._displayXform = self.eventArray1._comboXform = \ - self.eventArray1._fullXform = mockXform + eventArray1._displayXform = eventArray1._comboXform = \ + eventArray1._fullXform = mockXform # for __getitem__ to work in getEventIndexNear elif section == 3: @@ -1166,7 +1202,7 @@ def mockXform(time, val, session=None, noBivariates=False): def mockGetBlockSampleTime(idx, start=None): return 1 - self.eventArray1._getBlockSampleTime = mockGetBlockSampleTime + eventArray1._getBlockSampleTime = mockGetBlockSampleTime def testGetItem(self): """ Test the getitem special method. """ From 88d911d9077db87366a25d0c2b1e481ab22f7de6 Mon Sep 17 00:00:00 2001 From: "MIDE\\cflanigan" Date: Thu, 24 Jun 2021 17:38:46 -0400 Subject: [PATCH 05/41] fixes eventArray tests --- idelib/dataset.py | 9 +- testing/test_dataset.py | 621 ++++++++++++++-------------------------- 2 files changed, 215 insertions(+), 415 deletions(-) diff --git a/idelib/dataset.py b/idelib/dataset.py index 2c25e530..c505c2f4 100644 --- a/idelib/dataset.py +++ b/idelib/dataset.py @@ -2581,10 +2581,11 @@ def getSampleTime(self, idx=None): :return: The time between samples (us) """ sr = self.parent.sampleRate - if idx is None and sr is not None: - return 1.0 / sr - else: - idx = 0 + if idx is None: + if sr is not None: + return 1.0/sr + else: + idx = 0 return self._getBlockSampleTime(self._getBlockIndexWithIndex(idx)) diff --git a/testing/test_dataset.py b/testing/test_dataset.py index fce28641..e39ba281 100644 --- a/testing/test_dataset.py +++ b/testing/test_dataset.py @@ -9,6 +9,7 @@ future. """ +from contextlib import nullcontext from io import StringIO, BytesIO import sys import unittest @@ -1229,8 +1230,8 @@ def testGetItem(self): parseBlock=(lambda block, start=None, end=None, step=1: np.array([[range(length)[block.id]]])) ) - - self.assertRaises(TypeError, EventArray.__getitem__, eventArray, 'd') + with pytest.raises(TypeError): + eventArray['d'] # # if the transform returns a none type, it should just skip through # # and return None @@ -1282,12 +1283,12 @@ def testGetItem(self): np.testing.assert_array_equal(EventArray.__getitem__(eventArray, 2), (0.02, 14)) np.testing.assert_array_equal(EventArray.__getitem__(eventArray, 3), (0.03, 21)) - def testIter(self): + def testIter(self, eventArray1): """ Test for iter special method. """ - self.eventArray1.iterSlice = self.mockIterSlice + eventArray1.iterSlice = self.mockIterSlice np.testing.assert_array_equal( - [x for x in self.eventArray1], - [x for x in self.eventArray1.iterSlice()] + [x for x in eventArray1], + [x for x in eventArray1.iterSlice()] ) # TODO talk to david about how to test these @@ -1338,62 +1339,27 @@ def testIterValues(self): ) # Run test - self.assertListEqual( + np.testing.assert_array_equal( list(EventArray.itervalues(eventArray)), [(0,), (7,), (14,), (21,)] ) - def testArrayValues(self): + @pytest.mark.parametrize('start, end, step', + [ + (None, None, 1), + (None, None, 5), + (10, 300, 3), + ], + ) + def testArrayValues(self, testIDE, start, end, step): """ Test for arrayValues method. """ - # Stub dependencies - length = 4 - eventArray = mock.Mock(spec=EventArray) - eventArray.configure_mock( - useAllTransforms=True, - __len__=lambda self: length, - _fullXform=Univariate((7, 0)), - _data=mock.Mock(), - _getBlockIndexWithIndex=(lambda idx, start=0, stop=None: - range(length)[idx]), - _getBlockIndexRange=lambda idx: [idx, idx+1], - _getBlockSampleTime=lambda idx: 0.01*idx, - _getBlockRollingMean=lambda blockIdx: (0,), - allowMeanRemoval=True, - removeMean=True, - hasMinMeanMax=True, - parent=mock.Mock(), - session=mock.sentinel.session, - noBivariates=mock.sentinel.noBivariates, - hasSubchannels=True, - arraySlice=( - lambda *a, **kw: - EventArray.arraySlice(eventArray, *a, **kw) - ), - _blockSlice=( - lambda *a, **kw: - EventArray._blockSlice(eventArray, *a, **kw) - ), - _makeBlockEventsFactory=( - lambda *a, **kw: - EventArray._makeBlockEventsFactory(eventArray, *a, **kw) - ), - ) - eventArray._data.configure_mock( - __getitem__=lambda self, i: mock.Mock( - id=i % length, numSamples=1, - startTime=eventArray._getBlockSampleTime(i), - ) - ) - eventArray.parent.configure_mock( - parseBlock=(lambda block, start=None, end=None, step=1: - np.array([[range(length)[block.id]]])) - ) - # Run test - np.testing.assert_array_equal( - EventArray.arrayValues(eventArray), - [[0, 7, 14, 21]] - ) + x = np.arange(*(slice(start, end, step).indices(1000)))/1000 + expected = np.floor(np.vstack((x, x**2, x**0.5))*1000 + 1e-6) + + actual = testIDE.channels[8].getSession().arrayValues(start, end, step) + + np.testing.assert_equal(actual, expected) def testIterSlice(self): """ Test for the iterSlice method. """ @@ -1443,52 +1409,23 @@ def testIterSlice(self): [(0.00, 0), (0.01, 7), (0.02, 14), (0.03, 21)] ) - def testArraySlice(self): + @pytest.mark.parametrize('start, end, step', + [ + (None, None, 1), + (None, None, 5), + (10, 300, 3), + ], + ) + def testArraySlice(self, testIDE, start, end, step): """ Test for the arraySlice method. """ - length = 4 - eventArray = mock.Mock(spec=EventArray) - eventArray.configure_mock( - useAllTransforms=True, - __len__=lambda self: length, - _fullXform=Univariate((7, 0)), - _data=mock.Mock(), - _getBlockIndexWithIndex=(lambda idx, start=0, stop=None: - range(length)[idx]), - _getBlockIndexRange=lambda idx: [idx, idx+1], - _getBlockSampleTime=lambda idx: 0.01*idx, - _getBlockRollingMean=lambda blockIdx: (0,), - allowMeanRemoval=True, - removeMean=True, - hasMinMeanMax=True, - parent=mock.Mock(), - session=mock.sentinel.session, - noBivariates=mock.sentinel.noBivariates, - hasSubchannels=True, - _blockSlice=( - lambda *a, **kw: - EventArray._blockSlice(eventArray, *a, **kw) - ), - _makeBlockEventsFactory=( - lambda *a, **kw: - EventArray._makeBlockEventsFactory(eventArray, *a, **kw) - ), - ) - eventArray._data.configure_mock( - __getitem__=lambda self, i: mock.Mock( - id=i % length, numSamples=1, - startTime=eventArray._getBlockSampleTime(i), - ) - ) - eventArray.parent.configure_mock( - parseBlock=(lambda block, start=None, end=None, step=1: - np.array([[range(length)[block.id]]])) - ) - # Run test - np.testing.assert_array_equal( - EventArray.arraySlice(eventArray), - [(0.00, 0.01, 0.02, 0.03), (0, 7, 14, 21)] - ) + x = np.arange(*(slice(start, end, step).indices(1000)))/1000 + expected = np.floor(np.vstack((x, x, x**2, x**0.5))*1000 + 1e-6) + expected[0] = np.arange(*(slice(start, end, step).indices(1000)))*1000 + + actual = testIDE.channels[8].getSession().arraySlice(start, end, step) + + np.testing.assert_equal(actual, expected) def testIterJitterySlice(self): """ Test for the iterJitterySlice method. """ @@ -1538,140 +1475,99 @@ def testIterJitterySlice(self): [(0.00, 0), (0.01, 7), (0.02, 14), (0.03, 21)] ) - def testArrayJitterySlice(self): + @pytest.mark.parametrize('jitter, step', [(0.5, 5), (0.5, 1), (0.1, 20), (0.1, 5)]) + def testArrayJitterySlice(self, testIDE, jitter, step): """ Test for the arrayJitterySlice method. """ - # Stub dependencies - length = 4 - eventArray = mock.Mock(spec=EventArray) - eventArray.configure_mock( - useAllTransforms=True, - __len__=lambda self: length, - _fullXform=Univariate((7, 0)), - _data=mock.Mock(), - _getBlockIndexWithIndex=(lambda idx, start=0, stop=None: - range(length)[idx]), - _getBlockIndexRange=lambda idx: [idx, idx+1], - _getBlockSampleTime=lambda idx: 0.01*idx, - _getBlockRollingMean=lambda blockIdx: (0,), - allowMeanRemoval=True, - removeMean=True, - hasMinMeanMax=True, - parent=mock.Mock(), - session=mock.sentinel.session, - noBivariates=mock.sentinel.noBivariates, - hasSubchannels=True, - _blockJitterySlice=( - lambda *a, **kw: - EventArray._blockJitterySlice(eventArray, *a, **kw) - ), - _makeBlockEventsFactory=( - lambda *a, **kw: - EventArray._makeBlockEventsFactory(eventArray, *a, **kw) - ), - ) - eventArray._data.configure_mock( - __getitem__=lambda self, i: mock.Mock( - id=i % length, numSamples=1, - startTime=eventArray._getBlockSampleTime(i), - ) - ) - eventArray.parent.configure_mock( - parseBlockByIndex=(lambda block, indices, subchannel=None: - np.array([[range(length)[block.id]]])) - ) - # Run test - np.testing.assert_array_equal( - EventArray.arrayJitterySlice(eventArray), - [(0.00, 0.01, 0.02, 0.03), (0, 7, 14, 21)] - ) + targetIdx = np.arange(0, 1000, step) - def testGetEventIndexBefore(self): - """ Test for getEventIndexBefore method. """ - self.mockData() + dt = np.diff(testIDE.channels[8].getSession()[:][0]).mean() + idx = testIDE.channels[8].getSession().arrayJitterySlice(None, None, step, jitter=jitter)[0]/dt - self.assertEqual(self.eventArray1.getEventIndexBefore(1), 1) - self.assertEqual(self.eventArray1.getEventIndexBefore(-1), -1) + np.testing.assert_array_less(np.abs(targetIdx - idx).round(), step/jitter) - def testGetEventIndexNear(self): - """ Test for getEventIndexNear method. """ + @pytest.mark.parametrize('t, expected', [(1, 0), (-1, -1), (1005, 1)]) + def testGetEventIndexBefore(self, testIDE, t, expected): + """ Test for getEventIndexBefore method. """ - # Stub data/methods - eventArray = mock.Mock(spec=EventArray) - eventArray._data = [mock.Mock(startTime=0, indexRange=[0, 4], - sampleTime=1, numSamples=1)] - eventArray.getEventIndexBefore = lambda t: int(t) - eventArray.__getitem__ = ( - lambda self, index: np.array([ - (i, i+1) - for i in range(index.start or 0, index.stop, - index.step or 1) - ]) - ) + assert testIDE.channels[8].getSession().getEventIndexBefore(t) == expected - # Tests - self.assertEqual(EventArray.getEventIndexNear(eventArray, -1), 0) - self.assertEqual(EventArray.getEventIndexNear(eventArray, 0), 0) - self.assertEqual(EventArray.getEventIndexNear(eventArray, 1), 1) - self.assertEqual(EventArray.getEventIndexNear(eventArray, -1), 0) + @pytest.mark.parametrize('t, expected', [(1, 0), (-1, 0), (1005, 1), (1e99, 1000)]) + def testGetEventIndexNear(self, testIDE, t, expected): + """ Test for getEventIndexNear method. """ - def testGetRangeIndices(self): + assert testIDE.channels[8].getSession().getEventIndexNear(t) == expected + + @pytest.mark.parametrize( + 'indices, expected, isSingleSample', + [ + ((1, 1500), (1, 2), False), + ((None, 1), (0, 1), False), + ((None, None), (0, 1000), False), + ((2, -51), (1, 0), False), + ((2, -51), (0, 1), True), + ((2, None), (0, 1000), True) + ], + ) + def testGetRangeIndices(self, testIDE, indices, expected, isSingleSample): """ Test for getRangeIndices method. """ - self.mockData() - - # input permutations for multi sample - self.assertEqual(self.eventArray1.getRangeIndices(1, 2), (2, 3)) - self.assertEqual(self.eventArray1.getRangeIndices(None, 2), (0, 3)) - self.assertEqual(self.eventArray1.getRangeIndices(None, None), (0, 4)) - self.assertEqual(self.eventArray1.getRangeIndices(2, -51), (3, 0)) + testIDE.channels[8].singleSample = isSingleSample + eventArray = testIDE.channels[8].getSession() - # input permutations for single sample - self.eventArray1.parent.singleSample = True - self.assertEqual(self.eventArray1.getRangeIndices(2, -51), (0, 1)) - self.assertEqual(self.eventArray1.getRangeIndices(2, None), (0, 4)) + assert eventArray.getRangeIndices(*indices) == expected - def testIterRange(self): + @pytest.mark.parametrize( + 'args, kwargs, expectedIdx', + [ + ((0, 10000, 1), {'display': False}, (None, 11, None)), + ((0, 99999999, 1), {'display': False}, (None, None, None)), + ], + ) + def testIterRange(self, testIDE, args, kwargs, expectedIdx): """ Test for iterRange method. """ - self.mockData() - self.eventArray1.iterSlice = lambda w, x, y, display: (w+1, x, y, display) - self.assertEqual( - self.eventArray1.iterRange(1, 4, 1, display=False), - self.eventArray1.iterSlice(2, 4, 1, display=False) - ) + np.testing.assert_array_almost_equal( + np.vstack(list(testIDE.channels[8].getSession().iterRange(*args, **kwargs))).T, + testIDE.channels[8].getSession().arraySlice(*expectedIdx), + ) - def testArrayRange(self): + @pytest.mark.parametrize( + 'args, kwargs, expectedIdx', + [ + ((0, 10000, 1), {'display': False}, (None, 11, None)), + ((0, 99999999, 1), {'display': False}, (None, None, None)), + ], + ) + def testArrayRange(self, testIDE, args, kwargs, expectedIdx): """ Test for arrayRange method. """ - self.mockData() - self.eventArray1.arraySlice = ( - lambda w, x, y, display: np.array([w+1, x, y, display]) - ) - np.testing.assert_array_equal( - self.eventArray1.arrayRange(1, 4, 1, display=False), - self.eventArray1.arraySlice(2, 4, 1, display=False) - ) + np.testing.assert_array_almost_equal( + testIDE.channels[8].getSession().arrayRange(*args, **kwargs), + testIDE.channels[8].getSession().arraySlice(*expectedIdx), + ) - def testGetRange(self): + def testGetRange(self, testIDE): """ Test for getRange method. """ - self.mockData() - self.eventArray1.arrayRange = lambda x, y, display: (x, y, display) - self.assertSequenceEqual(self.eventArray1.getRange(), - [None, None, False]) + eventArray = testIDE.channels[8].getSession() + + np.testing.assert_array_almost_equal( + eventArray.getRange(), + eventArray.arraySlice(), + ) @unittest.skip('failing, poorly formed') def testIterMinMeanMax(self): """ Test for iterMinMeanMax method. """ self.mockData() - self.eventArray1._data[0].minMeanMax = 1 - self.eventArray1._data[0].blockIndex = 2 - self.eventArray1._data[0].min = [3] - self.eventArray1._data[0].mean = [4] - self.eventArray1._data[0].max = [5] + eventArray1._data[0].minMeanMax = 1 + eventArray1._data[0].blockIndex = 2 + eventArray1._data[0].min = [3] + eventArray1._data[0].mean = [4] + eventArray1._data[0].max = [5] self.assertListEqual( - [x for x in self.eventArray1.iterMinMeanMax()], + [x for x in eventArray1.iterMinMeanMax()], [((0, 3), (0, 4), (0, 5))] ) @@ -1691,43 +1587,41 @@ def testArrayMinMeanMax(self): result = EventArray.arrayMinMeanMax(eventArray) np.testing.assert_array_equal(result, np.moveaxis(statsStub, 0, -1)) - def testGetMinMeanMax(self): + def testGetMinMeanMax(self, testIDE): """ Test getMinMeanMax. """ - # Stub data/methods - eventArray = mock.Mock(spec=EventArray) - eventArray.arrayMinMeanMax = mock.Mock( - spec=EventArray.arrayMinMeanMax, - return_value=mock.sentinel.return_value - ) - args = (mock.sentinel.startTime, mock.sentinel.endTime, - mock.sentinel.padding, mock.sentinel.times, - mock.sentinel.display, mock.sentinel.iterator) + eventArray = testIDE.channels[8].getSession() + eventArray.hasMinMeanMax = False + + times = [d.startTime for d in eventArray._data] + mins_ = [d.min for d in eventArray._data] + means = [d.mean for d in eventArray._data] + maxes = [d.max for d in eventArray._data] + + arrayMins_ = np.stack([np.concatenate(([t], m)) for t, m in zip(times, mins_)]).T + arrayMeans = np.stack([np.concatenate(([t], m)) for t, m in zip(times, means)]).T + arrayMaxes = np.stack([np.concatenate(([t], m)) for t, m in zip(times, maxes)]).T - self.assertEqual(EventArray.getMinMeanMax(eventArray, *args), - mock.sentinel.return_value) - self.assertEqual(eventArray.arrayMinMeanMax.call_args, (args,)) np.testing.assert_array_equal( - self.dataset.channels[32].getSession().getMinMeanMax(), - np.array([]) - ) + np.stack(eventArray.getMinMeanMax()), + np.stack([arrayMins_, arrayMeans, arrayMaxes]), + ) - def testGetRangeMinMeanMax(self): + def testGetRangeMinMeanMax(self, testIDE): """ Test for getRangeMinMeanMax method. """ - # TODO fix - eventArray = mock.Mock(spec=EventArray, hasSubchannels=True) - statsStub = np.array([ - ((3, 10), (4., 11.)), - ((5, 12), (6., 13.)), - ((7, 14), (8., 15.)), - ]) - eventArray.arrayMinMeanMax = mock.Mock(spec=EventArray.arrayMinMeanMax, - return_value=statsStub) + eventArray = testIDE.channels[8].getSession() + eventArray.hasMinMeanMax = False + + mmm = eventArray.getMinMeanMax() + _min = mmm[0][1:].min() + _mean = np.median(mmm[1][1:], axis=-1).mean() + _max = mmm[2][1:].max() np.testing.assert_array_equal( - EventArray.getRangeMinMeanMax(eventArray), (3, 9, 15) - ) + eventArray.getRangeMinMeanMax(), + [_min, _mean, _max], + ) def testGetMax(self): """ Test for getMax method. """ @@ -1774,210 +1668,115 @@ def testGetMin(self): # Run test np.testing.assert_array_equal(EventArray.getMin(eventArray), (0, 3)) - self.assertEqual(eventArray._computeMinMeanMax.call_count, 1) - - def testGetSampleTime(self): + assert eventArray._computeMinMeanMax.call_count == 1 + + @pytest.mark.parametrize( + 'kwargs, expected', + [ + ({}, 1e-3), + ({'idx': 1}, 1000.), + ({'idx': 404}, 1000.), + ], + ) + def testGetSampleTime(self, testIDE, kwargs, expected): """ Test for getSampleTime method. """ - self.mockData() - - self.assertEqual(self.eventArray1.getSampleTime(), 1) - self.assertEqual(self.eventArray1.getSampleTime(1), 1) - self.assertEqual( - self.dataset.channels[32].getSession().getSampleTime(), -1 - ) - self.assertEqual( - self.dataset.channels[32].getSession().getSampleTime(1), -1 - ) - - def testGetSampleRate(self): + testIDE.channels[8].sampleRate = 1000. + eventArray = testIDE.channels[8].getSession() + + assert eventArray.getSampleTime(**kwargs) == expected + + @pytest.mark.parametrize( + 'sr, idx, expected', + [ + (None, None, 1000.), + (100., None, 100.), + (None, 1, 1000.), + (None, 8, 1000.), + ] + ) + def testGetSampleRate(self, testIDE, sr, idx, expected): """ Test for getSampleRate method. """ - self.mockData() - self.eventArray1._data[0].sampleRate = 5 - self.dataset.channels[32].sampleRate = 3 - self.dataset.channels[32].getSession()._data = self.eventArray1._data - self.assertEqual(self.eventArray1.getSampleRate(), 5) - self.assertEqual(self.eventArray1.getSampleRate(0), 5) - self.assertEqual(self.dataset.channels[32].getSession().getSampleRate(), 3) - self.assertEqual(self.dataset.channels[32].getSession().getSampleRate(0), 5) + eventArray = testIDE.channels[8].getSession() + eventArray.parent.sampleRate = sr - def testGetValueAt(self): - """ Test for getValueAt method. """ - # Stub dependencies - length = 4 - eventArray = mock.Mock(spec=EventArray) - eventArray.configure_mock( - useAllTransforms=True, - __len__=lambda self: length, - _fullXform=Univariate((7, 0)), - _data=mock.Mock(), - getEventIndexBefore=lambda at: min(max(-1, int(at//0.01)), length-1), - _getBlockIndexWithIndex=lambda idx: range(length)[idx], - _getBlockIndexRange=lambda idx: [idx, idx+1], - _getBlockSampleTime=lambda idx: 0.01*idx, - _getBlockRollingMean=lambda blockIdx: None, - parent=mock.Mock(), - session=mock.sentinel.session, - noBivariates=mock.sentinel.noBivariates, - hasSubchannels=True, - __getitem__=lambda self, *a, **kw: EventArray.__getitem__(eventArray, *a, **kw), - ) - eventArray._data.configure_mock( - __getitem__=lambda self, i: mock.Mock( - id=i % length, startTime=eventArray._getBlockSampleTime(i) - ) - ) - eventArray.parent.configure_mock( - parseBlock=(lambda block, start=None, end=None, step=1: - np.array([[range(length)[block.id]]])) - ) - eventArray.parent.types.__len__ = lambda self: 1 + assert eventArray.getSampleRate(idx) == expected - # Run test - self.assertRaises(IndexError, EventArray.getValueAt, eventArray, -0.01) - np.testing.assert_array_equal( - EventArray.getValueAt(eventArray, -0.01, outOfRange=True), - (0.00, 0) - ) - np.testing.assert_array_equal( - EventArray.getValueAt(eventArray, 0.00), (0.00, 0) - ) - np.testing.assert_array_equal( - EventArray.getValueAt(eventArray, 0.01), (0.01, 7) - ) - np.testing.assert_array_equal( - EventArray.getValueAt(eventArray, 0.02), (0.02, 14) - ) - np.testing.assert_array_equal( - EventArray.getValueAt(eventArray, 0.03), (0.03, 21) - ) - np.testing.assert_array_equal( - EventArray.getValueAt(eventArray, 0.04, outOfRange=True), - (0.03, 21) - ) - self.assertRaises(IndexError, EventArray.getValueAt, eventArray, 0.04) + @pytest.mark.parametrize( + 'at, expected, raises', + [ + (0, (0, 0, 0, 0), nullcontext()), + (10, (10, 0, 0, 0), nullcontext()), + (2000, (2000, 0, 0, 0), nullcontext()), + (9500, (9500, 0, 0, 0), nullcontext()), + (-1, None, pytest.raises(IndexError)), + ], + ) + def testGetValueAt(self, testIDE, at, expected, raises): + """ Test for getValueAt method. """ - def testGetMeanNear(self): + eventArray = testIDE.channels[8].getSession() + with raises: + assert eventArray.getValueAt(at) == expected + + @pytest.mark.parametrize( + 't, expected', + [ + (0, (499., 332., 666.)), + (10, (499., 332., 666.)), + (2000, (499., 332., 666.)), + (9500, (499., 332., 666.)), + (-1, (499., 332., 666.)), + ], + ) + def testGetMeanNear(self, testIDE, t, expected): """ Test for getMeanNear method. """ - # Stub dependencies - length = 4 - eventArray = mock.Mock(spec=EventArray) - eventArray.configure_mock( - __len__=lambda self: length, - _comboXform=(lambda time, val, session=None, noBivariates=False: - (time, tuple(7*i for i in val))), - _data=mock.Mock(), - _getBlockIndexWithTime=lambda at: min(max(-1, int((at+0.005)//0.01)), length-1), - _getBlockRollingMean=lambda blockIdx, force=False: (range(length)[blockIdx],), - hasSubchannels=True, - ) - eventArray._data.configure_mock( - __len__=eventArray.__len__, - ) - # Run test - self.assertEqual(EventArray.getMeanNear(eventArray, 0.00), (0,)) - self.assertEqual(EventArray.getMeanNear(eventArray, 0.01), (7,)) - self.assertEqual(EventArray.getMeanNear(eventArray, 0.02), (14,)) - self.assertEqual(EventArray.getMeanNear(eventArray, 0.03), (21,)) + eventArray = testIDE.channels[8].getSession() + assert eventArray.getMeanNear(t) == expected - def testIterResampledRange(self): + def testIterResampledRange(self, testIDE): """ Test for iterResampledRange method. """ - # Stub data/methods - eventArray = mock.Mock(spec=EventArray) - eventArray.configure_mock( - __len__=lambda self: 100, - ) - eventArray.getRangeIndices.return_value = 0, 105 - eventArray.iterSlice.return_value = mock.sentinel.a - eventArray.iterJitterySlice.return_value = mock.sentinel.b + eventArray = testIDE.channels[8].getSession() - startTime = mock.sentinel.startTime - stopTime = mock.sentinel.stopTime - maxPoints = 43 + dat = eventArray.arraySlice() # Run tests - self.assertEqual( - EventArray.iterResampledRange(eventArray, startTime, stopTime, - maxPoints), - mock.sentinel.a - ) - startIdx, stopIdx, step = ( - eventArray.iterSlice.call_args[0] - ) - self.assertTrue(startIdx >= 0) - self.assertTrue(stopIdx <= len(eventArray)) - self.assertTrue(len(range(startIdx, stopIdx, step)) <= maxPoints) - - self.assertEqual( - EventArray.iterResampledRange(eventArray, startTime, stopTime, - maxPoints, jitter=0.1,), - mock.sentinel.b - ) - startIdx, stopIdx, step, jitter = ( - eventArray.iterJitterySlice.call_args[0] - ) - self.assertTrue(startIdx >= 0) - self.assertTrue(stopIdx <= len(eventArray)) - self.assertTrue(len(range(startIdx, stopIdx, step)) <= maxPoints) + np.testing.assert_array_almost_equal( + np.stack(list(eventArray.iterResampledRange(0, 1e6, 9))).T, + dat[:, [0, 111, 211, 311, 411, 511, 611, 711, 811]], + ) - def testArrayResampledRange(self): + def testArrayResampledRange(self, testIDE): """ Test for arrayResampledRange method. """ - # Stub data/methods - eventArray = mock.Mock(spec=EventArray) - eventArray.configure_mock( - __len__=lambda self: 100, - ) - eventArray.getRangeIndices.return_value = 0, 105 - eventArray.arraySlice.return_value = mock.sentinel.a - eventArray.arrayJitterySlice.return_value = mock.sentinel.b + eventArray = testIDE.channels[8].getSession() - startTime = mock.sentinel.startTime - stopTime = mock.sentinel.stopTime - maxPoints = 43 + dat = eventArray.arraySlice() # Run tests - self.assertEqual( - EventArray.arrayResampledRange(eventArray, startTime, stopTime, - maxPoints), - mock.sentinel.a - ) - startIdx, stopIdx, step = ( - eventArray.arraySlice.call_args[0] - ) - self.assertTrue(startIdx >= 0) - self.assertTrue(stopIdx <= len(eventArray)) - self.assertTrue(len(range(startIdx, stopIdx, step)) <= maxPoints) - - self.assertEqual( - EventArray.arrayResampledRange(eventArray, startTime, stopTime, - maxPoints, jitter=0.1,), - mock.sentinel.b - ) - startIdx, stopIdx, step, jitter = ( - eventArray.arrayJitterySlice.call_args[0] - ) - self.assertTrue(startIdx >= 0) - self.assertTrue(stopIdx <= len(eventArray)) - self.assertTrue(len(range(startIdx, stopIdx, step)) <= maxPoints) + np.testing.assert_array_almost_equal( + eventArray.arrayResampledRange(0, 1e6, 9), + dat[:, [0, 112, 224, 336, 448, 560, 672, 784, 896]], + ) + @pytest.mark.skip("this doesn't actually do anything") def testExportCSV(self): """ Test for exportCsv method.""" self.mockData() - self.eventArray1._data[0].minMeanMax = 1 - self.eventArray1._data[0].blockIndex = 2 - self.eventArray1._data[0].min = [3] - self.eventArray1._data[0].mean = [4] - self.eventArray1._data[0].max = [5] + eventArray1._data[0].minMeanMax = 1 + eventArray1._data[0].blockIndex = 2 + eventArray1._data[0].min = [3] + eventArray1._data[0].mean = [4] + eventArray1._data[0].max = [5] #=============================================================================== # #=============================================================================== -class PlotTestCase(unittest.TestCase): +class TestPlot(unittest.TestCase): """ Unit test for the Plot class. """ def setUp(self): @@ -2054,7 +1853,7 @@ def testGetRange(self): #--- Data test cases #=============================================================================== -class DataTestCase(unittest.TestCase): +class TestData(unittest.TestCase): """ Basic tests of data fidelity against older, "known good" CSV exports. Exports were generated using the library as of the release of 1.8.0. From e8c1a8a6dc07d084acf86fad7eff0b886b200534 Mon Sep 17 00:00:00 2001 From: "MIDE\\cflanigan" Date: Thu, 24 Jun 2021 17:43:34 -0400 Subject: [PATCH 06/41] update travis with working directory --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 607432e8..c42a05b0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ before_install: install: - pip install .[test] script: - - python -m pytest ./testing/ --cov=idelib + - python -m pytest ./testing/ --cov=idelib --rootdir=. - sphinx-build -W -b html docs/source docs/html after_success: - bash <(curl -s https://codecov.io/bash) From 5819f3437a9f00395d0e105c646efbfeaef0bd94 Mon Sep 17 00:00:00 2001 From: "MIDE\\cflanigan" Date: Thu, 24 Jun 2021 17:45:55 -0400 Subject: [PATCH 07/41] this is odd --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index c42a05b0..4468c27d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ before_install: install: - pip install .[test] script: + - ls - python -m pytest ./testing/ --cov=idelib --rootdir=. - sphinx-build -W -b html docs/source docs/html after_success: From 5ff33e1ac912096c358857e120b302ff1c1342a8 Mon Sep 17 00:00:00 2001 From: "MIDE\\cflanigan" Date: Thu, 24 Jun 2021 17:48:22 -0400 Subject: [PATCH 08/41] revert travis changes and update file path --- .travis.yml | 3 +-- testing/test_dataset.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4468c27d..607432e8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,8 +4,7 @@ before_install: install: - pip install .[test] script: - - ls - - python -m pytest ./testing/ --cov=idelib --rootdir=. + - python -m pytest ./testing/ --cov=idelib - sphinx-build -W -b html docs/source docs/html after_success: - bash <(curl -s https://codecov.io/bash) diff --git a/testing/test_dataset.py b/testing/test_dataset.py index e39ba281..6f83b743 100644 --- a/testing/test_dataset.py +++ b/testing/test_dataset.py @@ -1862,7 +1862,7 @@ class TestData(unittest.TestCase): """ def setUp(self): - self.dataset = importer.importFile('./testing/SSX_Data.IDE') + self.dataset = importer.importFile('./testing/SSX_Data.ide') self.delta = 0.0015 From 598c4fb3c4ffb047a6827a9c317b261d14969af3 Mon Sep 17 00:00:00 2001 From: "MIDE\\cflanigan" Date: Thu, 24 Jun 2021 17:50:14 -0400 Subject: [PATCH 09/41] actually fixes file path --- testing/test_dataset.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/test_dataset.py b/testing/test_dataset.py index 6f83b743..e3109c51 100644 --- a/testing/test_dataset.py +++ b/testing/test_dataset.py @@ -59,7 +59,7 @@ def _load_file(filePath): @pytest.fixture def testIDE(): - doc = idelib.importer.openFile(_load_file('./test.IDE')) + doc = idelib.importer.openFile(_load_file('./test.ide')) idelib.importer.readData(doc) return doc @@ -1862,7 +1862,7 @@ class TestData(unittest.TestCase): """ def setUp(self): - self.dataset = importer.importFile('./testing/SSX_Data.ide') + self.dataset = importer.importFile('./testing/SSX_Data.IDE') self.delta = 0.0015 From b0d97b32ba176cd4bc89b2b110d1746a32137dc2 Mon Sep 17 00:00:00 2001 From: "MIDE\\cflanigan" Date: Fri, 25 Jun 2021 09:57:35 -0400 Subject: [PATCH 10/41] fixes the plot tests --- testing/test_dataset.py | 272 +++++++++++++++++++--------------------- testing/utils.py | 9 ++ 2 files changed, 138 insertions(+), 143 deletions(-) create mode 100644 testing/utils.py diff --git a/testing/test_dataset.py b/testing/test_dataset.py index e3109c51..33d8f6e6 100644 --- a/testing/test_dataset.py +++ b/testing/test_dataset.py @@ -9,7 +9,6 @@ future. """ -from contextlib import nullcontext from io import StringIO, BytesIO import sys import unittest @@ -36,6 +35,8 @@ from idelib import importer from idelib import parsers +from testing.utils import nullcontext + import numpy as np # type: ignore from .file_streams import makeStreamLike @@ -71,6 +72,13 @@ def SSX70065IDE(): return doc +@pytest.fixture +def SSX_DataIDE(): + doc = idelib.importer.openFile(_load_file('./testing/SSX_Data.IDE')) + idelib.importer.readData(doc) + return doc + + #=============================================================================== # #=============================================================================== @@ -1776,84 +1784,63 @@ def testExportCSV(self): # #=============================================================================== -class TestPlot(unittest.TestCase): +class TestPlot: """ Unit test for the Plot class. """ - - def setUp(self): - self.dataset = importer.importFile('./testing/SSX70065.IDE') - self.dataset.addSession(0, 1, 2) - self.dataset.addSensor(0) - - self.fakeParser = GenericObject() - self.fakeParser.types = [0] - self.fakeParser.format = [] - - self.channel1 = Channel( - self.dataset, channelId=0, name="channel1", parser=self.fakeParser, - displayRange=[0]) - self.eventList1 = EventArray(self.channel1, session=self.dataset.sessions[0]) - - self.channel1.addSubChannel(subchannelId=0) - - self.subChannel1 = SubChannel(self.channel1, 0) - - self.plot1 = Plot(self.eventList1, 0, name="Plot1") - - - def tearDown(self): - self.dataset.close() - self.dataset = None - - self.fakeParser = None - self.channel1 = None - self.eventList1 = None - self.subChannel1 = None - self.plot1 = None - - - def mockData(self): - """ mock up a bit of fake data so I don't have to worry that external - classes are working during testing. - """ - fakeData = GenericObject() - fakeData.startTime = 0 - fakeData.indexRange = [0, 3] - fakeData.sampleTime = 1 - fakeData.numSamples = 1 - self.eventList1._data = [fakeData] - - - def testConstructor(self): + + @pytest.fixture + def channel32(self, SSX70065IDE): + return SSX70065IDE.channels[32] + + @pytest.fixture + def eventArray(self, channel32): + return channel32.getSession() + + @pytest.fixture + def plot1(self, eventArray): + return Plot(eventArray, 0, name='Plot1') + + def testConstructor(self, plot1, eventArray): """ Test for the constructor. """ - self.assertEqual(self.plot1.source, self.eventList1) - self.assertEqual(self.plot1.id, 0) - self.assertEqual(self.plot1.session, self.eventList1.session) - self.assertEqual(self.plot1.dataset, self.eventList1.dataset) - self.assertEqual(self.plot1.name, "Plot1") - self.assertEqual(self.plot1.units, self.eventList1.units) - self.assertEqual(self.plot1.attributes, None) - + + plotParams = ( + plot1.source, + plot1.id, + plot1.session, + plot1.dataset, + plot1.name, + plot1.units, + plot1.attributes, + ) + + targetParams = ( + eventArray, + 0, + eventArray.session, + eventArray.dataset, + 'Plot1', + eventArray.units, + None, + ) + + assert plotParams == targetParams - def testGetEventIndexBefore(self): + @pytest.mark.parametrize('t', [0, 1, 10, 100, 1000, 10000, 100000]) + def testGetEventIndexBefore(self, eventArray, plot1, t): """ Test for getEventIndexBefore method. """ - self.mockData() - - self.assertEqual( - self.plot1.getEventIndexBefore(0), - self.eventList1.getEventIndexBefore(0)) - + + assert plot1.getEventIndexBefore(t) == eventArray.getEventIndexBefore(t) + @pytest.mark.skip('not implemented') def testGetRange(self): """ Test for getRange method. """ - print('gotta get to this')#self.plot1.getRange(0, 1)) - # TODO + pass #=============================================================================== #--- Data test cases #=============================================================================== -class TestData(unittest.TestCase): +class TestData: """ Basic tests of data fidelity against older, "known good" CSV exports. Exports were generated using the library as of the release of 1.8.0. @@ -1861,127 +1848,126 @@ class TestData(unittest.TestCase): errors. """ - def setUp(self): - self.dataset = importer.importFile('./testing/SSX_Data.IDE') - self.delta = 0.0015 + @pytest.fixture + def dataset(self, SSX_DataIDE): + return SSX_DataIDE + + @pytest.fixture + def channel8(self, dataset): + return dataset.channels[8] + + @pytest.fixture + def accelArray(self, channel8): + return channel8.getSession() + + @pytest.fixture + def out(self): + return StringIO() + @staticmethod + def generateCsvArray(filestream, eventArray): + eventArray.exportCsv(filestream) + filestream.seek(0) + return np.genfromtxt(filestream, delimiter=', ').T - def testCalibratedExport(self): + def testCalibratedExport(self, accelArray, out): """ Test regular export, with bivariate polynomials applied. """ - out = StringIO() - accel = self.dataset.channels[8].getSession() - accel.exportCsv(out) - out.seek(0) - new = np.genfromtxt(out, delimiter=', ').T - old = accel.__getitem__(slice(None), display=True) + new = self.generateCsvArray(out, accelArray) + old = accelArray.__getitem__(slice(None), display=True) old = np.round(1e6*old)/1e6 - np.testing.assert_allclose(new, old, rtol=1e-4) + np.testing.assert_equal(new[1:], old[1:]) - - def testUncalibratedExport(self): + def testUncalibratedExport(self, accelArray, out): """ Test export with no per-channel polynomials.""" + new = self.generateCsvArray(out, accelArray) + old = accelArray.__getitem__(slice(None), display=True) + old = np.round(1e6*old)/1e6 - out = StringIO() - accel = self.dataset.channels[8].getSession() - - accel.exportCsv(out) - out.seek(0) - new = np.genfromtxt(out, delimiter=', ') - old = accel.__getitem__(slice(None), display=True) - - np.testing.assert_allclose(new.T, old, rtol=1e-4) - + np.testing.assert_equal(new[1:], old[1:]) - def testNoBivariates(self): + def testNoBivariates(self, accelArray, out): """ Test export with bivariate polynomial references disabled (values only offset, not temperature-corrected). """ - out = StringIO() - accel = self.dataset.channels[8].getSession() - accel.noBivariates = True - accel.exportCsv(out) - out.seek(0) - new = np.genfromtxt(out, delimiter=', ') - old = accel.__getitem__(slice(None), display=True) + accelArray.noBivariates = True - np.testing.assert_allclose(new.T, old, rtol=1e-4) + new = self.generateCsvArray(out, accelArray) + old = accelArray.__getitem__(slice(None), display=True) + old = np.round(1e6*old)/1e6 + np.testing.assert_equal(new[1:], old[1:]) - def testRollingMeanRemoval(self): + def testRollingMeanRemoval(self, accelArray, out): """ Test regular export, with the rolling mean removed from the data. """ - - out = StringIO() - accel = self.dataset.channels[8].getSession() - accel.removeMean = True - accel.rollingMeanSpan = 5000000 - accel.exportCsv(out) - out.seek(0) - new = np.genfromtxt(out, delimiter=', ') - old = accel.__getitem__(slice(None), display=True) + accelArray.removeMean = True + accelArray.rollingMeanSpan = 5000000 - np.testing.assert_allclose(new.T, old, rtol=1e-4) + new = self.generateCsvArray(out, accelArray) + old = accelArray.__getitem__(slice(None), display=True) + old = np.round(1e6*old)/1e6 + np.testing.assert_equal(new[1:], old[1:]) - def testTotalMeanRemoval(self): + def testTotalMeanRemoval(self, accelArray, out): """ Test regular export, calibrated, with the total mean removed from the data. """ - out = StringIO() - accel = self.dataset.channels[8].getSession() - accel.removeMean = True - accel.rollingMeanSpan = -1 + accelArray.removeMean = True + accelArray.rollingMeanSpan = -1 - accel.exportCsv(out) - out.seek(0) - new = np.genfromtxt(out, delimiter=', ') - old = accel.__getitem__(slice(None), display=True) + new = self.generateCsvArray(out, accelArray) + old = accelArray.__getitem__(slice(None), display=True) + old = np.round(1e6*old)/1e6 - np.testing.assert_allclose(new.T, old, rtol=1e-4) - + np.testing.assert_equal(new[1:], old[1:]) - def testCalibratedRollingMeanRemoval(self): + def testCalibratedRollingMeanRemoval(self, accelArray, out): """ Test regular export, calibrated, with the rolling mean removed from the data. """ - out = StringIO() - accel = self.dataset.channels[8].getSession() - accel.removeMean = True - accel.rollingMeanSpan = 5000000 - accel.exportCsv(out) - out.seek(0) - new = np.genfromtxt(out, delimiter=', ') - old = accel.__getitem__(slice(None), display=True) + accelArray.removeMean = True + accelArray.rollingMeanSpan = 5000000 + + new = self.generateCsvArray(out, accelArray) + old = accelArray.__getitem__(slice(None), display=True) + old = np.round(1e6*old)/1e6 - np.testing.assert_allclose(new.T, old, rtol=1e-4) + np.testing.assert_equal(new[1:], old[1:]) - - def testCalibratedTotalMeanRemoval(self): + def testCalibratedTotalMeanRemoval(self, accelArray, out): """ Test regular export, with the total mean removed from the data. """ - out = StringIO() - accel = self.dataset.channels[8].getSession() - accel.removeMean = True - accel.rollingMeanSpan = -1 - accel.exportCsv(out) - out.seek(0) - new = np.genfromtxt(out, delimiter=', ') - old = accel.__getitem__(slice(None), display=True) + accelArray.removeMean = True + accelArray.rollingMeanSpan = -1 + + new = self.generateCsvArray(out, accelArray) + old = accelArray.__getitem__(slice(None), display=True) + old = np.round(1e6*old)/1e6 + + np.testing.assert_equal(new[1:], old[1:]) - np.testing.assert_allclose(new.T, old, rtol=1e-4) + def testTimestamps(self, accelArray, out): + """ Tests the timestamps, which are the same on all exports + """ + + new = self.generateCsvArray(out, accelArray) + old = accelArray[:] + + np.testing.assert_equal(new[0], old[0]) -#=============================================================================== +# =============================================================================== # -#=============================================================================== +# ============================================================================== DEFAULTS = { "sensors": { diff --git a/testing/utils.py b/testing/utils.py new file mode 100644 index 00000000..4297468a --- /dev/null +++ b/testing/utils.py @@ -0,0 +1,9 @@ +class nullcontext: + """ A replacement for `contextlib.nullcontext` for python versions before 3.7 + """ + + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_val, exc_tb): + pass From 0017957a8c3eb97b61becb2e4ace9de3dccf8d86 Mon Sep 17 00:00:00 2001 From: "MIDE\\cflanigan" Date: Fri, 25 Jun 2021 10:09:16 -0400 Subject: [PATCH 11/41] fix some issues caught by unit tests --- idelib/dataset.py | 11 +++++++++-- testing/test_dataset.py | 5 ++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/idelib/dataset.py b/idelib/dataset.py index c505c2f4..54a7c2df 100644 --- a/idelib/dataset.py +++ b/idelib/dataset.py @@ -1685,7 +1685,7 @@ def __getitem__(self, idx, display=False): else: xform = self._fullXform - if isinstance(idx, int): + if isinstance(idx, (int, np.integer)): if idx >= len(self): raise IndexError("EventArray index out of range") @@ -1867,7 +1867,10 @@ def arrayValues(self, start=None, end=None, step=1, subchannels=True, for i, (k, _) in enumerate(rawData.dtype.descr): xform.polys[i].inplace(rawData[k], out=out[i]) - return out + if subchannels is True: + return out + else: + return out[list(subchannels)] def _blockSlice(self, start=None, end=None, step=1, display=False): @@ -2082,6 +2085,10 @@ def arrayJitterySlice(self, start=None, end=None, step=1, jitter=0.5, :return: a structured array of events in the specified index range. """ + if not isinstance(start, slice): + start = slice(start, end, step) + start, end, step = start.indices(len(self)) + # Check for non-jittered cases if jitter is True: jitter = 0.5 diff --git a/testing/test_dataset.py b/testing/test_dataset.py index 33d8f6e6..19110c3d 100644 --- a/testing/test_dataset.py +++ b/testing/test_dataset.py @@ -17,6 +17,8 @@ import pytest +import numpy as np # type: ignore + import idelib from idelib.dataset import (Cascading, Channel, @@ -29,7 +31,6 @@ Transformable, WarningRange, ) - from idelib.transforms import Transform, CombinedPoly, PolyPoly from idelib.transforms import AccelTransform, Univariate from idelib import importer @@ -37,8 +38,6 @@ from testing.utils import nullcontext -import numpy as np # type: ignore - from .file_streams import makeStreamLike From 6e5d433db14a74c026810d22a6688119b69cbfc3 Mon Sep 17 00:00:00 2001 From: "MIDE\\cflanigan" Date: Fri, 25 Jun 2021 10:35:12 -0400 Subject: [PATCH 12/41] replacing iterslice and itervalues code with references to arrayslice and arrayvalues --- idelib/dataset.py | 27 +- testing/test_dataset.py | 542 ++++++++++++++++++---------------------- 2 files changed, 247 insertions(+), 322 deletions(-) diff --git a/idelib/dataset.py b/idelib/dataset.py index 54a7c2df..24a5401f 100644 --- a/idelib/dataset.py +++ b/idelib/dataset.py @@ -1810,20 +1810,11 @@ def itervalues(self, start=None, end=None, step=1, subchannels=True, :return: an iterable of structured array value blocks in the specified index range. """ - # TODO: Optimize; times don't need to be computed since they aren't used - iterBlockValues = ( - np.stack(values) - for _, values in self._blockSlice(start, end, step, display) - ) - if self.hasSubchannels and subchannels is not True: - chIdx = np.asarray(subchannels) - return (vals - for blockVals in iterBlockValues - for vals in blockVals[chIdx].T) - else: - return (vals - for blockVals in iterBlockValues - for vals in blockVals.T) + + out = self.arrayValues(start=start, end=end, step=step) + + for evt in out.T: + yield evt def arrayValues(self, start=None, end=None, step=1, subchannels=True, @@ -1934,10 +1925,10 @@ def iterSlice(self, start=None, end=None, step=1, display=False): 'display' transform) will be applied to the data. :return: an iterable of events in the specified index range. """ - for times, values in self._blockSlice(start, end, step, display): - blockEvents = np.append(times[np.newaxis], values, axis=0) - for event in blockEvents.T: - yield event + + out = self.arraySlice(start=start, end=end, step=step, display=display) + for evt in out.T: + yield evt def arraySlice(self, start=None, end=None, step=1, display=False): diff --git a/testing/test_dataset.py b/testing/test_dataset.py index 19110c3d..b08a69ef 100644 --- a/testing/test_dataset.py +++ b/testing/test_dataset.py @@ -96,20 +96,20 @@ def __init__(self): self.startTime = 0 self.sampleTime = 0 self.numSamples = 1 - - + + def __getitem__(self, index): return self.data[index] - - + + def __len__(self): return len(self.data) - - + + def updateTransforms(self): self.isUpdated = True - - + + def parseWith(self, x, start, end, step, subchannel): return (x, start, end, step, subchannel) @@ -124,37 +124,37 @@ def parseByIndexWith(self, parser, indices, subchannel): class TestCascading(unittest.TestCase): """ Test case for methods in the Cascading class. """ - + def setUp(self): self.casc1 = Cascading() self.casc1.name = 'parent' self.casc2 = Cascading() self.casc2.name = 'child' self.casc2.parent = self.casc1 - - + + def tearDown(self): self.casc1 = None self.casc2 = None - - + + def testHierarchy(self): """ Test for hierarchy method. """ self.assertEqual(self.casc2.hierarchy(), [self.casc1, self.casc2]) - - + + def testPath(self): """ Test for path method. """ self.assertEqual(self.casc1.path(), 'parent') self.assertEqual(self.casc2.path(), 'parent:child') self.casc1.path = lambda : None self.assertEqual(self.casc2.path(), 'child') - - + + def testRepr(self): """ Test that casting to a string creates the correct string. """ self.assertIn(" Date: Fri, 25 Jun 2021 10:37:41 -0400 Subject: [PATCH 13/41] modifying the timestamp check to allow immensely small differences --- testing/test_dataset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testing/test_dataset.py b/testing/test_dataset.py index b08a69ef..39c51aad 100644 --- a/testing/test_dataset.py +++ b/testing/test_dataset.py @@ -1895,8 +1895,9 @@ def testTimestamps(self, accelArray, out): new = self.generateCsvArray(out, accelArray) old = accelArray[:] + old = np.round(1e6*old)/1e6 - np.testing.assert_equal(new[0], old[0]) + np.testing.assert_allclose(new[0], old[0], rtol=1e-10) # =============================================================================== From 80f1792620172bc5581c5c14abab085c858ca419 Mon Sep 17 00:00:00 2001 From: "MIDE\\cflanigan" Date: Fri, 25 Jun 2021 12:55:21 -0400 Subject: [PATCH 14/41] fix subchannels and minmeanmax methods. It looks like they had been returning times before, which should be incorrect --- idelib/dataset.py | 39 +++++--- testing/test_dataset.py | 207 ++++++++++++++++++++++++---------------- 2 files changed, 152 insertions(+), 94 deletions(-) diff --git a/idelib/dataset.py b/idelib/dataset.py index 24a5401f..b98205a1 100644 --- a/idelib/dataset.py +++ b/idelib/dataset.py @@ -1208,18 +1208,20 @@ def __init__(self, parentChannel, session=None, parentList=None): self._mean = None - format = self.parent.parser.format - if len(format) == 0: + _format = self.parent.parser.format + if isinstance(self.parent, SubChannel): + self._npType = self.parent.parent.getSession()._npType[self.subchannelId] + elif len(_format) == 0: self._npType = np.uint8 else: - if isinstance(format, bytes): - format = format.decode() + if isinstance(_format, bytes): + _format = _format.decode() - if format[0] in ['<', '>', '=']: - endian = format[0] - dtypes = [endian + ChannelDataBlock.TO_NP_TYPESTR[x] for x in format[1:]] + if _format[0] in ['<', '>', '=']: + endian = _format[0] + dtypes = [endian + ChannelDataBlock.TO_NP_TYPESTR[x] for x in _format[1:]] else: - dtypes = [ChannelDataBlock.TO_NP_TYPESTR[x] for x in str(format)] + dtypes = [ChannelDataBlock.TO_NP_TYPESTR[x] for x in str(_format)] self._npType = np.dtype([(str(i), dtype) for i, dtype in enumerate(dtypes)]) @@ -2371,9 +2373,24 @@ def arrayMinMeanMax(self, startTime=None, endTime=None, padding=0, and max, respectively). """ - return np.moveaxis([i for i in iterator(self.iterMinMeanMax( - startTime, endTime, padding, times, display - ))], 0, -1) + startBlock, endBlock = self._getBlockRange(startTime, endTime) + shape = (3, max(1, len(self._npType)), endBlock - startBlock) + scid = self.subchannelId + isSubchannel = isinstance(self.parent, SubChannel) + + out = np.zeros(shape) + + for i, d in enumerate(self._data[startBlock:endBlock]): + if isSubchannel: + out[0, 0, i] = d.min[scid] + out[1, 0, i] = d.mean[scid] + out[2, 0, i] = d.max[scid] + else: + out[0, :, i] = d.min + out[1, :, i] = d.mean + out[2, :, i] = d.max + + return out def getMinMeanMax(self, startTime=None, endTime=None, padding=0, diff --git a/testing/test_dataset.py b/testing/test_dataset.py index 39c51aad..39ad0259 100644 --- a/testing/test_dataset.py +++ b/testing/test_dataset.py @@ -8,7 +8,7 @@ `calibration.AccelTransform`. These classes may be refactored out in the future. """ - +import struct from io import StringIO, BytesIO import sys import unittest @@ -744,9 +744,63 @@ def testUpdateTransforms(self): # #=============================================================================== -class TestSubChannel(unittest.TestCase): +class TestSubChannel: """ Test case for methods in the SubChannel class. """ + @pytest.fixture + def dataset(self, SSX70065IDE): + SSX70065IDE.addSensor(0) + return SSX70065IDE + + @pytest.fixture + def fakeParser(self): + return struct.Struct(b' Date: Fri, 25 Jun 2021 15:17:23 -0400 Subject: [PATCH 15/41] marginally improve efficiency. --- idelib/dataset.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/idelib/dataset.py b/idelib/dataset.py index b98205a1..360b4614 100644 --- a/idelib/dataset.py +++ b/idelib/dataset.py @@ -2958,7 +2958,9 @@ def _inplaceTime(self, start, end, step, out=None): nSamples = len(self) samplePeriod = (arrayEnd - arrayStart)/(nSamples - 1) - out[:] = np.arange(start, end, step)*samplePeriod + arrayStart + out[:] = np.arange(start, end, step) + out *= samplePeriod + out += arrayStart return out From 4b1cc979b3285f0f6267cb986809d83527b82d81 Mon Sep 17 00:00:00 2001 From: "MIDE\\cflanigan" Date: Fri, 25 Jun 2021 16:47:31 -0400 Subject: [PATCH 16/41] fixes `arrayJitterySlice` --- idelib/dataset.py | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/idelib/dataset.py b/idelib/dataset.py index 360b4614..66c006bd 100644 --- a/idelib/dataset.py +++ b/idelib/dataset.py @@ -2055,12 +2055,12 @@ def iterJitterySlice(self, start=None, end=None, step=1, jitter=0.5, :return: an iterable of events in the specified index range. """ self._computeMinMeanMax() + + data = self.arrayJitterySlice(start=start, end=end, step=step, jitter=jitter, display=display) + + for evt in data.T: + yield evt - for times, values in self._blockJitterySlice(start, end, step, jitter, - display): - blockEvents = np.append(times[np.newaxis], values, axis=0) - for event in blockEvents.T: - yield event def arrayJitterySlice(self, start=None, end=None, step=1, jitter=0.5, @@ -2105,7 +2105,7 @@ def arrayJitterySlice(self, start=None, end=None, step=1, jitter=0.5, xform = self._comboXform # grab all raw data - rawData = self._accessCache(start, end, 1) + rawData = self._accessCache(None, None, 1) # slightly janky way of enforcing output length if isinstance(self.parent, SubChannel): @@ -2125,10 +2125,7 @@ def arrayJitterySlice(self, start=None, end=None, step=1, jitter=0.5, rawData = rawData[indices] # now times - times = self._inplaceTime(start, end, 1) - times = times[indices] - out[0] = times[:] - del times + self._inplaceTimeFromIndices(indices, out=out[0]) if isinstance(self.parent, SubChannel): xform.polys[self.subchannelId].inplace(rawData, out=out[1], timestamp=out[0]) @@ -2963,6 +2960,21 @@ def _inplaceTime(self, start, end, step, out=None): out += arrayStart return out + def _inplaceTimeFromIndices(self, indices, out=None): + if out is None: + out = indices.astype(np.float64) + + out[:] = indices + + arrayStart = float(self._data[0].startTime) + arrayEnd = float(self._data[-1].endTime) + nSamples = len(self) + samplePeriod = (arrayEnd - arrayStart)/(nSamples - 1) + + out *= samplePeriod + out += arrayStart + return out + #=============================================================================== # From a1f096d36f8b283a5c3509f56d10fed761230b2b Mon Sep 17 00:00:00 2001 From: "MIDE\\cflanigan" Date: Thu, 1 Jul 2021 15:16:59 -0400 Subject: [PATCH 17/41] switching to caching while reading. --- idelib/dataset.py | 46 ++++++++++++++++++++++++++++++++------------ idelib/importer.py | 48 +++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 77 insertions(+), 17 deletions(-) diff --git a/idelib/dataset.py b/idelib/dataset.py index 66c006bd..7afd420d 100644 --- a/idelib/dataset.py +++ b/idelib/dataset.py @@ -492,11 +492,11 @@ def updateTransforms(self): for ch in self.channels.values(): ch.updateTransforms() - def allocateCaches(self): + def allocateCaches(self, sizes): for channel in self.channels.values(): for ea in channel.sessions.values(): with self._channelDataLock: - ea.allocateCache() + ea.allocateCache(sizes[channel]) def fillCaches(self): for channel in self.channels.values(): @@ -1233,6 +1233,7 @@ def __init__(self, parentChannel, session=None, parentList=None): self._cacheEnd = None self._cacheBlockStart = None self._cacheBlockEnd = None + self._cacheLen = 0 def updateTransforms(self, recurse=True): @@ -1391,6 +1392,13 @@ def append(self, block): self._data.append(block) self._length += block.numSamples + blockPayload = block._payloadEl.dump() + plLen = len(blockPayload) + + self._cacheBytes[self._cacheLen:self._cacheLen + plLen] = blockPayload + self._cacheLen += plLen + + @property def _firstTime(self): return self._data[0].startTime if self._data else None @@ -2371,7 +2379,7 @@ def arrayMinMeanMax(self, startTime=None, endTime=None, padding=0, """ startBlock, endBlock = self._getBlockRange(startTime, endTime) - shape = (3, max(1, len(self._npType)), endBlock - startBlock) + shape = (3, max(1, len(self._npType)) + int(times), endBlock - startBlock) scid = self.subchannelId isSubchannel = isinstance(self.parent, SubChannel) @@ -2379,13 +2387,25 @@ def arrayMinMeanMax(self, startTime=None, endTime=None, padding=0, for i, d in enumerate(self._data[startBlock:endBlock]): if isSubchannel: - out[0, 0, i] = d.min[scid] - out[1, 0, i] = d.mean[scid] - out[2, 0, i] = d.max[scid] + if times: + out[:, 0, i] = d.startTime + out[0, 1, i] = d.min[scid] + out[1, 1, i] = d.mean[scid] + out[2, 1, i] = d.max[scid] + else: + out[0, 0, i] = d.min[scid] + out[1, 0, i] = d.mean[scid] + out[2, 0, i] = d.max[scid] else: - out[0, :, i] = d.min - out[1, :, i] = d.mean - out[2, :, i] = d.max + if times: + out[:, 0, i] = d.startTime + out[0, 1:, i] = d.min + out[1, 1:, i] = d.mean + out[2, 1:, i] = d.max + else: + out[0, :, i] = d.min + out[1, :, i] = d.mean + out[2, :, i] = d.max return out @@ -2885,10 +2905,12 @@ def exportCsv(self, stream, start=None, stop=None, step=1, subchannels=True, return num+1, datetime.now() - t0 - def allocateCache(self): + def allocateCache(self, size): - payloadLen = sum((d.payloadSize for d in self._data)) - self._cacheBytes = np.zeros(payloadLen, dtype=np.uint8) + if (size/self._npType.itemsize) % 1 == 0.: + self._cacheBytes = np.zeros(size, dtype=np.uint8) + else: + a = 1 self._cacheArray = self._cacheBytes.view(self._npType) diff --git a/idelib/importer.py b/idelib/importer.py index f8a82e0a..b5edaa81 100644 --- a/idelib/importer.py +++ b/idelib/importer.py @@ -8,12 +8,15 @@ from time import time as time_time from time import sleep +import numpy as np import struct from . import transforms from .dataset import Dataset from . import parsers +import tqdm + #=============================================================================== # #=============================================================================== @@ -427,14 +430,51 @@ def readData(doc, source=None, updater=nullUpdater, numUpdates=500, updateInterv timeOffset = 0 maxPause = getattr(updater, "maxPause", maxPause) - + # Actual importing --------------------------------------------------------- if source is None: source = doc try: # This just skips 'header' elements. It could be more efficient, but # the size of the header isn't significantly large; savings are minimal. - for r in source.ebmldoc: + + # elementList = [x for x in tqdm.tqdm(source.ebmldoc)] + + idType = source.ebmldoc.children[0xA1].children[0xB0] + payloadType = source.ebmldoc.children[0xA1].children[0xB2] + + channelSize = {k: 0 for k in source.channels.keys()} + for r in tqdm.tqdm(source.ebmldoc): + + doc.loadCancelled = getattr(updater, "cancelled", False) + if doc.loadCancelled: + break + + if updater.paused: + # Pause or throttle import. + pauseTime = time_time() + while updater.paused: + sleep(0.125) + if maxPause and time_time() - pauseTime > maxPause: + break + + elementList.append(r) + if not isinstance(r, source.ebmldoc.children[0xA1]): + continue + + rd = {type(rc): rc for rc in r} + + chId = rd[idType].dump() + payloadSize = rd[payloadType].size + + channelSize[chId] += payloadSize + + print(channelSize) + + for ch, ea in doc.channels.items(): + ea.getSession().allocateCache(channelSize[ch]) + + for r in tqdm.tqdm(source.ebmldoc): r_name = r.name @@ -504,9 +544,7 @@ def readData(doc, source=None, updater=nullUpdater, numUpdates=500, updateInterv # (typically the last) doc.fileDamaged = True - doc.allocateCaches() - - doc.fillCaches() + # doc.fillCaches() doc.loading = False updater(done=True) From 69f5aea6d688f5ffad5e5d248c4093e8b2aa6935 Mon Sep 17 00:00:00 2001 From: "MIDE\\cflanigan" Date: Thu, 1 Jul 2021 16:28:50 -0400 Subject: [PATCH 18/41] new strategy: Rather than pre-allocate or anticipate, just do the caching in the blocks. When accessing the cache, just concatenate all the data arrays in the blocks. After loading all data, set the cache as the concatenated array. Then make the blocks' payloads views on the full array. --- idelib/dataset.py | 64 ++++++++++++++-------------------------------- idelib/importer.py | 40 ++--------------------------- 2 files changed, 21 insertions(+), 83 deletions(-) diff --git a/idelib/dataset.py b/idelib/dataset.py index 7afd420d..48d25c2a 100644 --- a/idelib/dataset.py +++ b/idelib/dataset.py @@ -1392,11 +1392,7 @@ def append(self, block): self._data.append(block) self._length += block.numSamples - blockPayload = block._payloadEl.dump() - plLen = len(blockPayload) - - self._cacheBytes[self._cacheLen:self._cacheLen + plLen] = blockPayload - self._cacheLen += plLen + block._payload = np.frombuffer(block._payloadEl.dump(), dtype=self._npType) @property @@ -2915,14 +2911,15 @@ def allocateCache(self, size): self._cacheArray = self._cacheBytes.view(self._npType) def fillCache(self): - idx = 0 - for d in self._data: - with self.dataset._channelDataLock: - payloadLen = len(d._payloadEl.value) - self._cacheBytes[idx:idx + payloadLen] = np.frombuffer(d._payloadEl.value, dtype=np.uint8) - d._payloadEl._value = self._cacheBytes[idx:idx + payloadLen] - d._payload = self._cacheBytes[idx:idx + payloadLen].view(self._npType) - idx += payloadLen + print(self) + with self.dataset._channelDataLock: + self._cacheArray = np.concatenate([d.payload for d in self._data]) + self._cacheBytes = self._cacheArray.view(np.uint8) + + idx = 0 + for d in self._data: + d._payload = self._cacheArray[idx:idx + len(d._payload)] + idx += len(d._payload) def _accessCache(self, start, end, step): """ Access cached data in a thread-safe way. If data has not fully @@ -2937,35 +2934,10 @@ def _accessCache(self, start, end, step): schKey = rawData.dtype.names[schId] return rawData[schKey] - if self.dataset.loading: - nSamples = int(np.ceil((end - start)/step)) - newBytes = np.zeros(nSamples*self._npType.itemsize, dtype=np.uint8) - newArray = newBytes.view(self._npType) - - nextStart = start - idx = 0 - for d in self._data: - with self.dataset._channelDataLock: - blockStart, blockEnd = d.indexRange - if blockStart > end: - return newArray - elif blockEnd < nextStart: - continue - - relativeStart = nextStart - blockStart - if end > blockEnd: - relativeEnd = blockEnd - blockStart - else: - relativeEnd = end - blockStart - - payloadView = d.payload.view(self._npType)[relativeStart:relativeEnd:step] - newArray[idx:idx + len(payloadView)] = payloadView - idx += len(payloadView) - nextStart = idx*step - - return newArray - else: - with self.dataset._channelDataLock: + with self.dataset._channelDataLock: + if self.dataset.loading: + return np.concatenate([d.payload for d in self._data])[start:end:step] + else: return self._cacheArray[start:end:step] def _inplaceTime(self, start, end, step, out=None): @@ -2977,9 +2949,11 @@ def _inplaceTime(self, start, end, step, out=None): nSamples = len(self) samplePeriod = (arrayEnd - arrayStart)/(nSamples - 1) - out[:] = np.arange(start, end, step) - out *= samplePeriod - out += arrayStart + # out = samplePeriod*(step*out + start) + arrayStart + # out = out*(samplePeriod*step) + (samplePeriod*start + arrayStart) + out[:] = np.arange(len(out)) + out *= samplePeriod*step + out += samplePeriod*start + arrayStart return out def _inplaceTimeFromIndices(self, indices, out=None): diff --git a/idelib/importer.py b/idelib/importer.py index b5edaa81..89dc9347 100644 --- a/idelib/importer.py +++ b/idelib/importer.py @@ -438,42 +438,6 @@ def readData(doc, source=None, updater=nullUpdater, numUpdates=500, updateInterv # This just skips 'header' elements. It could be more efficient, but # the size of the header isn't significantly large; savings are minimal. - # elementList = [x for x in tqdm.tqdm(source.ebmldoc)] - - idType = source.ebmldoc.children[0xA1].children[0xB0] - payloadType = source.ebmldoc.children[0xA1].children[0xB2] - - channelSize = {k: 0 for k in source.channels.keys()} - for r in tqdm.tqdm(source.ebmldoc): - - doc.loadCancelled = getattr(updater, "cancelled", False) - if doc.loadCancelled: - break - - if updater.paused: - # Pause or throttle import. - pauseTime = time_time() - while updater.paused: - sleep(0.125) - if maxPause and time_time() - pauseTime > maxPause: - break - - elementList.append(r) - if not isinstance(r, source.ebmldoc.children[0xA1]): - continue - - rd = {type(rc): rc for rc in r} - - chId = rd[idType].dump() - payloadSize = rd[payloadType].size - - channelSize[chId] += payloadSize - - print(channelSize) - - for ch, ea in doc.channels.items(): - ea.getSession().allocateCache(channelSize[ch]) - for r in tqdm.tqdm(source.ebmldoc): r_name = r.name @@ -527,7 +491,7 @@ def readData(doc, source=None, updater=nullUpdater, numUpdates=500, updateInterv if thisTime > nextUpdateTime or thisOffset > nextUpdatePos: # Update progress bar updater(count=eventsRead+samplesRead, - percent=(thisOffset-firstDataPos+0.0)/dataSize) + percent=(1/3) + (2/3)*(thisOffset-firstDataPos)/dataSize) nextUpdatePos = thisOffset + ticSize nextUpdateTime = thisTime + updateInterval @@ -544,7 +508,7 @@ def readData(doc, source=None, updater=nullUpdater, numUpdates=500, updateInterv # (typically the last) doc.fileDamaged = True - # doc.fillCaches() + doc.fillCaches() doc.loading = False updater(done=True) From 6169d64cb547db765b37c2d0352e56a381908e57 Mon Sep 17 00:00:00 2001 From: "MIDE\\cflanigan" Date: Thu, 1 Jul 2021 16:54:45 -0400 Subject: [PATCH 19/41] remove some prints --- idelib/dataset.py | 3 +-- idelib/importer.py | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/idelib/dataset.py b/idelib/dataset.py index 48d25c2a..09771d93 100644 --- a/idelib/dataset.py +++ b/idelib/dataset.py @@ -2906,12 +2906,11 @@ def allocateCache(self, size): if (size/self._npType.itemsize) % 1 == 0.: self._cacheBytes = np.zeros(size, dtype=np.uint8) else: - a = 1 + raise Exception("I haven't gotten to this yet, whoops") self._cacheArray = self._cacheBytes.view(self._npType) def fillCache(self): - print(self) with self.dataset._channelDataLock: self._cacheArray = np.concatenate([d.payload for d in self._data]) self._cacheBytes = self._cacheArray.view(np.uint8) diff --git a/idelib/importer.py b/idelib/importer.py index 89dc9347..6a9ce778 100644 --- a/idelib/importer.py +++ b/idelib/importer.py @@ -15,8 +15,6 @@ from .dataset import Dataset from . import parsers -import tqdm - #=============================================================================== # #=============================================================================== @@ -438,7 +436,7 @@ def readData(doc, source=None, updater=nullUpdater, numUpdates=500, updateInterv # This just skips 'header' elements. It could be more efficient, but # the size of the header isn't significantly large; savings are minimal. - for r in tqdm.tqdm(source.ebmldoc): + for r in source.ebmldoc: r_name = r.name From 48cd1235013d8785e6b7ae2be44616ab04897b68 Mon Sep 17 00:00:00 2001 From: "MIDE\\cflanigan" Date: Thu, 1 Jul 2021 16:55:12 -0400 Subject: [PATCH 20/41] rework a few test functions, adding time back to ones that mistakenly had them removed --- testing/test_dataset.py | 82 +++++++++++------------------------------ 1 file changed, 21 insertions(+), 61 deletions(-) diff --git a/testing/test_dataset.py b/testing/test_dataset.py index 39ad0259..dda81a75 100644 --- a/testing/test_dataset.py +++ b/testing/test_dataset.py @@ -1411,53 +1411,16 @@ def testArraySlice(self, testIDE, start, end, step): np.testing.assert_equal(actual, expected) - def testIterJitterySlice(self): + @pytest.mark.parametrize('jitter, step', [(0.5, 5), (0.5, 1), (0.1, 20), (0.1, 5)]) + def testIterJitterySlice(self, testIDE, jitter, step): """ Test for the iterJitterySlice method. """ - # Stub dependencies - length = 4 - eventArray = mock.Mock(spec=EventArray) - eventArray.configure_mock( - useAllTransforms=True, - __len__=lambda self: length, - _fullXform=Univariate((7, 0)), - _data=mock.Mock(), - _getBlockIndexWithIndex=(lambda idx, start=0, stop=None: - range(length)[idx]), - _getBlockIndexRange=lambda idx: [idx, idx+1], - _getBlockSampleTime=lambda idx: 0.01*idx, - _getBlockRollingMean=lambda blockIdx: (0,), - allowMeanRemoval=True, - removeMean=True, - hasMinMeanMax=True, - parent=mock.Mock(), - session=mock.sentinel.session, - noBivariates=mock.sentinel.noBivariates, - hasSubchannels=True, - _blockJitterySlice=( - lambda *a, **kw: - EventArray._blockJitterySlice(eventArray, *a, **kw) - ), - _makeBlockEventsFactory=( - lambda *a, **kw: - EventArray._makeBlockEventsFactory(eventArray, *a, **kw) - ), - ) - eventArray._data.configure_mock( - __getitem__=lambda self, i: mock.Mock( - id=i % length, numSamples=1, - startTime=eventArray._getBlockSampleTime(i), - ) - ) - eventArray.parent.configure_mock( - parseBlockByIndex=(lambda block, indices, subchannel=None: - np.array([[range(length)[block.id]]])) - ) - # Run test - np.testing.assert_array_equal( - list(EventArray.iterJitterySlice(eventArray)), - [(0.00, 0), (0.01, 7), (0.02, 14), (0.03, 21)] - ) + targetIdx = np.arange(0, 1000, step) + + dt = np.diff(testIDE.channels[8].getSession()[:][0]).mean() + idx = np.array([x[0] for x in testIDE.channels[8].getSession().iterJitterySlice(None, None, step, jitter=jitter)])/dt + + np.testing.assert_array_less(np.abs(targetIdx - idx).round(), step/jitter) @pytest.mark.parametrize('jitter, step', [(0.5, 5), (0.5, 1), (0.1, 20), (0.1, 5)]) def testArrayJitterySlice(self, testIDE, jitter, step): @@ -1558,16 +1521,18 @@ def testIterMinMeanMax(self): def testArrayMinMeanMax(self, eventArray): """ Test arrayMinMeanMax. """ - expected = np.zeros((3, 3, 10)) - expected[1, 0, :] = 499 - expected[1, 1, :] = 332 - expected[1, 2, :] = 666 - expected[2, 0, :] = 999 - expected[2, 1, :] = 998 - expected[2, 2, :] = 999 + expected = np.zeros((3, 4, 10)) + expected[:, 0, :] = np.linspace(0, 900000, 10) + expected[1, 1, :] = 499 + expected[1, 2, :] = 332 + expected[1, 3, :] = 666 + expected[2, 1, :] = 999 + expected[2, 2, :] = 998 + expected[2, 3, :] = 999 # Run tests result = eventArray.arrayMinMeanMax() + print(result[:, 0, :]) np.testing.assert_array_equal(result, expected) def testGetMinMeanMax(self, testIDE): @@ -1576,18 +1541,13 @@ def testGetMinMeanMax(self, testIDE): eventArray = testIDE.channels[8].getSession() eventArray.hasMinMeanMax = False - times = [d.startTime for d in eventArray._data] - mins_ = [d.min for d in eventArray._data] - means = [d.mean for d in eventArray._data] - maxes = [d.max for d in eventArray._data] - - arrayMins_ = np.stack([m for t, m in zip(times, mins_)]).T - arrayMeans = np.stack([m for t, m in zip(times, means)]).T - arrayMaxes = np.stack([m for t, m in zip(times, maxes)]).T + mins_ = [np.concatenate(((d.startTime,), d.min)) for d in eventArray._data] + means = [np.concatenate(((d.startTime,), d.mean)) for d in eventArray._data] + maxes = [np.concatenate(((d.startTime,), d.max)) for d in eventArray._data] np.testing.assert_array_equal( np.stack(eventArray.getMinMeanMax()), - np.stack([arrayMins_, arrayMeans, arrayMaxes]), + np.moveaxis(np.stack([mins_, means, maxes]), 1, -1), ) def testGetRangeMinMeanMax(self, testIDE): From 5b26afd8345bec2f4da28b69637ea593473e7655 Mon Sep 17 00:00:00 2001 From: "MIDE\\cflanigan" Date: Fri, 9 Jul 2021 16:32:44 -0400 Subject: [PATCH 21/41] testing an FFT based mean removal which does not work well. --- idelib/dataset.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/idelib/dataset.py b/idelib/dataset.py index 09771d93..1aa74550 100644 --- a/idelib/dataset.py +++ b/idelib/dataset.py @@ -1948,6 +1948,9 @@ def arraySlice(self, start=None, end=None, step=1, display=False): 'display' transform) will be applied to the data. :return: a structured array of events in the specified index range. """ + + import tqdm + if not isinstance(start, slice): start = slice(start, end, step) start, end, step = start.indices(len(self)) @@ -1974,6 +1977,28 @@ def arraySlice(self, start=None, end=None, step=1, display=False): for i, (k, _) in enumerate(rawData.dtype.descr): xform.polys[i].inplace(rawData[k], out=out[i + 1], timestamp=out[0]) + if not self.removeMean: + return out + + if self.rollingMeanSpan == -1: + out[1:] -= out[1:].mean(axis=1) + else: + spanTime = self.rollingMeanSpan*1e-6 + span = int(np.round(spanTime*self.getSampleRate())/10) + if span % 2 == 0: + span += 1 + kernel = -np.ones((span,))/span + kernel[int(span/2)] += 1 + + s = kernel.shape[0] + out.shape[1] - 1 + startIdx = (s - out.shape[1])//2 + endIdx = startIdx + out.shape[1] + + kernel = np.fft.rfft(kernel, s) + + for i in tqdm.trange(1, out.shape[0]): + out[i] = np.fft.irfft(kernel*np.fft.rfft(out[i], s))[startIdx:endIdx] + return out From 19cab4cdcef5c95d83fc1a40e448b229ca5b89c9 Mon Sep 17 00:00:00 2001 From: "MIDE\\cflanigan" Date: Tue, 13 Jul 2021 11:17:04 -0400 Subject: [PATCH 22/41] fixes mean removal stuff, adds tests for it --- idelib/dataset.py | 58 +++++++++++++++++++++++---------- testing/test_dataset.py | 71 +++++++++++++++++++++++++++++++++-------- 2 files changed, 99 insertions(+), 30 deletions(-) diff --git a/idelib/dataset.py b/idelib/dataset.py index 1aa74550..7dde9853 100644 --- a/idelib/dataset.py +++ b/idelib/dataset.py @@ -1981,23 +1981,49 @@ def arraySlice(self, start=None, end=None, step=1, display=False): return out if self.rollingMeanSpan == -1: - out[1:] -= out[1:].mean(axis=1) + out[1:] -= out[1:].mean(axis=1)[:, np.newaxis] else: - spanTime = self.rollingMeanSpan*1e-6 - span = int(np.round(spanTime*self.getSampleRate())/10) - if span % 2 == 0: - span += 1 - kernel = -np.ones((span,))/span - kernel[int(span/2)] += 1 - - s = kernel.shape[0] + out.shape[1] - 1 - startIdx = (s - out.shape[1])//2 - endIdx = startIdx + out.shape[1] - - kernel = np.fft.rfft(kernel, s) - - for i in tqdm.trange(1, out.shape[0]): - out[i] = np.fft.irfft(kernel*np.fft.rfft(out[i], s))[startIdx:endIdx] + with self._channelDataLock: + timeMean = np.zeros( + (len(self._data),), + [ + ('startTime', np.uint64), + ('endTime', np.uint64), + ('startIdx', np.uint64), + ('endIdx', np.uint64), + ('means', np.float64, (len(self._data[0].mean))), + ], + ) + for i, d in enumerate(self._data): + timeMean['startTime'][i] = d.startTime + timeMean['endTime'][i] = d.endTime + timeMean['startIdx'][i] = d.indexRange[0] + timeMean['endIdx'][i] = d.indexRange[1] + timeMean['means'][i] = d.mean + + for i, tm in enumerate(timeMean): + blockData = out[1:, tm['startIdx']:tm['endIdx']] + spanStart = tm['startTime'] - self.rollingMeanSpan + spanEnd = tm['endTime'] + self.rollingMeanSpan + + firstBlock = 0 + if i != 0: + for j, block in zip(range(i - 1, -1, -1), timeMean[i - 1::-1]): + if block['endTime'] < spanStart: + firstBlock = j + 1 + break + + lastBlock = len(timeMean) + if i != (len(timeMean) - 1): + for j, block in zip(range(i + 1, len(timeMean)), timeMean[i + 1:]): + if block['startTime'] > spanEnd: + lastBlock = j + break + + if firstBlock == lastBlock: + blockData -= tm['means'][:, np.newaxis] + else: + blockData -= timeMean['means'][firstBlock:lastBlock].mean(axis=0)[:, np.newaxis] return out diff --git a/testing/test_dataset.py b/testing/test_dataset.py index dda81a75..ec14f633 100644 --- a/testing/test_dataset.py +++ b/testing/test_dataset.py @@ -1714,6 +1714,37 @@ def testExportCSV(self): eventArray1._data[0].mean = [4] eventArray1._data[0].max = [5] + def testMeanRemovalSingleBlock(self, testIDE): + + eventArray = testIDE.channels[8].getSession() + eventArray.removeMean = False + + unremovedData = eventArray[:] + + eventArray.rollingMeanSpan = 1 + eventArray.removeMean = True + + for d in eventArray._data: + unremovedData[1:, slice(*d.indexRange)] -= d.mean[:, np.newaxis] + + removedData = eventArray[:] + + np.testing.assert_array_equal(removedData, unremovedData) + + def testMeanRemovalFullFile(self, testIDE): + + eventArray = testIDE.channels[8].getSession() + eventArray.removeMean = False + + unremovedData = eventArray[:] + unremovedData[1:] -= unremovedData[1:].mean(axis=1)[:, np.newaxis] + + eventArray.rollingMeanSpan = -1 + eventArray.removeMean = True + + removedData = eventArray[:] + + np.testing.assert_array_equal(removedData, unremovedData) #=============================================================================== # @@ -1800,8 +1831,8 @@ def out(self): return StringIO() @staticmethod - def generateCsvArray(filestream, eventArray): - eventArray.exportCsv(filestream) + def generateCsvArray(filestream, eventArray, **kwargs): + eventArray.exportCsv(filestream, **kwargs) filestream.seek(0) return np.genfromtxt(filestream, delimiter=', ').T @@ -1840,10 +1871,13 @@ def testRollingMeanRemoval(self, accelArray, out): """ Test regular export, with the rolling mean removed from the data. """ - accelArray.removeMean = True - accelArray.rollingMeanSpan = 5000000 + removeMean = True + meanSpan = 5000000 - new = self.generateCsvArray(out, accelArray) + accelArray.removeMean = removeMean + accelArray.rollingMeanSpan = meanSpan + + new = self.generateCsvArray(out, accelArray, removeMean=removeMean, meanSpan=meanSpan) old = accelArray.__getitem__(slice(None), display=True) old = np.round(1e6*old)/1e6 @@ -1854,10 +1888,13 @@ def testTotalMeanRemoval(self, accelArray, out): the data. """ - accelArray.removeMean = True - accelArray.rollingMeanSpan = -1 + removeMean = True + meanSpan = -1 - new = self.generateCsvArray(out, accelArray) + accelArray.removeMean = removeMean + accelArray.rollingMeanSpan = meanSpan + + new = self.generateCsvArray(out, accelArray, removeMean=removeMean, meanSpan=meanSpan) old = accelArray.__getitem__(slice(None), display=True) old = np.round(1e6*old)/1e6 @@ -1868,10 +1905,13 @@ def testCalibratedRollingMeanRemoval(self, accelArray, out): the data. """ - accelArray.removeMean = True - accelArray.rollingMeanSpan = 5000000 + removeMean = True + meanSpan = 5000000 - new = self.generateCsvArray(out, accelArray) + accelArray.removeMean = removeMean + accelArray.rollingMeanSpan = meanSpan + + new = self.generateCsvArray(out, accelArray, removeMean=removeMean, meanSpan=meanSpan) old = accelArray.__getitem__(slice(None), display=True) old = np.round(1e6*old)/1e6 @@ -1881,10 +1921,13 @@ def testCalibratedTotalMeanRemoval(self, accelArray, out): """ Test regular export, with the total mean removed from the data. """ - accelArray.removeMean = True - accelArray.rollingMeanSpan = -1 + removeMean = True + meanSpan = -1 - new = self.generateCsvArray(out, accelArray) + accelArray.removeMean = removeMean + accelArray.rollingMeanSpan = meanSpan + + new = self.generateCsvArray(out, accelArray, removeMean=removeMean, meanSpan=meanSpan) old = accelArray.__getitem__(slice(None), display=True) old = np.round(1e6*old)/1e6 From dbbe719179fb0f1f559cfad6d09b990f166f17ae Mon Sep 17 00:00:00 2001 From: "MIDE\\cflanigan" Date: Tue, 13 Jul 2021 11:26:19 -0400 Subject: [PATCH 23/41] Adds docstrings to new tests --- testing/test_dataset.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testing/test_dataset.py b/testing/test_dataset.py index ec14f633..3a2fbdcf 100644 --- a/testing/test_dataset.py +++ b/testing/test_dataset.py @@ -1715,6 +1715,7 @@ def testExportCSV(self): eventArray1._data[0].max = [5] def testMeanRemovalSingleBlock(self, testIDE): + """ Testing mean removal for spans less than one block """ eventArray = testIDE.channels[8].getSession() eventArray.removeMean = False @@ -1732,6 +1733,7 @@ def testMeanRemovalSingleBlock(self, testIDE): np.testing.assert_array_equal(removedData, unremovedData) def testMeanRemovalFullFile(self, testIDE): + """ Testing mean removal spanning the full file """ eventArray = testIDE.channels[8].getSession() eventArray.removeMean = False From 2a1605c8b800f318ffe0c2d38cda48e23f95de79 Mon Sep 17 00:00:00 2001 From: "MIDE\\cflanigan" Date: Tue, 13 Jul 2021 11:30:42 -0400 Subject: [PATCH 24/41] removing import that was for testing. --- idelib/dataset.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/idelib/dataset.py b/idelib/dataset.py index 7dde9853..bd6b8749 100644 --- a/idelib/dataset.py +++ b/idelib/dataset.py @@ -1949,8 +1949,6 @@ def arraySlice(self, start=None, end=None, step=1, display=False): :return: a structured array of events in the specified index range. """ - import tqdm - if not isinstance(start, slice): start = slice(start, end, step) start, end, step = start.indices(len(self)) From 9dc8c7402827b9aff010c58f15d014b87338d91e Mon Sep 17 00:00:00 2001 From: "MIDE\\cflanigan" Date: Tue, 13 Jul 2021 13:49:56 -0400 Subject: [PATCH 25/41] adding warnings to iter methods for future deprecation --- examples/read_ide_file.py | 7 ++++++- idelib/dataset.py | 23 +++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/examples/read_ide_file.py b/examples/read_ide_file.py index 9c182331..5fe4cb72 100644 --- a/examples/read_ide_file.py +++ b/examples/read_ide_file.py @@ -34,7 +34,8 @@ # Loading an ide file is very simple, the supporting files are defined by # default in idelib. This function returns a Document object which contains # the data for the given file -doc = idelib.importFile('../test.ide') +# doc = idelib.importFile('../test.ide') +doc = idelib.importFile(r'C:\Users\cflanigan\Documents\assorted data\Software_Test_Recordings\20000_Hz_3200_Hz_ran_out_memory.IDE') # The channels in a document are contained in an easily accessed dictionary @@ -50,11 +51,15 @@ # Channel 8 is the accelerometer data, so we'll start with that. # First, we get the EventArray ch8EventArray = doc.channels[8].getSession() +ch8EventArray.removeMean = True + # The EventArray object has several methods to access data, but the simplest is # EventArray.arraySlice, which returns a numpy ndarray where the first row is # the time in microseconds, and the following rows are the subchannels in order ch8Data = ch8EventArray.arraySlice() +print(ch8Data[1:].mean()) +exit(0) ch8Time = ch8Data[0, :]/1e6 ch8NSubchannels = len(doc.channels[8].subchannels) diff --git a/idelib/dataset.py b/idelib/dataset.py index bd6b8749..aa970be9 100644 --- a/idelib/dataset.py +++ b/idelib/dataset.py @@ -50,6 +50,7 @@ from collections.abc import Iterable, Sequence from datetime import datetime from threading import Lock +import warnings from functools import partial import os.path @@ -1817,6 +1818,9 @@ def itervalues(self, start=None, end=None, step=1, subchannels=True, specified index range. """ + warnings.warn(DeprecationWarning('iter methods should be expecte to be ' + 'removed in future versions of idelib')) + out = self.arrayValues(start=start, end=end, step=step) for evt in out.T: @@ -1932,6 +1936,9 @@ def iterSlice(self, start=None, end=None, step=1, display=False): :return: an iterable of events in the specified index range. """ + warnings.warn(DeprecationWarning('iter methods should be expecte to be ' + 'removed in future versions of idelib')) + out = self.arraySlice(start=start, end=end, step=step, display=display) for evt in out.T: yield evt @@ -2107,6 +2114,10 @@ def iterJitterySlice(self, start=None, end=None, step=1, jitter=0.5, 'display' transform) will be applied to the data. :return: an iterable of events in the specified index range. """ + + warnings.warn(DeprecationWarning('iter methods should be expecte to be ' + 'removed in future versions of idelib')) + self._computeMinMeanMax() data = self.arrayJitterySlice(start=start, end=end, step=step, jitter=jitter, display=display) @@ -2276,6 +2287,10 @@ def iterRange(self, startTime=None, endTime=None, step=1, display=False): :keyword endTime: The second time, or `None` to use the end of the session. """ + + warnings.warn(DeprecationWarning('iter methods should be expecte to be ' + 'removed in future versions of idelib')) + startIdx, endIdx = self.getRangeIndices(startTime, endTime) return self.iterSlice(startIdx,endIdx,step,display=display) @@ -2327,6 +2342,10 @@ def iterMinMeanMax(self, startTime=None, endTime=None, padding=0, :return: An iterator producing sets of three events (min, mean, and max, respectively). """ + + warnings.warn(DeprecationWarning('iter methods should be expecte to be ' + 'removed in future versions of idelib')) + if not self.hasMinMeanMax: self._computeMinMeanMax() @@ -2798,6 +2817,10 @@ def iterResampledRange(self, startTime, stopTime, maxPoints, padding=0, :todo: Optimize iterResampledRange(); not very efficient, particularly not with single-sample blocks. """ + + warnings.warn(DeprecationWarning('iter methods should be expecte to be ' + 'removed in future versions of idelib')) + startIdx, stopIdx = self.getRangeIndices(startTime, stopTime) numPoints = (stopIdx - startIdx) startIdx = max(startIdx-padding, 0) From a97794e2aa1557ccbacacab22e52d3a4e3ce504e Mon Sep 17 00:00:00 2001 From: Becker A Date: Tue, 20 Jul 2021 10:05:24 -0400 Subject: [PATCH 26/41] fixed typo in warning text (#56) --- idelib/dataset.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/idelib/dataset.py b/idelib/dataset.py index ec6f8204..62047b9a 100644 --- a/idelib/dataset.py +++ b/idelib/dataset.py @@ -1858,7 +1858,7 @@ def itervalues(self, start=None, end=None, step=1, subchannels=True, specified index range. """ - warnings.warn(DeprecationWarning('iter methods should be expecte to be ' + warnings.warn(DeprecationWarning('iter methods should be expected to be ' 'removed in future versions of idelib')) out = self.arrayValues(start=start, end=end, step=step) @@ -1976,7 +1976,7 @@ def iterSlice(self, start=None, end=None, step=1, display=False): :return: an iterable of events in the specified index range. """ - warnings.warn(DeprecationWarning('iter methods should be expecte to be ' + warnings.warn(DeprecationWarning('iter methods should be expected to be ' 'removed in future versions of idelib')) out = self.arraySlice(start=start, end=end, step=step, display=display) @@ -2155,7 +2155,7 @@ def iterJitterySlice(self, start=None, end=None, step=1, jitter=0.5, :return: an iterable of events in the specified index range. """ - warnings.warn(DeprecationWarning('iter methods should be expecte to be ' + warnings.warn(DeprecationWarning('iter methods should be expected to be ' 'removed in future versions of idelib')) self._computeMinMeanMax() @@ -2328,7 +2328,7 @@ def iterRange(self, startTime=None, endTime=None, step=1, display=False): the session. """ - warnings.warn(DeprecationWarning('iter methods should be expecte to be ' + warnings.warn(DeprecationWarning('iter methods should be expected to be ' 'removed in future versions of idelib')) startIdx, endIdx = self.getRangeIndices(startTime, endTime) @@ -2383,7 +2383,7 @@ def iterMinMeanMax(self, startTime=None, endTime=None, padding=0, and max, respectively). """ - warnings.warn(DeprecationWarning('iter methods should be expecte to be ' + warnings.warn(DeprecationWarning('iter methods should be expected to be ' 'removed in future versions of idelib')) if not self.hasMinMeanMax: @@ -2859,7 +2859,7 @@ def iterResampledRange(self, startTime, stopTime, maxPoints, padding=0, particularly not with single-sample blocks. """ - warnings.warn(DeprecationWarning('iter methods should be expecte to be ' + warnings.warn(DeprecationWarning('iter methods should be expected to be ' 'removed in future versions of idelib')) startIdx, stopIdx = self.getRangeIndices(startTime, stopTime) From 2358781f20caf5e771de917ce7c92497f2bfb791 Mon Sep 17 00:00:00 2001 From: Becker Awqatty Date: Tue, 20 Jul 2021 19:02:50 -0400 Subject: [PATCH 27/41] removed unnecessary imports --- testing/test_dataset.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/testing/test_dataset.py b/testing/test_dataset.py index 11391ea6..2186c0c0 100644 --- a/testing/test_dataset.py +++ b/testing/test_dataset.py @@ -13,13 +13,11 @@ import sys import unittest import mock -import os import pytest import numpy as np # type: ignore -import idelib from idelib.dataset import (Cascading, Channel, Dataset, @@ -59,22 +57,22 @@ def _load_file(filePath): @pytest.fixture def testIDE(): - doc = idelib.importer.openFile(_load_file('./test.ide')) - idelib.importer.readData(doc) + doc = importer.openFile(_load_file('./test.ide')) + importer.readData(doc) return doc @pytest.fixture def SSX70065IDE(): - doc = idelib.importer.openFile(_load_file('./testing/SSX70065.IDE')) - idelib.importer.readData(doc) + doc = importer.openFile(_load_file('./testing/SSX70065.IDE')) + importer.readData(doc) return doc @pytest.fixture def SSX_DataIDE(): - doc = idelib.importer.openFile(_load_file('./testing/SSX_Data.IDE')) - idelib.importer.readData(doc) + doc = importer.openFile(_load_file('./testing/SSX_Data.IDE')) + importer.readData(doc) return doc From 2b1848729a07a7c560f0c9418d3a9310b77e08ea Mon Sep 17 00:00:00 2001 From: Becker Awqatty Date: Tue, 20 Jul 2021 19:11:13 -0400 Subject: [PATCH 28/41] add missing parameter to unused tests --- testing/test_dataset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/test_dataset.py b/testing/test_dataset.py index 2186c0c0..e4a894c2 100644 --- a/testing/test_dataset.py +++ b/testing/test_dataset.py @@ -1118,7 +1118,7 @@ def testCopy(self, eventArray1): @unittest.skip('failing, poorly formed') - def testAppend(self): + def testAppend(self, eventArray1): """ Test the append method. """ fakeData = GenericObject() fakeData.numSamples = 1 @@ -1502,7 +1502,7 @@ def testGetRange(self, testIDE): ) @unittest.skip('failing, poorly formed') - def testIterMinMeanMax(self): + def testIterMinMeanMax(self, eventArray1): """ Test for iterMinMeanMax method. """ self.mockData() eventArray1._data[0].minMeanMax = 1 @@ -1703,7 +1703,7 @@ def testArrayResampledRange(self, testIDE): ) @pytest.mark.skip("this doesn't actually do anything") - def testExportCSV(self): + def testExportCSV(self, eventArray1): """ Test for exportCsv method.""" self.mockData() eventArray1._data[0].minMeanMax = 1 From 5122914cf4365d9757cbe0fdc06dda515eefe27a Mon Sep 17 00:00:00 2001 From: Becker Awqatty Date: Tue, 20 Jul 2021 18:48:12 -0400 Subject: [PATCH 29/41] removed unused test function --- testing/test_dataset.py | 66 ----------------------------------------- 1 file changed, 66 deletions(-) diff --git a/testing/test_dataset.py b/testing/test_dataset.py index e4a894c2..b2d64a00 100644 --- a/testing/test_dataset.py +++ b/testing/test_dataset.py @@ -1194,72 +1194,6 @@ def testGetInterval(self, dataset): accel.dataset.loading = True assert accel.getInterval() == (3, 1) - def mockForGetItem(self, section): - """ Mock different things for testGetItem. """ - - # mock with xforms - if section == 0: - - mockBlockIndex = [0, 1, 2, 3] - - def mockGetBlockIndexWithIndex(idx, start=None): - return mockBlockIndex[idx] - - def mockGetBlockIndexRange(idx): - return mockBlockIndex - - def mockXform(time, val, session=None, noBivariates=False): - if type(val) is tuple: - return time, val - return time, [val.id] - - def mockParseBlock(block, start=None, end=None, step=None): - return [(block,)] - - eventArray1._getBlockIndexRange = mockGetBlockIndexRange - eventArray1._getBlockIndexWithIndex = mockGetBlockIndexWithIndex - eventArray1.parent.parseBlock = mockParseBlock - eventArray1._displayXform = eventArray1._comboXform = \ - eventArray1._fullXform = mockXform - - eventArray1._data = [GenericObject() for _ in range(4)] - - for i, datum in enumerate(eventArray1._data): - datum.id = i - - # mock without xforms - elif section == 1: - - def mockXform(time, val, session=None, noBivariates=False): - return None - - eventArray1._displayXform = eventArray1._comboXform = \ - eventArray1._fullXform = mockXform - - # with blockRollingMean - elif section == 2: - - self.mockForGetItem(0) - - eventArray1._getBlockRollingMean = lambda x: [1] - - def mockXform(time, val, session=None, noBivariates=False): - if type(val) is tuple: - return time, val - return time, [val.id] - - eventArray1._displayXform = eventArray1._comboXform = \ - eventArray1._fullXform = mockXform - - # for __getitem__ to work in getEventIndexNear - elif section == 3: - self.mockForGetItem(0) - - def mockGetBlockSampleTime(idx, start=None): - return 1 - - eventArray1._getBlockSampleTime = mockGetBlockSampleTime - def testGetItem(self): """ Test the getitem special method. """ length = 4 From d9e2d04fc59e1ca97e0dea74cfe4ddccb14fcb2b Mon Sep 17 00:00:00 2001 From: "MIDE\\cflanigan" Date: Tue, 27 Jul 2021 15:41:23 -0400 Subject: [PATCH 30/41] remove unused array allocation methods. --- idelib/dataset.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/idelib/dataset.py b/idelib/dataset.py index 62047b9a..a5685c91 100644 --- a/idelib/dataset.py +++ b/idelib/dataset.py @@ -493,12 +493,6 @@ def updateTransforms(self): for ch in self.channels.values(): ch.updateTransforms() - def allocateCaches(self, sizes): - for channel in self.channels.values(): - for ea in channel.sessions.values(): - with self._channelDataLock: - ea.allocateCache(sizes[channel]) - def fillCaches(self): for channel in self.channels.values(): for ea in channel.sessions.values(): @@ -3014,15 +3008,6 @@ def exportCsv(self, stream, start=None, stop=None, step=1, subchannels=True, return num+1, datetime.now() - t0 - def allocateCache(self, size): - - if (size/self._npType.itemsize) % 1 == 0.: - self._cacheBytes = np.zeros(size, dtype=np.uint8) - else: - raise Exception("I haven't gotten to this yet, whoops") - - self._cacheArray = self._cacheBytes.view(self._npType) - def fillCache(self): with self.dataset._channelDataLock: self._cacheArray = np.concatenate([d.payload for d in self._data]) From 6b8ad29f7fbdb3ea2db35d5fe54723c5d460dc33 Mon Sep 17 00:00:00 2001 From: "MIDE\\cflanigan" Date: Tue, 27 Jul 2021 15:43:16 -0400 Subject: [PATCH 31/41] use `yield from x` idiom --- idelib/dataset.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/idelib/dataset.py b/idelib/dataset.py index a5685c91..b9b4fa59 100644 --- a/idelib/dataset.py +++ b/idelib/dataset.py @@ -1857,8 +1857,7 @@ def itervalues(self, start=None, end=None, step=1, subchannels=True, out = self.arrayValues(start=start, end=end, step=step) - for evt in out.T: - yield evt + yield from out.T def arrayValues(self, start=None, end=None, step=1, subchannels=True, @@ -1974,8 +1973,8 @@ def iterSlice(self, start=None, end=None, step=1, display=False): 'removed in future versions of idelib')) out = self.arraySlice(start=start, end=end, step=step, display=display) - for evt in out.T: - yield evt + + yield from out.T def arraySlice(self, start=None, end=None, step=1, display=False): @@ -2156,8 +2155,7 @@ def iterJitterySlice(self, start=None, end=None, step=1, jitter=0.5, data = self.arrayJitterySlice(start=start, end=end, step=step, jitter=jitter, display=display) - for evt in data.T: - yield evt + yield from data.T From 1f2e7faec39cf4ec6ded42d95575dd623956d64e Mon Sep 17 00:00:00 2001 From: "MIDE\\cflanigan" Date: Tue, 27 Jul 2021 15:54:36 -0400 Subject: [PATCH 32/41] replace `np.ones` with `np.empty` --- idelib/dataset.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/idelib/dataset.py b/idelib/dataset.py index b9b4fa59..38b41703 100644 --- a/idelib/dataset.py +++ b/idelib/dataset.py @@ -1891,9 +1891,9 @@ def arrayValues(self, start=None, end=None, step=1, subchannels=True, rawData = self._accessCache(start, end, step) if isinstance(self.parent, SubChannel): - out = np.zeros((1, len(rawData))) + out = np.empty((1, len(rawData))) else: - out = np.zeros((len(rawData.dtype), len(rawData))) + out = np.empty((len(rawData.dtype), len(rawData))) if isinstance(self.parent, SubChannel): xform.polys[self.subchannelId].inplace(rawData, out=out) @@ -2003,9 +2003,9 @@ def arraySlice(self, start=None, end=None, step=1, display=False): rawData = self._accessCache(start, end, step) if isinstance(self.parent, SubChannel): - out = np.zeros((2, len(rawData))) + out = np.empty((2, len(rawData))) else: - out = np.zeros((len(rawData.dtype) + 1, len(rawData))) + out = np.empty((len(rawData.dtype) + 1, len(rawData))) self._inplaceTime(start, end, step, out=out[0]) @@ -2022,7 +2022,7 @@ def arraySlice(self, start=None, end=None, step=1, display=False): out[1:] -= out[1:].mean(axis=1)[:, np.newaxis] else: with self._channelDataLock: - timeMean = np.zeros( + timeMean = np.empty( (len(self._data),), [ ('startTime', np.uint64), @@ -2205,9 +2205,9 @@ def arrayJitterySlice(self, start=None, end=None, step=1, jitter=0.5, # slightly janky way of enforcing output length if isinstance(self.parent, SubChannel): - out = np.zeros((2, len(self._accessCache(start, end, step)))) + out = np.empty((2, len(self._accessCache(start, end, step)))) else: - out = np.zeros((len(rawData.dtype) + 1, len(self._accessCache(start, end, step)))) + out = np.empty((len(rawData.dtype) + 1, len(self._accessCache(start, end, step)))) # save on space by being really clever and storing indices in timestamps indices = out[0].view(np.int64) @@ -2479,7 +2479,7 @@ def arrayMinMeanMax(self, startTime=None, endTime=None, padding=0, scid = self.subchannelId isSubchannel = isinstance(self.parent, SubChannel) - out = np.zeros(shape) + out = np.empty(shape) for i, d in enumerate(self._data[startBlock:endBlock]): if isSubchannel: @@ -3037,7 +3037,7 @@ def _accessCache(self, start, end, step): def _inplaceTime(self, start, end, step, out=None): if out is None: - out = np.zeros((int(np.ceil((end - start)/step)),)) + out = np.empty((int(np.ceil((end - start)/step)),)) arrayStart = float(self._data[0].startTime) arrayEnd = float(self._data[-1].endTime) From 94cbcc2ccb5af9f2dea61190b760432863e9ce40 Mon Sep 17 00:00:00 2001 From: "MIDE\\cflanigan" Date: Wed, 28 Jul 2021 15:14:13 -0400 Subject: [PATCH 33/41] fixes time stamps. Not ideal, but functional. --- idelib/dataset.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/idelib/dataset.py b/idelib/dataset.py index 38b41703..c91ddbf6 100644 --- a/idelib/dataset.py +++ b/idelib/dataset.py @@ -3039,16 +3039,26 @@ def _inplaceTime(self, start, end, step, out=None): if out is None: out = np.empty((int(np.ceil((end - start)/step)),)) - arrayStart = float(self._data[0].startTime) - arrayEnd = float(self._data[-1].endTime) - nSamples = len(self) - samplePeriod = (arrayEnd - arrayStart)/(nSamples - 1) + if self._singleSample: + out[:] = [d.startTime for d in self._data[start:end:step]] + return out + + vals = np.empty((self._data[-1].indexRange[-1],)) + + for d in self._data: + arrayStart = d.startTime + arrayEnd = d.endTime + startIdx, endIdx = d.indexRange + samplePeriod = (arrayEnd - arrayStart)/(d.numSamples - 1) + + # out = samplePeriod*(step*out + start) + arrayStart + # out = out*(samplePeriod*step) + (samplePeriod*start + arrayStart) + vals[startIdx:endIdx] = np.arange(d.numSamples) + vals[startIdx:endIdx] *= samplePeriod + vals[startIdx:endIdx] += arrayStart + + out[:] = vals[start:end:step] - # out = samplePeriod*(step*out + start) + arrayStart - # out = out*(samplePeriod*step) + (samplePeriod*start + arrayStart) - out[:] = np.arange(len(out)) - out *= samplePeriod*step - out += samplePeriod*start + arrayStart return out def _inplaceTimeFromIndices(self, indices, out=None): From 4e32eea76f9d89777c8d7f702f591a7aeeed4580 Mon Sep 17 00:00:00 2001 From: "MIDE\\cflanigan" Date: Wed, 28 Jul 2021 15:21:18 -0400 Subject: [PATCH 34/41] officially deprecated rolling mean removal. --- idelib/dataset.py | 61 ++++++++++++----------------------------------- 1 file changed, 15 insertions(+), 46 deletions(-) diff --git a/idelib/dataset.py b/idelib/dataset.py index c91ddbf6..0316cd75 100644 --- a/idelib/dataset.py +++ b/idelib/dataset.py @@ -1222,6 +1222,7 @@ def __init__(self, parentChannel, session=None, parentList=None): self.removeMean = False self.hasMinMeanMax = True + self._rollingMeanSpan = None self.rollingMeanSpan = self.DEFAULT_MEAN_SPAN self.transform = None @@ -1267,6 +1268,19 @@ def __init__(self, parentChannel, session=None, parentList=None): self._cacheLen = 0 + @property + def rollingMeanSpan(self): + return self._rollingMeanSpan + + + @rollingMeanSpan.setter + def rollingMeanSpan(self, value): + if value != -1: + warnings.warn('Rolling mean has been deprecated, this behavior has ' + 'been replaced with total mean removal.') + self._rollingMeanSpan = value + + def updateTransforms(self, recurse=True): """ (Re-)Build and (re-)apply the transformation functions. """ @@ -2015,53 +2029,8 @@ def arraySlice(self, start=None, end=None, step=1, display=False): for i, (k, _) in enumerate(rawData.dtype.descr): xform.polys[i].inplace(rawData[k], out=out[i + 1], timestamp=out[0]) - if not self.removeMean: - return out - - if self.rollingMeanSpan == -1: + if self.removeMean: out[1:] -= out[1:].mean(axis=1)[:, np.newaxis] - else: - with self._channelDataLock: - timeMean = np.empty( - (len(self._data),), - [ - ('startTime', np.uint64), - ('endTime', np.uint64), - ('startIdx', np.uint64), - ('endIdx', np.uint64), - ('means', np.float64, (len(self._data[0].mean))), - ], - ) - for i, d in enumerate(self._data): - timeMean['startTime'][i] = d.startTime - timeMean['endTime'][i] = d.endTime - timeMean['startIdx'][i] = d.indexRange[0] - timeMean['endIdx'][i] = d.indexRange[1] - timeMean['means'][i] = d.mean - - for i, tm in enumerate(timeMean): - blockData = out[1:, tm['startIdx']:tm['endIdx']] - spanStart = tm['startTime'] - self.rollingMeanSpan - spanEnd = tm['endTime'] + self.rollingMeanSpan - - firstBlock = 0 - if i != 0: - for j, block in zip(range(i - 1, -1, -1), timeMean[i - 1::-1]): - if block['endTime'] < spanStart: - firstBlock = j + 1 - break - - lastBlock = len(timeMean) - if i != (len(timeMean) - 1): - for j, block in zip(range(i + 1, len(timeMean)), timeMean[i + 1:]): - if block['startTime'] > spanEnd: - lastBlock = j - break - - if firstBlock == lastBlock: - blockData -= tm['means'][:, np.newaxis] - else: - blockData -= timeMean['means'][firstBlock:lastBlock].mean(axis=0)[:, np.newaxis] return out From f678e30bcfdc18dd0174ad35e6ac72f29ef4f388 Mon Sep 17 00:00:00 2001 From: "MIDE\\cflanigan" Date: Wed, 28 Jul 2021 15:22:22 -0400 Subject: [PATCH 35/41] added mean removal to `arrayValues` --- idelib/dataset.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/idelib/dataset.py b/idelib/dataset.py index 0316cd75..ae6b26d0 100644 --- a/idelib/dataset.py +++ b/idelib/dataset.py @@ -1915,6 +1915,9 @@ def arrayValues(self, start=None, end=None, step=1, subchannels=True, for i, (k, _) in enumerate(rawData.dtype.descr): xform.polys[i].inplace(rawData[k], out=out[i]) + if self.removeMean: + out[1:] -= out[1:].mean(axis=1)[:, np.newaxis] + if subchannels is True: return out else: From 2ae5b33fc1a54ff82ff798cffbc5a7e3135c931e Mon Sep 17 00:00:00 2001 From: "MIDE\\cflanigan" Date: Wed, 28 Jul 2021 15:24:16 -0400 Subject: [PATCH 36/41] reverting changes to read_ide_file.py --- examples/read_ide_file.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/examples/read_ide_file.py b/examples/read_ide_file.py index 5fe4cb72..9c182331 100644 --- a/examples/read_ide_file.py +++ b/examples/read_ide_file.py @@ -34,8 +34,7 @@ # Loading an ide file is very simple, the supporting files are defined by # default in idelib. This function returns a Document object which contains # the data for the given file -# doc = idelib.importFile('../test.ide') -doc = idelib.importFile(r'C:\Users\cflanigan\Documents\assorted data\Software_Test_Recordings\20000_Hz_3200_Hz_ran_out_memory.IDE') +doc = idelib.importFile('../test.ide') # The channels in a document are contained in an easily accessed dictionary @@ -51,15 +50,11 @@ # Channel 8 is the accelerometer data, so we'll start with that. # First, we get the EventArray ch8EventArray = doc.channels[8].getSession() -ch8EventArray.removeMean = True - # The EventArray object has several methods to access data, but the simplest is # EventArray.arraySlice, which returns a numpy ndarray where the first row is # the time in microseconds, and the following rows are the subchannels in order ch8Data = ch8EventArray.arraySlice() -print(ch8Data[1:].mean()) -exit(0) ch8Time = ch8Data[0, :]/1e6 ch8NSubchannels = len(doc.channels[8].subchannels) From 349ee886e2f3971b87ef0cb97f4df62889bb7ae6 Mon Sep 17 00:00:00 2001 From: "MIDE\\cflanigan" Date: Thu, 29 Jul 2021 09:24:34 -0400 Subject: [PATCH 37/41] fixes a test broken with the removal of rolling mean removal. --- testing/test_dataset.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/testing/test_dataset.py b/testing/test_dataset.py index 11391ea6..7775b69c 100644 --- a/testing/test_dataset.py +++ b/testing/test_dataset.py @@ -1725,8 +1725,9 @@ def testMeanRemovalSingleBlock(self, testIDE): eventArray.rollingMeanSpan = 1 eventArray.removeMean = True - for d in eventArray._data: - unremovedData[1:, slice(*d.indexRange)] -= d.mean[:, np.newaxis] + # for d in eventArray._data: + # unremovedData[1:, slice(*d.indexRange)] -= d.mean[:, np.newaxis] + unremovedData[1:] -= unremovedData[1:].mean(axis=1)[:, np.newaxis] removedData = eventArray[:] From 51a83a6b5a1f0a292bf78df571a3d767660759d2 Mon Sep 17 00:00:00 2001 From: Becker Awqatty Date: Fri, 30 Jul 2021 17:40:55 -0400 Subject: [PATCH 38/41] refactored to use parameter `mean(keepdims=True)` --- idelib/dataset.py | 4 ++-- testing/test_dataset.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/idelib/dataset.py b/idelib/dataset.py index ae6b26d0..1781a4dc 100644 --- a/idelib/dataset.py +++ b/idelib/dataset.py @@ -1916,7 +1916,7 @@ def arrayValues(self, start=None, end=None, step=1, subchannels=True, xform.polys[i].inplace(rawData[k], out=out[i]) if self.removeMean: - out[1:] -= out[1:].mean(axis=1)[:, np.newaxis] + out[1:] -= out[1:].mean(axis=1, keepdims=True) if subchannels is True: return out @@ -2033,7 +2033,7 @@ def arraySlice(self, start=None, end=None, step=1, display=False): xform.polys[i].inplace(rawData[k], out=out[i + 1], timestamp=out[0]) if self.removeMean: - out[1:] -= out[1:].mean(axis=1)[:, np.newaxis] + out[1:] -= out[1:].mean(axis=1, keepdims=True) return out diff --git a/testing/test_dataset.py b/testing/test_dataset.py index b4eabf15..511e5fa8 100644 --- a/testing/test_dataset.py +++ b/testing/test_dataset.py @@ -1659,7 +1659,7 @@ def testMeanRemovalSingleBlock(self, testIDE): # for d in eventArray._data: # unremovedData[1:, slice(*d.indexRange)] -= d.mean[:, np.newaxis] - unremovedData[1:] -= unremovedData[1:].mean(axis=1)[:, np.newaxis] + unremovedData[1:] -= unremovedData[1:].mean(axis=1, keepdims=True) removedData = eventArray[:] From 095b569f3715c991b806eebbaf53fbff679d8532 Mon Sep 17 00:00:00 2001 From: "MIDE\\cflanigan" Date: Mon, 2 Aug 2021 10:05:02 -0400 Subject: [PATCH 39/41] removed unused unit test. --- testing/test_dataset.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/testing/test_dataset.py b/testing/test_dataset.py index 511e5fa8..f57216ea 100644 --- a/testing/test_dataset.py +++ b/testing/test_dataset.py @@ -1047,10 +1047,6 @@ def testConstructor(self, eventArray1, channel1, dataset): assert eventArray1._blockIndicesArray.size == 0 assert eventArray1._blockTimesArray.size == 0 - @pytest.mark.skip('not yet implemented') - def testJoinTimesValues(self): - pass - def testUpdateTransformsNoRecursion(self, eventArray1, channel1, dataset): """ Test the updateTransforms method. """ # update transforms without recursion From ee50adbff9a10278087d897ed6091226e0118add Mon Sep 17 00:00:00 2001 From: "MIDE\\cflanigan" Date: Tue, 3 Aug 2021 11:54:42 -0400 Subject: [PATCH 40/41] uses `_channelDataLock` in `EventArray.append` --- idelib/dataset.py | 139 +++++++++++++++++++++++----------------------- 1 file changed, 70 insertions(+), 69 deletions(-) diff --git a/idelib/dataset.py b/idelib/dataset.py index 2e4404a7..64fe1c24 100644 --- a/idelib/dataset.py +++ b/idelib/dataset.py @@ -1373,78 +1373,79 @@ def append(self, block): :attention: Added elements must be in chronological order! """ - if block.numSamples is None: - block.numSamples = block.getNumSamples(self.parent.parser) + with self._channelDataLock: + if block.numSamples is None: + block.numSamples = block.getNumSamples(self.parent.parser) + + # Set the session first/last times if they aren't already set. + # Possibly redundant if all sessions are 'closed.' + if self.session.firstTime is None: + self.session.firstTime = block.startTime + else: + self.session.firstTime = min(self.session.firstTime, block.startTime) - # Set the session first/last times if they aren't already set. - # Possibly redundant if all sessions are 'closed.' - if self.session.firstTime is None: - self.session.firstTime = block.startTime - else: - self.session.firstTime = min(self.session.firstTime, block.startTime) + if self.session.lastTime is None: + self.session.lastTime = block.endTime + else: + self.session.lastTime = max(self.session.lastTime, block.endTime) + + # Check that the block actually contains at least one sample. + if block.numSamples < 1: + # Ignore blocks with empty payload. Could occur in FW <17. + # TODO: Make sure this doesn't hide too many errors! + logger.warning("Ignoring block with bad payload size for %r" % self) + return + + block.cache = self.parent.cache + oldLength = self._length + + block.blockIndex = len(self._data) + block.indexRange = (oldLength, oldLength + block.numSamples) + + # _singleSample hint not explicitly set; set it based on this block. + # There will be problems if the first block has only one sample, but + # future ones don't. This shouldn't happen, though. + if self._singleSample is None: + self._singleSample = block.numSamples == 1 + if self._parentList is not None: + self._parentList._singleSample = self._singleSample + if self.parent.singleSample is None: + self.parent.singleSample = self._singleSample + if self.parent.parent is not None: + self.parent.parent.singleSample = self._singleSample + + # HACK (somewhat): Single-sample-per-block channels get min/mean/max + # which is just the same as the value of the sample. Set the values, + # but don't set hasMinMeanMax. + if self._singleSample is True:# and not self.hasMinMeanMax: + block.minMeanMax = np.tile(block.payload, 3) + block.parseMinMeanMax(self.parent.parser) + self.hasMinMeanMax = False + elif block.minMeanMax is not None: + block.parseMinMeanMax(self.parent.parser) + self.hasMinMeanMax = True #self.hasMinMeanMax and True + else: + # XXX: Attempt to calculate min/mean/max here instead of + # in _computeMinMeanMax(). Causes issues with pressure for some + # reason - it starts removing mean and won't plot. + vals = self.parseBlock(block) + block.min = vals.min(axis=-1) + block.mean = vals.mean(axis=-1) + block.max = vals.max(axis=-1) + self.hasMinMeanMax = True + # self.hasMinMeanMax = False + # self.allowMeanRemoval = False - if self.session.lastTime is None: - self.session.lastTime = block.endTime - else: - self.session.lastTime = max(self.session.lastTime, block.endTime) + # Cache the index range for faster searching + self._blockIndices.append(oldLength) + self._blockTimes.append(block.startTime) - # Check that the block actually contains at least one sample. - if block.numSamples < 1: - # Ignore blocks with empty payload. Could occur in FW <17. - # TODO: Make sure this doesn't hide too many errors! - logger.warning("Ignoring block with bad payload size for %r" % self) - return - - block.cache = self.parent.cache - oldLength = self._length - - block.blockIndex = len(self._data) - block.indexRange = (oldLength, oldLength + block.numSamples) - - # _singleSample hint not explicitly set; set it based on this block. - # There will be problems if the first block has only one sample, but - # future ones don't. This shouldn't happen, though. - if self._singleSample is None: - self._singleSample = block.numSamples == 1 - if self._parentList is not None: - self._parentList._singleSample = self._singleSample - if self.parent.singleSample is None: - self.parent.singleSample = self._singleSample - if self.parent.parent is not None: - self.parent.parent.singleSample = self._singleSample - - # HACK (somewhat): Single-sample-per-block channels get min/mean/max - # which is just the same as the value of the sample. Set the values, - # but don't set hasMinMeanMax. - if self._singleSample is True:# and not self.hasMinMeanMax: - block.minMeanMax = np.tile(block.payload, 3) - block.parseMinMeanMax(self.parent.parser) - self.hasMinMeanMax = False - elif block.minMeanMax is not None: - block.parseMinMeanMax(self.parent.parser) - self.hasMinMeanMax = True #self.hasMinMeanMax and True - else: - # XXX: Attempt to calculate min/mean/max here instead of - # in _computeMinMeanMax(). Causes issues with pressure for some - # reason - it starts removing mean and won't plot. - vals = self.parseBlock(block) - block.min = vals.min(axis=-1) - block.mean = vals.mean(axis=-1) - block.max = vals.max(axis=-1) - self.hasMinMeanMax = True -# self.hasMinMeanMax = False -# self.allowMeanRemoval = False - - # Cache the index range for faster searching - self._blockIndices.append(oldLength) - self._blockTimes.append(block.startTime) - - self._hasSubsamples = self._hasSubsamples or block.numSamples > 1 - - self._data.append(block) - self._length += block.numSamples - - block._payload = np.frombuffer(block._payloadEl.dump(), dtype=self._npType) + self._hasSubsamples = self._hasSubsamples or block.numSamples > 1 + + self._data.append(block) + self._length += block.numSamples + + block._payload = np.frombuffer(block._payloadEl.dump(), dtype=self._npType) @property From 0799b893d3b4f7f3c70b82f7f5ef44b040a3e87a Mon Sep 17 00:00:00 2001 From: "MIDE\\cflanigan" Date: Tue, 3 Aug 2021 16:19:25 -0400 Subject: [PATCH 41/41] fixes the getSampleAt test, which was broken. --- testing/test_dataset.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/testing/test_dataset.py b/testing/test_dataset.py index f57216ea..73ef70f3 100644 --- a/testing/test_dataset.py +++ b/testing/test_dataset.py @@ -1574,21 +1574,30 @@ def testGetSampleRate(self, testIDE, sr, idx, expected): assert eventArray.getSampleRate(idx) == expected @pytest.mark.parametrize( - 'at, expected, raises', + 'at, raises', [ - (0, (0, 0, 0, 0), nullcontext()), - (10, (10, 0, 0, 0), nullcontext()), - (2000, (2000, 0, 0, 0), nullcontext()), - (9500, (9500, 0, 0, 0), nullcontext()), - (-1, None, pytest.raises(IndexError)), + (0, nullcontext()), + (10, nullcontext()), + (2000, nullcontext()), + (9500, nullcontext()), + (-1, pytest.raises(IndexError)), ], ) - def testGetValueAt(self, testIDE, at, expected, raises): + def testGetValueAt(self, testIDE, at, raises): """ Test for getValueAt method. """ + if raises == nullcontext(): + expected = None + else: + x = np.arange(1000) + vals = np.floor(np.array([x*1000, x, 1000.*(x/1000)**2, 1000*(x/1000)**0.5])) + expected = np.zeros([4]) + expected[0] = at + for i in range(1, 4): + expected[i] = np.interp([at], vals[0], vals[i]) eventArray = testIDE.channels[8].getSession() with raises: - assert eventArray.getValueAt(at) == expected + np.testing.assert_equal(eventArray.getValueAt(at), expected) @pytest.mark.parametrize( 't, expected',