From d68e9141ecac4cada89fd53fd2c35b57ad016159 Mon Sep 17 00:00:00 2001 From: eendebakpt Date: Wed, 11 May 2016 16:16:48 +0200 Subject: [PATCH 01/13] wait for data to complete --- qcodes/data/data_set.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/qcodes/data/data_set.py b/qcodes/data/data_set.py index 60f2f4ed3752..d8897c2d9749 100644 --- a/qcodes/data/data_set.py +++ b/qcodes/data/data_set.py @@ -1,6 +1,7 @@ from enum import Enum from datetime import datetime import time +import logging from .manager import get_data_manager, NoData from .gnuplot_format import GNUPlotFormat @@ -355,6 +356,20 @@ def sync(self): self.read() return False + def complete(self, delay=0.2): + logging.info('waiting for data to complete') + try: + nloops=0 + while True: + logging.info('waiting for data to complete (loop %d)' % nloops) + if self.sync()==False: + break + time.sleep(delay) + nloops=nloops+1 + except Exception as ex: + return False + return True + def get_changes(self, synced_index): changes = {} From d72a895204bc04a17e099d0bd1d49452b34f50f9 Mon Sep 17 00:00:00 2001 From: eendebakpt Date: Wed, 11 May 2016 16:21:37 +0200 Subject: [PATCH 02/13] update windows during complete() --- qcodes/data/data_set.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qcodes/data/data_set.py b/qcodes/data/data_set.py index d8897c2d9749..0cad25b40604 100644 --- a/qcodes/data/data_set.py +++ b/qcodes/data/data_set.py @@ -2,6 +2,7 @@ from datetime import datetime import time import logging +import pyqtgraph from .manager import get_data_manager, NoData from .gnuplot_format import GNUPlotFormat @@ -366,6 +367,7 @@ def complete(self, delay=0.2): break time.sleep(delay) nloops=nloops+1 + pyqtgraph.QtGui.QApplication.instance().processEvents() except Exception as ex: return False return True From dec774179c68e36760c4b82f5c443130fc5b3350 Mon Sep 17 00:00:00 2001 From: Pieter Date: Fri, 13 May 2016 10:45:31 +0200 Subject: [PATCH 03/13] add generic callback mechanism for .complete() --- qcodes/data/data_set.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/qcodes/data/data_set.py b/qcodes/data/data_set.py index 0cad25b40604..22e6907f949c 100644 --- a/qcodes/data/data_set.py +++ b/qcodes/data/data_set.py @@ -2,7 +2,7 @@ from datetime import datetime import time import logging -import pyqtgraph +import traceback from .manager import get_data_manager, NoData from .gnuplot_format import GNUPlotFormat @@ -199,6 +199,9 @@ class DataSet(DelegateAttributes): default_io = DiskIO('.') default_formatter = GNUPlotFormat() location_provider = TimestampLocation() + + # functions to be called when operating in background mode + background_functions=dict() def __init__(self, location=None, mode=DataMode.LOCAL, arrays=None, data_manager=None, formatter=None, io=None, write_period=5): @@ -367,8 +370,12 @@ def complete(self, delay=0.2): break time.sleep(delay) nloops=nloops+1 - pyqtgraph.QtGui.QApplication.instance().processEvents() + + for key, fn in self.background_functions.items(): + logging.debug('calling %s: %s' % (key, fn)) + fn() except Exception as ex: + print(traceback.format_exc(ex)) return False return True From 96d683e58869582232866b9bd031cd53edb5a0f9 Mon Sep 17 00:00:00 2001 From: eendebakpt Date: Tue, 17 May 2016 13:07:23 +0200 Subject: [PATCH 04/13] print fraction complete --- qcodes/data/data_set.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/qcodes/data/data_set.py b/qcodes/data/data_set.py index 22e6907f949c..74aef55712fa 100644 --- a/qcodes/data/data_set.py +++ b/qcodes/data/data_set.py @@ -1,6 +1,7 @@ from enum import Enum from datetime import datetime import time +import numpy import logging import traceback @@ -360,12 +361,23 @@ def sync(self): self.read() return False - def complete(self, delay=0.2): - logging.info('waiting for data to complete') + def fraction_complete(self): + try: + first_param = next(iter(self.arrays.keys())) + A=getattr(self, first_param) + sz=numpy.prod(A.size) + fraction = (sz-numpy.isnan(A).sum())/sz + except Exception as ex: + logging.debug(traceback.format_exc(ex)) + fraction = numpy.NaN + return fraction + def complete(self, delay=1.5): + logging.info('waiting for DataSet to complete') + try: nloops=0 while True: - logging.info('waiting for data to complete (loop %d)' % nloops) + logging.info('waiting for DataSet to complete (fraction %.2f)' % (self.fraction_complete(),) ) if self.sync()==False: break time.sleep(delay) From dfe9553d20a99b6a81bed364026be54aa4b5dfeb Mon Sep 17 00:00:00 2001 From: alexcjohnson Date: Tue, 5 Jul 2016 10:27:05 +0200 Subject: [PATCH 05/13] style: Lint data_set --- qcodes/data/data_set.py | 54 ++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/qcodes/data/data_set.py b/qcodes/data/data_set.py index 3d204d9f5952..767de0a04d23 100644 --- a/qcodes/data/data_set.py +++ b/qcodes/data/data_set.py @@ -2,14 +2,10 @@ from enum import Enum import time -<<<<<<< HEAD -import numpy +import numpy as np import logging -import traceback -||||||| merged common ancestors -======= +from traceback import format_exc from copy import deepcopy ->>>>>>> master from .manager import get_data_manager, NoData from .gnuplot_format import GNUPlotFormat @@ -244,16 +240,10 @@ class DataSet(DelegateAttributes): default_io = DiskIO('.') default_formatter = GNUPlotFormat() -<<<<<<< HEAD - location_provider = TimestampLocation() - - # functions to be called when operating in background mode - background_functions=dict() -||||||| merged common ancestors - location_provider = TimestampLocation() -======= location_provider = FormatLocation() ->>>>>>> master + + # functions to be called when operating in background mode + background_functions = {} def __init__(self, location=None, mode=DataMode.LOCAL, arrays=None, data_manager=None, formatter=None, io=None, write_period=5): @@ -417,33 +407,37 @@ def sync(self): def fraction_complete(self): try: first_param = next(iter(self.arrays.keys())) - A=getattr(self, first_param) - sz=numpy.prod(A.size) - fraction = (sz-numpy.isnan(A).sum())/sz - except Exception as ex: - logging.debug(traceback.format_exc(ex)) - fraction = numpy.NaN - return fraction + A = getattr(self, first_param) + sz = np.prod(A.size) + fraction = (sz - np.isnan(A).sum()) / sz + except: + logging.debug(format_exc()) + fraction = np.NaN + return fraction + def complete(self, delay=1.5): logging.info('waiting for DataSet to complete') try: - nloops=0 + nloops = 0 while True: - logging.info('waiting for DataSet to complete (fraction %.2f)' % (self.fraction_complete(),) ) - if self.sync()==False: + logging.info( + 'waiting for DataSet to complete (fraction %.2f)' % + self.fraction_complete()) + if self.sync() is False: break + time.sleep(delay) - nloops=nloops+1 - + nloops += 1 + for key, fn in self.background_functions.items(): logging.debug('calling %s: %s' % (key, fn)) fn() - except Exception as ex: - print(traceback.format_exc(ex)) + except: + print(format_exc()) return False return True - + def get_changes(self, synced_index): changes = {} From 7a764e29c69dcaad349100b1377c32e4f55f8888 Mon Sep 17 00:00:00 2001 From: alexcjohnson Date: Tue, 5 Jul 2016 11:20:11 +0200 Subject: [PATCH 06/13] refactor: make DataArray.fraction_complete to use in DataSet.fraction_complete And document the new stuff in this branch --- qcodes/data/data_array.py | 23 +++++++++++++++++++ qcodes/data/data_set.py | 47 +++++++++++++++++++++++++++++++-------- 2 files changed, 61 insertions(+), 9 deletions(-) diff --git a/qcodes/data/data_array.py b/qcodes/data/data_array.py index 328918767013..c81879bf894e 100644 --- a/qcodes/data/data_array.py +++ b/qcodes/data/data_array.py @@ -332,3 +332,26 @@ def snapshot(self, update=False): snap[attr] = getattr(self, attr) return snap + + def fraction_complete(self): + """ + Get the fraction of this array which has data in it. + + Or more specifically, the fraction of the latest point in the array + where we have touched it. + + Returns: + float: fraction of array which is complete, from 0.0 to 1.0 + """ + if self.ndarray is None: + return 0.0 + + last_index = -1 + if self.last_saved_index is not None: + last_index = max(last_index, self.last_saved_index) + if self.modified_range is not None: + last_index = max(last_index, self.modified_range[1]) + if getattr(self, 'synced_index', None) is not None: + last_index = max(last_index, self.synced_index) + + return (last_index + 1) / self.ndarray.size diff --git a/qcodes/data/data_set.py b/qcodes/data/data_set.py index 767de0a04d23..5c97c05282c5 100644 --- a/qcodes/data/data_set.py +++ b/qcodes/data/data_set.py @@ -233,6 +233,15 @@ class DataSet(DelegateAttributes): between saves to disk. If not ``LOCAL``, the ``DataServer`` handles this and generally writes more often. Use None to disable writing from calls to ``self.store``. Default 5. + + Attributes: + background_functions (Dict[callable]): Class attribute, ``{key: fn}``, + ``fn`` is a callable accepting no arguments, and ``key`` is a name + to identify the function and help you attach and remove it. + In ``DataSet.complete`` we will call each of these periodically. + Note that because this is a class attribute, the functions will + apply to every DataSet. If you want specific functions for one + DataSet you can override this with an instance attribute. """ # ie data_set.arrays['vsd'] === data_set.vsd @@ -405,17 +414,37 @@ def sync(self): return False def fraction_complete(self): - try: - first_param = next(iter(self.arrays.keys())) - A = getattr(self, first_param) - sz = np.prod(A.size) - fraction = (sz - np.isnan(A).sum()) / sz - except: - logging.debug(format_exc()) - fraction = np.NaN - return fraction + """ + Get the fraction of this DataSet which has data in it. + + Returns: + float: the average of all measured (not setpoint) arrays' + ``fraction_complete()`` values, independent of the individual + array sizes. If there are no measured arrays, returns zero. + """ + array_count, total = 0, 0 + + for array in self.arrays.values(): + if not array.is_setpoint: + array_count += 1 + total += array.fraction_complete() + + return total / (array_count or 1) def complete(self, delay=1.5): + """ + Periodically sync the DataSet and display percent complete status. + + Also, each period, execute functions stored in (class attribute) + ``self.background_functions`` + + Args: + delay (float): seconds between status messages + + Returns: + bool: True if we managed to wait until the DataSet finished, + False if something went wrong and it it not yet complete. + """ logging.info('waiting for DataSet to complete') try: From c9d0a56298aef3355686bd89c8cab079b8ad39e4 Mon Sep 17 00:00:00 2001 From: alexcjohnson Date: Tue, 5 Jul 2016 11:26:23 +0200 Subject: [PATCH 07/13] lint: remove unused numpy import in DataSet --- qcodes/data/data_set.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qcodes/data/data_set.py b/qcodes/data/data_set.py index 5c97c05282c5..2dec6152edfb 100644 --- a/qcodes/data/data_set.py +++ b/qcodes/data/data_set.py @@ -2,7 +2,6 @@ from enum import Enum import time -import numpy as np import logging from traceback import format_exc from copy import deepcopy From 99762498bb752b1d609645fb68fbfd65fdc2f947 Mon Sep 17 00:00:00 2001 From: alexcjohnson Date: Tue, 5 Jul 2016 14:46:34 +0200 Subject: [PATCH 08/13] clean up _set in tests and synchronize DataSet and DataArray --- qcodes/data/data_set.py | 6 ++-- qcodes/tests/data_mocks.py | 24 +++++++++----- qcodes/tests/test_data.py | 46 ++++++++++++++++++++++----- qcodes/tests/test_format.py | 63 +++++++++++++++++++------------------ qcodes/tests/test_loop.py | 1 + 5 files changed, 91 insertions(+), 49 deletions(-) diff --git a/qcodes/data/data_set.py b/qcodes/data/data_set.py index 2dec6152edfb..fadb1ddece68 100644 --- a/qcodes/data/data_set.py +++ b/qcodes/data/data_set.py @@ -505,9 +505,9 @@ def _clean_array_ids(self, arrays): action_indices = [array.action_indices for array in arrays] for array in arrays: name = array.full_name - if array.is_setpoint: - if name: - name += '_set' + if array.is_setpoint and name and not name.endswith('_set'): + name += '_set' + array.array_id = name array_ids = set([array.array_id for array in arrays]) for name in array_ids: diff --git a/qcodes/tests/data_mocks.py b/qcodes/tests/data_mocks.py index 27824cc336d0..5ce27353debe 100644 --- a/qcodes/tests/data_mocks.py +++ b/qcodes/tests/data_mocks.py @@ -93,7 +93,12 @@ def init_data(self): def DataSet1D(location=None): # DataSet with one 1D array with 5 points - x = DataArray(name='x', label='X', preset_data=(1., 2., 3., 4., 5.)) + + # TODO: since y lists x as a set_array, it should automatically + # set is_setpoint=True for x, shouldn't it? Any reason we woundn't + # want that? + x = DataArray(name='x', label='X', preset_data=(1., 2., 3., 4., 5.), + is_setpoint=True) y = DataArray(name='y', label='Y', preset_data=(3., 4., 5., 6., 7.), set_arrays=(x,)) return new_data(arrays=(x, y), location=location) @@ -105,15 +110,16 @@ def DataSet2D(location=None): zz = xx**2+yy**2 # outer setpoint should be 1D xx = xx[:, 0] - x = DataArray(name='x', label='X', preset_data=xx) - y = DataArray(name='y', label='Y', preset_data=yy, set_arrays=(x,)) + x = DataArray(name='x', label='X', preset_data=xx, is_setpoint=True) + y = DataArray(name='y', label='Y', preset_data=yy, set_arrays=(x,), + is_setpoint=True) z = DataArray(name='z', label='Z', preset_data=zz, set_arrays=(x, y)) return new_data(arrays=(x, y, z), location=location) def file_1d(): return '\n'.join([ - '# x\ty', + '# x_set\ty', '# "X"\t"Y"', '# 5', '1\t3', @@ -125,13 +131,15 @@ def file_1d(): def DataSetCombined(location=None): # Complex DataSet with two 1D and two 2D arrays - x = DataArray(name='x', label='X!', preset_data=(16., 17.)) + x = DataArray(name='x', label='X!', preset_data=(16., 17.), + is_setpoint=True) y1 = DataArray(name='y1', label='Y1 value', preset_data=(18., 19.), set_arrays=(x,)) y2 = DataArray(name='y2', label='Y2 value', preset_data=(20., 21.), set_arrays=(x,)) - yset = DataArray(name='yset', label='Y', preset_data=(22., 23., 24.)) + yset = DataArray(name='y', label='Y', preset_data=(22., 23., 24.), + is_setpoint=True) yset.nest(2, 0, x) z1 = DataArray(name='z1', label='Z1', preset_data=((25., 26., 27.), (28., 29., 30.)), @@ -145,14 +153,14 @@ def DataSetCombined(location=None): def files_combined(): return [ '\n'.join([ - '# x\ty1\ty2', + '# x_set\ty1\ty2', '# "X!"\t"Y1 value"\t"Y2 value"', '# 2', '16\t18\t20', '17\t19\t21', '']), '\n'.join([ - '# x\tyset\tz1\tz2', + '# x_set\ty_set\tz1\tz2', '# "X!"\t"Y"\t"Z1"\t"Z2"', '# 2\t3', '16\t22\t25\t31', diff --git a/qcodes/tests/test_data.py b/qcodes/tests/test_data.py index 7fe373053814..490cb5f01216 100644 --- a/qcodes/tests/test_data.py +++ b/qcodes/tests/test_data.py @@ -13,7 +13,7 @@ from .data_mocks import (MockDataManager, MockFormatter, MatchIO, MockLive, MockArray, DataSet2D, DataSet1D, - RecordingMockFormatter) + DataSetCombined, RecordingMockFormatter) from .common import strip_qc @@ -250,6 +250,31 @@ def test_data_set_property(self): data.data_set = mock_data_set2 self.assertEqual(data.data_set, mock_data_set2) + def test_fraction_complete(self): + data = DataArray(shape=(5, 10)) + self.assertIsNone(data.ndarray) + self.assertEqual(data.fraction_complete(), 0.0) + + data.init_data() + self.assertEqual(data.fraction_complete(), 0.0) + + # index = 1 * 10 + 7 - add 1 (for index 0) and you get 18 + # each index is 2% of the total, so this is 36% + data[1, 7] = 1 + self.assertEqual(data.fraction_complete(), 18/50) + + # add a last_saved_index but modified_range is still bigger + data.mark_saved(13) + self.assertEqual(data.fraction_complete(), 18/50) + + # now last_saved_index wins + data.mark_saved(19) + self.assertEqual(data.fraction_complete(), 20/50) + + # now pretend we get more info from syncing + data.synced_index = 22 + self.assertEqual(data.fraction_complete(), 23/50) + class TestLoadData(TestCase): @@ -495,9 +520,9 @@ def test_write_copy(self): mr = (2, 3) mr_full = (0, 4) lsi = 1 - data.x.modified_range = mr + data.x_set.modified_range = mr data.y.modified_range = mr - data.x.last_saved_index = lsi + data.x_set.last_saved_index = lsi data.y.last_saved_index = lsi with self.assertRaises(TypeError): @@ -517,13 +542,13 @@ def test_write_copy(self): [(None, '/some/abs/path', False)]) # check that the formatter gets called as if nothing has been saved self.assertEqual(data.formatter.modified_ranges, - [{'x': mr_full, 'y': mr_full}]) + [{'x_set': mr_full, 'y': mr_full}]) self.assertEqual(data.formatter.last_saved_indices, - [{'x': None, 'y': None}]) + [{'x_set': None, 'y': None}]) # but the dataset afterward has its original mods back - self.assertEqual(data.x.modified_range, mr) + self.assertEqual(data.x_set.modified_range, mr) self.assertEqual(data.y.modified_range, mr) - self.assertEqual(data.x.last_saved_index, lsi) + self.assertEqual(data.x_set.last_saved_index, lsi) self.assertEqual(data.y.last_saved_index, lsi) # recreate the formatter to clear the calls attributes @@ -554,3 +579,10 @@ def test_pickle_dataset(self): # If the data_manager is set to None, then the object should pickle. m = DataSet2D() pickle.dumps(m) + + def test_fraction_complete(self): + empty_data = new_data(arrays=(), location=False) + self.assertEqual(empty_data.fraction_complete(), 0.0) + + data = DataSetCombined(location=False) + diff --git a/qcodes/tests/test_format.py b/qcodes/tests/test_format.py index 287144ac0fa4..d3f9cc5bc100 100644 --- a/qcodes/tests/test_format.py +++ b/qcodes/tests/test_format.py @@ -63,7 +63,7 @@ def read_metadata(self, data_set): formatter = MyFormatter() data = DataSet1D(location) - data.x.ndarray = None + data.x_set.ndarray = None data.y.ndarray = None os.makedirs(os.path.dirname(path), exist_ok=True) @@ -78,7 +78,7 @@ def read_metadata(self, data_set): self.assertEqual(data.files_read, [os.path.abspath(path)]) expected_array_repr = repr([float('nan')] * 5) - self.assertEqual(repr(data.x.tolist()), expected_array_repr) + self.assertEqual(repr(data.x_set.tolist()), expected_array_repr) self.assertEqual(repr(data.y.tolist()), expected_array_repr) def test_group_arrays(self): @@ -93,14 +93,14 @@ def test_group_arrays(self): g1d, g2d = groups self.assertEqual(g1d.shape, (2,)) - self.assertEqual(g1d.set_arrays, (data.x,)) + self.assertEqual(g1d.set_arrays, (data.x_set,)) self.assertEqual(g1d.data, (data.y1, data.y2)) - self.assertEqual(g1d.name, 'x') + self.assertEqual(g1d.name, 'x_set') self.assertEqual(g2d.shape, (2, 3)) - self.assertEqual(g2d.set_arrays, (data.x, data.yset)) + self.assertEqual(g2d.set_arrays, (data.x_set, data.y_set)) self.assertEqual(g2d.data, (data.z1, data.z2)) - self.assertEqual(g2d.name, 'x_yset') + self.assertEqual(g2d.name, 'x_set_y_set') def test_match_save_range(self): formatter = Formatter() @@ -111,7 +111,7 @@ def test_match_save_range(self): # no matter what else, if nothing is listed as modified # then save_range is None for lsi_x in [None, 0, 3]: - data.x.last_saved_index = lsi_x + data.x_set.last_saved_index = lsi_x for lsi_y in [None, 1, 4]: data.y.last_saved_index = lsi_y for fe in [True, False]: @@ -123,12 +123,12 @@ def test_match_save_range(self): # modified range, or if file does not exist, we need to overwrite # otherwise start just after last_saved_index for lsi, start in [(None, 0), (0, 1), (1, 2), (2, 3), (3, 0), (4, 0)]: - data.x.last_saved_index = data.y.last_saved_index = lsi + data.x_set.last_saved_index = data.y.last_saved_index = lsi # inconsistent modified_range: expands to greatest extent # so these situations are identical for xmr, ymr in ([(4, 4), (3, 3)], [(3, 4), None], [None, (3, 4)]): - data.x.modified_range = xmr + data.x_set.modified_range = xmr data.y.modified_range = ymr save_range = formatter.match_save_range(group, @@ -140,7 +140,7 @@ def test_match_save_range(self): self.assertEqual(save_range, (start, 4)) # inconsistent last_saved_index: need to overwrite no matter what - data.x.last_saved_index = 1 + data.x_set.last_saved_index = 1 data.y.last_saved_index = 2 save_range = formatter.match_save_range(group, file_exists=True) self.assertEqual(save_range, (0, 4)) @@ -149,7 +149,7 @@ def test_match_save_range(self): # but this will only back up one point! data.y[4] = float('nan') data.y[3] = float('nan') - data.x.last_saved_index = data.y.last_saved_index = 2 + data.x_set.last_saved_index = data.y.last_saved_index = 2 save_range = formatter.match_save_range(group, file_exists=True) self.assertEqual(save_range, (3, 3)) @@ -196,7 +196,7 @@ def test_full_write(self): formatter.write(data, data.io, data.location) - with open(location + '/x.dat', 'r') as f: + with open(location + '/x_set.dat', 'r') as f: self.assertEqual(f.read(), file_1d()) # check that we can add comment lines randomly into the file @@ -207,7 +207,7 @@ def test_full_write(self): lines[1] = lines[1].replace('"', '') lines[3:3] = ['# this data is awesome!'] lines[6:6] = ['# the next point is my favorite.'] - with open(location + '/x.dat', 'w') as f: + with open(location + '/x_set.dat', 'w') as f: f.write('\n'.join(lines)) # normally this would be just done by data2 = load_data(location) @@ -215,14 +215,14 @@ def test_full_write(self): data2 = DataSet(location=location) formatter.read(data2) - self.checkArraysEqual(data2.x, data.x) + self.checkArraysEqual(data2.x_set, data.x_set) self.checkArraysEqual(data2.y, data.y) # while we're here, check some errors on bad reads # first: trying to read into a dataset that already has the # wrong size - x = DataArray(name='x', label='X', preset_data=(1., 2.)) + x = DataArray(name='x_set', label='X', preset_data=(1., 2.)) y = DataArray(name='y', label='Y', preset_data=(3., 4.), set_arrays=(x,)) data3 = new_data(arrays=(x, y), location=location + 'XX') @@ -236,10 +236,10 @@ def test_full_write(self): # no problem reading again if only data has changed, it gets # overwritten with the disk copy - data2.x[2] = 42 + data2.x_set[2] = 42 data2.y[2] = 99 formatter.read(data2) - self.assertEqual(data2.x[2], 3) + self.assertEqual(data2.x_set[2], 3) self.assertEqual(data2.y[2], 5) def test_format_options(self): @@ -263,7 +263,7 @@ def test_format_options(self): # back to '\n' on read. So I'm tempted to just take out terminator # as an option rather than turn this feature off. odd_format = '\n'.join([ - '?:x y', + '?:x_set y', '?:"X" "Y"', '?:5', ' 1.00 3.00', @@ -272,7 +272,7 @@ def test_format_options(self): ' 4.00 6.00', ' 5.00 7.00', '']) - with open(location + '/x.splat', 'r') as f: + with open(location + '/x_set.splat', 'r') as f: self.assertEqual(f.read(), odd_format) def add_star(self, path): @@ -286,14 +286,14 @@ def test_incremental_write(self): formatter = GNUPlotFormat() location = self.locations[0] data = DataSet1D(location) - path = location + '/x.dat' + path = location + '/x_set.dat' data_copy = DataSet1D(False) # empty the data and mark it as unmodified - data.x[:] = float('nan') + data.x_set[:] = float('nan') data.y[:] = float('nan') - data.x.modified_range = None + data.x_set.modified_range = None data.y.modified_range = None # simulate writing after every value comes in, even within @@ -302,8 +302,8 @@ def test_incremental_write(self): # in the right places afterward, ie we don't write any given # row until it's done and we never totally rewrite the file self.stars_before_write = 0 - for i, (x, y) in enumerate(zip(data_copy.x, data_copy.y)): - data.x[i] = x + for i, (x, y) in enumerate(zip(data_copy.x_set, data_copy.y)): + data.x_set[i] = x formatter.write(data, data.io, data.location) self.add_star(path) @@ -312,7 +312,7 @@ def test_incremental_write(self): self.add_star(path) starred_file = '\n'.join([ - '# x\ty', + '# x_set\ty', '# "X"\t"Y"', '# 5', '1\t3', @@ -348,7 +348,7 @@ def test_read_errors(self): location = self.locations[0] data = DataSet(location=location) os.makedirs(location, exist_ok=True) - with open(location + '/x.dat', 'w') as f: + with open(location + '/x_set.dat', 'w') as f: f.write('1\t2\n' + file_1d()) with LogCapture() as logs: formatter.read(data) @@ -359,8 +359,9 @@ def test_read_errors(self): location = self.locations[1] data = DataSet(location=location) os.makedirs(location, exist_ok=True) - with open(location + '/x.dat', 'w') as f: - f.write('\n'.join(['# x\ty', '# "X"\t"Y"', '# 2', '1\t2', '3\t4'])) + with open(location + '/x_set.dat', 'w') as f: + f.write('\n'.join(['# x_set\ty', + '# "X"\t"Y"', '# 2', '1\t2', '3\t4'])) with open(location + '/q.dat', 'w') as f: f.write('\n'.join(['# q\ty', '# "Q"\t"Y"', '# 2', '1\t2', '3\t4'])) with LogCapture() as logs: @@ -382,14 +383,14 @@ def test_multifile(self): filex, filexy = files_combined() - with open(location + '/x.dat', 'r') as f: + with open(location + '/x_set.dat', 'r') as f: self.assertEqual(f.read(), filex) - with open(location + '/x_yset.dat', 'r') as f: + with open(location + '/x_set_y_set.dat', 'r') as f: self.assertEqual(f.read(), filexy) data2 = DataSet(location=location) formatter.read(data2) - for array_id in ('x', 'y1', 'y2', 'yset', 'z1', 'z2'): + for array_id in ('x_set', 'y1', 'y2', 'y_set', 'z1', 'z2'): self.checkArraysEqual(data2.arrays[array_id], data.arrays[array_id]) diff --git a/qcodes/tests/test_loop.py b/qcodes/tests/test_loop.py index 99653daeafbd..4d9a95a93c30 100644 --- a/qcodes/tests/test_loop.py +++ b/qcodes/tests/test_loop.py @@ -261,6 +261,7 @@ def test_nesting(self): self.assertEqual(data.p3.tolist(), [[[5, 6]] * 2] * 2) def test_repr(self): + self.maxDiff=None loop2 = Loop(self.p2[3:5:1], 0.001).each(self.p2) loop = Loop(self.p1[1:3:1], 0.001).each(self.p3, self.p2, From c29b41b076396b6243400aae8cdfabc43690c3a7 Mon Sep 17 00:00:00 2001 From: alexcjohnson Date: Wed, 6 Jul 2016 00:44:38 +0200 Subject: [PATCH 09/13] fix: Correctly mark newly created or read arrays as modified or saved - If you read in a DataSet (ie with load_data) then it is marked as saved up to whatever data you read in. The `Formatter` handles this. - If you create a DataSet with preset data, then it is marked as 100% modified - ie it's entirely data that has not been saved. the DataArray constructor and nest method handle this. --- qcodes/data/data_array.py | 56 +++++++++++++++++++++++++---------- qcodes/data/gnuplot_format.py | 10 +++++++ qcodes/tests/test_data.py | 8 +++-- qcodes/tests/test_format.py | 43 ++++++++++++++++----------- 4 files changed, 82 insertions(+), 35 deletions(-) diff --git a/qcodes/data/data_array.py b/qcodes/data/data_array.py index c81879bf894e..5171be9d98af 100644 --- a/qcodes/data/data_array.py +++ b/qcodes/data/data_array.py @@ -66,7 +66,18 @@ def __init__(self, parameter=None, name=None, full_name=None, label=None, self.array_id = array_id self.is_setpoint = is_setpoint self.action_indices = action_indices + self.set_arrays = set_arrays + + self._preset = False + + # store a reference up to the containing DataSet + # this also lets us make sure a DataArray is only in one DataSet + self._data_set = None + + self.last_saved_index = None + self.modified_range = None + self.ndarray = None if snapshot is None: snapshot = {} self._snapshot_input = {} @@ -95,22 +106,11 @@ def __init__(self, parameter=None, name=None, full_name=None, label=None, if not self.label: self.label = self.name - self.set_arrays = set_arrays - self._preset = False - - # store a reference up to the containing DataSet - # this also lets us make sure a DataArray is only in one DataSet - self._data_set = None - - self.ndarray = None if preset_data is not None: self.init_data(preset_data) elif shape is None: self.shape = () - self.last_saved_index = None - self.modified_range = None - @property def data_set(self): return self._data_set @@ -157,6 +157,9 @@ def nest(self, size, action_index=None, set_array=None): for i in range(size): self.ndarray[i] = inner_data + # update modified_range so the entire array still looks modified + self.modified_range = (0, self.ndarray.size - 1) + self._set_index_bounds() return self @@ -184,6 +187,10 @@ def init_data(self, data=None): data.shape, self.shape) self.ndarray = data self._preset = True + + # mark the entire array as modified + self.modified_range = (0, data.size - 1) + elif self.ndarray is not None: if self.ndarray.shape != self.shape: raise ValueError('data has already been initialized, ' @@ -231,8 +238,8 @@ def __setitem__(self, loop_indices, value): max_indices[i] = start + ( ((stop - start - 1)//step) * step) - min_li = self._flat_index(min_indices, self._min_indices) - max_li = self._flat_index(max_indices, self._max_indices) + min_li = self.flat_index(min_indices, self._min_indices) + max_li = self.flat_index(max_indices, self._max_indices) self._update_modified_range(min_li, max_li) self.ndarray.__setitem__(loop_indices, value) @@ -249,8 +256,27 @@ def __len__(self): """ return len(self.ndarray) - def _flat_index(self, indices, index_fill): - indices = indices + index_fill[len(indices):] + def flat_index(self, indices, index_fill=None): + """ + Generate the raveled index for the given indices. + + This is the index you would have if the array is reshaped to 1D, + looping over the indices from inner to outer. + + Args: + indices (sequence): indices of an element or slice of this array. + + index_fill (sequence, optional): extra indices to use if + ``indices`` has less dimensions than the array, ie it points + to a slice rather than a single element. Use zeros to get the + beginning of this slice, and [d - 1 for d in shape] to get the + end of the slice. + + Returns: + int: the resulting flat index. + """ + if len(indices) < len(self.shape): + indices = indices + index_fill[len(indices):] return np.ravel_multi_index(tuple(zip(indices)), self.shape)[0] def _update_modified_range(self, low, high): diff --git a/qcodes/data/gnuplot_format.py b/qcodes/data/gnuplot_format.py index c4cc27ddaa23..f1101b5a514d 100644 --- a/qcodes/data/gnuplot_format.py +++ b/qcodes/data/gnuplot_format.py @@ -200,11 +200,21 @@ def read_one_file(self, data_set, f, ids_read): myindices, indices) for value, data_array in zip(values[ndim:], data_arrays): + # set .ndarray directly to avoid the overhead of __setitem__ + # which updates modified_range on every call data_array.ndarray[tuple(indices)] = value indices[-1] += 1 first_point = False + # Since we skipped __setitem__, back up to the last read point and + # mark it as saved that far. + # Using mark_saved is better than directly setting last_saved_index + # because it also ensures modified_range is set correctly. + indices[-1] -= 1 + for array in set_arrays + tuple(data_arrays): + array.mark_saved(array.flat_index(indices[:array.ndim])) + def _is_comment(self, line): return line[:self.comment_len] == self.comment_chars diff --git a/qcodes/tests/test_data.py b/qcodes/tests/test_data.py index 490cb5f01216..9af81d61c4d5 100644 --- a/qcodes/tests/test_data.py +++ b/qcodes/tests/test_data.py @@ -140,7 +140,7 @@ def test_edit_and_mark(self): self.assertEqual(data[0].tolist(), [1, 2]) self.assertEqual(data[0, 1], 2) - self.assertIsNone(data.modified_range) + data.modified_range = None self.assertIsNone(data.last_saved_index) self.assertEqual(len(data), 2) @@ -169,7 +169,7 @@ def test_edit_and_mark_slice(self): data = DataArray(preset_data=[[1] * 5] * 6) self.assertEqual(data.shape, (6, 5)) - self.assertEqual(data.modified_range, None) + data.modified_range = None data[:4:2, 2:] = 2 self.assertEqual(data.tolist(), [ @@ -229,6 +229,10 @@ def test_nest_preset(self): self.assertEqual(data.action_indices, ()) self.assertEqual(data.set_arrays, (data,)) + # test that the modified range gets correctly set to + # (0, 2*3-1 = 5) + self.assertEqual(data.modified_range, (0, 5)) + # you need a set array for all but the inner nesting with self.assertRaises(TypeError): data.nest(4) diff --git a/qcodes/tests/test_format.py b/qcodes/tests/test_format.py index d3f9cc5bc100..c4d67c1fd1cf 100644 --- a/qcodes/tests/test_format.py +++ b/qcodes/tests/test_format.py @@ -4,7 +4,7 @@ from qcodes.data.format import Formatter from qcodes.data.gnuplot_format import GNUPlotFormat from qcodes.data.data_array import DataArray -from qcodes.data.data_set import DataSet, new_data +from qcodes.data.data_set import DataSet, new_data, load_data from qcodes.utils.helpers import LogCapture from .data_mocks import DataSet1D, file_1d, DataSetCombined, files_combined @@ -110,6 +110,7 @@ def test_match_save_range(self): # no matter what else, if nothing is listed as modified # then save_range is None + data.x_set.modified_range = data.y.modified_range = None for lsi_x in [None, 0, 3]: data.x_set.last_saved_index = lsi_x for lsi_y in [None, 1, 4]: @@ -188,12 +189,6 @@ def test_full_write(self): location = self.locations[0] data = DataSet1D(location) - # mark the data set as modified by... modifying it! - # without actually changing it :) - # TODO - are there cases we should automatically mark the data as - # modified on construction? - data.y[4] = data.y[4] - formatter.write(data, data.io, data.location) with open(location + '/x_set.dat', 'r') as f: @@ -218,6 +213,12 @@ def test_full_write(self): self.checkArraysEqual(data2.x_set, data.x_set) self.checkArraysEqual(data2.y, data.y) + # data has been saved + self.assertEqual(data.y.last_saved_index, 4) + # data2 has been read back in, should show the same + # last_saved_index + self.assertEqual(data2.y.last_saved_index, 4) + # while we're here, check some errors on bad reads # first: trying to read into a dataset that already has the @@ -249,12 +250,6 @@ def test_format_options(self): location = self.locations[0] data = DataSet1D(location) - # mark the data set as modified by... modifying it! - # without actually changing it :) - # TODO - are there cases we should automatically mark the data as - # modified on construction? - data.y[4] = data.y[4] - formatter.write(data, data.io, data.location) # TODO - Python3 uses universal newlines for read and write... @@ -285,6 +280,7 @@ def add_star(self, path): def test_incremental_write(self): formatter = GNUPlotFormat() location = self.locations[0] + location2 = self.locations[1] # use 2nd location for reading back in data = DataSet1D(location) path = location + '/x_set.dat' @@ -305,12 +301,28 @@ def test_incremental_write(self): for i, (x, y) in enumerate(zip(data_copy.x_set, data_copy.y)): data.x_set[i] = x formatter.write(data, data.io, data.location) + formatter.write(data, data.io, location2) self.add_star(path) data.y[i] = y formatter.write(data, data.io, data.location) + data.x_set.clear_save() + data.y.clear_save() + formatter.write(data, data.io, location2) self.add_star(path) + # we wrote to a second location without the stars, so we can read + # back in and make sure that we get the right last_saved_index + # for the amount of data we've read. + reread_data = load_data(location=location2, data_manager=False, + formatter=formatter, io=data.io) + self.assertEqual(repr(reread_data.x_set.tolist()), + repr(data.x_set.tolist())) + self.assertEqual(repr(reread_data.y.tolist()), + repr(data.y.tolist())) + self.assertEqual(reread_data.x_set.last_saved_index, i) + self.assertEqual(reread_data.y.last_saved_index, i) + starred_file = '\n'.join([ '# x_set\ty', '# "X"\t"Y"', @@ -374,11 +386,6 @@ def test_multifile(self): location = self.locations[1] data = DataSetCombined(location) - # mark one array in each file as completely modified - # that should cause the whole files to be written, even though - # the other data and setpoint arrays are not marked as modified - data.y1[:] += 0 - data.z1[:, :] += 0 formatter.write(data, data.io, data.location) filex, filexy = files_combined() From 6f1bbf244f1c002617d7d579f4adf0233db63819 Mon Sep 17 00:00:00 2001 From: alexcjohnson Date: Wed, 6 Jul 2016 00:53:04 +0200 Subject: [PATCH 10/13] finish testing DataSet.fraction_complete --- qcodes/tests/test_data.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/qcodes/tests/test_data.py b/qcodes/tests/test_data.py index 9af81d61c4d5..ac8c566f0fd3 100644 --- a/qcodes/tests/test_data.py +++ b/qcodes/tests/test_data.py @@ -589,4 +589,17 @@ def test_fraction_complete(self): self.assertEqual(empty_data.fraction_complete(), 0.0) data = DataSetCombined(location=False) - + self.assertEqual(data.fraction_complete(), 1.0) + + # alter only the measured arrays, check that only these are used + # to calculate fraction_complete + data.y1.modified_range = (0, 0) # 1 of 2 + data.y2.modified_range = (0, 0) # 1 of 2 + data.z1.modified_range = (0, 2) # 3 of 6 + data.z2.modified_range = (0, 2) # 3 of 6 + self.assertEqual(data.fraction_complete(), 0.5) + + # mark more things complete using last_saved_index and synced_index + data.y1.last_saved_index = 1 # 2 of 2 + data.z1.synced_index = 5 # 6 of 6 + self.assertEqual(data.fraction_complete(), 0.75) From d1fa42ed6e084c01486b2a0cf08d3910d4a60d94 Mon Sep 17 00:00:00 2001 From: alexcjohnson Date: Wed, 6 Jul 2016 09:54:32 +0200 Subject: [PATCH 11/13] style: remove debug cruft --- qcodes/tests/test_loop.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qcodes/tests/test_loop.py b/qcodes/tests/test_loop.py index 4d9a95a93c30..99653daeafbd 100644 --- a/qcodes/tests/test_loop.py +++ b/qcodes/tests/test_loop.py @@ -261,7 +261,6 @@ def test_nesting(self): self.assertEqual(data.p3.tolist(), [[[5, 6]] * 2] * 2) def test_repr(self): - self.maxDiff=None loop2 = Loop(self.p2[3:5:1], 0.001).each(self.p2) loop = Loop(self.p1[1:3:1], 0.001).each(self.p3, self.p2, From c32f823156e278a208b032c5ea13b4a979d2f08b Mon Sep 17 00:00:00 2001 From: alexcjohnson Date: Wed, 6 Jul 2016 13:42:47 +0200 Subject: [PATCH 12/13] fix: Clean up and test DataSet.complete() Now it doesn't return anything, and only catches errors from the background_functions, removing functions which fail twice in a row Any other errors (like from sync()) will not be caught so will stop future code from running when it shouldn't --- qcodes/data/data_set.py | 75 +++++++++++++++++++++++---------------- qcodes/tests/test_data.py | 60 +++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 31 deletions(-) diff --git a/qcodes/data/data_set.py b/qcodes/data/data_set.py index fadb1ddece68..f2a39c0b2f3a 100644 --- a/qcodes/data/data_set.py +++ b/qcodes/data/data_set.py @@ -5,6 +5,7 @@ import logging from traceback import format_exc from copy import deepcopy +from collections import OrderedDict from .manager import get_data_manager, NoData from .gnuplot_format import GNUPlotFormat @@ -234,10 +235,14 @@ class DataSet(DelegateAttributes): from calls to ``self.store``. Default 5. Attributes: - background_functions (Dict[callable]): Class attribute, ``{key: fn}``, - ``fn`` is a callable accepting no arguments, and ``key`` is a name - to identify the function and help you attach and remove it. - In ``DataSet.complete`` we will call each of these periodically. + background_functions (OrderedDict[callable]): Class attribute, + ``{key: fn}``: ``fn`` is a callable accepting no arguments, and + ``key`` is a name to identify the function and help you attach and + remove it. + + In ``DataSet.complete`` we call each of these periodically, in the + order that they were attached. + Note that because this is a class attribute, the functions will apply to every DataSet. If you want specific functions for one DataSet you can override this with an instance attribute. @@ -250,8 +255,7 @@ class DataSet(DelegateAttributes): default_formatter = GNUPlotFormat() location_provider = FormatLocation() - # functions to be called when operating in background mode - background_functions = {} + background_functions = OrderedDict() def __init__(self, location=None, mode=DataMode.LOCAL, arrays=None, data_manager=None, formatter=None, io=None, write_period=5): @@ -435,36 +439,45 @@ def complete(self, delay=1.5): Periodically sync the DataSet and display percent complete status. Also, each period, execute functions stored in (class attribute) - ``self.background_functions`` + ``self.background_functions``. If a function fails, we log its + traceback and continue on. If any one function fails twice in + a row, it gets removed. Args: - delay (float): seconds between status messages - - Returns: - bool: True if we managed to wait until the DataSet finished, - False if something went wrong and it it not yet complete. + delay (float): seconds between iterations. Default 1.5 """ - logging.info('waiting for DataSet to complete') + logging.info( + 'waiting for DataSet <{}> to complete'.format(self.location)) - try: - nloops = 0 - while True: - logging.info( - 'waiting for DataSet to complete (fraction %.2f)' % - self.fraction_complete()) - if self.sync() is False: - break - - time.sleep(delay) - nloops += 1 - - for key, fn in self.background_functions.items(): - logging.debug('calling %s: %s' % (key, fn)) + failing = {key: False for key in self.background_functions} + + nloops = 0 + completed = False + while not completed: + logging.info('DataSet: {:.0f}% complete'.format( + self.fraction_complete() * 100)) + + time.sleep(delay) + nloops += 1 + + if self.sync() is False: + completed = True + + for key, fn in list(self.background_functions.items()): + try: + logging.debug('calling {}: {}'.format(key, repr(fn))) fn() - except: - print(format_exc()) - return False - return True + failing[key] = False + except Exception: + logging.info(format_exc()) + if failing[key]: + logging.warning( + 'background function {} failed twice in a row, ' + 'removing it'.format(key)) + del self.background_functions[key] + failing[key] = True + + logging.info('DataSet <{}> is complete'.format(self.location)) def get_changes(self, synced_index): changes = {} diff --git a/qcodes/tests/test_data.py b/qcodes/tests/test_data.py index ac8c566f0fd3..ec23039e2d4b 100644 --- a/qcodes/tests/test_data.py +++ b/qcodes/tests/test_data.py @@ -3,12 +3,14 @@ import numpy as np import os import pickle +import logging from qcodes.data.data_array import DataArray from qcodes.data.manager import get_data_manager, NoData from qcodes.data.io import DiskIO from qcodes.data.data_set import load_data, new_data, DataMode, DataSet from qcodes.process.helpers import kill_processes +from qcodes.utils.helpers import LogCapture from qcodes import active_children from .data_mocks import (MockDataManager, MockFormatter, MatchIO, @@ -603,3 +605,61 @@ def test_fraction_complete(self): data.y1.last_saved_index = 1 # 2 of 2 data.z1.synced_index = 5 # 6 of 6 self.assertEqual(data.fraction_complete(), 0.75) + + def mock_sync(self): + # import pdb; pdb.set_trace() + i = self.sync_index + self.syncing_array[i] = i + self.sync_index = i + 1 + return self.sync_index < self.syncing_array.size + + def failing_func(self): + raise RuntimeError('it is called failing_func for a reason!') + + def logging_func(self): + logging.info('background at index {}'.format(self.sync_index)) + + def test_complete(self): + array = DataArray(name='y', shape=(5,)) + array.init_data() + data = new_data(arrays=(array,), location=False) + self.syncing_array = array + self.sync_index = 0 + data.sync = self.mock_sync + DataSet.background_functions.update({ + 'fail': self.failing_func, + 'log': self.logging_func + }) + + with LogCapture() as logs: + # grab info and warnings but not debug messages + logging.getLogger().setLevel(logging.INFO) + data.complete(delay=0.001) + + logs = logs.value + + expected_logs = [ + 'waiting for DataSet to complete', + 'DataSet: 0% complete', + 'RuntimeError: it is called failing_func for a reason!', + 'background at index 1', + 'DataSet: 20% complete', + 'RuntimeError: it is called failing_func for a reason!', + 'background function fail failed twice in a row, removing it', + 'background at index 2', + 'DataSet: 40% complete', + 'background at index 3', + 'DataSet: 60% complete', + 'background at index 4', + 'DataSet: 80% complete', + 'background at index 5', + 'DataSet is complete' + ] + + log_index = 0 + for line in expected_logs: + self.assertIn(line, logs, logs) + log_index = logs.index(line, log_index) + self.assertTrue(log_index >= 0, logs) + log_index += len(line) + 1 # +1 for \n + self.assertEqual(log_index, len(logs), logs) From d8912d1bafe27aab6f1bf30118bf2e3709ebc4f6 Mon Sep 17 00:00:00 2001 From: alexcjohnson Date: Wed, 6 Jul 2016 14:05:18 +0200 Subject: [PATCH 13/13] test: Oops, unsaved change to DataSet.complete test --- qcodes/tests/test_data.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/qcodes/tests/test_data.py b/qcodes/tests/test_data.py index ec23039e2d4b..8e0d1a120c18 100644 --- a/qcodes/tests/test_data.py +++ b/qcodes/tests/test_data.py @@ -626,10 +626,9 @@ def test_complete(self): self.syncing_array = array self.sync_index = 0 data.sync = self.mock_sync - DataSet.background_functions.update({ - 'fail': self.failing_func, - 'log': self.logging_func - }) + bf = DataSet.background_functions + bf['fail'] = self.failing_func + bf['log'] = self.logging_func with LogCapture() as logs: # grab info and warnings but not debug messages