From 0beec548cdcd3b46443df71c453d91cf841b1a97 Mon Sep 17 00:00:00 2001 From: Rok Roskar Date: Tue, 27 Aug 2019 09:07:48 +0200 Subject: [PATCH 1/3] fix: always add commit to dataset if possible addresses #646 --- renku/api/datasets.py | 6 +++--- renku/models/_jsonld.py | 12 +++++------- renku/models/datasets.py | 10 +++++++++- renku/models/provenance/entities.py | 8 ++++++++ tests/test_dataset.py | 1 - 5 files changed, 25 insertions(+), 12 deletions(-) diff --git a/renku/api/datasets.py b/renku/api/datasets.py index 54fc10ae19..8a3c68ed1d 100644 --- a/renku/api/datasets.py +++ b/renku/api/datasets.py @@ -50,7 +50,7 @@ class DatasetsApiMixin(object): @property def renku_datasets_path(self): """Return a ``Path`` instance of Renku dataset metadata folder.""" - return self.renku_path.joinpath(self.DATASETS) + return Path(self.renku_home).joinpath(self.DATASETS) def datasets_from_commit(self, commit=None): """Return datasets defined in a commit.""" @@ -80,11 +80,11 @@ def datasets(self): result[path] = self.get_dataset(path) return result - def get_dataset(self, path): + def get_dataset(self, path, commit=None): """Return a dataset from a given path.""" if not path.is_absolute(): path = self.path / path - return Dataset.from_yaml(path, client=self) + return Dataset.from_yaml(path, client=self, commit=commit) def dataset_path(self, name): """Get dataset path from name.""" diff --git a/renku/models/_jsonld.py b/renku/models/_jsonld.py index dbb71a8589..8aeebf8435 100644 --- a/renku/models/_jsonld.py +++ b/renku/models/_jsonld.py @@ -313,9 +313,9 @@ class JSONLDMixin(ReferenceMixin): def from_jsonld( cls, data, - client=None, __reference__=None, __source__=None, + **kwargs, ): """Instantiate a JSON-LD class from data.""" if isinstance(data, cls): @@ -331,7 +331,7 @@ def from_jsonld( ) != type_: new_cls = cls.__type_registry__[type_] if cls != new_cls: - return new_cls.from_jsonld(data, client=client) + return new_cls.from_jsonld(data, **kwargs) if cls._jsonld_translate: data = ld.compact(data, {'@context': cls._jsonld_translate}) @@ -352,9 +352,7 @@ def from_jsonld( fields = cls._jsonld_fields - data_ = {} - if client: - data_['client'] = client + data_ = kwargs for k, v in compacted.items(): if k in fields: @@ -372,7 +370,7 @@ def from_jsonld( return self @classmethod - def from_yaml(cls, path, client=None): + def from_yaml(cls, path, **kwargs): """Return an instance from a YAML file.""" import yaml @@ -380,9 +378,9 @@ def from_yaml(cls, path, client=None): source = yaml.safe_load(fp) or {} self = cls.from_jsonld( source, - client=client, __reference__=path, __source__=deepcopy(source), + **kwargs ) return self diff --git a/renku/models/datasets.py b/renku/models/datasets.py index b3a05b01b9..6cd6e19440 100644 --- a/renku/models/datasets.py +++ b/renku/models/datasets.py @@ -467,7 +467,7 @@ def unlink_file(self, file_path): return self.files.pop(index) def __attrs_post_init__(self): - """Post-Init hook to set _id field.""" + """Post-Init hook.""" self._id = self.identifier if not self._label: @@ -487,3 +487,11 @@ def __attrs_post_init__(self): ) datasetfile.client = client + + try: + self.commit = self.client.find_previous_commit( + self.path, revision=self.commit or 'HEAD' + ) + except KeyError: + # if with_dataset is used, the dataset is not committed yet + pass diff --git a/renku/models/provenance/entities.py b/renku/models/provenance/entities.py index 7a8f0a6985..99f6fa994e 100644 --- a/renku/models/provenance/entities.py +++ b/renku/models/provenance/entities.py @@ -18,6 +18,7 @@ """Represent provenance entities.""" import weakref +from pathlib import Path import attr @@ -77,6 +78,13 @@ def default_project(self): if self.client: return self.client.project + def __attrs_post_init__(self): + """Post-init hook.""" + if self.path: + path = Path(self.path) + if path.is_absolute(): + self.path = str(path.relative_to(self.client.path)) + @jsonld.s( type=[ diff --git a/tests/test_dataset.py b/tests/test_dataset.py index d1c9115a6d..88cf074145 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -88,7 +88,6 @@ def test_data_add( assert not os.access( 'data/dataset/file', stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH ) - # assert os.stat('data/dataset/file/metadata.yml') # check the linking if scheme in ('', 'file://'): From 1bc390ae6122499297592d5054ae20016bed56fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20Ro=C5=A1kar?= Date: Tue, 27 Aug 2019 13:50:00 +0200 Subject: [PATCH 2/3] Update renku/models/_jsonld.py Co-Authored-By: Sam --- renku/models/_jsonld.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renku/models/_jsonld.py b/renku/models/_jsonld.py index 8aeebf8435..c012ec1920 100644 --- a/renku/models/_jsonld.py +++ b/renku/models/_jsonld.py @@ -315,7 +315,7 @@ def from_jsonld( data, __reference__=None, __source__=None, - **kwargs, + **kwargs ): """Instantiate a JSON-LD class from data.""" if isinstance(data, cls): From 2e1e82ecb62e4aacb8c892c99383f4a250158099 Mon Sep 17 00:00:00 2001 From: Rok Roskar Date: Tue, 27 Aug 2019 18:18:22 +0200 Subject: [PATCH 3/3] expand kwargs --- renku/models/_jsonld.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/renku/models/_jsonld.py b/renku/models/_jsonld.py index c012ec1920..b7cf384c47 100644 --- a/renku/models/_jsonld.py +++ b/renku/models/_jsonld.py @@ -313,9 +313,10 @@ class JSONLDMixin(ReferenceMixin): def from_jsonld( cls, data, + client=None, + commit=None, __reference__=None, __source__=None, - **kwargs ): """Instantiate a JSON-LD class from data.""" if isinstance(data, cls): @@ -331,7 +332,9 @@ def from_jsonld( ) != type_: new_cls = cls.__type_registry__[type_] if cls != new_cls: - return new_cls.from_jsonld(data, **kwargs) + return new_cls.from_jsonld( + data, client=client, commit=commit + ) if cls._jsonld_translate: data = ld.compact(data, {'@context': cls._jsonld_translate}) @@ -352,7 +355,10 @@ def from_jsonld( fields = cls._jsonld_fields - data_ = kwargs + data_ = {} + if client: + data_['client'] = client + data_['commit'] = commit for k, v in compacted.items(): if k in fields: @@ -370,7 +376,7 @@ def from_jsonld( return self @classmethod - def from_yaml(cls, path, **kwargs): + def from_yaml(cls, path, client=None, commit=None): """Return an instance from a YAML file.""" import yaml @@ -378,11 +384,11 @@ def from_yaml(cls, path, **kwargs): source = yaml.safe_load(fp) or {} self = cls.from_jsonld( source, + client=client, + commit=commit, __reference__=path, - __source__=deepcopy(source), - **kwargs + __source__=deepcopy(source) ) - return self def asjsonld(self):