From a2d9a172cf52fc4a2868fdaa2ad87cd7223a0018 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 11 Aug 2022 10:15:48 -0400 Subject: [PATCH 1/3] get all datasets in dvconfig.py.sample to load at least #30 Replace "CC0" with "CC0 1.0". Remove `"license": "NONE"` entirely as it's no longer supported. --- .../reproductive-health-laws-around-the-world.json | 2 +- data/dataverses/cms/datasets/cmssampledata/cmssampledata.json | 2 +- .../this-is-my-test-dataset/this-is-my-test-dataset.json | 2 +- .../king/datasets/cause-of-death/cause-of-death.json | 3 +-- .../manchester/datasets/test-dataset/test-dataset.json | 2 +- data/dataverses/pums/datasets/2000pums5/2000pums5.json | 2 +- .../datasets/diabeticconnect/diabeticconnect.json | 4 ++-- .../dataverses/jopd/datasets/bafacalo/bafacalo.json | 1 - .../flynn-effect-in-estonia/flynn-effect-in-estonia.json | 3 +-- 9 files changed, 9 insertions(+), 12 deletions(-) diff --git a/data/dataverses/HCPDS/datasets/reproductive-health-laws-around-the-world/reproductive-health-laws-around-the-world.json b/data/dataverses/HCPDS/datasets/reproductive-health-laws-around-the-world/reproductive-health-laws-around-the-world.json index 7574a346..f2e1585e 100644 --- a/data/dataverses/HCPDS/datasets/reproductive-health-laws-around-the-world/reproductive-health-laws-around-the-world.json +++ b/data/dataverses/HCPDS/datasets/reproductive-health-laws-around-the-world/reproductive-health-laws-around-the-world.json @@ -15,7 +15,7 @@ "lastUpdateTime": "2017-08-31T05:24:19Z", "releaseTime": "2017-08-31T05:24:19Z", "createTime": "2015-04-20T10:14:02Z", - "license": "CC0", + "license": "CC0 1.0", "termsOfUse": "CC0 Waiver", "termsOfAccess": "You need to request for access. ", "metadataBlocks": { diff --git a/data/dataverses/cms/datasets/cmssampledata/cmssampledata.json b/data/dataverses/cms/datasets/cmssampledata/cmssampledata.json index 9217faee..4f1145b9 100644 --- a/data/dataverses/cms/datasets/cmssampledata/cmssampledata.json +++ b/data/dataverses/cms/datasets/cmssampledata/cmssampledata.json @@ -17,7 +17,7 @@ "lastUpdateTime":"2019-08-01T16:46:02Z", "releaseTime":"2019-08-01T16:46:02Z", "createTime":"2019-08-01T16:45:51Z", - "license":"CC0", + "license":"CC0 1.0", "termsOfUse":"CC0 Waiver", "fileAccessRequest":false, "metadataBlocks":{ diff --git a/data/dataverses/ecastro/datasets/this-is-my-test-dataset/this-is-my-test-dataset.json b/data/dataverses/ecastro/datasets/this-is-my-test-dataset/this-is-my-test-dataset.json index 537f8ea9..7e5c97cd 100644 --- a/data/dataverses/ecastro/datasets/this-is-my-test-dataset/this-is-my-test-dataset.json +++ b/data/dataverses/ecastro/datasets/this-is-my-test-dataset/this-is-my-test-dataset.json @@ -15,7 +15,7 @@ "lastUpdateTime": "2015-04-20T13:58:35Z", "releaseTime": "2015-04-20T13:58:35Z", "createTime": "2015-04-20T13:57:32Z", - "license": "CC0", + "license": "CC0 1.0", "termsOfUse": "CC0 Waiver", "termsOfAccess": "You need to request for access.", "metadataBlocks": { diff --git a/data/dataverses/king/datasets/cause-of-death/cause-of-death.json b/data/dataverses/king/datasets/cause-of-death/cause-of-death.json index 81c222be..3cc09780 100644 --- a/data/dataverses/king/datasets/cause-of-death/cause-of-death.json +++ b/data/dataverses/king/datasets/cause-of-death/cause-of-death.json @@ -9,7 +9,6 @@ "lastUpdateTime":"2016-11-17T18:31:33Z", "releaseTime":"2016-11-17T18:31:33Z", "createTime":"2016-11-17T17:50:28Z", - "license":"NONE", "metadataBlocks":{ "citation":{ "displayName":"Citation Metadata", @@ -167,4 +166,4 @@ }, "citation":"Girosi, Frederico; King, Gary, 2007, \"Cause of Death Data\", hdl:1902.1/UOVMCPSWOL, Harvard Dataverse, V5" } -} \ No newline at end of file +} diff --git a/data/dataverses/manchester/datasets/test-dataset/test-dataset.json b/data/dataverses/manchester/datasets/test-dataset/test-dataset.json index 3c1274f7..92c055dd 100644 --- a/data/dataverses/manchester/datasets/test-dataset/test-dataset.json +++ b/data/dataverses/manchester/datasets/test-dataset/test-dataset.json @@ -17,7 +17,7 @@ "lastUpdateTime": "2018-08-12T03:49:12Z", "releaseTime": "2018-08-12T03:49:12Z", "createTime": "2016-04-12T16:57:26Z", - "license": "CC0", + "license": "CC0 1.0", "termsOfUse": "CC0 Waiver", "termsOfAccess": "Terms: Must be a member of Manchester Dataverse group.", "metadataBlocks": { diff --git a/data/dataverses/pums/datasets/2000pums5/2000pums5.json b/data/dataverses/pums/datasets/2000pums5/2000pums5.json index f134bc45..382af97f 100644 --- a/data/dataverses/pums/datasets/2000pums5/2000pums5.json +++ b/data/dataverses/pums/datasets/2000pums5/2000pums5.json @@ -11,7 +11,7 @@ "lastUpdateTime": "2021-09-20T18:38:32Z", "releaseTime": "2021-09-20T18:38:32Z", "createTime": "2021-09-20T18:16:38Z", - "license": "CC0", + "license": "CC0 1.0", "termsOfUse": "CC0 Waiver", "fileAccessRequest": false, "metadataBlocks": { diff --git a/data/dataverses/scholcommlab/datasets/diabeticconnect/diabeticconnect.json b/data/dataverses/scholcommlab/datasets/diabeticconnect/diabeticconnect.json index f7d3c828..028e82c8 100644 --- a/data/dataverses/scholcommlab/datasets/diabeticconnect/diabeticconnect.json +++ b/data/dataverses/scholcommlab/datasets/diabeticconnect/diabeticconnect.json @@ -18,7 +18,7 @@ "lastUpdateTime": "2019-07-11T14:58:52Z", "releaseTime": "2019-07-11T14:58:52Z", "createTime": "2019-07-04T18:48:01Z", - "license": "CC0", + "license": "CC0 1.0", "termsOfUse": "CC0 Waiver", "fileAccessRequest": false, "metadataBlocks": { @@ -206,4 +206,4 @@ }, "citation": "Didegah; Ghaseminik, Zahra; Alperin, Juan Pablo, 2019, \"Text data for comparing the language used in Wikipedia, the DiabeticConnect Forum, and Research Articles\", https://doi.org/10.7910/DVN/X27DSE, Harvard Dataverse, V1, UNF:6:pE4SFkzhKvY+PwYHdhT3sg== [fileUNF]" } -} \ No newline at end of file +} diff --git a/data/dataverses/ubiquity-press/dataverses/jopd/datasets/bafacalo/bafacalo.json b/data/dataverses/ubiquity-press/dataverses/jopd/datasets/bafacalo/bafacalo.json index 7d8483be..4b5ee27b 100644 --- a/data/dataverses/ubiquity-press/dataverses/jopd/datasets/bafacalo/bafacalo.json +++ b/data/dataverses/ubiquity-press/dataverses/jopd/datasets/bafacalo/bafacalo.json @@ -10,7 +10,6 @@ "lastUpdateTime":"2013-12-13T12:27:43Z", "releaseTime":"2013-12-13T00:00:00Z", "createTime":"2013-11-01T20:22:37Z", - "license":"NONE", "termsOfUse":"All data uploaded to this Dataverse are licensed under the CC0 licence. Please follow this link for further information: http://creativecommons.org/publicdomain/zero/1.0/", "metadataBlocks":{ "citation":{ diff --git a/data/dataverses/ubiquity-press/dataverses/jopd/datasets/flynn-effect-in-estonia/flynn-effect-in-estonia.json b/data/dataverses/ubiquity-press/dataverses/jopd/datasets/flynn-effect-in-estonia/flynn-effect-in-estonia.json index bb9cd0fd..51de5c47 100644 --- a/data/dataverses/ubiquity-press/dataverses/jopd/datasets/flynn-effect-in-estonia/flynn-effect-in-estonia.json +++ b/data/dataverses/ubiquity-press/dataverses/jopd/datasets/flynn-effect-in-estonia/flynn-effect-in-estonia.json @@ -9,7 +9,6 @@ "lastUpdateTime":"2015-08-17T08:27:43Z", "releaseTime":"2015-08-17T08:27:43Z", "createTime":"2015-08-12T07:59:03Z", - "license":"NONE", "termsOfUse":"All data uploaded to this Dataverse are licensed under the CC0 licence. Please follow this link for further information: http://creativecommons.org/publicdomain/zero/1.0/", "metadataBlocks":{ "citation":{ @@ -159,4 +158,4 @@ }, "citation":"Must, Olev; Must, Aasa, 2014, \"“Changes in test-taking patterns over time” concerning the Flynn Effect in Estonia\", doi:10.7910/DVN/23791, Harvard Dataverse, V2" } -} \ No newline at end of file +} From 25c04865a90d3ad7635700820b0d9a1eef2314d7 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 11 Aug 2022 11:59:58 -0400 Subject: [PATCH 2/3] reformat dataverseno dataset with jq --- .../tabular-sample-data.json | 215 +++++++++++++++++- 1 file changed, 214 insertions(+), 1 deletion(-) diff --git a/data/dataverses/dataverseno/datasets/tabular-sample-data/tabular-sample-data.json b/data/dataverses/dataverseno/datasets/tabular-sample-data/tabular-sample-data.json index 6b6e6030..d0518d54 100644 --- a/data/dataverses/dataverseno/datasets/tabular-sample-data/tabular-sample-data.json +++ b/data/dataverses/dataverseno/datasets/tabular-sample-data/tabular-sample-data.json @@ -1 +1,214 @@ -{"id":1101620,"identifier":"FK2/0MVCBM","persistentUrl":"https://doi.org/10.70122/FK2/0MVCBM","protocol":"doi","authority":"10.70122","publisher":"Demo Dataverse","publicationDate":"2020-06-08","storageIdentifier":"file://10.70122/FK2/0MVCBM","datasetVersion":{"id":134195,"datasetId":1101620,"datasetPersistentId":"doi:10.70122/FK2/0MVCBM","storageIdentifier":"file://10.70122/FK2/0MVCBM","versionNumber":1,"versionMinorNumber":0,"versionState":"RELEASED","UNF":"UNF:6:eFXLLKw9zMMIcjPRZOLhPg==","lastUpdateTime":"2020-06-08T05:52:18Z","releaseTime":"2020-06-08T05:52:18Z","createTime":"2020-06-04T09:06:36Z","license":"CC0","termsOfUse":"CC0 Waiver","fileAccessRequest":false,"metadataBlocks":{"citation":{"displayName":"Citation Metadata","fields":[{"typeName":"title","multiple":false,"typeClass":"primitive","value":"Testing Tabular File Ingest - Excel and tab-separated .txt"},{"typeName":"author","multiple":true,"typeClass":"compound","value":[{"authorName":{"typeName":"authorName","multiple":false,"typeClass":"primitive","value":"Test User 1, UiT"},"authorAffiliation":{"typeName":"authorAffiliation","multiple":false,"typeClass":"primitive","value":"UiT The Arctic University of Norway"}}]},{"typeName":"datasetContact","multiple":true,"typeClass":"compound","value":[{"datasetContactName":{"typeName":"datasetContactName","multiple":false,"typeClass":"primitive","value":"Test User 1, UiT"},"datasetContactAffiliation":{"typeName":"datasetContactAffiliation","multiple":false,"typeClass":"primitive","value":"UiT The Arctic University of Norway"},"datasetContactEmail":{"typeName":"datasetContactEmail","multiple":false,"typeClass":"primitive","value":"philipp.conzett@uit.no"}}]},{"typeName":"dsDescription","multiple":true,"typeClass":"compound","value":[{"dsDescriptionValue":{"typeName":"dsDescriptionValue","multiple":false,"typeClass":"primitive","value":"Testing ingest of tabular files in Dataverse."},"dsDescriptionDate":{"typeName":"dsDescriptionDate","multiple":false,"typeClass":"primitive","value":"2020-06-04"}}]},{"typeName":"subject","multiple":true,"typeClass":"controlledVocabulary","value":["Social Sciences"]},{"typeName":"keyword","multiple":true,"typeClass":"compound","value":[{"keywordValue":{"typeName":"keywordValue","multiple":false,"typeClass":"primitive","value":"testing"}},{"keywordValue":{"typeName":"keywordValue","multiple":false,"typeClass":"primitive","value":"file ingest"}}]},{"typeName":"depositor","multiple":false,"typeClass":"primitive","value":"Test User 1, UiT"},{"typeName":"dateOfDeposit","multiple":false,"typeClass":"primitive","value":"2020-06-04"}]},"geospatial":{"displayName":"Geospatial Metadata","fields":[]},"journal":{"displayName":"Journal Metadata","fields":[]}},"files":[{"description":"Excel file","label":"Tabular_Sample_Data.tab","restricted":false,"version":3,"datasetVersionId":134195,"dataFile":{"id":1104456,"persistentId":"doi:10.70122/FK2/0MVCBM/LISFFL","pidURL":"https://doi.org/10.70122/FK2/0MVCBM/LISFFL","filename":"Tabular_Sample_Data.tab","contentType":"text/tab-separated-values","filesize":2200,"description":"Excel file","storageIdentifier":"file://17282ef3127-d8ed28281ac7","originalFileFormat":"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet","originalFormatLabel":"MS Excel Spreadsheet","originalFileSize":12751,"UNF":"UNF:6:eFXLLKw9zMMIcjPRZOLhPg==","rootDataFileId":-1,"md5":"45a5a81537c772485a640a030b12bd33","checksum":{"type":"MD5","value":"45a5a81537c772485a640a030b12bd33"},"creationDate":"2020-06-05"}},{"description":"Tab-separated plain text file","label":"Tabular_Sample_Data.txt","restricted":false,"version":1,"datasetVersionId":134195,"dataFile":{"id":1104457,"persistentId":"doi:10.70122/FK2/0MVCBM/IYDYQR","pidURL":"https://doi.org/10.70122/FK2/0MVCBM/IYDYQR","filename":"Tabular_Sample_Data.txt","contentType":"text/plain","filesize":2060,"description":"Tab-separated plain text file","storageIdentifier":"file://17282ee8e22-382bc0edc2d6","rootDataFileId":-1,"md5":"b2a1624bcf972094c48d639d4aa67c9e","checksum":{"type":"MD5","value":"b2a1624bcf972094c48d639d4aa67c9e"},"creationDate":"2020-06-05"}}],"citation":"Test User 1, UiT, 2020, \"Testing Tabular File Ingest - Excel and tab-separated .txt\", https://doi.org/10.70122/FK2/0MVCBM, Demo Dataverse, V1, UNF:6:eFXLLKw9zMMIcjPRZOLhPg== [fileUNF]"}} \ No newline at end of file +{ + "id": 1101620, + "identifier": "FK2/0MVCBM", + "persistentUrl": "https://doi.org/10.70122/FK2/0MVCBM", + "protocol": "doi", + "authority": "10.70122", + "publisher": "Demo Dataverse", + "publicationDate": "2020-06-08", + "storageIdentifier": "file://10.70122/FK2/0MVCBM", + "datasetVersion": { + "id": 134195, + "datasetId": 1101620, + "datasetPersistentId": "doi:10.70122/FK2/0MVCBM", + "storageIdentifier": "file://10.70122/FK2/0MVCBM", + "versionNumber": 1, + "versionMinorNumber": 0, + "versionState": "RELEASED", + "UNF": "UNF:6:eFXLLKw9zMMIcjPRZOLhPg==", + "lastUpdateTime": "2020-06-08T05:52:18Z", + "releaseTime": "2020-06-08T05:52:18Z", + "createTime": "2020-06-04T09:06:36Z", + "license": "CC0", + "termsOfUse": "CC0 Waiver", + "fileAccessRequest": false, + "metadataBlocks": { + "citation": { + "displayName": "Citation Metadata", + "fields": [ + { + "typeName": "title", + "multiple": false, + "typeClass": "primitive", + "value": "Testing Tabular File Ingest - Excel and tab-separated .txt" + }, + { + "typeName": "author", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "authorName": { + "typeName": "authorName", + "multiple": false, + "typeClass": "primitive", + "value": "Test User 1, UiT" + }, + "authorAffiliation": { + "typeName": "authorAffiliation", + "multiple": false, + "typeClass": "primitive", + "value": "UiT The Arctic University of Norway" + } + } + ] + }, + { + "typeName": "datasetContact", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "datasetContactName": { + "typeName": "datasetContactName", + "multiple": false, + "typeClass": "primitive", + "value": "Test User 1, UiT" + }, + "datasetContactAffiliation": { + "typeName": "datasetContactAffiliation", + "multiple": false, + "typeClass": "primitive", + "value": "UiT The Arctic University of Norway" + }, + "datasetContactEmail": { + "typeName": "datasetContactEmail", + "multiple": false, + "typeClass": "primitive", + "value": "philipp.conzett@uit.no" + } + } + ] + }, + { + "typeName": "dsDescription", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "dsDescriptionValue": { + "typeName": "dsDescriptionValue", + "multiple": false, + "typeClass": "primitive", + "value": "Testing ingest of tabular files in Dataverse." + }, + "dsDescriptionDate": { + "typeName": "dsDescriptionDate", + "multiple": false, + "typeClass": "primitive", + "value": "2020-06-04" + } + } + ] + }, + { + "typeName": "subject", + "multiple": true, + "typeClass": "controlledVocabulary", + "value": [ + "Social Sciences" + ] + }, + { + "typeName": "keyword", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "keywordValue": { + "typeName": "keywordValue", + "multiple": false, + "typeClass": "primitive", + "value": "testing" + } + }, + { + "keywordValue": { + "typeName": "keywordValue", + "multiple": false, + "typeClass": "primitive", + "value": "file ingest" + } + } + ] + }, + { + "typeName": "depositor", + "multiple": false, + "typeClass": "primitive", + "value": "Test User 1, UiT" + }, + { + "typeName": "dateOfDeposit", + "multiple": false, + "typeClass": "primitive", + "value": "2020-06-04" + } + ] + }, + "geospatial": { + "displayName": "Geospatial Metadata", + "fields": [] + }, + "journal": { + "displayName": "Journal Metadata", + "fields": [] + } + }, + "files": [ + { + "description": "Excel file", + "label": "Tabular_Sample_Data.tab", + "restricted": false, + "version": 3, + "datasetVersionId": 134195, + "dataFile": { + "id": 1104456, + "persistentId": "doi:10.70122/FK2/0MVCBM/LISFFL", + "pidURL": "https://doi.org/10.70122/FK2/0MVCBM/LISFFL", + "filename": "Tabular_Sample_Data.tab", + "contentType": "text/tab-separated-values", + "filesize": 2200, + "description": "Excel file", + "storageIdentifier": "file://17282ef3127-d8ed28281ac7", + "originalFileFormat": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "originalFormatLabel": "MS Excel Spreadsheet", + "originalFileSize": 12751, + "UNF": "UNF:6:eFXLLKw9zMMIcjPRZOLhPg==", + "rootDataFileId": -1, + "md5": "45a5a81537c772485a640a030b12bd33", + "checksum": { + "type": "MD5", + "value": "45a5a81537c772485a640a030b12bd33" + }, + "creationDate": "2020-06-05" + } + }, + { + "description": "Tab-separated plain text file", + "label": "Tabular_Sample_Data.txt", + "restricted": false, + "version": 1, + "datasetVersionId": 134195, + "dataFile": { + "id": 1104457, + "persistentId": "doi:10.70122/FK2/0MVCBM/IYDYQR", + "pidURL": "https://doi.org/10.70122/FK2/0MVCBM/IYDYQR", + "filename": "Tabular_Sample_Data.txt", + "contentType": "text/plain", + "filesize": 2060, + "description": "Tab-separated plain text file", + "storageIdentifier": "file://17282ee8e22-382bc0edc2d6", + "rootDataFileId": -1, + "md5": "b2a1624bcf972094c48d639d4aa67c9e", + "checksum": { + "type": "MD5", + "value": "b2a1624bcf972094c48d639d4aa67c9e" + }, + "creationDate": "2020-06-05" + } + } + ], + "citation": "Test User 1, UiT, 2020, \"Testing Tabular File Ingest - Excel and tab-separated .txt\", https://doi.org/10.70122/FK2/0MVCBM, Demo Dataverse, V1, UNF:6:eFXLLKw9zMMIcjPRZOLhPg== [fileUNF]" + } +} From 7f4af08e171f3b2f97e85b293221f57058def631 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 11 Aug 2022 12:00:46 -0400 Subject: [PATCH 3/3] change CC0 to CC0 1.0 #30 --- .../datasets/tabular-sample-data/tabular-sample-data.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/dataverses/dataverseno/datasets/tabular-sample-data/tabular-sample-data.json b/data/dataverses/dataverseno/datasets/tabular-sample-data/tabular-sample-data.json index d0518d54..acd32d5c 100644 --- a/data/dataverses/dataverseno/datasets/tabular-sample-data/tabular-sample-data.json +++ b/data/dataverses/dataverseno/datasets/tabular-sample-data/tabular-sample-data.json @@ -19,7 +19,7 @@ "lastUpdateTime": "2020-06-08T05:52:18Z", "releaseTime": "2020-06-08T05:52:18Z", "createTime": "2020-06-04T09:06:36Z", - "license": "CC0", + "license": "CC0 1.0", "termsOfUse": "CC0 Waiver", "fileAccessRequest": false, "metadataBlocks": {