diff --git a/conf/docker-aio/c7.dockerfile b/conf/docker-aio/c7.dockerfile index 74aa6398084..982f682e296 100644 --- a/conf/docker-aio/c7.dockerfile +++ b/conf/docker-aio/c7.dockerfile @@ -12,8 +12,8 @@ COPY testdata/solrconfig.xml /tmp/dv # IPv6 and localhost appears to be related to some of the intermittant connection issues COPY disableipv6.conf /etc/sysctl.d/ RUN rm /etc/httpd/conf/* -COPY httpd.conf /etc/httpd/conf -RUN cd /opt ; tar zxf /tmp/dv/deps/solr-7.3.1dv.tgz +COPY httpd.conf /etc/httpd/conf +RUN cd /opt ; tar zxf /tmp/dv/deps/solr-7.3.1dv.tgz RUN cd /opt ; tar zxf /tmp/dv/deps/glassfish4dv.tgz # this copy of domain.xml is the result of running `asadmin set server.monitoring-service.module-monitoring-levels.jvm=LOW` on a default glassfish installation (aka - enable the glassfish REST monitir endpoint for the jvm` @@ -24,6 +24,7 @@ RUN sudo -u postgres /usr/pgsql-9.6/bin/initdb -D /var/lib/pgsql/data # copy configuration related files RUN cp /tmp/dv/pg_hba.conf /var/lib/pgsql/data/ +RUN cp /tmp/dv/postgresql.conf /var/lib/pgsql/data/ RUN cp -r /opt/solr-7.3.1/server/solr/configsets/_default /opt/solr-7.3.1/server/solr/collection1 RUN cp /tmp/dv/schema.xml /opt/solr-7.3.1/server/solr/collection1/conf/schema.xml RUN cp /tmp/dv/solrconfig.xml /opt/solr-7.3.1/server/solr/collection1/conf/solrconfig.xml @@ -71,7 +72,7 @@ ENV doi_username=${doi_username} ENV doi_password=${doi_password} COPY configure_doi.bash /opt/dv -# healthcheck for glassfish only (assumes modified domain.xml); +# healthcheck for glassfish only (assumes modified domain.xml); # does not check dataverse application status. HEALTHCHECK CMD curl --fail http://localhost:4848/monitoring/domain/server.json || exit 1 CMD ["/opt/dv/entrypoint.bash"] diff --git a/conf/docker-aio/configure_doi.bash b/conf/docker-aio/configure_doi.bash index 24ed6005b95..c082fb9c6df 100755 --- a/conf/docker-aio/configure_doi.bash +++ b/conf/docker-aio/configure_doi.bash @@ -7,7 +7,7 @@ if [ ! -z "${DoiProvider}" ]; then curl -X PUT -d ${DoiProvider} http://localhost:8080/api/admin/settings/:DoiProvider fi if [ ! -z "${doi_username}" ]; then - bin/asadmin create-jvm-options "-Ddoi.username=${doi_password}" + bin/asadmin create-jvm-options "-Ddoi.username=${doi_username}" fi if [ ! -z "${doi_password}" ]; then bin/asadmin create-jvm-options "-Ddoi.password=${doi_password}" diff --git a/conf/docker-aio/dv/postgresql.conf b/conf/docker-aio/dv/postgresql.conf new file mode 100644 index 00000000000..22ab7ab85e8 --- /dev/null +++ b/conf/docker-aio/dv/postgresql.conf @@ -0,0 +1,185 @@ + # (change requires restart) + # (change requires restart) + # (change requires restart) + # (change requires restart) +listen_addresses = '*' # what IP address(es) to listen on; + # comma-separated list of addresses; + # defaults to 'localhost'; use '*' for all + # (change requires restart) +max_connections = 100 # (change requires restart) + # (change requires restart) + # (change requires restart) + # (change requires restart) + # (change requires restart) + # (change requires restart) + # 0 selects the system default + # 0 selects the system default + # 0 selects the system default +shared_buffers = 128MB # min 128kB + # (change requires restart) + # (change requires restart) + # (change requires restart) +dynamic_shared_memory_type = posix # the default is the first option + # supported by the operating system: + # posix + # sysv + # windows + # mmap + # use none to disable dynamic shared memory + # (change requires restart) + # in kB, or -1 for no limit + # (change requires restart) + # (change requires restart) + # (change requires restart) + # (turning this off can cause + # unrecoverable data corruption) + # off, local, remote_write, remote_apply, or on + # supported by the operating system: + # open_datasync + # fdatasync (default on Linux) + # fsync + # fsync_writethrough + # open_sync + # (change requires restart) + # (change requires restart) + # (change requires restart) + # placeholders: %p = path of file to archive + # %f = file name only + # e.g. 'test ! -f /mnt/server/archivedir/%f && cp %p /mnt/server/archivedir/%f' + # number of seconds; 0 disables + # (change requires restart) + # (change requires restart) + # (change requires restart) + # number of sync standbys and comma-separated list of application_name + # from standby(s); '*' = all + # (change requires restart) + # when reading WAL from archive; + # -1 allows indefinite delay + # when reading streaming WAL; + # -1 allows indefinite delay + # 0 disables + # query conflicts + # communication from master + # in milliseconds; 0 disables + # retrieve WAL after a failed attempt + # JOIN clauses +log_destination = 'stderr' # Valid values are combinations of + # stderr, csvlog, syslog, and eventlog, + # depending on platform. csvlog + # requires logging_collector to be on. +logging_collector = on # Enable capturing of stderr and csvlog + # into log files. Required to be on for + # csvlogs. + # (change requires restart) +log_directory = 'pg_log' # directory where log files are written, + # can be absolute or relative to PGDATA +log_filename = 'postgresql-%a.log' # log file name pattern, + # can include strftime() escapes + # begin with 0 to use octal notation +log_truncate_on_rotation = on # If on, an existing log file with the + # same name as the new log file will be + # truncated rather than appended to. + # But such truncation only occurs on + # time-driven rotation, not on restarts + # or size-driven rotation. Default is + # off, meaning append to existing files + # in all cases. +log_rotation_age = 1d # Automatic rotation of logfiles will + # happen after that time. 0 disables. +log_rotation_size = 0 # Automatic rotation of logfiles will + # happen after that much log output. + # 0 disables. + # debug5 + # debug4 + # debug3 + # debug2 + # debug1 + # log + # notice + # warning + # error + # debug5 + # debug4 + # debug3 + # debug2 + # debug1 + # info + # notice + # warning + # error + # log + # fatal + # panic + # debug5 + # debug4 + # debug3 + # debug2 + # debug1 + # info + # notice + # warning + # error + # log + # fatal + # panic (effectively off) + # and their durations, > 0 logs only + # statements running at least this number + # of milliseconds +log_line_prefix = '< %m > ' # special values: + # %a = application name + # %u = user name + # %d = database name + # %r = remote host and port + # %h = remote host + # %p = process ID + # %t = timestamp without milliseconds + # %m = timestamp with milliseconds + # %n = timestamp with milliseconds (as a Unix epoch) + # %i = command tag + # %e = SQL state + # %c = session ID + # %l = session line number + # %s = session start timestamp + # %v = virtual transaction ID + # %x = transaction ID (0 if none) + # %q = stop here in non-session + # processes + # %% = '%' + # e.g. '<%u%%%d> ' + # than the specified size in kilobytes; + # -1 disables, 0 logs all temp files +log_timezone = 'UTC' + # (change requires restart) + # requires track_counts to also be on. + # their durations, > 0 logs only + # actions running at least this number + # of milliseconds. + # (change requires restart) + # vacuum + # analyze + # (change requires restart) + # before forced vacuum + # (change requires restart) + # autovacuum, in milliseconds; + # -1 means use vacuum_cost_delay + # autovacuum, -1 means use + # vacuum_cost_limit + # only default tablespace +datestyle = 'iso, mdy' +timezone = 'UTC' + # abbreviations. Currently, there are + # Default + # Australia (historical usage) + # India + # You can create your own file in + # share/timezonesets/. + # encoding +lc_messages = 'C' # locale for system error message + # strings +lc_monetary = 'C' # locale for monetary formatting +lc_numeric = 'C' # locale for number formatting +lc_time = 'C' # locale for time formatting +default_text_search_config = 'pg_catalog.english' + # (change requires restart) + # (change requires restart) + # directory 'conf.d' diff --git a/conf/docker-aio/entrypoint.bash b/conf/docker-aio/entrypoint.bash index da01ee56153..baefee9e029 100755 --- a/conf/docker-aio/entrypoint.bash +++ b/conf/docker-aio/entrypoint.bash @@ -2,6 +2,7 @@ export LANG=en_US.UTF-8 #sudo -u postgres /usr/bin/postgres -D /var/lib/pgsql/data & sudo -u postgres /usr/pgsql-9.6/bin/postgres -D /var/lib/pgsql/data & +sleep 15 cd /opt/solr-7.3.1/ # TODO: Run Solr as non-root and remove "-force". bin/solr start -force @@ -11,7 +12,7 @@ bin/solr create_core -c collection1 -d server/solr/collection1/conf -force apachectl -DFOREGROUND & # TODO: Run Glassfish as non-root. -cd /opt/glassfish4 -bin/asadmin start-domain --debug +cd /opt/glassfish4/ +bin/asadmin start-domain sleep infinity diff --git a/conf/docker-aio/prep_it.bash b/conf/docker-aio/prep_it.bash index e707fe6a76c..85858a633a2 100755 --- a/conf/docker-aio/prep_it.bash +++ b/conf/docker-aio/prep_it.bash @@ -10,7 +10,7 @@ n_retries=10 # glassfish healthy/ready retries n_wait=5 -cd conf/docker-aio +#cd conf/docker-aio ./0prep_deps.sh ./1prep.sh docker build -t dv0 -f c7.dockerfile . @@ -21,7 +21,7 @@ do # cleanup from previous runs if necessary docker rm -f dv # start container - docker run -d -p 8084:80 -p 8083:8080 -p 9010:9009 --name dv dv0 + docker run -d -p 5432:5432 -p 80:80 -p 8080:8080 --name dv dv0 # wait for glassfish to be healthy i_wait=0 @@ -35,7 +35,7 @@ do sleep $d_wait fi i_wait=$(( $i_wait + 1 )) - + done # try setupIT.bash @@ -69,6 +69,5 @@ docker exec -it dv /opt/dv/configure_doi.bash cd ../.. echo "docker-aio ready to run integration tests ($i_retry)" -curl http://localhost:8084/api/info/version +curl http://localhost:8080/api/info/version echo $? - diff --git a/conf/docker-aio/setJvmOptions.bash b/conf/docker-aio/setJvmOptions.bash new file mode 100755 index 00000000000..90940843ddf --- /dev/null +++ b/conf/docker-aio/setJvmOptions.bash @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -a +. $1 +set +a + +echo "SITE_URL=${SITE_URL}" +echo "DOI_USERNAME=${DOI_USERNAME}" +echo "DOI_PASSWORD=${DOI_PASSWORD}" +echo "DOI_BASEURL=${DOI_BASEURL}" + + +docker exec -it dv /usr/local/glassfish4/bin/asadmin create-jvm-options "\"-Ddataverse.siteUrl=${SITE_URL}\"" +sleep 15 +docker exec -it dv /usr/local/glassfish4/bin/asadmin create-jvm-options "\"-Ddoi.username=${DOI_USERNAME}\"" +sleep 15 +docker exec -it dv /usr/local/glassfish4/bin/asadmin create-jvm-options "\"-Ddoi.password=${DOI_PASSWORD}\"" +sleep 15 +docker exec -it dv /usr/local/glassfish4/bin/asadmin create-jvm-options "\"-Ddoi.baseurlstring=${DOI_BASEURL}\"" diff --git a/doc/Architecture/TRSA.adoc b/doc/Architecture/TRSA.adoc new file mode 100644 index 00000000000..4928b2ff64c --- /dev/null +++ b/doc/Architecture/TRSA.adoc @@ -0,0 +1,2 @@ +=== major changes to version 4.8.6 +* ImportType.IMPORT_METADATA_ONLY was added to ImportUtil \ No newline at end of file diff --git a/pom.xml b/pom.xml index e21ba43ef80..a13c2b26a70 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 edu.harvard.iq dataverse @@ -26,12 +26,12 @@ This seems to play well with NetBeans 8.2, IDEA 2018.1 and mvn including compatibility with JaCoCo. --> -Duser.timezone=${project.timezone} -Dfile.encoding=${project.build.sourceEncoding} -Duser.language=${project.language} -Duser.region=${project.region} - + 1.11.172 2.9.6 1.2 4.5.5 - + 4.12 5.3.1 5.3.1 @@ -75,7 +75,7 @@ file://${project.basedir}/local_lib - + @@ -106,7 +106,7 @@ - + + org.javaswift joss @@ -497,27 +497,27 @@ scribejava-apis 3.1.0 - - - - - - - - com.lyncode - xoai-common - 4.1.0-header-patch - - - com.lyncode - xoai-data-provider - 4.1.0-header-patch - - - com.lyncode - xoai-service-provider - 4.1.0-header-patch - + + + + + + + + com.lyncode + xoai-common + 4.1.0-header-patch + + + com.lyncode + xoai-data-provider + 4.1.0-header-patch + + + com.lyncode + xoai-service-provider + 4.1.0-header-patch + com.google.auto.service @@ -528,16 +528,16 @@ - org.glassfish.jersey.containers - jersey-container-servlet - 2.23.2 - + org.glassfish.jersey.containers + jersey-container-servlet + 2.23.2 + - - org.glassfish.jersey.media - jersey-media-multipart - 2.23.2 - + + org.glassfish.jersey.media + jersey-media-multipart + 2.23.2 + com.mashape.unirest unirest-java @@ -582,7 +582,7 @@ logback-classic - + commons-cli @@ -595,6 +595,15 @@ tika-parsers 1.19 + + + + com.thoughtworks.xstream + xstream + 1.4.10 + + + @@ -663,7 +672,7 @@ maven-war-plugin 2.3 - true + true false diff --git a/scripts/database/trsa_registry.sql b/scripts/database/trsa_registry.sql new file mode 100644 index 00000000000..b6c60a56efd --- /dev/null +++ b/scripts/database/trsa_registry.sql @@ -0,0 +1,24 @@ +-- Drop table + +DROP TABLE public.trsa_registry + +CREATE TABLE public.trsa_registry ( + id serial NOT NULL, + installation varchar(255) NOT NULL, + email varchar(255) NOT NULL, + dataverseurl varchar(255) NOT NULL, + apitoken varchar(255) NOT NULL, + datastoragelocation varchar(255) NOT NULL, + dataaccessinfo varchar(255) NOT NULL, + notaryserviceurl varchar(255) NOT NULL, + safeserviceurl varchar(255) NOT NULL, + registertime timestamp NULL, + expiretime timestamp NULL, + disabled bool NULL, + CONSTRAINT trsa_registry_pkey null +); + +-- Permissions + +ALTER TABLE public.trsa_registry OWNER TO dvnapp; +GRANT ALL ON TABLE public.trsa_registry TO dvnapp; diff --git a/src/main/java/Bundle_trsa_registry.properties b/src/main/java/Bundle_trsa_registry.properties new file mode 100644 index 00000000000..6eb1761fabc --- /dev/null +++ b/src/main/java/Bundle_trsa_registry.properties @@ -0,0 +1,143 @@ +PersistenceErrorOccured=A persistence error occurred. +Create=Create +View=View +Edit=Edit +Delete=Delete +Close=Close +Cancel=Cancel +Save=Save +SelectOneMessage=Select One... +Home=Home +Maintenance=Maintenance +AppName=dataverse + +TrsaRegistryCreated=TrsaRegistry was successfully created. +TrsaRegistryUpdated=TrsaRegistry was successfully updated. +TrsaRegistryDeleted=TrsaRegistry was successfully deleted. +CreateTrsaRegistryTitle=Create New TrsaRegistry +CreateTrsaRegistrySaveLink=Save +CreateTrsaRegistryShowAllLink=Show All TrsaRegistry Items +CreateTrsaRegistryIndexLink=Index +CreateTrsaRegistryLabel_installation=Installation: +CreateTrsaRegistryRequiredMessage_installation=The Installation field is required. +CreateTrsaRegistryTitle_installation=Installation +CreateTrsaRegistryLabel_email=Email: +CreateTrsaRegistryRequiredMessage_email=The Email field is required. +CreateTrsaRegistryTitle_email=Email +CreateTrsaRegistryLabel_dataverseurl=Dataverseurl: +CreateTrsaRegistryRequiredMessage_dataverseurl=The Dataverseurl field is required. +CreateTrsaRegistryTitle_dataverseurl=Dataverseurl +CreateTrsaRegistryLabel_apitoken=Apitoken: +CreateTrsaRegistryRequiredMessage_apitoken=The Apitoken field is required. +CreateTrsaRegistryTitle_apitoken=Apitoken +CreateTrsaRegistryLabel_datastoragelocation=Datastoragelocation: +CreateTrsaRegistryRequiredMessage_datastoragelocation=The Datastoragelocation field is required. +CreateTrsaRegistryTitle_datastoragelocation=Datastoragelocation +CreateTrsaRegistryLabel_dataaccessinfo=Dataaccessinfo: +CreateTrsaRegistryRequiredMessage_dataaccessinfo=The Dataaccessinfo field is required. +CreateTrsaRegistryTitle_dataaccessinfo=Dataaccessinfo +CreateTrsaRegistryLabel_notaryserviceurl=Notaryserviceurl: +CreateTrsaRegistryRequiredMessage_notaryserviceurl=The Notaryserviceurl field is required. +CreateTrsaRegistryTitle_notaryserviceurl=Notaryserviceurl +CreateTrsaRegistryLabel_safeserviceurl=Safeserviceurl: +CreateTrsaRegistryRequiredMessage_safeserviceurl=The Safeserviceurl field is required. +CreateTrsaRegistryTitle_safeserviceurl=Safeserviceurl +CreateTrsaRegistryLabel_registertime=Registertime: +CreateTrsaRegistryTitle_registertime=Registertime +CreateTrsaRegistryLabel_disabled=Disabled: +CreateTrsaRegistryTitle_disabled=Disabled +CreateTrsaRegistryLabel_expiretime=Expiretime: +CreateTrsaRegistryTitle_expiretime=Expiretime +CreateTrsaRegistryLabel_id=Id: +CreateTrsaRegistryRequiredMessage_id=The Id field is required. +CreateTrsaRegistryTitle_id=Id + + +EditTrsaRegistryTitle=Edit TrsaRegistry +EditTrsaRegistrySaveLink=Save +EditTrsaRegistryViewLink=View +EditTrsaRegistryShowAllLink=Show All TrsaRegistry Items +EditTrsaRegistryIndexLink=Index +EditTrsaRegistryLabel_installation=Installation: +EditTrsaRegistryRequiredMessage_installation=The Installation field is required. +EditTrsaRegistryTitle_installation=Installation +EditTrsaRegistryLabel_email=Email: +EditTrsaRegistryRequiredMessage_email=The Email field is required. +EditTrsaRegistryTitle_email=Email +EditTrsaRegistryLabel_dataverseurl=Dataverseurl: +EditTrsaRegistryRequiredMessage_dataverseurl=The Dataverseurl field is required. +EditTrsaRegistryTitle_dataverseurl=Dataverseurl +EditTrsaRegistryLabel_apitoken=Apitoken: +EditTrsaRegistryRequiredMessage_apitoken=The Apitoken field is required. +EditTrsaRegistryTitle_apitoken=Apitoken +EditTrsaRegistryLabel_datastoragelocation=Datastoragelocation: +EditTrsaRegistryTitle_datastoragelocation=Datastoragelocation +EditTrsaRegistryLabel_dataaccessinfo=Dataaccessinfo: +EditTrsaRegistryRequiredMessage_dataaccessinfo=The Dataaccessinfo field is required. +EditTrsaRegistryTitle_dataaccessinfo=Dataaccessinfo +EditTrsaRegistryLabel_notaryserviceurl=Notaryserviceurl: +EditTrsaRegistryTitle_notaryserviceurl=Notaryserviceurl +EditTrsaRegistryLabel_safeserviceurl=Safeserviceurl: +EditTrsaRegistryTitle_safeserviceurl=Safeserviceurl +EditTrsaRegistryLabel_registertime=Registertime: +EditTrsaRegistryTitle_registertime=Registertime +EditTrsaRegistryLabel_disabled=Disabled: +EditTrsaRegistryTitle_disabled=Disabled +EditTrsaRegistryLabel_expiretime=Expiretime: +EditTrsaRegistryTitle_expiretime=Expiretime +EditTrsaRegistryLabel_id=Id: +EditTrsaRegistryRequiredMessage_id=The Id field is required. +EditTrsaRegistryTitle_id=Id +ViewTrsaRegistryTitle=View +ViewTrsaRegistryDestroyLink=Destroy +ViewTrsaRegistryEditLink=Edit +ViewTrsaRegistryCreateLink=Create New TrsaRegistry +ViewTrsaRegistryShowAllLink=Show All TrsaRegistry Items +ViewTrsaRegistryIndexLink=Index +ViewTrsaRegistryLabel_installation=Installation: +ViewTrsaRegistryTitle_installation=Installation +ViewTrsaRegistryLabel_email=Email: +ViewTrsaRegistryTitle_email=Email +ViewTrsaRegistryLabel_dataverseurl=Dataverseurl: +ViewTrsaRegistryTitle_dataverseurl=Dataverseurl +ViewTrsaRegistryLabel_apitoken=Apitoken: +ViewTrsaRegistryTitle_apitoken=Apitoken +ViewTrsaRegistryLabel_datastoragelocation=Datastoragelocation: +ViewTrsaRegistryTitle_datastoragelocation=Datastoragelocation +ViewTrsaRegistryLabel_dataaccessinfo=Dataaccessinfo: +ViewTrsaRegistryTitle_dataaccessinfo=Dataaccessinfo +ViewTrsaRegistryLabel_notaryserviceurl=Notaryserviceurl: +ViewTrsaRegistryTitle_notaryserviceurl=Notaryserviceurl +ViewTrsaRegistryLabel_safeserviceurl=Safeserviceurl: +ViewTrsaRegistryTitle_safeserviceurl=Safeserviceurl +ViewTrsaRegistryLabel_registertime=Registertime: +ViewTrsaRegistryTitle_registertime=Registertime +ViewTrsaRegistryLabel_disabled=Disabled: +ViewTrsaRegistryTitle_disabled=Disabled +ViewTrsaRegistryLabel_expiretime=Expiretime: +ViewTrsaRegistryTitle_expiretime=Expiretime +ViewTrsaRegistryLabel_id=Id: +ViewTrsaRegistryTitle_id=Id +ListTrsaRegistryTitle=List +ListTrsaRegistryEmpty=(No TrsaRegistry Items Found) +ListTrsaRegistryDestroyLink=Destroy +ListTrsaRegistryEditLink=Edit +ListTrsaRegistryViewLink=View +ListTrsaRegistryCreateLink=Create New TrsaRegistry +ListTrsaRegistryIndexLink=Index +ListTrsaRegistryTitle_installation=Installation +ListTrsaRegistryTitle_email=Email +ListTrsaRegistryTitle_dataverseurl=Dataverseurl +ListTrsaRegistryTitle_apitoken=Apitoken +ListTrsaRegistryTitle_datastoragelocation=Datastoragelocation +ListTrsaRegistryTitle_dataaccessinfo=Dataaccessinfo +ListTrsaRegistryTitle_notaryserviceurl=Notaryserviceurl +ListTrsaRegistryTitle_safeserviceurl=Safeserviceurl +ListTrsaRegistryTitle_registertime=Registertime +ListTrsaRegistryTitle_disabled=Disabled +ListTrsaRegistryTitle_expiretime=Expiretime +ListTrsaRegistryTitle_id=Id + + +dashboardCardTrsaRegistryHeader=TRSA(Trusted Remote Storage Agent) Registry +dashboardCardTrsaRegistryManage=Manage TRSAs \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/api/BatchImport.java b/src/main/java/edu/harvard/iq/dataverse/api/BatchImport.java index 7b44d920fbe..57b8c13ef47 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/BatchImport.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/BatchImport.java @@ -1,5 +1,7 @@ package edu.harvard.iq.dataverse.api; +import com.thoughtworks.xstream.XStream; +import com.thoughtworks.xstream.io.json.JsonHierarchicalStreamDriver; import edu.harvard.iq.dataverse.api.imports.ImportServiceBean; import edu.harvard.iq.dataverse.DatasetFieldServiceBean; import edu.harvard.iq.dataverse.DatasetServiceBean; @@ -8,19 +10,26 @@ import edu.harvard.iq.dataverse.MetadataBlockServiceBean; import edu.harvard.iq.dataverse.api.imports.ImportException; +import edu.harvard.iq.dataverse.api.imports.ImportUtil.ImportFileType; import edu.harvard.iq.dataverse.api.imports.ImportUtil.ImportType; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import java.io.IOException; +import java.io.InputStream; import java.io.PrintWriter; +import java.util.logging.Level; +import java.util.logging.Logger; import javax.ejb.EJB; import javax.ejb.Stateless; import javax.json.JsonObjectBuilder; +import javax.ws.rs.Consumes; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.QueryParam; +import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import org.glassfish.jersey.media.multipart.FormDataParam; @Stateless @Path("batch") @@ -41,10 +50,13 @@ public class BatchImport extends AbstractApiBean { @EJB BatchServiceBean batchService; + private static final Logger logger = Logger.getLogger(BatchImport.class.getName()); + static XStream xstream = new XStream(new JsonHierarchicalStreamDriver()); + @GET @Path("harvest") public Response harvest(@QueryParam("path") String fileDir, @QueryParam("dv") String parentIdtf, @QueryParam("createDV") Boolean createDV, @QueryParam("key") String apiKey) throws IOException { - return startBatchJob(fileDir, parentIdtf, apiKey, ImportType.HARVEST, createDV); + return startBatchJob(fileDir, parentIdtf, apiKey, ImportType.HARVEST, createDV, ImportFileType.XML); } @@ -84,6 +96,55 @@ public Response postImport(String body, @QueryParam("dv") String parentIdtf, @Qu } } + + + + + /** + * Import a new Dataset with JSON data posted in the request + * + * @param body the JSON + * @param parentIdtf the dataverse to import into (id or alias) + * @param apiKey user's api key + * @return import status (including id of the dataset created) + */ + @POST + @Path("importJson") + public Response postImportJson(String body, @QueryParam("dv") String parentIdtf, @QueryParam("key") String apiKey) { + + + DataverseRequest dataverseRequest; + try { + dataverseRequest = createDataverseRequest(findAuthenticatedUserOrDie()); + } catch (WrappedResponse wr) { + return wr.getResponse(); + } + + if (parentIdtf == null) { + parentIdtf = "root"; + } else { + logger.log(Level.INFO, "dvId={0}", parentIdtf); + } + Dataverse owner = findDataverse(parentIdtf); + logger.log(Level.INFO, "Dataverse:alias={0}", owner.getAlias()); + if (owner == null) { + return error(Response.Status.NOT_FOUND, "Can't find dataverse with identifier='" + parentIdtf + "'"); + } + try { + PrintWriter cleanupLog = null; // Cleanup log isn't needed for ImportType == NEW. We don't do any data cleanup in this mode. + String filename = null; // Since this is a single input from a POST, there is no file that we are reading from. + JsonObjectBuilder status = importService.doImportJson(dataverseRequest, owner, body, filename, ImportType.NEW, cleanupLog); + return this.ok(status); + } catch (ImportException | IOException e) { + return this.error(Response.Status.BAD_REQUEST, e.getMessage()); + } + } + + + + + + /** * Import single or multiple datasets that are in the local filesystem * @@ -97,11 +158,34 @@ public Response postImport(String body, @QueryParam("dv") String parentIdtf, @Qu @Path("import") public Response getImport(@QueryParam("path") String fileDir, @QueryParam("dv") String parentIdtf, @QueryParam("createDV") Boolean createDV, @QueryParam("key") String apiKey) { - return startBatchJob(fileDir, parentIdtf, apiKey, ImportType.NEW, createDV); + return startBatchJob(fileDir, parentIdtf, apiKey, ImportType.NEW, createDV, ImportFileType.XML); } - private Response startBatchJob(String fileDir, String parentIdtf, String apiKey, ImportType importType, Boolean createDV) { + + + + /** + * Import single or multiple datasets that are in the local filesystem + * + * @param fileDir the absolute path of the file or directory (all files + * within the directory will be imported + * @param parentIdtf the dataverse to import into (id or alias) + * @param apiKey user's api key + * @return import status (including id's of the datasets created) + */ + @GET + @Path("importJson") + public Response getImportJson(@QueryParam("path") String fileDir, @QueryParam("dv") String parentIdtf, @QueryParam("createDV") Boolean createDV, @QueryParam("key") String apiKey) { + + return startBatchJob(fileDir, parentIdtf, apiKey, ImportType.NEW, createDV, ImportFileType.JSON); + + } + + + + + private Response startBatchJob(String fileDir, String parentIdtf, String apiKey, ImportType importType, Boolean createDV, ImportFileType importFileType) { if (createDV == null) { createDV = Boolean.FALSE; } @@ -123,7 +207,7 @@ private Response startBatchJob(String fileDir, String parentIdtf, String apiKey, return error(Response.Status.NOT_FOUND, "Can't find dataverse with identifier='" + parentIdtf + "'"); } } - batchService.processFilePath(fileDir, parentIdtf, dataverseRequest, owner, importType, createDV); + batchService.processFilePath(fileDir, parentIdtf, dataverseRequest, owner, importType, createDV, importFileType); } catch (ImportException e) { return this.error(Response.Status.BAD_REQUEST, "Import Exception, " + e.getMessage()); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/BatchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/BatchServiceBean.java index 8fe58298481..e6e4365dc2d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/BatchServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/BatchServiceBean.java @@ -5,6 +5,7 @@ import edu.harvard.iq.dataverse.DataverseServiceBean; import edu.harvard.iq.dataverse.api.imports.ImportException; import edu.harvard.iq.dataverse.api.imports.ImportUtil; +import edu.harvard.iq.dataverse.api.imports.ImportUtil.ImportFileType; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import java.io.File; import java.io.FileWriter; @@ -36,7 +37,7 @@ public class BatchServiceBean { @Asynchronous - public void processFilePath(String fileDir, String parentIdtf, DataverseRequest dataverseRequest, Dataverse owner, ImportUtil.ImportType importType, Boolean createDV) { + public void processFilePath(String fileDir, String parentIdtf, DataverseRequest dataverseRequest, Dataverse owner, ImportUtil.ImportType importType, Boolean createDV, ImportFileType importFileType) { logger.info("BEGIN IMPORT"); PrintWriter validationLog = null; PrintWriter cleanupLog = null; @@ -54,13 +55,13 @@ public void processFilePath(String fileDir, String parentIdtf, DataverseRequest if (!file.isHidden()) { if (file.isDirectory()) { try { - status.add(handleDirectory(dataverseRequest, file, importType, validationLog, cleanupLog, createDV)); + status.add(handleDirectory(dataverseRequest, file, importType, validationLog, cleanupLog, createDV, importFileType)); } catch (ImportException e) { logger.log(Level.SEVERE, "Exception in handleDirectory() for "+ file.getName(),e); } } else { try { - status.add(importService.handleFile(dataverseRequest, owner, file, importType, validationLog, cleanupLog)); + status.add(importService.handleFile(dataverseRequest, owner, file, importType, validationLog, cleanupLog, importFileType)); } catch(ImportException e) { logger.log(Level.SEVERE, "Exception in handleFile() for "+ file.getName(),e); } @@ -69,7 +70,7 @@ public void processFilePath(String fileDir, String parentIdtf, DataverseRequest } } } else { - status.add(importService.handleFile(dataverseRequest, owner, dir, importType, validationLog, cleanupLog)); + status.add(importService.handleFile(dataverseRequest, owner, dir, importType, validationLog, cleanupLog, importFileType)); } } @@ -83,7 +84,7 @@ public void processFilePath(String fileDir, String parentIdtf, DataverseRequest } - public JsonArrayBuilder handleDirectory(DataverseRequest dataverseRequest, File dir, ImportUtil.ImportType importType, PrintWriter validationLog, PrintWriter cleanupLog, Boolean createDV) throws ImportException{ + public JsonArrayBuilder handleDirectory(DataverseRequest dataverseRequest, File dir, ImportUtil.ImportType importType, PrintWriter validationLog, PrintWriter cleanupLog, Boolean createDV, ImportFileType importFileType) throws ImportException{ JsonArrayBuilder status = Json.createArrayBuilder(); Dataverse owner = dataverseService.findByAlias(dir.getName()); if (owner == null ) { @@ -97,7 +98,7 @@ public JsonArrayBuilder handleDirectory(DataverseRequest dataverseRequest, File for (File file : dir.listFiles()) { if (!file.isHidden()) { try { - JsonObjectBuilder fileStatus = importService.handleFile(dataverseRequest, owner, file, importType, validationLog, cleanupLog); + JsonObjectBuilder fileStatus = importService.handleFile(dataverseRequest, owner, file, importType, validationLog, cleanupLog, importFileType); status.add(fileStatus); } catch (ImportException | IOException e) { status.add(Json.createObjectBuilder().add("importStatus", "Exception importing " + file.getName() + ", message = " + e.getMessage())); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 4f868d90ae7..b436db1a025 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -1,5 +1,7 @@ package edu.harvard.iq.dataverse.api; +import com.thoughtworks.xstream.XStream; +import com.thoughtworks.xstream.io.json.JsonHierarchicalStreamDriver; import edu.harvard.iq.dataverse.ControlledVocabularyValue; import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.DataFileServiceBean; @@ -80,6 +82,7 @@ import edu.harvard.iq.dataverse.util.EjbUtil; import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.json.JsonParseException; +import edu.harvard.iq.dataverse.util.json.JsonPrinter; import edu.harvard.iq.dataverse.search.IndexServiceBean; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.*; import java.io.IOException; @@ -105,6 +108,7 @@ import javax.json.JsonArrayBuilder; import javax.json.JsonObject; import javax.json.JsonObjectBuilder; +import javax.json.stream.JsonParsingException; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; @@ -125,7 +129,7 @@ public class Datasets extends AbstractApiBean { private static final Logger logger = Logger.getLogger(Datasets.class.getCanonicalName()); - + static XStream xstream = new XStream(new JsonHierarchicalStreamDriver()); @Inject DataverseSession session; @EJB @@ -410,7 +414,7 @@ public Response updateDatasetPIDMetadataAll() { @PUT @Path("{id}/versions/{versionId}") public Response updateDraftVersion( String jsonBody, @PathParam("id") String id, @PathParam("versionId") String versionId ){ - + logger.log(Level.INFO, "updateDraftVersion:jsonBody={0}", jsonBody); if ( ! ":draft".equals(versionId) ) { return error( Response.Status.BAD_REQUEST, "Only the :draft version can be updated"); } @@ -418,9 +422,42 @@ public Response updateDraftVersion( String jsonBody, @PathParam("id") String id, try ( StringReader rdr = new StringReader(jsonBody) ) { DataverseRequest req = createDataverseRequest(findUserOrDie()); Dataset ds = findDatasetOrDie(id); + + if (ds.getCreateDate()!=null){ + logger.log(Level.INFO, "dataset:creationdate={0}", ds.getCreateDate()); + } else { + logger.log(Level.INFO, "dataset:creationdate is not set"); + } + + + + logger.log(Level.INFO, "target dataset is retrieved={0}", + xstream.toXML(ds)); + JsonObject json = Json.createReader(rdr).readObject(); + logger.log(Level.INFO, "calling 1-arg-jsonParser().parseDatasetVersion()"); DatasetVersion incomingVersion = jsonParser().parseDatasetVersion(json); + + + if (incomingVersion.getDataset() !=null){ + logger.log(Level.INFO, "this case has a dataset: copy datafile info"); + for (DataFile df: incomingVersion.getDataset().getFiles()){ + df.setOwner(ds); + } + + // datafile info was attached + ds.setFiles(incomingVersion.getDataset().getFiles()); + logger.log(Level.INFO, "updated dataset with datafiles={0}", + xstream.toXML(ds)); + } + logger.log(Level.INFO, "incomingVersion:updateDraftVersion={0}", + xstream.toXML(incomingVersion)); + + + + + // clear possibly stale fields from the incoming dataset version. // creation and modification dates are updated by the commands. incomingVersion.setId(null); @@ -428,18 +465,23 @@ public Response updateDraftVersion( String jsonBody, @PathParam("id") String id, incomingVersion.setMinorVersionNumber(null); incomingVersion.setVersionState(DatasetVersion.VersionState.DRAFT); incomingVersion.setDataset(ds); + + + incomingVersion.setCreateTime(null); incomingVersion.setLastUpdateTime(null); boolean updateDraft = ds.getLatestVersion().isDraft(); DatasetVersion managedVersion; if ( updateDraft ) { + logger.log(Level.INFO, "updateDraft: yes"); final DatasetVersion editVersion = ds.getEditVersion(); editVersion.setDatasetFields(incomingVersion.getDatasetFields()); editVersion.setTermsOfUseAndAccess( incomingVersion.getTermsOfUseAndAccess() ); Dataset managedDataset = execCommand(new UpdateDatasetVersionCommand(ds, req)); managedVersion = managedDataset.getEditVersion(); } else { + logger.log(Level.INFO, "updateDraft: no"); managedVersion = execCommand(new CreateDatasetVersionCommand(req, ds, incomingVersion)); } // DatasetVersion managedVersion = execCommand( updateDraft @@ -616,7 +658,7 @@ private String getCompoundDisplayValue (DatasetFieldCompoundValue dscv){ @PUT @Path("{id}/editMetadata") public Response editVersionMetadata(String jsonBody, @PathParam("id") String id, @QueryParam("replace") Boolean replace) throws WrappedResponse{ - + logger.log(Level.INFO, "jsonBody:editVersionMetadata ={0}", jsonBody); Boolean replaceData = replace != null; DataverseRequest req = createDataverseRequest(findUserOrDie()); @@ -630,8 +672,9 @@ private Response processDatasetUpdate(String jsonBody, String id, DataverseReque Dataset ds = findDatasetOrDie(id); JsonObject json = Json.createReader(rdr).readObject(); + logger.log(Level.INFO, "json:processDatasetUpdate={0}", xstream.toXML(json)); DatasetVersion dsv = ds.getEditVersion(); - + logger.log(Level.INFO, "DatasetVersion:processDatasetUpdate={0}", xstream.toXML(dsv)); List fields = new LinkedList<>(); DatasetField singleField = null; @@ -1350,6 +1393,107 @@ public Response addFileToDataset(@PathParam("id") String idSupplied, } // end: addFileToDataset + + /** + * Add DataFile-related Metadata to an existing Dataset without invoking + * an ingest request + * + * @param jsonBody + * @param datasetId + * @return + */ + @POST + @Path("{id}/addFileMetadata") + public Response addFileMetadataToDataset(String jsonBody, @PathParam("id") String datasetId + ){ + try ( StringReader rdr = new StringReader(jsonBody) ) { + logger.log(Level.INFO, "addFileMetadataToDataset:received jsonBody={0}", jsonBody); + DataverseRequest req = createDataverseRequest(findUserOrDie()); + // retrieve dataset by id + logger.log(Level.INFO, "datasetId={0}", datasetId); + Dataset ds = findDatasetOrDie(datasetId); + logger.log(Level.INFO, "dataset display name={0}", ds.getDisplayName()); + // parse jsonbody and create metadata instances + JsonObject json = Json.createReader(rdr).readObject(); + + // step_060 + // set datafile's owner + // add each datafile to the list of datafile of dataset + logger.log(Level.INFO, "dataset: editversion={0}", ds.getEditVersion().getVersion()); + // step_070 + DatasetVersion incomingVersion = jsonParser().parseDatasetVersion(json, ds.getEditVersion()); + + // check datafile is owned by the dataset + List dataFiles = incomingVersion.getDataset().getFiles(); + if (dataFiles != null) { + + + if (dataFiles.isEmpty()){ + throw new NoFilesException("newlyAddedFiles is empty!"); + } + for (DataFile df : dataFiles) { + if (df.getOwner() == null){ + // df.setOwner(ds); + logger.log(Level.INFO, "df is not owned by any dataset"); + } else { + logger.log(Level.INFO, "df is owned by dataset whose id={0}", df.getOwner().getId()); + } + } + + + + } else { + logger.log(Level.INFO, "no datafile was parsed"); + throw new NullPointerException("newlyAddedFiles is null!"); + + + } + // call UpdateDatasetVersion command + + ds = execCommand(new UpdateDatasetVersionCommand(ds, req)); + + // save datafiles + // from addFileToDataset case + dataFiles = ds.getFiles(); + + if (dataFiles != null) { + for (DataFile df : dataFiles) { + if (df.getOwner() == null){ + // df.setOwner(ds); + logger.log(Level.INFO, "df is not owned by any dataset"); + + df=fileService.save(df); + } else { + logger.log(Level.INFO, "df is owned by dataset whose id={0}", df.getOwner().getId()); + } + } + + + + } else { + logger.log(Level.INFO, "no datafile was parsed"); + dataFiles= new ArrayList<>(); + } + + logger.log(Level.INFO, "newlyAddedFiles: how many={0}", dataFiles.size()); + logger.log(Level.INFO, "newlyAddedFiles={0}", xstream.toXML(dataFiles)); + + return ok( JsonPrinter.jsonDataFileList(dataFiles)); + } catch (JsonParseException ex) { + logger.log(Level.SEVERE, "Semantic error parsing dataset version Json: " + ex.getMessage(), ex); + return error( Response.Status.BAD_REQUEST, "Error parsing dataset: " + ex.getMessage() ); + + } catch (NoFilesException ex) { + logger.log(Level.SEVERE, null, ex); + return error(Response.Status.BAD_REQUEST, "Datafile was not found: " + ex.getMessage() ); + } catch (WrappedResponse ex) { + logger.log(Level.SEVERE, null, ex); + return ex.getResponse(); + + } + } + + private void msg(String m){ //System.out.println(m); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 094ffbc06b2..986b5f48a05 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1,5 +1,7 @@ package edu.harvard.iq.dataverse.api; +import com.thoughtworks.xstream.XStream; +import com.thoughtworks.xstream.io.json.JsonHierarchicalStreamDriver; import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetFieldType; @@ -112,7 +114,7 @@ public class Dataverses extends AbstractApiBean { private static final Logger logger = Logger.getLogger(Dataverses.class.getCanonicalName()); - + static XStream xstream = new XStream(new JsonHierarchicalStreamDriver()); @EJB ExplicitGroupServiceBean explicitGroupSvc; @@ -213,7 +215,9 @@ public Response createDataset(String jsonBody, @PathParam("identifier") String p try { User u = findUserOrDie(); Dataverse owner = findDataverseOrDie(parentIdtf); + logger.log(Level.INFO, "jsonBody:createDataset={0}", jsonBody); Dataset ds = parseDataset(jsonBody); + logger.log(Level.INFO, "dataset={0}", xstream.toXML(ds)); ds.setOwner(owner); if (ds.getVersions().isEmpty()) { @@ -247,9 +251,11 @@ public Response createDataset(String jsonBody, @PathParam("identifier") String p @Path("{identifier}/datasets/:import") public Response importDataset(String jsonBody, @PathParam("identifier") String parentIdtf, @QueryParam("pid") String pidParam, @QueryParam("release") String releaseParam) { try { + logger.log(Level.INFO, "jsonBody:importDataset={0}", jsonBody); User u = findUserOrDie(); Dataverse owner = findDataverseOrDie(parentIdtf); Dataset ds = parseDataset(jsonBody); + logger.log(Level.INFO, "dataset:importDataset={0}", xstream.toXML(ds)); ds.setOwner(owner); if (ds.getVersions().isEmpty()) { @@ -379,7 +385,9 @@ public Response importDatasetDdi(String xml, @PathParam("identifier") String par } private Dataset parseDataset(String datasetJson) throws WrappedResponse { + logger.log(Level.INFO, "parseDataset is called"); try (StringReader rdr = new StringReader(datasetJson)) { + logger.log(Level.INFO, "calling jsonParser().parseDataset()"); return jsonParser().parseDataset(Json.createReader(rdr).readObject()); } catch (JsonParsingException | JsonParseException jpe) { logger.log(Level.SEVERE, "Error parsing dataset json. Json: {0}", datasetJson); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java index 62a213ecf39..f8dcf25a4d7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java @@ -5,7 +5,7 @@ import edu.harvard.iq.dataverse.DatasetFieldType; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.DatasetVersion.VersionState; -import edu.harvard.iq.dataverse.api.dto.*; +import edu.harvard.iq.dataverse.api.dto.*; import edu.harvard.iq.dataverse.api.dto.FieldDTO; import edu.harvard.iq.dataverse.api.dto.MetadataBlockDTO; import edu.harvard.iq.dataverse.api.imports.ImportUtil.ImportType; @@ -33,15 +33,17 @@ * * @author ellenk */ -// TODO: -// does this need to be a service bean/stateless? - could be transformed into -// a util with static methods. +// TODO: +// does this need to be a service bean/stateless? - could be transformed into +// a util with static methods. // (it would need to be passed the fields service beans as arguments) // -- L.A. 4.5 @Stateless public class ImportDDIServiceBean { + + private static final Logger logger = Logger.getLogger(ImportDDIServiceBean.class.getName()); public static final String SOURCE_DVN_3_0 = "DVN_3_0"; - + public static final String NAMING_PROTOCOL_HANDLE = "hdl"; public static final String NAMING_PROTOCOL_DOI = "doi"; public static final String AGENCY_HANDLE = "handle"; @@ -54,7 +56,7 @@ public class ImportDDIServiceBean { public static final String CAT_STAT_TYPE_FREQUENCY = "freq"; public static final String VAR_FORMAT_TYPE_NUMERIC = "numeric"; public static final String VAR_FORMAT_SCHEMA_ISO = "ISO"; - + public static final String EVENT_START = "start"; public static final String EVENT_END = "end"; @@ -84,7 +86,7 @@ public class ImportDDIServiceBean { public static final String NOTE_TYPE_ARCHIVE_DATE = "DVN:ARCHIVE_DATE"; public static final String NOTE_SUBJECT_ARCHIVE_DATE= "Archive Date"; - + public static final String NOTE_TYPE_EXTENDED_METADATA = "DVN:EXTENDED_METADATA"; public static final String NOTE_TYPE_LOCKSS_CRAWL = "LOCKSS:CRAWLING"; @@ -93,13 +95,23 @@ public class ImportDDIServiceBean { public static final String NOTE_TYPE_REPLICATION_FOR = "DVN:REPLICATION_FOR"; private static final String HARVESTED_FILE_STORAGE_PREFIX = "http://"; private XMLInputFactory xmlInputFactory = null; - private static final Logger logger = Logger.getLogger(ImportDDIServiceBean.class.getName()); - + @EJB CustomFieldServiceBean customFieldService; - + @EJB DatasetFieldServiceBean datasetFieldService; - - + + + // required field flags + // subject + boolean isSubjectAdded=false; + String subjectText; + String defaultSubjectText="Other"; + // email + boolean isContactEmailAdded = false; + String defaultEmailAddress = "emailAddressNotFound@dataverse.org"; + boolean isAbstractAdded = false; + String defaultAbsractText="Required description text(abstract tag) was not found"; + // TODO: stop passing the xml source as a string; (it could be huge!) -- L.A. 4.5 // TODO: what L.A. Said. public DatasetDTO doImport(ImportType importType, String xmlToParse) throws XMLStreamException, ImportException { @@ -107,23 +119,23 @@ public DatasetDTO doImport(ImportType importType, String xmlToParse) throws XMLS xmlInputFactory.setProperty("javax.xml.stream.isCoalescing", java.lang.Boolean.TRUE); DatasetDTO datasetDTO = this.initializeDataset(); // Read docDescr and studyDesc into DTO objects. - // TODO: the fileMap is likely not needed. + // TODO: the fileMap is likely not needed. Map fileMap = mapDDI(importType, xmlToParse, datasetDTO); return datasetDTO; } - + public void importFileMetadata(DatasetVersion dv, String xmlToParse) { - - } - + + } + private boolean isHarvestImport(ImportType importType) { return importType.equals(ImportType.HARVEST); } - + private boolean isNewImport(ImportType importType) { return importType.equals(ImportType.NEW); } - + public Map mapDDI(ImportType importType, String xmlToParse, DatasetDTO datasetDTO) throws XMLStreamException, ImportException { Map filesMap = new HashMap<>(); @@ -136,8 +148,8 @@ public Map mapDDI(ImportType importType, String xmlToParse, Data return filesMap; } - - + + public Map mapDDI(ImportType importType, File ddiFile, DatasetDTO datasetDTO ) throws ImportException { FileInputStream in = null; XMLStreamReader xmlr = null; @@ -165,9 +177,9 @@ public Map mapDDI(ImportType importType, File ddiFile, DatasetD return filesMap; } - + private void processDDI(ImportType importType, XMLStreamReader xmlr, DatasetDTO datasetDTO, Map filesMap) throws XMLStreamException, ImportException { - + // make sure we have a codeBook //while ( xmlr.next() == XMLStreamConstants.COMMENT ); // skip pre root comments xmlr.nextTag(); @@ -179,27 +191,41 @@ private void processDDI(ImportType importType, XMLStreamReader xmlr, DatasetDTO // in a harvested DDI). String codeBookLevelId = xmlr.getAttributeValue(null, "ID"); - - // (but first we will parse and process the entire DDI - and only - // then add this codeBook-level id to the list of identifiers; i.e., + + // (but first we will parse and process the entire DDI - and only + // then add this codeBook-level id to the list of identifiers; i.e., // we don't want it to be the first on the list, if one or more - // ids are available in the studyDscr section - those should take + // ids are available in the studyDscr section - those should take // precedence!) - // In fact, we should only use these IDs when no ID is available down - // in the study description section! - + // In fact, we should only use these IDs when no ID is available down + // in the study description section! + processCodeBook(importType, xmlr, datasetDTO, filesMap); MetadataBlockDTO citationBlock = datasetDTO.getDatasetVersion().getMetadataBlocks().get("citation"); - - if (codeBookLevelId != null && !codeBookLevelId.isEmpty()) { + + if (StringUtils.isNotBlank(codeBookLevelId)) { if (citationBlock.getField("otherId")==null) { - // this means no ids were found during the parsing of the - // study description section. we'll use the one we found in + logger.log(Level.INFO, "field:otherId does not exist"); + List> otherIds = new ArrayList<>(); + HashSet set = new HashSet<>(); + addToSet(set, "otherIdValue", codeBookLevelId); + if (!set.isEmpty()){ + otherIds.add(set); + } + citationBlock.addField(FieldDTO.createMultipleCompoundFieldDTO("otherId", otherIds)); + // this means no ids were found during the parsing of the + // study description section. we'll use the one we found in // the codeBook entry: - FieldDTO otherIdValue = FieldDTO.createPrimitiveFieldDTO("otherIdValue", codeBookLevelId); - FieldDTO otherId = FieldDTO.createCompoundFieldDTO("otherId", otherIdValue); - citationBlock.getFields().add(otherId); - + // FieldDTO otherIdValue = FieldDTO.createPrimitiveFieldDTO("otherIdValue", codeBookLevelId); + // FieldDTO otherId = FieldDTO.createCompoundFieldDTO("otherId", otherIdValue); + // citationBlock.getFields().add(otherId); + + + + + + } else { + logger.log(Level.INFO, "field:otherId already exits"); } } @@ -210,7 +236,7 @@ private void processDDI(ImportType importType, XMLStreamReader xmlr, DatasetDTO else { datasetDTO.getDatasetVersion().setVersionState(VersionState.DRAFT); } - + } public DatasetDTO initializeDataset() { @@ -219,18 +245,18 @@ public DatasetDTO initializeDataset() { datasetDTO.setDatasetVersion(datasetVersionDTO); HashMap metadataBlocks = new HashMap<>(); datasetVersionDTO.setMetadataBlocks(metadataBlocks); - + datasetVersionDTO.getMetadataBlocks().put("citation", new MetadataBlockDTO()); datasetVersionDTO.getMetadataBlocks().get("citation").setFields(new ArrayList<>()); datasetVersionDTO.getMetadataBlocks().put("socialscience", new MetadataBlockDTO()); datasetVersionDTO.getMetadataBlocks().get("socialscience").setFields(new ArrayList<>()); datasetVersionDTO.getMetadataBlocks().put("geospatial", new MetadataBlockDTO()); datasetVersionDTO.getMetadataBlocks().get("geospatial").setFields(new ArrayList<>()); - + return datasetDTO; - + } - + // Read the XMLStream, and populate datasetDTO and filesMap private void processCodeBook(ImportType importType, XMLStreamReader xmlr, DatasetDTO datasetDTO, Map filesMap) throws XMLStreamException, ImportException { for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) { @@ -241,42 +267,42 @@ private void processCodeBook(ImportType importType, XMLStreamReader xmlr, Datase processStdyDscr(importType, xmlr, datasetDTO); } else if (xmlr.getLocalName().equals("otherMat") && (isNewImport(importType) || isHarvestImport(importType)) ) { processOtherMat(xmlr, datasetDTO); - } else if (xmlr.getLocalName().equals("fileDscr") && isHarvestImport(importType)) { - // If this is a harvesting import, we'll attempt to extract some minimal - // file-level metadata information from the fileDscr sections as well. + } else if (xmlr.getLocalName().equals("fileDscr") && isHarvestImport(importType)) { + // If this is a harvesting import, we'll attempt to extract some minimal + // file-level metadata information from the fileDscr sections as well. // TODO: add more info here... -- 4.6 processFileDscrMinimal(xmlr, datasetDTO); } else if (xmlr.getLocalName().equals("fileDscr") && isNewImport(importType)) { - // this is a "full" fileDscr section - Dataverses use it + // this is a "full" fileDscr section - Dataverses use it // to encode *tabular* files only. It will contain the information - // about variables, observations, etc. It will be complemented - // by a number of entries in the dataDscr section. - // Dataverses do not use this section for harvesting exports, since - // we don't harvest tabular metadata. And all the "regular" - // file-level metadata is encoded in otherMat sections. - // The goal is to one day be able to import such tabular - // metadata using the direct (non-harvesting) import API. + // about variables, observations, etc. It will be complemented + // by a number of entries in the dataDscr section. + // Dataverses do not use this section for harvesting exports, since + // we don't harvest tabular metadata. And all the "regular" + // file-level metadata is encoded in otherMat sections. + // The goal is to one day be able to import such tabular + // metadata using the direct (non-harvesting) import API. // EMK TODO: add this back in for ImportType.NEW //processFileDscr(xmlr, datasetDTO, filesMap); - } + } } else if (event == XMLStreamConstants.END_ELEMENT) { if (xmlr.getLocalName().equals("codeBook")) return; } } } - + private void processDocDscr(XMLStreamReader xmlr, DatasetDTO datasetDTO) throws XMLStreamException { for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) { if (event == XMLStreamConstants.START_ELEMENT) { - + if (xmlr.getLocalName().equals("IDNo") && StringUtil.isEmpty(datasetDTO.getIdentifier()) ) { // this will set a StudyId if it has not yet been set; it will get overridden by a metadata // id in the StudyDscr section, if one exists if ( AGENCY_HANDLE.equals( xmlr.getAttributeValue(null, "agency") ) ) { parseStudyIdHandle( parseText(xmlr), datasetDTO ); - } - // EMK TODO: we need to save this somewhere when we add harvesting infrastructure + } + // EMK TODO: we need to save this somewhere when we add harvesting infrastructure } /*else if ( xmlr.getLocalName().equals("holdings") && StringUtil.isEmpty(datasetDTO..getHarvestHoldings()) ) { metadata.setHarvestHoldings( xmlr.getAttributeValue(null, "URI") ); }*/ @@ -284,7 +310,7 @@ private void processDocDscr(XMLStreamReader xmlr, DatasetDTO datasetDTO) throws if (xmlr.getLocalName().equals("docDscr")) return; } } - } + } private String parseText(XMLStreamReader xmlr) throws XMLStreamException { return parseText(xmlr,true); } @@ -302,7 +328,7 @@ private String parseDate (XMLStreamReader xmlr, String endTag) throws XMLStreamE date = parseText(xmlr); } return date; - } + } /* We had to add this method because the ref getElementText has a bug where it * would append a null before the text, if there was an escaped apostrophe; it appears * that the code finds an null ENTITY_REFERENCE in this case which seems like a bug; @@ -335,20 +361,20 @@ private String getElementText(XMLStreamReader xmlr) throws XMLStreamException { } return content.toString(); } - + private void processStdyDscr(ImportType importType, XMLStreamReader xmlr, DatasetDTO datasetDTO) throws XMLStreamException, ImportException { - + for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) { if (event == XMLStreamConstants.START_ELEMENT) { if (xmlr.getLocalName().equals("citation")) processCitation(importType, xmlr, datasetDTO); else if (xmlr.getLocalName().equals("stdyInfo")) processStdyInfo(xmlr, datasetDTO.getDatasetVersion()); else if (xmlr.getLocalName().equals("method")) processMethod(xmlr, datasetDTO.getDatasetVersion()); - - else if (xmlr.getLocalName().equals("dataAccs")) processDataAccs(xmlr, datasetDTO.getDatasetVersion()); - + + else if (xmlr.getLocalName().equals("dataAccs")) processDataAccs(xmlr, datasetDTO.getDatasetVersion()); + else if (xmlr.getLocalName().equals("othrStdyMat")) processOthrStdyMat(xmlr, datasetDTO.getDatasetVersion()); else if (xmlr.getLocalName().equals("notes")) processNotes(xmlr, datasetDTO.getDatasetVersion()); - + } else if (event == XMLStreamConstants.END_ELEMENT) { if (xmlr.getLocalName().equals("stdyDscr")) return; } @@ -384,7 +410,7 @@ private void processOthrStdyMat(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) t relMaterial.add(parseText(xmlr, "relMat")); getCitation(dvDTO).addField(FieldDTO.createMultiplePrimitiveFieldDTO(DatasetFieldConstant.relatedMaterial, relMaterial)); } - } + } else if (xmlr.getLocalName().equals("relStdy")) { List relStudy = new ArrayList<>(); relStudy.add(parseText(xmlr, "relStdy")); @@ -406,7 +432,7 @@ else if (xmlr.getLocalName().equals("relStdy")) { // rp.setIdType((String) rpMap.get("idType")); // rp.setIdNumber((String) rpMap.get("idNumber")); // rp.setUrl((String) rpMap.get("url")); - // TODO: ask about where/whether we want to save this + // TODO: ask about where/whether we want to save this // if (!replicationForFound && rpMap.get("replicationData") != null) { // rp.setReplicationData(true); /// replicationForFound = true; @@ -446,8 +472,8 @@ private void processCitation(ImportType importType, XMLStreamReader xmlr, Datase else if (xmlr.getLocalName().equals("prodStmt")) processProdStmt(xmlr,citation); else if (xmlr.getLocalName().equals("distStmt")) { if (distStatementProcessed) { - // We've already encountered one Distribution Statement in - // this citation, we'll just skip any consecutive ones. + // We've already encountered one Distribution Statement in + // this citation, we'll just skip any consecutive ones. // This is a defensive check against duplicate distStmt // in some DDIs (notably, from ICPSR) } else { @@ -462,7 +488,7 @@ else if (xmlr.getLocalName().equals("notes")) { if (_note != null) { datasetDTO.getDatasetVersion().setUNF( parseUNF( _note ) ); } else { - + processNotes(xmlr,dvDTO); } } @@ -471,53 +497,97 @@ else if (xmlr.getLocalName().equals("notes")) { } } } - - + + /** - * - * + * + * * @param xmlr * @param citation - * @throws XMLStreamException - */ + * @throws XMLStreamException + */ private void processStdyInfo(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) throws XMLStreamException { List> descriptions = new ArrayList<>(); - + for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) { if (event == XMLStreamConstants.START_ELEMENT) { if (xmlr.getLocalName().equals("subject")) { - processSubject(xmlr, getCitation(dvDTO)); +// subjectText = parseText(xmlr); +// if (StringUtils.isBlank(subjectText)){ +// subjectText = defaultSubjectText; +// logger.log(Level.INFO, "subject is set to default"); +// } else { +// logger.log(Level.INFO, "subjectText={0}", subjectText); +// } + + processSubject(xmlr, getCitation(dvDTO)); } else if (xmlr.getLocalName().equals("abstract")) { HashSet set = new HashSet<>(); addToSet(set,"dsDescriptionDate", xmlr.getAttributeValue(null, "date")); Map dsDescriptionDetails = parseCompoundText(xmlr, "abstract"); addToSet(set,"dsDescriptionValue", dsDescriptionDetails.get("name")); if (!set.isEmpty()) { + isAbstractAdded=true; descriptions.add(set); } - - } else if (xmlr.getLocalName().equals("sumDscr")) processSumDscr(xmlr, dvDTO); - - else if (xmlr.getLocalName().equals("notes")) processNotes(xmlr,dvDTO); + + } else if (xmlr.getLocalName().equals("sumDscr")) { + processSumDscr(xmlr, dvDTO); + } else if (xmlr.getLocalName().equals("notes")) { + processNotes(xmlr, dvDTO); + + } else if (xmlr.getLocalName().equals("contact")) { + HashSet set = new HashSet<>(); + + String emailText = xmlr.getAttributeValue(null, "email"); + if (StringUtils.isBlank(emailText)) { + emailText = defaultEmailAddress; + } + + addToSet(set, "datasetContactEmail", emailText); + addToSet(set, "datasetContactAffiliation", xmlr.getAttributeValue(null, "affiliation")); + addToSet(set, "datasetContactName", parseText(xmlr)); + if (!set.isEmpty()) { + + if (getCitation(dvDTO).getField("datasetContact") == null) { + List> datasetContacts = new ArrayList<>(); + datasetContacts.add(set); + getCitation(dvDTO).addField(FieldDTO.createMultipleCompoundFieldDTO("datasetContact", datasetContacts)); + } else { + getCitation(dvDTO).getField("datasetContact").getMultipleCompound().add(set); + } + } + } } else if (event == XMLStreamConstants.END_ELEMENT) { if (xmlr.getLocalName().equals("stdyInfo") ) { - if (descriptions.size()>0) { + if (!isAbstractAdded) { + logger.log(Level.INFO, "abstract tag was not found; default text is supplied because this is one of the required fields"); + HashSet set = new HashSet<>(); + addToSet(set, "dsDescriptionDate", null); + addToSet(set, "dsDescriptionValue", defaultAbsractText); + if (!set.isEmpty()) { + descriptions.add(set); + } + } + if (descriptions.size() > 0) { getCitation(dvDTO).getFields().add(FieldDTO.createMultipleCompoundFieldDTO("dsDescription", descriptions)); } return; } } } - } + } private void processSubject(XMLStreamReader xmlr, MetadataBlockDTO citation) throws XMLStreamException { + logger.log(Level.INFO, "processSubject is called"); + List subjects = new ArrayList<>(); List> keywords = new ArrayList<>(); List> topicClasses = new ArrayList<>(); for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) { if (event == XMLStreamConstants.START_ELEMENT) { - + if (xmlr.getLocalName().equals("keyword")) { HashSet set = new HashSet<>(); - addToSet(set,"keywordVocabulary", xmlr.getAttributeValue(null, "vocab")); + addToSet(set,"keywordVocabulary", xmlr.getAttributeValue(null, "vocab")); addToSet(set, "keywordVocabularyURI", xmlr.getAttributeValue(null, "vocabURI") ); addToSet(set,"keywordValue", parseText(xmlr)); if (!set.isEmpty()) { @@ -525,16 +595,21 @@ private void processSubject(XMLStreamReader xmlr, MetadataBlockDTO citation) thr } } else if (xmlr.getLocalName().equals("topcClas")) { HashSet set = new HashSet<>(); - addToSet(set,"topicClassVocab", xmlr.getAttributeValue(null, "vocab")); + addToSet(set,"topicClassVocab", xmlr.getAttributeValue(null, "vocab")); addToSet(set,"topicClassVocabURI", xmlr.getAttributeValue(null, "vocabURI") ); - addToSet(set,"topicClassValue",parseText(xmlr)); + addToSet(set,"topicClassValue",parseText(xmlr)); if (!set.isEmpty()) { topicClasses.add(set); } - + } } else if (event == XMLStreamConstants.END_ELEMENT) { if (xmlr.getLocalName().equals("subject")) { + subjectText=defaultSubjectText; + subjects.add(subjectText); + citation.getFields().add(FieldDTO.createMultipleVocabFieldDTO("subject", subjects)); + + if (keywords.size()>0) { citation.getFields().add(FieldDTO.createMultipleCompoundFieldDTO("keyword", keywords)); } @@ -545,20 +620,20 @@ private void processSubject(XMLStreamReader xmlr, MetadataBlockDTO citation) thr } } else { // citation.getFields().add(FieldDTO.createPrimitiveFieldDTO( "subject",xmlr.getElementText())); - + } } } - + /** * Process the notes portion of the DDI doc -- if there is one * Return a formatted string - * + * * @param xmlr - * @return + * @return */ private String formatNotesfromXML(XMLStreamReader xmlr) throws XMLStreamException{ - + if (xmlr==null){ throw new NullPointerException("XMLStreamReader xmlr cannot be null"); } @@ -572,25 +647,25 @@ private String formatNotesfromXML(XMLStreamReader xmlr) throws XMLStreamExceptio if (attrVal != null){ noteValues.add("Subject: " + attrVal); } - + // Check for "type" attrVal = xmlr.getAttributeValue(null, "type"); if (attrVal != null){ noteValues.add("Type: " + attrVal); } - + // Add notes, if they exist attrVal = parseText(xmlr, "notes"); if ((attrVal != null) && (!attrVal.isEmpty())){ noteValues.add("Notes: " + attrVal); - } - + } + // Nothing to add if (noteValues.isEmpty()){ //System.out.println("nuthin'"); return null; } - + //System.out.println(StringUtils.join(noteValues, " ") + ";"); return StringUtils.join(noteValues, " ") + ";"; @@ -601,7 +676,7 @@ private String formatNotesfromXML(XMLStreamReader xmlr) throws XMLStreamExceptio Note Text 2 Note Text 3 */ - + /* // Original, changed b/c of string 'null' appearing in final output String note = " Subject: "+xmlr.getAttributeValue(null, "subject")+" " @@ -610,17 +685,17 @@ private String formatNotesfromXML(XMLStreamReader xmlr) throws XMLStreamExceptio addNote(note, dvDTO); */ } - - + + private void processNotes (XMLStreamReader xmlr, DatasetVersionDTO dvDTO) throws XMLStreamException { - + String formattedNotes = this.formatNotesfromXML(xmlr); - + if (formattedNotes != null){ this.addNote(formattedNotes, dvDTO); } } - + private void addNote(String noteText, DatasetVersionDTO dvDTO ) { MetadataBlockDTO citation = getCitation(dvDTO); FieldDTO field = citation.getField("notesText"); @@ -632,7 +707,7 @@ private void addNote(String noteText, DatasetVersionDTO dvDTO ) { noteValue+= noteText; field.setSinglePrimitive(noteValue); } - + private void processSumDscr(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) throws XMLStreamException { List geoUnit = new ArrayList<>(); List unitOfAnalysis = new ArrayList<>(); @@ -648,13 +723,13 @@ private void processSumDscr(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) throw for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) { if (event == XMLStreamConstants.START_ELEMENT) { if (xmlr.getLocalName().equals("timePrd")) { - + String eventAttr = xmlr.getAttributeValue(null, "event"); if (eventAttr == null || EVENT_SINGLE.equalsIgnoreCase(eventAttr) || EVENT_START.equalsIgnoreCase(eventAttr)) { timePeriodStart = FieldDTO.createPrimitiveFieldDTO("timePeriodCoveredStart", parseDate(xmlr, "timePrd")); } else if (EVENT_END.equals(eventAttr)) { timePeriodEnd = FieldDTO.createPrimitiveFieldDTO("timePeriodCoveredEnd", parseDate(xmlr, "timePrd")); - } + } } else if (xmlr.getLocalName().equals("collDate")) { String eventAttr = xmlr.getAttributeValue(null, "event"); if (eventAttr == null || EVENT_SINGLE.equalsIgnoreCase(eventAttr) || EVENT_START.equalsIgnoreCase(eventAttr)) { @@ -662,7 +737,7 @@ private void processSumDscr(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) throw } else if (EVENT_END.equals(eventAttr)) { dateOfCollectionEnd = FieldDTO.createPrimitiveFieldDTO("dateOfCollectionEnd", parseDate(xmlr, "collDate")); } - + } else if (xmlr.getLocalName().equals("nation")) { HashSet set = new HashSet<>(); set.add(FieldDTO.createVocabFieldDTO("country", parseText(xmlr))); @@ -690,7 +765,7 @@ private void processSumDscr(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) throw if (dateOfCollectionStart!=null || dateOfCollectionEnd!=null) { getCitation(dvDTO).addField(FieldDTO.createMultipleCompoundFieldDTO("dateOfCollection", dateOfCollectionStart, dateOfCollectionEnd)); } - + if (geoUnit.size() > 0) { getGeospatial(dvDTO).addField(FieldDTO.createMultiplePrimitiveFieldDTO("geographicUnit", geoUnit)); } @@ -714,9 +789,9 @@ private void processSumDscr(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) throw } } } - - - + + + private HashSet processGeoBndBox(XMLStreamReader xmlr) throws XMLStreamException { HashSet set = new HashSet<>(); @@ -732,7 +807,7 @@ private HashSet processGeoBndBox(XMLStreamReader xmlr) throws XMLStrea addToSet(set,"northLongitude", parseText(xmlr)); } } else if (event == XMLStreamConstants.END_ELEMENT) { - if (xmlr.getLocalName().equals("geoBndBox")) break; + if (xmlr.getLocalName().equals("geoBndBox")) break; } } return set; @@ -743,14 +818,14 @@ private void processMethod(XMLStreamReader xmlr, DatasetVersionDTO dvDTO ) throw if (xmlr.getLocalName().equals("dataColl")) { processDataColl(xmlr, dvDTO); } else if (xmlr.getLocalName().equals("notes")) { - + String noteType = xmlr.getAttributeValue(null, "type"); if (NOTE_TYPE_EXTENDED_METADATA.equalsIgnoreCase(noteType) ) { - processCustomField(xmlr, dvDTO); + processCustomField(xmlr, dvDTO); } else { processNotes(xmlr, dvDTO); // addNote("Subject: Study Level Error Note, Notes: "+ parseText( xmlr,"notes" ) +";", dvDTO); - + } } else if (xmlr.getLocalName().equals("anlyInfo")) { processAnlyInfo(xmlr, getSocialScience(dvDTO)); @@ -760,7 +835,7 @@ private void processMethod(XMLStreamReader xmlr, DatasetVersionDTO dvDTO ) throw } } } - + private void processCustomField(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) throws XMLStreamException, ImportException { String subject = xmlr.getAttributeValue(null, "subject"); if (!subject.isEmpty()) { @@ -770,10 +845,10 @@ private void processCustomField(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) t String template = subject.substring(subject.indexOf(":") + 1, subject.indexOf(";")); String sourceField = subject.substring(subject.lastIndexOf(":") + 1); String fieldValue = parseText(xmlr); - + CustomFieldMap map = customFieldService.findByTemplateField(template.trim(), sourceField.trim()); - - if (map == null) { + + if (map == null) { throw new ImportException("Did not find mapping for template: "+template+", sourceField: "+sourceField); } if (map.getTargetDatasetField().endsWith("#IGNORE")) { @@ -781,7 +856,7 @@ private void processCustomField(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) t // copy this field from 3.6 to 4.0 return; } - + // 1. Get datasetFieldType for the targetField // 2. find the metadatablock for this field type // 3. If this metadatablock doesn't exist in DTO, create it @@ -822,10 +897,10 @@ private void processCustomField(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) t } } } - + private void handleChildField(MetadataBlockDTO customBlock, DatasetFieldType dsfType, String fieldValue) throws ImportException { DatasetFieldType parent = dsfType.getParentDatasetFieldType(); - + logger.log(Level.INFO, "fieldValue={0}", fieldValue); // Create child Field FieldDTO child = null; if (dsfType.isAllowControlledVocabulary()) { @@ -843,9 +918,9 @@ private void handleChildField(MetadataBlockDTO customBlock, DatasetFieldType dsf compound = FieldDTO.createCompoundFieldDTO(parent.getName(), child); } customBlock.addField(compound); - + } - + private void processSources(XMLStreamReader xmlr, MetadataBlockDTO citation) throws XMLStreamException { for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) { if (event == XMLStreamConstants.START_ELEMENT) { @@ -871,7 +946,7 @@ private void processSources(XMLStreamReader xmlr, MetadataBlockDTO citation) thr // citation accessToSources } else if (xmlr.getLocalName().equals("srcDocu")) { parsedText = parseText( xmlr, "srcDocu" ); - if (!parsedText.isEmpty()) { + if (!parsedText.isEmpty()) { citation.getFields().add(FieldDTO.createPrimitiveFieldDTO("accessToSources", parsedText)); } } @@ -886,10 +961,10 @@ private void processAnlyInfo(XMLStreamReader xmlr, MetadataBlockDTO socialScienc // socialscience responseRate if (xmlr.getLocalName().equals("respRate")) { socialScience.getFields().add(FieldDTO.createPrimitiveFieldDTO("responseRate", parseText( xmlr, "respRate" ))); - // socialscience samplingErrorEstimates + // socialscience samplingErrorEstimates } else if (xmlr.getLocalName().equals("EstSmpErr")) { socialScience.getFields().add(FieldDTO.createPrimitiveFieldDTO("samplingErrorEstimates", parseText( xmlr, "EstSmpErr" ))); - // socialscience otherDataAppraisal + // socialscience otherDataAppraisal } else if (xmlr.getLocalName().equals("dataAppr")) { socialScience.getFields().add(FieldDTO.createPrimitiveFieldDTO("otherDataAppraisal", parseText( xmlr, "dataAppr" ))); } @@ -901,12 +976,12 @@ private void processAnlyInfo(XMLStreamReader xmlr, MetadataBlockDTO socialScienc private void processDataColl(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) throws XMLStreamException { MetadataBlockDTO socialScience =getSocialScience(dvDTO); - + String collMode = ""; String timeMeth = ""; String weight = ""; String dataCollector = ""; - + for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) { if (event == XMLStreamConstants.START_ELEMENT) { //timeMethod @@ -928,7 +1003,7 @@ private void processDataColl(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) thro } dataCollector = dataCollector.concat(thisValue); } - // frequencyOfDataCollection + // frequencyOfDataCollection } else if (xmlr.getLocalName().equals("frequenc")) { socialScience.getFields().add(FieldDTO.createPrimitiveFieldDTO("frequencyOfDataCollection", parseText( xmlr, "frequenc" ))); //samplingProcedure @@ -949,7 +1024,7 @@ private void processDataColl(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) thro } collMode = collMode.concat(thisValue); } - //socialScience.getFields().add(FieldDTO.createPrimitiveFieldDTO("collectionMode", parseText( xmlr, "collMode" ))); + //socialScience.getFields().add(FieldDTO.createPrimitiveFieldDTO("collectionMode", parseText( xmlr, "collMode" ))); //researchInstrument } else if (xmlr.getLocalName().equals("resInstru")) { socialScience.getFields().add(FieldDTO.createPrimitiveFieldDTO("researchInstrument", parseText( xmlr, "resInstru" ))); @@ -1028,12 +1103,12 @@ private void processTargetSampleSize(XMLStreamReader xmlr, MetadataBlockDTO soci DDI's that we are migrating should have one and only one DVN version statement */ private void processVerStmt(ImportType importType, XMLStreamReader xmlr, DatasetVersionDTO dvDTO) throws XMLStreamException { - if ( isHarvestImport(importType) ) { + if ( isHarvestImport(importType) ) { if (!"DVN".equals(xmlr.getAttributeValue(null, "source"))) { for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) { if (event == XMLStreamConstants.START_ELEMENT) { if (xmlr.getLocalName().equals("version")) { - addNote("Version Date: "+ xmlr.getAttributeValue(null, "date"),dvDTO); + addNote("Version Date: "+ xmlr.getAttributeValue(null, "date"),dvDTO); addNote("Version Text: "+ parseText(xmlr),dvDTO); } else if (xmlr.getLocalName().equals("notes")) { processNotes(xmlr, dvDTO); } } else if (event == XMLStreamConstants.END_ELEMENT) { @@ -1045,7 +1120,7 @@ private void processVerStmt(ImportType importType, XMLStreamReader xmlr, Dataset for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) { if (event == XMLStreamConstants.START_ELEMENT) { if (xmlr.getLocalName().equals("version")) { - dvDTO.setReleaseDate(xmlr.getAttributeValue(null, "date")); + dvDTO.setReleaseDate(xmlr.getAttributeValue(null, "date")); String versionState =xmlr.getAttributeValue(null,"type"); if (versionState!=null ) { if( versionState.equals("ARCHIVED")) { @@ -1054,23 +1129,23 @@ private void processVerStmt(ImportType importType, XMLStreamReader xmlr, Dataset versionState = DatasetVersion.VersionState.DRAFT.toString(); dvDTO.setInReview(true); } - dvDTO.setVersionState(Enum.valueOf(VersionState.class, versionState)); - } + dvDTO.setVersionState(Enum.valueOf(VersionState.class, versionState)); + } parseVersionNumber(dvDTO,parseText(xmlr)); } } else if(event == XMLStreamConstants.END_ELEMENT) { if (xmlr.getLocalName().equals("verStmt")) return; } } - } - + } + } if (isNewImport(importType)) { // If this is a new, Draft version, versionNumber and minor versionNumber are null. dvDTO.setVersionState(VersionState.DRAFT); } } - + private void processDataAccs(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) throws XMLStreamException { for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) { if (event == XMLStreamConstants.START_ELEMENT) { @@ -1095,7 +1170,7 @@ private void processDataAccs(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) thro } } } - + private void processSetAvail(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) throws XMLStreamException { for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) { if (event == XMLStreamConstants.START_ELEMENT) { @@ -1105,7 +1180,7 @@ private void processSetAvail(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) thro dvDTO.setOriginalArchive( parseText( xmlr, "origArch" ) ); } else if (xmlr.getLocalName().equals("avlStatus")) { dvDTO.setAvailabilityStatus( parseText( xmlr, "avlStatus" ) ); - } else if (xmlr.getLocalName().equals("collSize")) { + } else if (xmlr.getLocalName().equals("collSize")) { dvDTO.setSizeOfCollection(parseText( xmlr, "collSize" ) ); } else if (xmlr.getLocalName().equals("complete")) { dvDTO.setStudyCompletion( parseText( xmlr, "complete" ) ); @@ -1144,11 +1219,11 @@ private void processUseStmt(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) throw } } /** - * Separate the versionNumber into two parts - before the first '.' + * Separate the versionNumber into two parts - before the first '.' * is the versionNumber, and after is the minorVersionNumber. * If no minorVersionNumber exists, set to "0". * @param dvDTO - * @param versionNumber + * @param versionNumber */ private void parseVersionNumber(DatasetVersionDTO dvDTO, String versionNumber) { int firstIndex = versionNumber.indexOf('.'); @@ -1159,10 +1234,10 @@ private void parseVersionNumber(DatasetVersionDTO dvDTO, String versionNumber) { dvDTO.setVersionNumber(Long.parseLong(versionNumber.substring(0, firstIndex - 1))); dvDTO.setMinorVersionNumber(versionNumber.substring(firstIndex + 1)); } - + } - + private void processSerStmt(XMLStreamReader xmlr, MetadataBlockDTO citation) throws XMLStreamException { FieldDTO seriesName=null; FieldDTO seriesInformation=null; @@ -1170,7 +1245,7 @@ private void processSerStmt(XMLStreamReader xmlr, MetadataBlockDTO citation) thr if (event == XMLStreamConstants.START_ELEMENT) { if (xmlr.getLocalName().equals("serName")) { seriesName = FieldDTO.createPrimitiveFieldDTO("seriesName", parseText(xmlr)); - + } else if (xmlr.getLocalName().equals("serInfo")) { seriesInformation=FieldDTO.createPrimitiveFieldDTO("seriesInformation", parseText(xmlr) ); } @@ -1201,7 +1276,13 @@ private void processDistStmt(XMLStreamReader xmlr, MetadataBlockDTO citation) th } else if (xmlr.getLocalName().equals("contact")) { HashSet set = new HashSet<>(); - addToSet(set, "datasetContactEmail", xmlr.getAttributeValue(null, "email")); + + String emailText = xmlr.getAttributeValue(null, "email"); + if (StringUtils.isBlank(emailText)){ + emailText=defaultEmailAddress; + } + //addToSet(set, "datasetContactEmail", xmlr.getAttributeValue(null, "email")); + addToSet(set, "datasetContactEmail", emailText); addToSet(set, "datasetContactAffiliation", xmlr.getAttributeValue(null, "affiliation")); addToSet(set, "datasetContactName", parseText(xmlr)); datasetContacts.add(set); @@ -1240,7 +1321,7 @@ private void processProdStmt(XMLStreamReader xmlr, MetadataBlockDTO citation) th HashSet set = new HashSet<>(); addToSet(set,"producerAbbreviation", xmlr.getAttributeValue(null, "abbr")); addToSet(set,"producerAffiliation", xmlr.getAttributeValue(null, "affiliation")); - + Map prodDetails = parseCompoundText(xmlr, "producer"); addToSet(set,"producerName", prodDetails.get("name")); addToSet(set,"producerURL", prodDetails.get("url" )); @@ -1279,7 +1360,7 @@ private void processProdStmt(XMLStreamReader xmlr, MetadataBlockDTO citation) th } if (grants.size()>0) { citation.addField(FieldDTO.createMultipleCompoundFieldDTO("grantNumber", grants)); - } + } if (producers.size()>0) { citation.getFields().add(FieldDTO.createMultipleCompoundFieldDTO("producer", producers)); } @@ -1288,11 +1369,11 @@ private void processProdStmt(XMLStreamReader xmlr, MetadataBlockDTO citation) th } } } - + private void processTitlStmt(XMLStreamReader xmlr, DatasetDTO datasetDTO) throws XMLStreamException, ImportException { MetadataBlockDTO citation = datasetDTO.getDatasetVersion().getMetadataBlocks().get("citation"); List> otherIds = new ArrayList<>(); - + for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) { if (event == XMLStreamConstants.START_ELEMENT) { if (xmlr.getLocalName().equals("titl")) { @@ -1310,13 +1391,13 @@ private void processTitlStmt(XMLStreamReader xmlr, DatasetDTO datasetDTO) throws } else if ( AGENCY_DOI.equals( xmlr.getAttributeValue(null, "agency") ) ) { parseStudyIdDOI( parseText(xmlr), datasetDTO ); } else if ( AGENCY_DARA.equals( xmlr.getAttributeValue(null, "agency"))) { - /* + /* da|ra - "Registration agency for social and economic data" (http://www.da-ra.de/en/home/) - ICPSR uses da|ra to register their DOIs; so they have agency="dara" - in their IDNo entries. - Also, their DOIs are formatted differently, without the - hdl: prefix. + ICPSR uses da|ra to register their DOIs; so they have agency="dara" + in their IDNo entries. + Also, their DOIs are formatted differently, without the + hdl: prefix. */ parseStudyIdDoiICPSRdara( parseText(xmlr), datasetDTO ); } else { @@ -1339,7 +1420,7 @@ private void processTitlStmt(XMLStreamReader xmlr, DatasetDTO datasetDTO) throws } } private void processRspStmt(XMLStreamReader xmlr, MetadataBlockDTO citation) throws XMLStreamException { - + List> authors = new ArrayList<>(); for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) { if (event == XMLStreamConstants.START_ELEMENT) { @@ -1357,7 +1438,7 @@ private void processRspStmt(XMLStreamReader xmlr, MetadataBlockDTO citation) thr FieldDTO author = FieldDTO.createMultipleCompoundFieldDTO("author", authors); citation.getFields().add(author); } - + return; } } @@ -1386,11 +1467,11 @@ private Map parseCompoundText (XMLStreamReader xmlr, String endTa returnMap.put( "name", text ); return returnMap; } - + private String parseText(XMLStreamReader xmlr, String endTag) throws XMLStreamException { return (String) parseTextNew(xmlr,endTag); } - + // #FIXME We should really type stabalize this. private Object parseTextNew(XMLStreamReader xmlr, String endTag) throws XMLStreamException { String returnString = ""; @@ -1436,16 +1517,16 @@ private Object parseTextNew(XMLStreamReader xmlr, String endTag) throws XMLStrea if (xmlr.getLocalName().equals(endTag)) break; } } - + if (returnMap != null) { // this is one of our new citation areas for DVN3.0 return returnMap; } - + // otherwise it's a standard section and just return the String like we always did return returnString.trim(); } - + private String parseNoteByType(XMLStreamReader xmlr, String type) throws XMLStreamException { if (type.equalsIgnoreCase(xmlr.getAttributeValue(null, "type"))) { return parseText(xmlr); @@ -1534,7 +1615,7 @@ private String parseText_citation (XMLStreamReader xmlr) throws XMLStreamExcepti return citation; } - + private String parseUNF(String unfString) { if (unfString.contains("UNF:")) { return unfString.substring( unfString.indexOf("UNF:") ); @@ -1542,22 +1623,22 @@ private String parseUNF(String unfString) { return null; } } - + private Map parseDVNCitation(XMLStreamReader xmlr) throws XMLStreamException { Map returnValues = new HashMap<>(); - + while (true) { int event = xmlr.next(); if (event == XMLStreamConstants.START_ELEMENT) { if (xmlr.getLocalName().equals("IDNo")) { returnValues.put("idType", xmlr.getAttributeValue(null, "agency") ); - returnValues.put("idNumber", parseText(xmlr) ); + returnValues.put("idNumber", parseText(xmlr) ); } else if (xmlr.getLocalName().equals("biblCit")) { - returnValues.put("text", parseText(xmlr) ); + returnValues.put("text", parseText(xmlr) ); } else if (xmlr.getLocalName().equals("holdings")) { - returnValues.put("url", xmlr.getAttributeValue(null, "URI") ); + returnValues.put("url", xmlr.getAttributeValue(null, "URI") ); } else if (xmlr.getLocalName().equals("notes")) { if (NOTE_TYPE_REPLICATION_FOR.equals(xmlr.getAttributeValue(null, "type")) ) { @@ -1567,11 +1648,11 @@ else if (xmlr.getLocalName().equals("notes")) { } else if (event == XMLStreamConstants.END_ELEMENT) { if (xmlr.getLocalName().equals("citation")) break; } - } - + } + return returnValues; - } - + } + private void parseStudyIdHandle(String _id, DatasetDTO datasetDTO) { int index1 = _id.indexOf(':'); @@ -1596,8 +1677,8 @@ private void parseStudyIdDOI(String _id, DatasetDTO datasetDTO) throws ImportExc int index2 = _id.indexOf('/'); if (index1==-1) { throw new EJBException("Error parsing (DOI) IdNo: "+_id+". ':' not found in string"); - } - + } + if (index2 == -1) { throw new ImportException("Error parsing (DOI) IdNo: "+_id+". '/' not found in string"); @@ -1605,47 +1686,47 @@ private void parseStudyIdDOI(String _id, DatasetDTO datasetDTO) throws ImportExc datasetDTO.setAuthority(_id.substring(index1+1, index2)); } datasetDTO.setProtocol("doi"); - + datasetDTO.setIdentifier(_id.substring(index2+1)); } - + private void parseStudyIdDoiICPSRdara(String _id, DatasetDTO datasetDTO) throws ImportException{ /* - dara/ICPSR DOIs are formatted without the hdl: prefix; for example - + dara/ICPSR DOIs are formatted without the hdl: prefix; for example - 10.3886/ICPSR06635.v1 - so we assume that everything before the "/" is the authority, + so we assume that everything before the "/" is the authority, and everything past it - the identifier: */ - - int index = _id.indexOf('/'); - + + int index = _id.indexOf('/'); + if (index == -1) { throw new ImportException("Error parsing ICPSR/dara DOI IdNo: "+_id+". '/' not found in string"); - } - + } + if (index == _id.length() - 1) { throw new ImportException("Error parsing ICPSR/dara DOI IdNo: "+_id+" ends with '/'"); } - + datasetDTO.setAuthority(_id.substring(0, index)); datasetDTO.setProtocol("doi"); - + datasetDTO.setIdentifier(_id.substring(index+1)); } // Helper methods private MetadataBlockDTO getCitation(DatasetVersionDTO dvDTO) { return dvDTO.getMetadataBlocks().get("citation"); } - + private MetadataBlockDTO getGeospatial(DatasetVersionDTO dvDTO) { return dvDTO.getMetadataBlocks().get("geospatial"); } - + private MetadataBlockDTO getSocialScience(DatasetVersionDTO dvDTO) { return dvDTO.getMetadataBlocks().get("socialscience"); } - - + + private void addToSet(HashSet set, String typeName, String value ) { if (value!=null && !value.trim().isEmpty()) { set.add(FieldDTO.createPrimitiveFieldDTO(typeName, value)); @@ -1655,7 +1736,7 @@ private void addToSet(HashSet set, String typeName, String value ) { // TODO : determine what is going on here ? private void processOtherMat(XMLStreamReader xmlr, DatasetDTO datasetDTO) throws XMLStreamException { FileMetadataDTO fmdDTO = new FileMetadataDTO(); - + if (datasetDTO.getDatasetVersion().getFileMetadatas() == null) { datasetDTO.getDatasetVersion().setFileMetadatas(new ArrayList<>()); } @@ -1666,7 +1747,7 @@ private void processOtherMat(XMLStreamReader xmlr, DatasetDTO datasetDTO) throws // datasetDTO.setDataFiles(new ArrayList<>()); //} //datasetDTO.getDataFiles().add(dfDTO); - + dfDTO.setStorageIdentifier( xmlr.getAttributeValue(null, "URI")); dfDTO.setPidURL(xmlr.getAttributeValue(null, "pidURL")); fmdDTO.setDataFile(dfDTO); @@ -1681,9 +1762,9 @@ private void processOtherMat(XMLStreamReader xmlr, DatasetDTO datasetDTO) throws for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) { if (event == XMLStreamConstants.START_ELEMENT) { if (xmlr.getLocalName().equals("labl")) { - // this is the file name: + // this is the file name: fmdDTO.setLabel( parseText(xmlr) ); - // TODO: in DVN3 we used to make an attempt to determine the file type + // TODO: in DVN3 we used to make an attempt to determine the file type // based on the file name. } else if (xmlr.getLocalName().equals("txt")) { fmdDTO.setDescription( parseText(xmlr) ); @@ -1704,7 +1785,7 @@ private void processOtherMat(XMLStreamReader xmlr, DatasetDTO datasetDTO) throws dfDTO.setContentType(contentType); } } - } + } } else if (event == XMLStreamConstants.END_ELEMENT) {// if (xmlr.getLocalName().equals("otherMat")) { // post process @@ -1720,13 +1801,13 @@ private void processOtherMat(XMLStreamReader xmlr, DatasetDTO datasetDTO) throws } // this method is for attempting to extract the minimal amount of file-level - // metadata from an ICPSR-supplied DDI. (they use the "fileDscr" instead of - // "otherMat" for general file metadata; the only field they populate is + // metadata from an ICPSR-supplied DDI. (they use the "fileDscr" instead of + // "otherMat" for general file metadata; the only field they populate is // "fileName". -- 4.6 - + private void processFileDscrMinimal(XMLStreamReader xmlr, DatasetDTO datasetDTO) throws XMLStreamException { FileMetadataDTO fmdDTO = new FileMetadataDTO(); - + if (datasetDTO.getDatasetVersion().getFileMetadatas() == null) { datasetDTO.getDatasetVersion().setFileMetadatas(new ArrayList<>()); } @@ -1735,11 +1816,11 @@ private void processFileDscrMinimal(XMLStreamReader xmlr, DatasetDTO datasetDTO) DataFileDTO dfDTO = new DataFileDTO(); dfDTO.setContentType("data/various-formats"); // reserved ICPSR content type identifier fmdDTO.setDataFile(dfDTO); - + for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) { if (event == XMLStreamConstants.START_ELEMENT) { if (xmlr.getLocalName().equals("fileName")) { - // this is the file name: + // this is the file name: String label = parseText(xmlr); // do some cleanup: int col = label.lastIndexOf(':'); @@ -1755,7 +1836,7 @@ private void processFileDscrMinimal(XMLStreamReader xmlr, DatasetDTO datasetDTO) // strip leading blanks: label = label.replaceFirst("^[ \t]*", ""); fmdDTO.setLabel(label); - } + } } else if (event == XMLStreamConstants.END_ELEMENT) { if (xmlr.getLocalName().equals("fileDscr")) { if (fmdDTO.getLabel() == null || fmdDTO.getLabel().trim().equals("") ) { @@ -1770,10 +1851,10 @@ private void processFileDscrMinimal(XMLStreamReader xmlr, DatasetDTO datasetDTO) } } } - + private void processFileDscr(XMLStreamReader xmlr, DatasetDTO datasetDTO, Map filesMap) throws XMLStreamException { FileMetadataDTO fmdDTO = new FileMetadataDTO(); - + datasetDTO.getDatasetVersion().getFileMetadatas().add(fmdDTO); //StudyFile sf = new OtherFile(studyVersion.getStudy()); // until we connect the sf and dt, we have to assume it's an other file @@ -1782,13 +1863,13 @@ private void processFileDscr(XMLStreamReader xmlr, DatasetDTO datasetDTO, Map if (xmlr.getLocalName().equals("fileDscr")) { @@ -1844,7 +1925,7 @@ else if (xmlr.getLocalName().equals("notes")) { } } } - + private String determineFileCategory(String catName, String icpsrDesc, String icpsrId) { if (catName == null) { catName = icpsrDesc; @@ -1864,7 +1945,7 @@ private String determineFileCategory(String catName, String icpsrDesc, String ic * @param fmdDTO * @param dtDTO * @return fmdDTO.label (ddiFileId) - * @throws XMLStreamException + * @throws XMLStreamException */ private String processFileTxt(XMLStreamReader xmlr, FileMetadataDTO fmdDTO, DataTableDTO dtDTO) throws XMLStreamException { String ddiFileId = null; @@ -1887,11 +1968,11 @@ private String processFileTxt(XMLStreamReader xmlr, FileMetadataDTO fmdDTO, Data } else if (xmlr.getLocalName().equals("dimensns")) processDimensns(xmlr, dtDTO); } else if (event == XMLStreamConstants.END_ELEMENT) { if (xmlr.getLocalName().equals("fileTxt")) { - // If we still don't know the content type of this file + // If we still don't know the content type of this file // (i.e., if there was no "" tag explicitly specifying - // the type), we can try and make an educated guess. We already - // now that this is a subsettable file. And now that the - // "" section has been parsed, we can further + // the type), we can try and make an educated guess. We already + // now that this is a subsettable file. And now that the + // "" section has been parsed, we can further // decide if it's a tab, or a fixed field: if (StringUtil.isEmpty(dfDTO.getContentType())) { String subsettableFileType = "text/tab-separated-values"; @@ -1901,19 +1982,19 @@ private String processFileTxt(XMLStreamReader xmlr, FileMetadataDTO fmdDTO, Data } //EMK TODO: ask Gustavo & Leonid what should be used here instead of setFileType // dfDTO.setFileType( subsettableFileType ); - + return ddiFileId; } } } return ddiFileId; - } - + } + /** * Set dtDTO. caseQuantity, varQuantity, recordsPerCase * @param xmlr * @param dtDTO - * @throws XMLStreamException + * @throws XMLStreamException */ private void processDimensns(XMLStreamReader xmlr, DataTableDTO dtDTO) throws XMLStreamException { for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) { @@ -1936,6 +2017,5 @@ private void processDimensns(XMLStreamReader xmlr, DataTableDTO dtDTO) throws XM } } } - -} +} diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java index 324af83afa2..75c71b60c3c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java @@ -7,6 +7,8 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import com.thoughtworks.xstream.XStream; +import com.thoughtworks.xstream.io.json.JsonHierarchicalStreamDriver; import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetField; @@ -22,6 +24,7 @@ import edu.harvard.iq.dataverse.EjbDataverseEngine; import edu.harvard.iq.dataverse.MetadataBlockServiceBean; import edu.harvard.iq.dataverse.api.dto.DatasetDTO; +import edu.harvard.iq.dataverse.api.imports.ImportUtil.ImportFileType; import edu.harvard.iq.dataverse.api.imports.ImportUtil.ImportType; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; @@ -38,6 +41,7 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; import java.io.PrintWriter; import java.io.StringReader; import java.nio.file.Files; @@ -76,9 +80,9 @@ public class ImportServiceBean { @PersistenceContext(unitName="VDCNet-ejbPU") private EntityManager em; - - private static final Logger logger = Logger.getLogger(ImportServiceBean.class.getCanonicalName()); + private static final Logger logger = Logger.getLogger(ImportServiceBean.class.getCanonicalName()); + static XStream xstream = new XStream(new JsonHierarchicalStreamDriver()); @EJB protected EjbDataverseEngine engineSvc; @EJB @@ -92,21 +96,21 @@ public class ImportServiceBean { MetadataBlockServiceBean metadataBlockService; @EJB SettingsServiceBean settingsService; - - @EJB + + @EJB ImportDDIServiceBean importDDIService; @EJB ImportGenericServiceBean importGenericService; - + @EJB IndexServiceBean indexService; /** - * This is just a convenience method, for testing migration. It creates + * This is just a convenience method, for testing migration. It creates * a dummy dataverse with the directory name as dataverse name & alias. * @param dvName * @param dataverseRequest * @return - * @throws ImportException + * @throws ImportException */ @TransactionAttribute(REQUIRES_NEW) public Dataverse createDataverse(String dvName, DataverseRequest dataverseRequest) throws ImportException { @@ -150,13 +154,18 @@ public Dataverse createDataverse(String dvName, DataverseRequest dataverseReques } @TransactionAttribute(REQUIRES_NEW) - public JsonObjectBuilder handleFile(DataverseRequest dataverseRequest, Dataverse owner, File file, ImportType importType, PrintWriter validationLog, PrintWriter cleanupLog) throws ImportException, IOException { + public JsonObjectBuilder handleFile(DataverseRequest dataverseRequest, Dataverse owner, File file, ImportType importType, PrintWriter validationLog, PrintWriter cleanupLog, ImportFileType importFileType) throws ImportException, IOException { System.out.println("handling file: " + file.getAbsolutePath()); - String ddiXMLToParse; + String fileToParse; + JsonObjectBuilder status; try { - ddiXMLToParse = new String(Files.readAllBytes(file.toPath())); - JsonObjectBuilder status = doImport(dataverseRequest, owner, ddiXMLToParse,file.getParentFile().getName() + "/" + file.getName(), importType, cleanupLog); + fileToParse = new String(Files.readAllBytes(file.toPath())); + if (importFileType == ImportUtil.ImportFileType.JSON) { + status = doImportJson(dataverseRequest, owner, fileToParse, file.getParentFile().getName() + "/" + file.getName(), importType, cleanupLog); + } else { + status = doImport(dataverseRequest, owner, fileToParse, file.getParentFile().getName() + "/" + file.getName(), importType, cleanupLog); + } status.add("file", file.getName()); logger.log(Level.INFO, "completed doImport {0}/{1}", new Object[]{file.getParentFile().getName(), file.getName()}); return status; @@ -178,16 +187,16 @@ public JsonObjectBuilder handleFile(DataverseRequest dataverseRequest, Dataverse } String msg = "Unexpected Error in handleFile(), file:" + file.getParentFile().getName() + "/" + file.getName(); if (e.getMessage()!=null) { - msg+= "message: " +e.getMessage(); + msg+= "message: " +e.getMessage(); } msg += ", caused by: " +causedBy; if (causedBy != null && causedBy.getMessage()!=null) { msg+=", caused by message: "+ causedBy.getMessage(); } msg += " at line: "+ stackLine; - - - + + + validationLog.println(msg); e.printStackTrace(); @@ -206,24 +215,24 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve DatasetDTO dsDTO = null; String json = null; - - // TODO: + + // TODO: // At the moment (4.5; the first official "export/harvest release"), there - // are 3 supported metadata formats: DDI, DC and native Dataverse metadata + // are 3 supported metadata formats: DDI, DC and native Dataverse metadata // encoded in JSON. The 2 XML formats are handled by custom implementations; - // each of the 2 implementations uses its own parsing approach. (see the - // ImportDDIServiceBean and ImportGenerciServiceBean for details). + // each of the 2 implementations uses its own parsing approach. (see the + // ImportDDIServiceBean and ImportGenerciServiceBean for details). // TODO: Need to create a system of standardized import plugins - similar to Stephen // Kraffmiller's export modules; replace the logic below with clean - // programmatic lookup of the import plugin needed. + // programmatic lookup of the import plugin needed. - if ("ddi".equalsIgnoreCase(metadataFormat) || "oai_ddi".equals(metadataFormat) + if ("ddi".equalsIgnoreCase(metadataFormat) || "oai_ddi".equals(metadataFormat) || metadataFormat.toLowerCase().matches("^oai_ddi.*")) { try { String xmlToParse = new String(Files.readAllBytes(metadataFile.toPath())); - // TODO: - // import type should be configurable - it should be possible to - // select whether you want to harvest with or without files, + // TODO: + // import type should be configurable - it should be possible to + // select whether you want to harvest with or without files, // ImportType.HARVEST vs. ImportType.HARVEST_WITH_FILES logger.fine("importing DDI "+metadataFile.getAbsolutePath()); dsDTO = importDDIService.doImport(ImportType.HARVEST, xmlToParse); @@ -239,17 +248,17 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve throw new ImportException("Failed to process Dublin Core XML record: "+ e.getClass() + " (" + e.getMessage() + ")"); } } else if ("dataverse_json".equals(metadataFormat)) { - // This is Dataverse metadata already formatted in JSON. + // This is Dataverse metadata already formatted in JSON. // Simply read it into a string, and pass to the final import further down: logger.fine("Attempting to import custom dataverse metadata from file "+metadataFile.getAbsolutePath()); - json = new String(Files.readAllBytes(metadataFile.toPath())); + json = new String(Files.readAllBytes(metadataFile.toPath())); } else { throw new ImportException("Unsupported import metadata format: " + metadataFormat); } if (json == null) { if (dsDTO != null ) { - // convert DTO to Json, + // convert DTO to Json, Gson gson = new GsonBuilder().setPrettyPrinting().create(); json = gson.toJson(dsDTO); logger.fine("JSON produced for the metadata harvested: "+json); @@ -257,10 +266,10 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve throw new ImportException("Failed to transform XML metadata format "+metadataFormat+" into a DatasetDTO"); } } - + JsonReader jsonReader = Json.createReader(new StringReader(json)); JsonObject obj = jsonReader.readObject(); - //and call parse Json to read it into a dataset + //and call parse Json to read it into a dataset try { JsonParser parser = new JsonParser(datasetfieldService, metadataBlockService, settingsService); parser.setLenient(true); @@ -296,7 +305,7 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve DatasetFieldValue f = v.getRootBean(); boolean fixed = false; boolean converted = false; - // TODO: Is this scrubbing something we want to continue doing? + // TODO: Is this scrubbing something we want to continue doing? if (settingsService.isTrueForKey(SettingsServiceBean.Key.ScrubMigrationData, false)) { fixed = processMigrationValidationError(f, cleanupLog, metadataFile.getName()); converted = true; @@ -316,7 +325,7 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve } } } - + // A Global ID is required, in order for us to be able to harvest and import // this dataset: if (StringUtils.isEmpty(ds.getGlobalIdString())) { @@ -325,7 +334,7 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve ds.setHarvestedFrom(harvestingClient); ds.setHarvestIdentifier(harvestIdentifier); - + Dataset existingDs = datasetService.findByGlobalId(ds.getGlobalIdString()); if (existingDs != null) { @@ -335,8 +344,8 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve throw new ImportException("The dataset with the global id "+ds.getGlobalIdString()+" already exists, in the dataverse "+existingDs.getOwner().getAlias()+", skipping."); } // And if we already have a dataset with this same id, in this same - // dataverse, but it is LOCAL dataset (can happen!), we're going to - // skip it also: + // dataverse, but it is LOCAL dataset (can happen!), we're going to + // skip it also: if (!existingDs.isHarvested()) { throw new ImportException("A LOCAL dataset with the global id "+ds.getGlobalIdString()+" already exists in this dataverse; skipping."); } @@ -345,25 +354,25 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve if (existingDs.getVersions().size() != 1) { throw new ImportException("Error importing Harvested Dataset, existing dataset has " + existingDs.getVersions().size() + " versions"); } - // Purge all the SOLR documents associated with this client from the - // index server: + // Purge all the SOLR documents associated with this client from the + // index server: indexService.deleteHarvestedDocuments(existingDs); - // files from harvested datasets are removed unceremoniously, - // directly in the database. no need to bother calling the + // files from harvested datasets are removed unceremoniously, + // directly in the database. no need to bother calling the // DeleteFileCommand on them. for (DataFile harvestedFile : existingDs.getFiles()) { DataFile merged = em.merge(harvestedFile); em.remove(merged); - harvestedFile = null; + harvestedFile = null; } - // TODO: - // Verify what happens with the indexed files in SOLR? + // TODO: + // Verify what happens with the indexed files in SOLR? // are they going to be overwritten by the reindexing of the dataset? existingDs.setFiles(null); Dataset merged = em.merge(existingDs); engineSvc.submit(new DestroyDatasetCommand(merged, dataverseRequest)); } - + importedDataset = engineSvc.submit(new CreateHarvestedDatasetCommand(ds, dataverseRequest)); } catch (JsonParseException | ImportException | CommandException ex) { @@ -400,24 +409,25 @@ public JsonObject ddiToJson(String xmlToParse) throws ImportException{ return obj; } - + +/* public JsonObjectBuilder doImport(DataverseRequest dataverseRequest, Dataverse owner, String xmlToParse, String fileName, ImportType importType, PrintWriter cleanupLog) throws ImportException, IOException { String status = ""; Long createdId = null; DatasetDTO dsDTO = null; try { - + dsDTO = importDDIService.doImport(importType, xmlToParse); } catch (XMLStreamException e) { throw new ImportException("XMLStreamException" + e); } - // convert DTO to Json, + // convert DTO to Json, Gson gson = new GsonBuilder().setPrettyPrinting().create(); String json = gson.toJson(dsDTO); JsonReader jsonReader = Json.createReader(new StringReader(json)); JsonObject obj = jsonReader.readObject(); - //and call parse Json to read it into a dataset + //and call parse Json to read it into a dataset try { JsonParser parser = new JsonParser(datasetfieldService, metadataBlockService, settingsService); parser.setLenient(!importType.equals(ImportType.NEW)); @@ -465,7 +475,7 @@ public JsonObjectBuilder doImport(DataverseRequest dataverseRequest, Dataverse o DatasetFieldValue f = v.getRootBean(); boolean fixed = false; boolean converted = false; - if ( importType.equals(ImportType.HARVEST) && + if ( importType.equals(ImportType.HARVEST) && settingsService.isTrueForKey(SettingsServiceBean.Key.ScrubMigrationData, false)) { fixed = processMigrationValidationError(f, cleanupLog, fileName); converted = true; @@ -508,7 +518,7 @@ public JsonObjectBuilder doImport(DataverseRequest dataverseRequest, Dataverse o engineSvc.submit(new DestroyDatasetCommand(existingDs, dataverseRequest)); Dataset managedDs = engineSvc.submit(new CreateHarvestedDatasetCommand(ds, dataverseRequest)); status = " updated dataset, id=" + managedDs.getId() + "."; - + } else { // If we are adding a new version to an existing dataset, // check that the version number isn't already in the dataset @@ -537,27 +547,188 @@ public JsonObjectBuilder doImport(DataverseRequest dataverseRequest, Dataverse o } return Json.createObjectBuilder().add("message", status); } - + +*/ + + public JsonObjectBuilder doImport(DataverseRequest dataverseRequest, Dataverse owner, String xmlToParse, String fileName, ImportType importType, PrintWriter cleanupLog) throws ImportException, IOException { + String status = ""; + Long createdId = null; + DatasetDTO dsDTO = null; + try { + + dsDTO = importDDIService.doImport(importType, xmlToParse); + } catch (XMLStreamException e) { + throw new ImportException("XMLStreamException" + e); + } + // convert DTO to Json, + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + String json = gson.toJson(dsDTO); + return doImportJson(dataverseRequest, owner, json, fileName, importType, cleanupLog); + } + + + public JsonObjectBuilder doImportJson(DataverseRequest dataverseRequest, Dataverse owner, String json, String fileName, ImportType importType, PrintWriter cleanupLog) throws ImportException, IOException { + + String status = ""; + Long createdId = null; +// DatasetDTO dsDTO = null; +// try { +// +// dsDTO = importDDIService.doImport(importType, xmlToParse); +// } catch (XMLStreamException e) { +// throw new ImportException("XMLStreamException" + e); +// } +// // convert DTO to Json, +// Gson gson = new GsonBuilder().setPrettyPrinting().create(); +// String json = gson.toJson(dsDTO); + JsonReader jsonReader = Json.createReader(new StringReader(json)); + JsonObject obj = jsonReader.readObject(); + //and call parse Json to read it into a dataset + try { + JsonParser parser = new JsonParser(datasetfieldService, metadataBlockService, settingsService); + parser.setLenient(!importType.equals(ImportType.NEW)); + Dataset ds = parser.parseDataset(obj); + + // For ImportType.NEW, if the user supplies a global identifier, and it's not a protocol + // we support, it will be rejected. + if (importType.equals(ImportType.NEW)) { + if (ds.getGlobalIdString() != null && !ds.getProtocol().equals(settingsService.getValueForKey(SettingsServiceBean.Key.Protocol, ""))) { + throw new ImportException("Could not register id " + ds.getGlobalIdString() + ", protocol not supported"); + } + } + + ds.setOwner(owner); + ds.getLatestVersion().setDatasetFields(ds.getLatestVersion().initDatasetFields()); + + // Check data against required contraints + List> violations = ds.getVersions().get(0).validateRequired(); + logger.log(Level.INFO, "DatasetField: how many violations:{0}", violations.size()); + if (!violations.isEmpty()) { + if ( importType.equals(ImportType.HARVEST) ) { + // For migration and harvest, add NA for missing required values + for (ConstraintViolation v : violations) { + DatasetField f = v.getRootBean(); + f.setSingleValue(DatasetField.NA_VALUE); + } + } else { + // when importing a new dataset, the import will fail + // if required values are missing. + String errMsg = "Error importing data:"; + for (ConstraintViolation v : violations) { + errMsg += " " + v.getMessage(); + } + throw new ImportException(errMsg); + } + } + + // Check data against validation constraints + // If we are migrating and "scrub migration data" is true we attempt to fix invalid data + // if the fix fails stop processing of this file by throwing exception + Set invalidViolations = ds.getVersions().get(0).validate(); + ValidatorFactory factory = Validation.buildDefaultValidatorFactory(); + Validator validator = factory.getValidator(); + if (!invalidViolations.isEmpty()) { + for (ConstraintViolation v : invalidViolations) { + DatasetFieldValue f = v.getRootBean(); + boolean fixed = false; + boolean converted = false; + if ( importType.equals(ImportType.HARVEST) && + settingsService.isTrueForKey(SettingsServiceBean.Key.ScrubMigrationData, false)) { + fixed = processMigrationValidationError(f, cleanupLog, fileName); + converted = true; + if (fixed) { + Set> scrubbedViolations = validator.validate(f); + if (!scrubbedViolations.isEmpty()) { + fixed = false; + } + } + } + if (!fixed) { + if (importType.equals(ImportType.HARVEST)) { + String msg = "Data modified - File: " + fileName + "; Field: " + f.getDatasetField().getDatasetFieldType().getDisplayName() + "; " + + "Invalid value: '" + f.getValue() + "'" + " Converted Value:'" + DatasetField.NA_VALUE + "'"; + cleanupLog.println(msg); + f.setValue(DatasetField.NA_VALUE); + + } else { + String msg = " Validation error for "; + if (converted) { + msg += "converted "; + } + msg += "value: " + f.getValue() + ", " + f.getValidationMessage(); + throw new ImportException(msg); + } + } + } + } + + + Dataset existingDs = datasetService.findByGlobalId(ds.getGlobalIdString()); + + if (existingDs != null) { + if (importType.equals(ImportType.HARVEST)) { + // For harvested datasets, there should always only be one version. + // We will replace the current version with the imported version. + if (existingDs.getVersions().size() != 1) { + throw new ImportException("Error importing Harvested Dataset, existing dataset has " + existingDs.getVersions().size() + " versions"); + } + engineSvc.submit(new DestroyDatasetCommand(existingDs, dataverseRequest)); + Dataset managedDs = engineSvc.submit(new CreateHarvestedDatasetCommand(ds, dataverseRequest)); + status = " updated dataset, id=" + managedDs.getId() + "."; + + } else { + // If we are adding a new version to an existing dataset, + // check that the version number isn't already in the dataset + for (DatasetVersion dsv : existingDs.getVersions()) { + if (dsv.getVersionNumber().equals(ds.getLatestVersion().getVersionNumber())) { + throw new ImportException("VersionNumber " + ds.getLatestVersion().getVersionNumber() + " already exists in dataset " + existingDs.getGlobalIdString()); + } + } + DatasetVersion dsv = engineSvc.submit(new CreateDatasetVersionCommand(dataverseRequest, existingDs, ds.getVersions().get(0))); + status = " created datasetVersion, for dataset "+ dsv.getDataset().getGlobalIdString(); + createdId = dsv.getId(); + } + + } else { + Dataset managedDs = engineSvc.submit(new CreateNewDatasetCommand(ds, dataverseRequest)); + status = " created dataset, id=" + managedDs.getId() + "."; + createdId = managedDs.getId(); + } + + } catch (JsonParseException ex) { + logger.log(Level.INFO, "Error parsing datasetVersion: {0}", ex.getMessage()); + throw new ImportException("Error parsing datasetVersion: " + ex.getMessage(), ex); + } catch (CommandException ex) { + logger.log(Level.INFO, "Error excuting Create dataset command: {0}", ex.getMessage()); + throw new ImportException("Error excuting dataverse command: " + ex.getMessage(), ex); + } + return Json.createObjectBuilder().add("message", status); + } + + + + + private boolean processMigrationValidationError(DatasetFieldValue f, PrintWriter cleanupLog, String fileName) { if (f.getDatasetField().getDatasetFieldType().getName().equals(DatasetFieldConstant.datasetContactEmail)) { //Try to convert it based on the errors we've seen String convertedVal = convertInvalidEmail(f.getValue()); if (!(convertedVal == null)) { String msg = "Data modified - File: " + fileName + "; Field: " + f.getDatasetField().getDatasetFieldType().getDisplayName() + "; " - + "Invalid value: '" + f.getValue() + "'" + " Converted Value:'" + convertedVal + "'"; + + "Invalid value: '" + f.getValue() + "'" + " Converted Value:'" + convertedVal + "'"; cleanupLog.println(msg); f.setValue(convertedVal); return true; } //if conversion fails set to NA - String msg = "Data modified - File: " + fileName + "; Field: Dataset Contact Email; " + "Invalid value: '" + f.getValue() + "'" + " Converted Value: 'NA'"; + String msg = "Data modified - File: " + fileName + "; Field: Dataset Contact Email; " + "Invalid value: '" + f.getValue() + "'" + " Converted Value: 'NA'"; cleanupLog.println(msg); f.setValue(DatasetField.NA_VALUE); return true; } if (f.getDatasetField().getDatasetFieldType().getName().equals(DatasetFieldConstant.producerURL)) { if (f.getValue().equals("PRODUCER URL")) { - String msg = "Data modified - File: " + fileName + "; Field: Producer URL; " + "Invalid value: '" + f.getValue() + "'" + " Converted Value: 'NA'"; + String msg = "Data modified - File: " + fileName + "; Field: Producer URL; " + "Invalid value: '" + f.getValue() + "'" + " Converted Value: 'NA'"; cleanupLog.println(msg); f.setValue(DatasetField.NA_VALUE); return true; @@ -566,7 +737,7 @@ private boolean processMigrationValidationError(DatasetFieldValue f, PrintWriter if (f.getDatasetField().getDatasetFieldType().getFieldType().equals(DatasetFieldType.FieldType.DATE)) { if(f.getValue().toUpperCase().equals("YYYY-MM-DD")){ String msg = "Data modified - File: " + fileName + "; Field:" + f.getDatasetField().getDatasetFieldType().getDisplayName() + "; " - + "Invalid value: '" + f.getValue() + "'" + " Converted Value: 'NA'"; + + "Invalid value: '" + f.getValue() + "'" + " Converted Value: 'NA'"; cleanupLog.println(msg); f.setValue(DatasetField.NA_VALUE); return true; @@ -576,20 +747,20 @@ private boolean processMigrationValidationError(DatasetFieldValue f, PrintWriter String msg = "Data modified - File: " + fileName + "; Field: " + f.getDatasetField().getDatasetFieldType().getDisplayName() + "" + " Converted Value:" + convertedVal + "; Invalid value: '" + f.getValue() + "'"; cleanupLog.println(msg); - f.setValue(convertedVal); + f.setValue(convertedVal); return true; - } + } } return false; } - - private String convertInvalidEmail(String inString){ + + private String convertInvalidEmail(String inString){ //First we'll see if the invalid email is a comma delimited list of email addresses //if so we'll return the first one - maybe try to get them all at some point? if (inString.contains(",")){ - String[] addresses = inString.split("\\,"); + String[] addresses = inString.split("\\,"); return addresses[0]; - } + } //This works on the specific error we've seen where the user has put in a link for the email address //as in ' IFPRI-Data@cgiar.org' @@ -597,25 +768,25 @@ private String convertInvalidEmail(String inString){ if (inString.indexOf(" -1){ try { String eMailAddress = inString.substring(inString.indexOf(">", 0) + 1, inString.indexOf("", inString.indexOf(">", 0))); - return eMailAddress.trim(); + return eMailAddress.trim(); } catch (Exception e){ return null; - } + } } return null; } - + private String convertInvalidDateString(String inString){ - + //converts XXXX0000 to XXXX for date purposes if (inString.trim().length() == 8){ if (inString.trim().endsWith("0000")){ return inString.replace("0000", "").trim(); } } - + //convert question marks to dashes and add brackets - + if (inString.contains("?")) { String testval = inString.replace("?", " ").replace("[", " ").replace("]", " "); if (StringUtils.isNumeric(testval.trim())) { @@ -634,57 +805,57 @@ private String convertInvalidDateString(String inString){ } } } - } - - //Convert string months to numeric - - + } + + //Convert string months to numeric + + if (inString.toUpperCase().contains("JANUARY")){ - return inString.toUpperCase().replace("JANUARY", "").replace(",", "").trim() + "-01"; + return inString.toUpperCase().replace("JANUARY", "").replace(",", "").trim() + "-01"; } - + if (inString.toUpperCase().contains("FEBRUARY")){ - return inString.toUpperCase().replace("FEBRUARY", "").replace(",", "").trim() + "-02"; + return inString.toUpperCase().replace("FEBRUARY", "").replace(",", "").trim() + "-02"; } - + if (inString.toUpperCase().contains("MARCH")){ - return inString.toUpperCase().replace("MARCH", "").replace(",", "").trim() + "-03"; + return inString.toUpperCase().replace("MARCH", "").replace(",", "").trim() + "-03"; } - + if (inString.toUpperCase().contains("APRIL")){ - return inString.toUpperCase().replace("APRIL", "").replace(",", "").trim() + "-04"; + return inString.toUpperCase().replace("APRIL", "").replace(",", "").trim() + "-04"; } - + if (inString.toUpperCase().contains("MAY")){ - return inString.toUpperCase().replace("MAY", "").replace(",", "").trim() + "-05"; + return inString.toUpperCase().replace("MAY", "").replace(",", "").trim() + "-05"; } - + if (inString.toUpperCase().contains("JUNE")){ - return inString.toUpperCase().replace("JUNE", "").replace(",", "").trim() + "-06"; + return inString.toUpperCase().replace("JUNE", "").replace(",", "").trim() + "-06"; } - + if (inString.toUpperCase().contains("JULY")){ - return inString.toUpperCase().replace("JULY", "").replace(",", "").trim() + "-07"; + return inString.toUpperCase().replace("JULY", "").replace(",", "").trim() + "-07"; } - + if (inString.toUpperCase().contains("AUGUST")){ - return inString.toUpperCase().replace("AUGUST", "").replace(",", "").trim() + "-08"; + return inString.toUpperCase().replace("AUGUST", "").replace(",", "").trim() + "-08"; } - + if (inString.toUpperCase().contains("SEPTEMBER")){ - return inString.toUpperCase().replace("SEPTEMBER", "").replace(",", "").trim() + "-09"; + return inString.toUpperCase().replace("SEPTEMBER", "").replace(",", "").trim() + "-09"; } - + if (inString.toUpperCase().contains("OCTOBER")){ - return inString.toUpperCase().replace("OCTOBER", "").replace(",", "").trim() + "-10"; + return inString.toUpperCase().replace("OCTOBER", "").replace(",", "").trim() + "-10"; } - + if (inString.toUpperCase().contains("NOVEMBER")){ - return inString.toUpperCase().replace("NOVEMBER", "").replace(",", "").trim() + "-11"; + return inString.toUpperCase().replace("NOVEMBER", "").replace(",", "").trim() + "-11"; } - + if (inString.toUpperCase().contains("DECEMBER")){ - return inString.toUpperCase().replace("DECEMBER", "").replace(",", "").trim() + "-12"; + return inString.toUpperCase().replace("DECEMBER", "").replace(",", "").trim() + "-12"; } return null; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportUtil.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportUtil.java index d47c20e4e12..43a9f3dab90 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportUtil.java @@ -20,5 +20,11 @@ public enum ImportType{ HARVEST }; + + public enum ImportFileType{ + XML, + JSON + }; + } \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java index 154399688cf..9b8b56f2119 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java @@ -276,7 +276,6 @@ public boolean runAddFileByDataset(Dataset chosenDataset, } - /** * After the constructor, this method is called to add a file * @@ -413,6 +412,9 @@ private boolean runAddReplaceFile(Dataset dataset, return runAddReplacePhase2(); } + + + /** * Note: UI replace is always a "force replace" which means @@ -511,7 +513,8 @@ private boolean runAddReplacePhase1(Dataset dataset, return true; } - + + public boolean runReplaceFromUI_Phase2(){ return runAddReplacePhase2(); } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java index 467ffd734c0..4e525eded3e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java @@ -1,5 +1,7 @@ package edu.harvard.iq.dataverse.engine.command.impl; +import com.thoughtworks.xstream.XStream; +import com.thoughtworks.xstream.io.json.JsonHierarchicalStreamDriver; import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; @@ -10,6 +12,7 @@ import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; import java.util.ArrayList; import java.util.List; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -20,6 +23,9 @@ public class UpdateDatasetVersionCommand extends AbstractDatasetCommand { private static final Logger logger = Logger.getLogger(UpdateDatasetVersionCommand.class.getCanonicalName()); + + static XStream xstream = new XStream(new JsonHierarchicalStreamDriver()); + private final List filesToDelete; private boolean validateLenient = false; private final DatasetVersion clone; @@ -87,11 +93,14 @@ public Dataset execute(CommandContext ctxt) throws CommandException { // Merge the new version into out JPA context, if needed. if ( editVersion.getId() == null || editVersion.getId() == 0L ) { + logger.log(Level.INFO, "editVersion Id is not set: persist data"); ctxt.em().persist(editVersion); } else { + logger.log(Level.INFO, "editVersion Id is already set: merge data"); ctxt.em().merge(editVersion); } + logger.log(Level.INFO, "datafiles attatched to ds:size={0}", getDataset().getFiles().size()); for (DataFile dataFile : getDataset().getFiles()) { if (dataFile.getCreateDate() == null) { dataFile.setCreateDate(getTimestamp()); @@ -124,10 +133,13 @@ public Dataset execute(CommandContext ctxt) throws CommandException { recalculateUNF = true; } } + logger.log(Level.INFO, "recalculateUNF:statsus={0}", recalculateUNF); // we have to merge to update the database but not flush because // we don't want to create two draft versions! Dataset tempDataset = ctxt.em().merge(getDataset()); + logger.log(Level.FINE, "UpdateDatasetVersionCommand:execute:tempDataset={0}", xstream.toXML(tempDataset)); + for (FileMetadata fmd : filesToDelete) { if (!fmd.getDataFile().isReleased()) { // if file is draft (ie. new to this version, delete; otherwise just remove filemetadata object) @@ -154,8 +166,10 @@ public Dataset execute(CommandContext ctxt) throws CommandException { tempDataset.getEditVersion().setLastUpdateTime(getTimestamp()); tempDataset.setModificationTime(getTimestamp()); - + Dataset savedDataset = ctxt.em().merge(tempDataset); + logger.log(Level.FINE, "UpdateDatasetVersionCommand:execute:savedDataset={0}", xstream.toXML(savedDataset)); + ctxt.em().flush(); updateDatasetUser(ctxt); diff --git a/src/main/java/edu/harvard/iq/dataverse/trsa/TrsaRegistries.java b/src/main/java/edu/harvard/iq/dataverse/trsa/TrsaRegistries.java new file mode 100644 index 00000000000..14b14b94b0c --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/trsa/TrsaRegistries.java @@ -0,0 +1,96 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package edu.harvard.iq.dataverse.trsa; + +import edu.harvard.iq.dataverse.api.AbstractApiBean; +import edu.harvard.iq.dataverse.trsa.TrsaRegistry; +import edu.harvard.iq.dataverse.trsa.TrsaRegistryServiceBean; +import java.util.List; +import javax.ejb.EJB; +import javax.ejb.Stateless; +import javax.json.Json; +import javax.json.JsonArrayBuilder; +import javax.persistence.EntityManager; +import javax.persistence.PersistenceContext; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +/** + * + * @author asone + */ + +@Path("admin/trsaRegistries") +public class TrsaRegistries extends AbstractApiBean { + + @EJB + TrsaRegistryServiceBean trsaRegistryServiceBean; + + + @POST + @Consumes({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON}) + public void create(TrsaRegistry entity) { + trsaRegistryServiceBean.create(entity); + } + + @PUT + @Path("{id}") + @Consumes({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON}) + public void edit(@PathParam("id") Long id, TrsaRegistry entity) { + trsaRegistryServiceBean.edit(entity); + } + + @DELETE + @Path("{id}") + public void remove(@PathParam("id") Long id) { + trsaRegistryServiceBean.remove(trsaRegistryServiceBean.find(id)); + } + + @GET + @Path("{id}") + @Produces({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON}) + public TrsaRegistry find(@PathParam("id") Long id) { + return trsaRegistryServiceBean.find(id); + } + + @GET + @Produces({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON}) + public Response getTrsaRegistries() { + JsonArrayBuilder jab = Json.createArrayBuilder(); + trsaRegistryServiceBean.findAll().forEach((trsaRegistry)->{ + jab.add(trsaRegistry.toJson()); + }); + return ok(jab); + } + + + + + + @GET + @Path("{from}/{to}") + @Produces({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON}) + public List findRange(@PathParam("from") Integer from, @PathParam("to") Integer to) { + return trsaRegistryServiceBean.findRange(new int[]{from, to}); + } + + @GET + @Path("count") + @Produces(MediaType.TEXT_PLAIN) + public String countREST() { + return String.valueOf(trsaRegistryServiceBean.count()); + } + + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/trsa/TrsaRegistry.java b/src/main/java/edu/harvard/iq/dataverse/trsa/TrsaRegistry.java new file mode 100644 index 00000000000..7ef5d7c02bf --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/trsa/TrsaRegistry.java @@ -0,0 +1,306 @@ +package edu.harvard.iq.dataverse.trsa; + +import com.ibm.icu.util.Calendar; +import com.ibm.icu.util.ULocale; +import java.io.Serializable; +import java.math.BigDecimal; +import java.sql.Timestamp; +import java.util.Date; +import java.util.logging.Logger; +import javax.json.Json; +import javax.json.JsonObjectBuilder; +import javax.persistence.Basic; +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.NamedQueries; +import javax.persistence.NamedQuery; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; +import javax.validation.constraints.NotNull; +import javax.validation.constraints.Size; +import javax.xml.bind.annotation.XmlRootElement; + +/** + * + * @author asone + */ +@Entity +@Table(name = "trsa_registry", catalog = "dvndb", schema = "public") +@XmlRootElement +@NamedQueries({ + @NamedQuery(name = "TrsaRegistry.findAll", query = "SELECT t FROM TrsaRegistry t"), + @NamedQuery(name = "TrsaRegistry.findByInstallation", query = "SELECT t FROM TrsaRegistry t WHERE t.installation = :installation"), + @NamedQuery(name = "TrsaRegistry.findByEmail", query = "SELECT t FROM TrsaRegistry t WHERE t.email = :email"), + @NamedQuery(name = "TrsaRegistry.findByDataverseurl", query = "SELECT t FROM TrsaRegistry t WHERE t.dataverseurl = :dataverseurl"), + @NamedQuery(name = "TrsaRegistry.findByApitoken", query = "SELECT t FROM TrsaRegistry t WHERE t.apitoken = :apitoken"), + @NamedQuery(name = "TrsaRegistry.findByDatastoragelocation", query = "SELECT t FROM TrsaRegistry t WHERE t.datastoragelocation = :datastoragelocation"), + @NamedQuery(name = "TrsaRegistry.findByDataaccessinfo", query = "SELECT t FROM TrsaRegistry t WHERE t.dataaccessinfo = :dataaccessinfo"), + @NamedQuery(name = "TrsaRegistry.findByNotaryserviceurl", query = "SELECT t FROM TrsaRegistry t WHERE t.notaryserviceurl = :notaryserviceurl"), + @NamedQuery(name = "TrsaRegistry.findBySafeserviceurl", query = "SELECT t FROM TrsaRegistry t WHERE t.safeserviceurl = :safeserviceurl"), + @NamedQuery(name = "TrsaRegistry.findByRegistertime", query = "SELECT t FROM TrsaRegistry t WHERE t.registertime = :registertime"), + @NamedQuery(name = "TrsaRegistry.findByDisabled", query = "SELECT t FROM TrsaRegistry t WHERE t.disabled = :disabled"), + @NamedQuery(name = "TrsaRegistry.findByExpiretime", query = "SELECT t FROM TrsaRegistry t WHERE t.expiretime = :expiretime"), + @NamedQuery(name = "TrsaRegistry.findById", query = "SELECT t FROM TrsaRegistry t WHERE t.id = :id")}) +public class TrsaRegistry implements Serializable { + + private static final Logger logger = Logger.getLogger(TrsaRegistry.class.getName()); + + public static Integer DEFAULT_VALID_PERIOD=1; + + private static final long serialVersionUID = 1L; + + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Basic(optional = false) + @Column(name = "ID") + private Long id; + + + @Basic(optional = false) + @NotNull + @Size(min = 1, max = 255) + @Column(nullable = false, length = 255) + private String installation; + // @Pattern(regexp="[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?", message="Invalid email")//if the field contains email address consider using this annotation to enforce field validation + + @Basic(optional = false) + @NotNull + @Size(min = 1, max = 255) + @Column(nullable = false, length = 255) + private String email; + + @Basic(optional = false) + @NotNull + @Size(min = 1, max = 255) + @Column(nullable = false, length = 255) + private String dataverseurl; + + @Basic(optional = false) + @NotNull + @Size(min = 1, max = 255) + @Column(nullable = false, length = 255) + private String apitoken; + + + @Basic(optional = false) + @NotNull + @Size(min=1, max = 255) + @Column(nullable = false, length = 255) + private String datastoragelocation; + + @Basic(optional = false) + @NotNull + @Size(min = 1, max = 255) + @Column(nullable = false, length = 255) + private String dataaccessinfo; + + @Basic(optional = false) + @NotNull + @Size(min = 1, max = 255) + @Column(nullable = false, length = 255) + private String notaryserviceurl; + + @Basic(optional = false) + @NotNull + @Size(max = 255) + @Column(nullable = false, length = 255) + private String safeserviceurl; + + @Temporal(TemporalType.TIMESTAMP) + private Date registertime; + + private Boolean disabled; + + @Temporal(TemporalType.TIMESTAMP) + private Date expiretime; + + + public TrsaRegistry() { + } + + public TrsaRegistry(Long id) { + this.id = id; + } + + public TrsaRegistry(Long id, String installation, String email, + String dataverseurl, String apitoken, String datastoragelocation, + String dataaccessinfo, String notaryserviceurl, + String safeserviceurl) { + this.id = id; + this.installation = installation; + this.email = email; + this.dataverseurl = dataverseurl; + this.apitoken = apitoken; + this.datastoragelocation = datastoragelocation; + this.dataaccessinfo = dataaccessinfo; + this.notaryserviceurl = notaryserviceurl; + this.registertime = new Timestamp(new Date().getTime()); + this.expiretime = generateExpireTimestamp(); + this.disabled=false; + } + + public String getInstallation() { + return installation; + } + + public void setInstallation(String installation) { + this.installation = installation; + } + + public String getEmail() { + return email; + } + + public void setEmail(String email) { + this.email = email; + } + + public String getDataverseurl() { + return dataverseurl; + } + + public void setDataverseurl(String dataverseurl) { + this.dataverseurl = dataverseurl; + } + + public String getApitoken() { + return apitoken; + } + + public void setApitoken(String apitoken) { + this.apitoken = apitoken; + } + + public String getDatastoragelocation() { + return datastoragelocation; + } + + public void setDatastoragelocation(String datastoragelocation) { + this.datastoragelocation = datastoragelocation; + } + + public String getDataaccessinfo() { + return dataaccessinfo; + } + + public void setDataaccessinfo(String dataaccessinfo) { + this.dataaccessinfo = dataaccessinfo; + } + + public String getNotaryserviceurl() { + return notaryserviceurl; + } + + public void setNotaryserviceurl(String notaryserviceurl) { + this.notaryserviceurl = notaryserviceurl; + } + + public String getSafeserviceurl() { + return safeserviceurl; + } + + public void setSafeserviceurl(String safeserviceurl) { + this.safeserviceurl = safeserviceurl; + } + + public Date getRegistertime() { + return registertime; + } + + public void setRegistertime(Date registertime) { + this.registertime = registertime; + } + + public Boolean getDisabled() { + return disabled; + } + + public void setDisabled(Boolean disabled) { + this.disabled = disabled; + } + + public Date getExpiretime() { + return expiretime; + } + + public void setExpiretime(Date expiretime) { + this.expiretime = expiretime; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + @Override + public int hashCode() { + int hash = 0; + hash += (id != null ? id.hashCode() : 0); + return hash; + } + + @Override + public boolean equals(Object object) { + // TODO: Warning - this method won't work in the case the id fields are not set + if (!(object instanceof TrsaRegistry)) { + return false; + } + TrsaRegistry other = (TrsaRegistry) object; + if ((this.id == null && other.id != null) || (this.id != null && !this.id.equals(other.id))) { + return false; + } + return true; + } + + @Override + public String toString() { + return "edu.harvard.iq.dataverse.trsa.TrsaRegistry[ id=" + id + " ]"; + } + + + public JsonObjectBuilder toJson() { + JsonObjectBuilder jab = Json.createObjectBuilder(); + return jab.add("id", getId()) + .add("installation", getInstallation()) + .add("email", getEmail()) + .add("dataverseurl", getDataverseurl()) + .add("apitoken", getApitoken()) + .add("datastoragelocation", getDatastoragelocation()) + .add("dataaccessinfo", getDataaccessinfo()) + .add("notaryserviceurl", getNotaryserviceurl()) + .add("registertime", getRegistertime().toString()) + .add("expiretime", getExpiretime().toString()) + ; + +// jab.add(DISPLAY_NAME, getDisplayName()); +// jab.add(DESCRIPTION, getDescription()); +// jab.add(TYPE, getType().text); +// jab.add(TOOL_URL, getToolUrl()); +// jab.add(TOOL_PARAMETERS, getToolParameters()); + } + + private Date generateExpireTimestamp(){ + return generateExpireTimestamp(null); + } + + + private Date generateExpireTimestamp(Integer year){ + if (year==null){ + year = DEFAULT_VALID_PERIOD; + } + Date baseline= this.registertime; + Calendar cal = Calendar.getInstance(); + cal.setTimeInMillis(baseline.getTime()); + cal.add(Calendar.YEAR, year); + return new Date(cal.getTime().getTime()); + + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/trsa/TrsaRegistryServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/trsa/TrsaRegistryServiceBean.java new file mode 100644 index 00000000000..a6b2d2ec703 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/trsa/TrsaRegistryServiceBean.java @@ -0,0 +1,88 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package edu.harvard.iq.dataverse.trsa; + +import java.util.List; +import java.util.logging.Logger; +import javax.ejb.Stateless; +import javax.inject.Named; +import javax.persistence.EntityManager; +import javax.persistence.NoResultException; +import javax.persistence.NonUniqueResultException; +import javax.persistence.PersistenceContext; +import javax.persistence.TypedQuery; + +/** + * + * @author asone + */ +@Stateless +@Named +public class TrsaRegistryServiceBean { + + private static final Logger logger = Logger.getLogger(TrsaRegistryServiceBean.class.getName()); + + @PersistenceContext(unitName = "VDCNet-ejbPU") + private EntityManager em; + + + public void create(TrsaRegistry entity) { + em.persist(entity); + } + + + public void edit(TrsaRegistry entity) { + em.merge(entity); + } + + + public void remove(TrsaRegistry entity) { + em.remove(em.merge(entity)); + } + + + public TrsaRegistry find(long id) { + return em.find(TrsaRegistry.class, id); + } + + + public List findAll() { + javax.persistence.criteria.CriteriaQuery cq = em.getCriteriaBuilder().createQuery(); + cq.select(cq.from(TrsaRegistry.class)); + return em.createQuery(cq).getResultList(); + } + + + public List findRange(int[] range) { + javax.persistence.criteria.CriteriaQuery cq = em.getCriteriaBuilder().createQuery(); + cq.select(cq.from(TrsaRegistry.class)); + javax.persistence.Query q = em.createQuery(cq); + q.setMaxResults(range[1] - range[0] + 1); + q.setFirstResult(range[0]); + return q.getResultList(); + } + + public long count() { + javax.persistence.criteria.CriteriaQuery cq = em.getCriteriaBuilder().createQuery(); + javax.persistence.criteria.Root rt = cq.from(TrsaRegistry.class); + cq.select(em.getCriteriaBuilder().count(rt)); + javax.persistence.Query q = em.createQuery(cq); + return (long) q.getSingleResult(); + } + + + public TrsaRegistry findById(long id) { + TypedQuery typedQuery = em.createQuery("SELECT OBJECT(o) FROM TrsaRegistry AS o WHERE o.id = :id", TrsaRegistry.class); + typedQuery.setParameter("id", id); + try { + TrsaRegistry trsaRegistry = typedQuery.getSingleResult(); + return trsaRegistry; + } catch (NoResultException | NonUniqueResultException ex) { + return null; + } + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/trsa/registry/AbstractFacade.java b/src/main/java/edu/harvard/iq/dataverse/trsa/registry/AbstractFacade.java new file mode 100644 index 00000000000..bd0ce7b56c3 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/trsa/registry/AbstractFacade.java @@ -0,0 +1,64 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package edu.harvard.iq.dataverse.trsa.registry; + +import java.util.List; +import javax.persistence.EntityManager; + +/** + * + * @author asone + */ +public abstract class AbstractFacade { + + private Class entityClass; + + public AbstractFacade(Class entityClass) { + this.entityClass = entityClass; + } + + protected abstract EntityManager getEntityManager(); + + public void create(T entity) { + getEntityManager().persist(entity); + } + + public void edit(T entity) { + getEntityManager().merge(entity); + } + + public void remove(T entity) { + getEntityManager().remove(getEntityManager().merge(entity)); + } + + public T find(Object id) { + return getEntityManager().find(entityClass, id); + } + + public List findAll() { + javax.persistence.criteria.CriteriaQuery cq = getEntityManager().getCriteriaBuilder().createQuery(); + cq.select(cq.from(entityClass)); + return getEntityManager().createQuery(cq).getResultList(); + } + + public List findRange(int[] range) { + javax.persistence.criteria.CriteriaQuery cq = getEntityManager().getCriteriaBuilder().createQuery(); + cq.select(cq.from(entityClass)); + javax.persistence.Query q = getEntityManager().createQuery(cq); + q.setMaxResults(range[1] - range[0] + 1); + q.setFirstResult(range[0]); + return q.getResultList(); + } + + public int count() { + javax.persistence.criteria.CriteriaQuery cq = getEntityManager().getCriteriaBuilder().createQuery(); + javax.persistence.criteria.Root rt = cq.from(entityClass); + cq.select(getEntityManager().getCriteriaBuilder().count(rt)); + javax.persistence.Query q = getEntityManager().createQuery(cq); + return ((Long) q.getSingleResult()).intValue(); + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/trsa/registry/TrsaRegistryController.java b/src/main/java/edu/harvard/iq/dataverse/trsa/registry/TrsaRegistryController.java new file mode 100644 index 00000000000..f71b5dfd39d --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/trsa/registry/TrsaRegistryController.java @@ -0,0 +1,164 @@ +package edu.harvard.iq.dataverse.trsa.registry; + +import edu.harvard.iq.dataverse.trsa.TrsaRegistry; +import edu.harvard.iq.dataverse.trsa.registry.util.JsfUtil; +import edu.harvard.iq.dataverse.trsa.registry.util.JsfUtil.PersistAction; + +import java.io.Serializable; +import java.util.List; +import java.util.ResourceBundle; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.ejb.EJB; +import javax.ejb.EJBException; +import javax.inject.Named; +import javax.enterprise.context.SessionScoped; +import javax.faces.component.UIComponent; +import javax.faces.context.FacesContext; +import javax.faces.convert.Converter; +import javax.faces.convert.FacesConverter; + +@Named("trsaRegistryController") +@SessionScoped +public class TrsaRegistryController implements Serializable { + + @EJB + private edu.harvard.iq.dataverse.trsa.registry.TrsaRegistryFacade ejbFacade; + private List items = null; + private TrsaRegistry selected; + + public TrsaRegistryController() { + } + + public TrsaRegistry getSelected() { + return selected; + } + + public void setSelected(TrsaRegistry selected) { + this.selected = selected; + } + + protected void setEmbeddableKeys() { + } + + protected void initializeEmbeddableKey() { + } + + private TrsaRegistryFacade getFacade() { + return ejbFacade; + } + + public TrsaRegistry prepareCreate() { + selected = new TrsaRegistry(); + initializeEmbeddableKey(); + return selected; + } + + public void create() { + persist(PersistAction.CREATE, ResourceBundle.getBundle("/Bundle_trsa_registry").getString("TrsaRegistryCreated")); + if (!JsfUtil.isValidationFailed()) { + items = null; // Invalidate list of items to trigger re-query. + } + } + + public void update() { + persist(PersistAction.UPDATE, ResourceBundle.getBundle("/Bundle_trsa_registry").getString("TrsaRegistryUpdated")); + } + + public void destroy() { + persist(PersistAction.DELETE, ResourceBundle.getBundle("/Bundle_trsa_registry").getString("TrsaRegistryDeleted")); + if (!JsfUtil.isValidationFailed()) { + selected = null; // Remove selection + items = null; // Invalidate list of items to trigger re-query. + } + } + + public List getItems() { + if (items == null) { + items = getFacade().findAll(); + } + return items; + } + + private void persist(PersistAction persistAction, String successMessage) { + if (selected != null) { + setEmbeddableKeys(); + try { + if (persistAction != PersistAction.DELETE) { + getFacade().edit(selected); + } else { + getFacade().remove(selected); + } + JsfUtil.addSuccessMessage(successMessage); + } catch (EJBException ex) { + String msg = ""; + Throwable cause = ex.getCause(); + if (cause != null) { + msg = cause.getLocalizedMessage(); + } + if (msg.length() > 0) { + JsfUtil.addErrorMessage(msg); + } else { + JsfUtil.addErrorMessage(ex, ResourceBundle.getBundle("/Bundle_trsa_registry").getString("PersistenceErrorOccured")); + } + } catch (Exception ex) { + Logger.getLogger(this.getClass().getName()).log(Level.SEVERE, null, ex); + JsfUtil.addErrorMessage(ex, ResourceBundle.getBundle("/Bundle_trsa_registry").getString("PersistenceErrorOccured")); + } + } + } + + public TrsaRegistry getTrsaRegistry(java.lang.Long id) { + return getFacade().find(id); + } + + public List getItemsAvailableSelectMany() { + return getFacade().findAll(); + } + + public List getItemsAvailableSelectOne() { + return getFacade().findAll(); + } + + @FacesConverter(forClass = TrsaRegistry.class) + public static class TrsaRegistryControllerConverter implements Converter { + + @Override + public Object getAsObject(FacesContext facesContext, UIComponent component, String value) { + if (value == null || value.length() == 0) { + return null; + } + TrsaRegistryController controller = (TrsaRegistryController) facesContext.getApplication().getELResolver(). + getValue(facesContext.getELContext(), null, "trsaRegistryController"); + return controller.getTrsaRegistry(getKey(value)); + } + + java.lang.Long getKey(String value) { + java.lang.Long key; + key = Long.valueOf(value); + return key; + } + + String getStringKey(java.lang.Long value) { + StringBuilder sb = new StringBuilder(); + sb.append(value); + return sb.toString(); + } + + @Override + public String getAsString(FacesContext facesContext, UIComponent component, Object object) { + if (object == null) { + return null; + } + if (object instanceof TrsaRegistry) { + TrsaRegistry o = (TrsaRegistry) object; + return getStringKey(o.getId()); + } else { + Logger.getLogger(this.getClass().getName()).log(Level.SEVERE, "object {0} is of type {1}; expected type: {2}", new Object[]{object, object.getClass().getName(), TrsaRegistry.class.getName()}); + return null; + } + } + + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/trsa/registry/TrsaRegistryFacade.java b/src/main/java/edu/harvard/iq/dataverse/trsa/registry/TrsaRegistryFacade.java new file mode 100644 index 00000000000..60c721cdf32 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/trsa/registry/TrsaRegistryFacade.java @@ -0,0 +1,32 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package edu.harvard.iq.dataverse.trsa.registry; + +import edu.harvard.iq.dataverse.trsa.TrsaRegistry; +import javax.ejb.Stateless; +import javax.persistence.EntityManager; +import javax.persistence.PersistenceContext; + +/** + * + * @author asone + */ +@Stateless +public class TrsaRegistryFacade extends AbstractFacade { + + @PersistenceContext(unitName = "VDCNet-ejbPU") + private EntityManager em; + + @Override + protected EntityManager getEntityManager() { + return em; + } + + public TrsaRegistryFacade() { + super(TrsaRegistry.class); + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/trsa/registry/util/JsfUtil.java b/src/main/java/edu/harvard/iq/dataverse/trsa/registry/util/JsfUtil.java new file mode 100644 index 00000000000..8881fe5bfa4 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/trsa/registry/util/JsfUtil.java @@ -0,0 +1,69 @@ +package edu.harvard.iq.dataverse.trsa.registry.util; + +import java.util.List; +import javax.faces.application.FacesMessage; +import javax.faces.component.UIComponent; +import javax.faces.context.FacesContext; +import javax.faces.convert.Converter; +import javax.faces.model.SelectItem; + +public class JsfUtil { + + public static SelectItem[] getSelectItems(List entities, boolean selectOne) { + int size = selectOne ? entities.size() + 1 : entities.size(); + SelectItem[] items = new SelectItem[size]; + int i = 0; + if (selectOne) { + items[0] = new SelectItem("", "---"); + i++; + } + for (Object x : entities) { + items[i++] = new SelectItem(x, x.toString()); + } + return items; + } + + public static boolean isValidationFailed() { + return FacesContext.getCurrentInstance().isValidationFailed(); + } + + public static void addErrorMessage(Exception ex, String defaultMsg) { + String msg = ex.getLocalizedMessage(); + if (msg != null && msg.length() > 0) { + addErrorMessage(msg); + } else { + addErrorMessage(defaultMsg); + } + } + + public static void addErrorMessages(List messages) { + for (String message : messages) { + addErrorMessage(message); + } + } + + public static void addErrorMessage(String msg) { + FacesMessage facesMsg = new FacesMessage(FacesMessage.SEVERITY_ERROR, msg, msg); + FacesContext.getCurrentInstance().addMessage(null, facesMsg); + } + + public static void addSuccessMessage(String msg) { + FacesMessage facesMsg = new FacesMessage(FacesMessage.SEVERITY_INFO, msg, msg); + FacesContext.getCurrentInstance().addMessage("successInfo", facesMsg); + } + + public static String getRequestParameter(String key) { + return FacesContext.getCurrentInstance().getExternalContext().getRequestParameterMap().get(key); + } + + public static Object getObjectFromRequestParameter(String requestParameterName, Converter converter, UIComponent component) { + String theId = JsfUtil.getRequestParameter(requestParameterName); + return converter.getAsObject(FacesContext.getCurrentInstance(), component, theId); + } + + public static enum PersistAction { + CREATE, + DELETE, + UPDATE + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java index 3b8efa17513..742b9c66050 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java @@ -1,9 +1,12 @@ package edu.harvard.iq.dataverse.util.json; import com.google.gson.Gson; +import com.thoughtworks.xstream.XStream; +import com.thoughtworks.xstream.io.json.JsonHierarchicalStreamDriver; import edu.harvard.iq.dataverse.ControlledVocabularyValue; import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.DataFileCategory; +import edu.harvard.iq.dataverse.DataTable; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetField; import edu.harvard.iq.dataverse.DatasetFieldConstant; @@ -25,6 +28,9 @@ import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddress; import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddressRange; import edu.harvard.iq.dataverse.datasetutility.OptionalFileParams; +import edu.harvard.iq.dataverse.datavariable.DataVariable; +import edu.harvard.iq.dataverse.datavariable.SummaryStatistic; +import edu.harvard.iq.dataverse.datavariable.VariableCategory; import edu.harvard.iq.dataverse.harvest.client.HarvestingClient; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.workflow.Workflow; @@ -41,14 +47,17 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.logging.Level; import java.util.logging.Logger; import javax.json.Json; import javax.json.JsonArray; +import javax.json.JsonNumber; import javax.json.JsonObject; import javax.json.JsonReader; import javax.json.JsonString; import javax.json.JsonValue; import javax.json.JsonValue.ValueType; +import org.apache.commons.lang.StringUtils; /** * Parses JSON objects into domain objects. @@ -58,7 +67,7 @@ public class JsonParser { private static final Logger logger = Logger.getLogger(JsonParser.class.getCanonicalName()); - + static XStream xstream = new XStream(new JsonHierarchicalStreamDriver()); DatasetFieldServiceBean datasetFieldSvc; MetadataBlockServiceBean blockService; SettingsServiceBean settingsService; @@ -247,10 +256,15 @@ public IpGroup parseIpGroup(JsonObject obj) { } public DatasetVersion parseDatasetVersion(JsonObject obj) throws JsonParseException { + logger.log(Level.INFO, "1-arg-parseDatasetVersion:obj is called"); + logger.log(Level.INFO, "obj:1-arg-parseDatasetVersion={0}",xstream.toXML(obj)); + logger.log(Level.INFO, "within 1-arg-parseDatasetVersion, 2-args-version is called"); return parseDatasetVersion(obj, new DatasetVersion()); } public Dataset parseDataset(JsonObject obj) throws JsonParseException { + logger.log(Level.INFO, "parseDataset is called"); + logger.log(Level.INFO, "obj: parseDataset={0}",xstream.toXML(obj)); Dataset dataset = new Dataset(); dataset.setAuthority(obj.getString("authority", null) == null ? settingsService.getValueForKey(SettingsServiceBean.Key.Authority) : obj.getString("authority")); @@ -259,6 +273,7 @@ public Dataset parseDataset(JsonObject obj) throws JsonParseException { DatasetVersion dsv = new DatasetVersion(); dsv.setDataset(dataset); + logger.log(Level.INFO, "calling 2-args-parseDatasetVersion method"); dsv = parseDatasetVersion(obj.getJsonObject("datasetVersion"), dsv); List versions = new ArrayList<>(1); versions.add(dsv); @@ -268,6 +283,8 @@ public Dataset parseDataset(JsonObject obj) throws JsonParseException { } public DatasetVersion parseDatasetVersion(JsonObject obj, DatasetVersion dsv) throws JsonParseException { + logger.log(Level.INFO, "2-args-parseDatasetVersion is called"); + logger.log(Level.INFO, "obj:2-args-parseDatasetVersion={0}",xstream.toXML(obj)); try { String archiveNote = obj.getString("archiveNote", null); @@ -326,6 +343,7 @@ public DatasetVersion parseDatasetVersion(JsonObject obj, DatasetVersion dsv) th filesJson = obj.getJsonArray("fileMetadatas"); } if (filesJson != null) { + logger.log(Level.INFO, "parseFiles to be called"); dsv.setFileMetadatas(parseFiles(filesJson, dsv)); } return dsv; @@ -345,7 +363,11 @@ private License parseLicense(String inString) { } public List parseMetadataBlocks(JsonObject json) throws JsonParseException { + logger.log(Level.INFO, "within parseMetadataBlocks"); + logger.log(Level.FINE, "json={0}",xstream.toXML(json)); + Set keys = json.keySet(); + logger.log(Level.INFO, "keys={0}", xstream.toXML(keys)); List fields = new LinkedList<>(); for (String blockName : keys) { @@ -371,6 +393,7 @@ public List parseMultipleFieldsForDelete(JsonObject json) throws J } private List parseFieldsFromArray(JsonArray fieldsArray, Boolean testType) throws JsonParseException { + logger.log(Level.INFO, "entering parseFieldsFromArray method:testType={0}", testType); List fields = new LinkedList<>(); for (JsonObject fieldJson : fieldsArray.getValuesAs(JsonObject.class)) { try { @@ -391,16 +414,19 @@ private List parseFieldsFromArray(JsonArray fieldsArray, Boolean t } public List parseFiles(JsonArray metadatasJson, DatasetVersion dsv) throws JsonParseException { + logger.log(Level.INFO, "parseFiles is called"); List fileMetadatas = new LinkedList<>(); if (metadatasJson != null) { for (JsonObject filemetadataJson : metadatasJson.getValuesAs(JsonObject.class)) { String label = filemetadataJson.getString("label"); + boolean restricted = filemetadataJson.getBoolean("restricted", false); String directoryLabel = filemetadataJson.getString("directoryLabel", null); String description = filemetadataJson.getString("description", null); FileMetadata fileMetadata = new FileMetadata(); fileMetadata.setLabel(label); + fileMetadata.setRestricted(restricted); fileMetadata.setDirectoryLabel(directoryLabel); fileMetadata.setDescription(description); fileMetadata.setDatasetVersion(dsv); @@ -408,6 +434,14 @@ public List parseFiles(JsonArray metadatasJson, DatasetVersion dsv if ( filemetadataJson.containsKey("dataFile") ) { DataFile dataFile = parseDataFile(filemetadataJson.getJsonObject("dataFile")); dataFile.getFileMetadatas().add(fileMetadata); + if (dsv.getDataset() == null) { + logger.log(Level.INFO, "datasetVersion does not have a dataset; attach a dataset to dsv"); + Dataset dataset = new Dataset(); + dsv.setDataset(dataset); + } + // the following line may cause NullpointerException + // if a Dataset is not attached to dsv + // therefore a null test is necessary dataFile.setOwner(dsv.getDataset()); fileMetadata.setDataFile(dataFile); if (dsv.getDataset().getFiles() == null) { @@ -426,6 +460,8 @@ public List parseFiles(JsonArray metadatasJson, DatasetVersion dsv } public DataFile parseDataFile(JsonObject datafileJson) { + logger.log(Level.INFO, "parseDataFile is called"); + logger.log(Level.INFO, "datafileJson={0}",xstream.toXML(datafileJson)); DataFile dataFile = new DataFile(); Timestamp timestamp = new Timestamp(new Date().getTime()); @@ -433,6 +469,10 @@ public DataFile parseDataFile(JsonObject datafileJson) { dataFile.setModificationTime(timestamp); dataFile.setPermissionModificationTime(timestamp); + // as of version 4.9.4, missing datafile-items that exist in JsonPrinter + // persistentId + // pidURL + if ( datafileJson.containsKey("filesize") ) { dataFile.setFilesize(datafileJson.getJsonNumber("filesize").longValueExact()); } @@ -442,6 +482,26 @@ public DataFile parseDataFile(JsonObject datafileJson) { contentType = "application/octet-stream"; } String storageIdentifier = datafileJson.getString("storageIdentifier", " "); + + // available items + // filename + String filename = datafileJson.getString("filename", null); + // filesize + JsonNumber filesizejsn = datafileJson.getJsonNumber("filesize"); + Long filesize = null; + if (filesizejsn != null){ + filesize = datafileJson.getJsonNumber("filesize").longValue(); + } + // originalFileFormat + String originalFileFormat = datafileJson.getString("originalFileFormat", null); + // originalFormatLabel + String originalFormatLabel = datafileJson.getString("originalFormatLabel", null); + // UNF + String UNF = datafileJson.getString("UNF", null); + // md5 + String MD5 = datafileJson.getString("md5", null); + + JsonObject checksum = datafileJson.getJsonObject("checksum"); if (checksum != null) { // newer style that allows for SHA-1 rather than MD5 @@ -476,13 +536,224 @@ public DataFile parseDataFile(JsonObject datafileJson) { } // TODO: + // the UNF of DataFile is called via DataTable and therefor no set method // unf (if available)... etc.? dataFile.setContentType(contentType); dataFile.setStorageIdentifier(storageIdentifier); + if (filesize != null){ + dataFile.setFilesize(filesize); + } + logger.log(Level.INFO, "parsing DataTable"); + // parse DataTable + JsonArray dataTablesJson = datafileJson.getJsonArray("dataTables"); + if ((dataTablesJson != null ) && (!dataTablesJson.isEmpty())){ + logger.log(Level.INFO, "dataTablesJson:size={0}", dataTablesJson.size()); + // get parsing results of a DataTable + List dataTables = parseDataTables(dataTablesJson); + logger.log(Level.INFO, "dataTables:size={0}", dataTables.size()); + logger.log(Level.FINE, "returned dataTables={0}", xstream.toXML(dataTables)); + dataFile.setDataTables(dataTables); + dataFile.setDataTable(dataTables.get(0)); + dataTables.get(0).setDataFile(dataFile); + dataTables.get(0).setOriginalFileFormat(originalFileFormat); + + } + logger.log(Level.INFO, "dataFile: parseDataFile={0}",xstream.toXML(dataFile)); return dataFile; } + + + + public List parseDataTables(JsonArray dataTablesJson){ + logger.log(Level.INFO, "parseDataTables is called"); + List dataTables = new LinkedList<>(); + if ((dataTablesJson !=null) && (!dataTablesJson.isEmpty())){ + logger.log(Level.INFO, "dataTables is not empty"); + for (JsonObject dataTableJsonL : dataTablesJson.getValuesAs(JsonObject.class)){ + JsonObject dataTableJson = dataTableJsonL.getJsonObject("dataTable"); + DataTable dataTable = new DataTable(); + // capture scalar items + // varQuantity + long varQuantity = dataTableJson.getJsonNumber("varQuantity").longValue(); + dataTable.setVarQuantity(varQuantity); + // caseQuantity + long caseQuantity = dataTableJson.getJsonNumber("caseQuantity").longValue(); + dataTable.setCaseQuantity(caseQuantity); + // UNF + String UNF = dataTableJson.getString("UNF", null); + dataTable.setUnf(UNF); + // call the method for pasring dataVariables array + List dataVariables = parseDataVariables(dataTableJson.getJsonArray("dataVariables"), dataTable); + logger.log(Level.INFO, "returned dataVariables list: size={0}", dataVariables.size()); + + dataTable.setDataVariables(dataVariables); + dataTables.add(dataTable); + logger.log(Level.INFO, " dataTables: current size={0}", dataTables.size()); + } + } + logger.log(Level.INFO, "dataTables: size(final)={0}", dataTables.size()); + return dataTables; + } + + + // note: the following method has not yet implemented a few of processXXX- + // calls in DataTableImportDDI#processVar + + public List parseDataVariables(JsonArray dataVariablesJson, DataTable dataTable){ + logger.log(Level.INFO, "parseDataVariables is called"); + List dataVariables = new LinkedList<>(); + if ((dataVariablesJson != null) && (!dataVariablesJson.isEmpty())) { + logger.log(Level.INFO, "dataVariablesJson is not empty:size={0}", dataVariablesJson.size()); + for (JsonObject dataVariableJson: dataVariablesJson.getValuesAs(JsonObject.class)){ + DataVariable dataVariable = new DataVariable(); + // capture scalar itemse. + // name + dataVariable.setName(dataVariableJson.getString("name", null)); + // label + dataVariable.setLabel(dataVariableJson.getString("label", null)); + // weighted + dataVariable.setWeighted(dataVariableJson.getBoolean("weighted", false)); + String variableIntervalType= dataVariableJson.getString("variableIntervalType", null); + if (variableIntervalType!=null){ + String variableIntervalTypeFinal = variableIntervalType.toUpperCase(); + if (variableIntervalType.equals("contin")){ + variableIntervalTypeFinal="CONTINUOUS"; + } + dataVariable.setInterval(DataVariable.VariableInterval.valueOf(variableIntervalTypeFinal)); + } + // variableFormatType + String variableFormatType = dataVariableJson.getString("variableFormatType", null); + if (variableFormatType!=null){ + dataVariable.setType(DataVariable.VariableType.valueOf(variableFormatType)); + } + // orderedFactor + dataVariable.setOrderedCategorical(dataVariableJson.getBoolean("orderedFactor", false)); + // fileOrder + dataVariable.setFileOrder(dataVariableJson.getInt("fileOrder")); + + // summaryStatistics + dataVariable.setSummaryStatistics(parseSummaryStatistics(dataVariableJson.getJsonObject("summaryStatistics"), dataVariable)); + // variableCategories + dataVariable.setCategories(parseVariableCategories(dataVariableJson.getJsonArray("variableCategories"), dataVariable)); + + // UNF + dataVariable.setUnf(dataVariableJson.getString("UNF", null)); + + // DataTable (do not forget this) + dataVariable.setDataTable(dataTable); + dataVariables.add(dataVariable); + logger.log(Level.INFO, "dataVariables: current size={0}", dataVariables.size()); + } + } + logger.log(Level.INFO, "dataVariables:final size={0}", dataVariables.size()); + return dataVariables; + } + + + public List parseSummaryStatistics(JsonObject summaryStatisticsJson, DataVariable dataVariable){ + logger.log(Level.INFO, "parseSummaryStatistics is called"); + List summaryStatistics = new LinkedList<>(); + if (summaryStatisticsJson !=null){ + // mean + String meanjsn = summaryStatisticsJson.getString("mean", null); + if (StringUtils.isNotBlank(meanjsn)){ + SummaryStatistic mean = new SummaryStatistic(); + mean.setType(SummaryStatistic.SummaryStatisticType.MEAN); + mean.setValue(meanjsn); + mean.setDataVariable(dataVariable); + summaryStatistics.add(mean); + } + // medn + String mednjsn = summaryStatisticsJson.getString("medn", null); + if (StringUtils.isNotBlank(mednjsn)){ + SummaryStatistic medn = new SummaryStatistic(); + medn.setType(SummaryStatistic.SummaryStatisticType.MEDN); + medn.setValue(mednjsn); + medn.setDataVariable(dataVariable); + summaryStatistics.add(medn); + } + // mode + String modejsn = summaryStatisticsJson.getString("mode", null); + if (StringUtils.isNotBlank(modejsn)){ + SummaryStatistic mode = new SummaryStatistic(); + mode.setType(SummaryStatistic.SummaryStatisticType.MODE); + mode.setValue(modejsn); + mode.setDataVariable(dataVariable); + summaryStatistics.add(mode); + } + // vald + String valdjsn = summaryStatisticsJson.getString("vald", null); + if (StringUtils.isNotBlank(valdjsn)){ + SummaryStatistic vald = new SummaryStatistic(); + vald.setType(SummaryStatistic.SummaryStatisticType.VALD); + vald.setValue(valdjsn); + vald.setDataVariable(dataVariable); + summaryStatistics.add(vald); + } + // invd + String invdjsn = summaryStatisticsJson.getString("invd", null); + if (StringUtils.isNotBlank(invdjsn)){ + SummaryStatistic invd = new SummaryStatistic(); + invd.setType(SummaryStatistic.SummaryStatisticType.INVD); + invd.setValue(invdjsn); + invd.setDataVariable(dataVariable); + summaryStatistics.add(invd); + } + // min + String minjsn = summaryStatisticsJson.getString("min", null); + if (StringUtils.isNotBlank(minjsn)){ + SummaryStatistic min = new SummaryStatistic(); + min.setType(SummaryStatistic.SummaryStatisticType.MIN); + min.setValue(minjsn); + min.setDataVariable(dataVariable); + summaryStatistics.add(min); + } + // max + String maxjsn = summaryStatisticsJson.getString("max", null); + if (StringUtils.isNotBlank(maxjsn)){ + SummaryStatistic max = new SummaryStatistic(); + max.setType(SummaryStatistic.SummaryStatisticType.MAX); + max.setValue(maxjsn); + max.setDataVariable(dataVariable); + summaryStatistics.add(max); + } + // stdev + String stdevjsn = summaryStatisticsJson.getString("stdev", null); + if (StringUtils.isNotBlank(stdevjsn)){ + SummaryStatistic stdev = new SummaryStatistic(); + stdev.setType(SummaryStatistic.SummaryStatisticType.STDEV); + stdev.setValue(stdevjsn); + stdev.setDataVariable(dataVariable); + summaryStatistics.add(stdev); + } + } + return summaryStatistics; + } + + + public List parseVariableCategories(JsonArray variableCategoriesJson, DataVariable dataVariable){ + logger.log(Level.INFO, "parseVariableCategories is called"); + List variableCategories = new LinkedList<>(); + if ((variableCategoriesJson != null) && (!variableCategoriesJson.isEmpty())){ + for (JsonObject variableCategoryJson : variableCategoriesJson.getValuesAs(JsonObject.class)){ + VariableCategory vc = new VariableCategory(); + // label + String label = variableCategoryJson.getString("label", ""); + vc.setLabel(label); + // value + String value = variableCategoryJson.getString("value", ""); + vc.setValue(value); + vc.setDataVariable(dataVariable); + variableCategories.add(vc); + } + } + return variableCategories; + } + + + /** * Special processing for GeographicCoverage compound field: * Handle parsing exceptions caused by invalid controlled vocabulary in the "country" field by @@ -537,17 +808,24 @@ public DatasetField parseField(JsonObject json) throws JsonParseException{ public DatasetField parseField(JsonObject json, Boolean testType) throws JsonParseException { + logger.log(Level.INFO, "parseField: testType={0}", testType); if (json == null) { return null; } DatasetField ret = new DatasetField(); DatasetFieldType type = datasetFieldSvc.findByNameOpt(json.getString("typeName", "")); - + if (type == null) { throw new JsonParseException("Can't find type '" + json.getString("typeName", "") + "'"); } + logger.log(Level.INFO, "DatasetFieldType:name={0}", type.getName()); + logger.log(Level.INFO, "testType={0}", testType); + logger.log(Level.INFO, "type.isAllowMultiples()={0}", type.isAllowMultiples()); + logger.log(Level.INFO, "json={0}", xstream.toXML(json)); + + if (testType && type.isAllowMultiples() != json.getBoolean("multiple")) { throw new JsonParseException("incorrect multiple for field " + json.getString("typeName", "")); } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 5f81dc554e2..a608eace98a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -3,6 +3,7 @@ import edu.harvard.iq.dataverse.ControlledVocabularyValue; import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.DataFileTag; +import edu.harvard.iq.dataverse.DataTable; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetDistributor; import edu.harvard.iq.dataverse.DatasetFieldType; @@ -33,6 +34,9 @@ import edu.harvard.iq.dataverse.authorization.providers.AuthenticationProviderRow; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.User; +import edu.harvard.iq.dataverse.datavariable.DataVariable; +import edu.harvard.iq.dataverse.datavariable.SummaryStatistic; +import edu.harvard.iq.dataverse.datavariable.VariableCategory; import edu.harvard.iq.dataverse.privateurl.PrivateUrl; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.DatasetFieldWalker; @@ -573,7 +577,7 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata) { .add("filesize", df.getFilesize()) .add("description", df.getDescription()) //.add("released", df.isReleased()) - //.add("restricted", df.isRestricted()) + .add("restricted", df.isRestricted()) .add("storageIdentifier", df.getStorageIdentifier()) .add("originalFileFormat", df.getOriginalFileFormat()) .add("originalFormatLabel", df.getOriginalFormatLabel()) @@ -592,9 +596,81 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata) { .add("md5", getMd5IfItExists(df.getChecksumType(), df.getChecksumValue())) .add("checksum", getChecksumTypeAndValue(df.getChecksumType(), df.getChecksumValue())) .add("tabularTags", getTabularFileTags(df)) + .add("dataTables", org.apache.commons.collections.CollectionUtils.isNotEmpty(df.getDataTables()) ? JsonPrinter.jsonDT(df.getDataTables()) : null) ; } + + public static JsonArrayBuilder jsonDT(List ldt) { + JsonArrayBuilder ldtArr = Json.createArrayBuilder(); + for(DataTable dt: ldt){ + ldtArr.add(jsonObjectBuilder().add("dataTable", JsonPrinter.json(dt))); + } + return ldtArr; + } + + public static JsonObjectBuilder json(DataTable dt) { + return jsonObjectBuilder() + .add("varQuantity", dt.getVarQuantity()) + .add("caseQuantity", dt.getCaseQuantity()) + .add("UNF", dt.getUnf()) + .add("dataVariables", JsonPrinter.jsonDV(dt.getDataVariables())) + ; + } + + public static JsonArrayBuilder jsonDV(List dvl) { + JsonArrayBuilder varArr = Json.createArrayBuilder(); + for (DataVariable dv: dvl){ + varArr.add(JsonPrinter.json(dv)); + } + return varArr; + } + + // TODO: add sumstat and variable categories, check formats + public static JsonObjectBuilder json(DataVariable dv) { + return jsonObjectBuilder() + .add("name", dv.getName()) + .add("label", dv.getLabel()) + .add("weighted", dv.isWeighted()) + .add("variableIntervalType", dv.getIntervalLabel()) + .add("variableFormatType", dv.getType().name()) // varFormat + .add("formatCategory", dv.getFormatCategory()) + .add("orderedFactor", dv.isOrderedCategorical()) + .add("fileOrder", dv.getFileOrder()) + .add("UNF",dv.getUnf()) + .add("summaryStatistics", org.apache.commons.collections.CollectionUtils.isNotEmpty(dv.getSummaryStatistics()) ? JsonPrinter.jsonSumStat(dv.getSummaryStatistics()) : null) + .add("variableCategories", org.apache.commons.collections.CollectionUtils.isNotEmpty(dv.getCategories()) ? JsonPrinter.jsonCatStat(dv.getCategories()) : null) + ; + } + + public static JsonObjectBuilder jsonSumStat(Collection sumStat){ + //JsonArrayBuilder sumStatArr = Json.createArrayBuilder(); + JsonObjectBuilder sumStatObj = Json.createObjectBuilder(); + for (SummaryStatistic stat: sumStat){ + sumStatObj.add(stat.getTypeLabel(), stat.getValue()); + } + return sumStatObj; + } + + + public static JsonArrayBuilder jsonCatStat(Collection catStat){ + JsonArrayBuilder catArr = Json.createArrayBuilder(); + + for (VariableCategory stat: catStat){ + JsonObjectBuilder catStatObj = Json.createObjectBuilder(); + catStatObj.add("label", stat.getLabel()) + .add("value", stat.getValue()) + //.add("frequency", stat.getFrequency()) // frequency is not calculated + ; + catArr.add(catStatObj); + } + return catArr; + } + + + + + public static String format(Date d) { return (d == null) ? null : Util.getDateTimeFormat().format(d); } diff --git a/src/main/webapp/WEB-INF/faces-config.xml b/src/main/webapp/WEB-INF/faces-config.xml index 2015ca55f5f..263d6e9f485 100644 --- a/src/main/webapp/WEB-INF/faces-config.xml +++ b/src/main/webapp/WEB-INF/faces-config.xml @@ -13,6 +13,10 @@ fr zh_CN + + /Bundle_trsa_registry + bundle_trsa_registry + diff --git a/src/main/webapp/dashboard.xhtml b/src/main/webapp/dashboard.xhtml index f81b0354f24..0d3a8aa4f7f 100644 --- a/src/main/webapp/dashboard.xhtml +++ b/src/main/webapp/dashboard.xhtml @@ -124,6 +124,21 @@ + + +
+
+

#{bundle_trsa_registry.dashboardCardTrsaRegistryHeader}

+
+
+

+ #{bundle_trsa_registry.dashboardCardTrsaRegistryManage} +

+
+
+
+
+ diff --git a/src/main/webapp/dataverse_template.xhtml b/src/main/webapp/dataverse_template.xhtml index e54980e7d9c..2c90527a857 100644 --- a/src/main/webapp/dataverse_template.xhtml +++ b/src/main/webapp/dataverse_template.xhtml @@ -45,6 +45,11 @@ + + + + + diff --git a/src/main/webapp/resources/css/jsfcrud.css b/src/main/webapp/resources/css/jsfcrud.css new file mode 100644 index 00000000000..fa75f5b0d6a --- /dev/null +++ b/src/main/webapp/resources/css/jsfcrud.css @@ -0,0 +1,82 @@ +root { + display: block; +} + +body { + font-family: Arial, Helvetica, sans-serif; + color: #3a4f54; + background-color: #dfecf1; + font-size: small; +} + +a { + color: #e33b06; +} + +table { + empty-cells: show; +} + +form.jsfcrud_list_form th, td th { + font-size: x-small; + color: #4e6a71; + border-top-style: solid; + border-bottom-style: solid; + border-left-style: solid; + border-right-style: solid; + border-top-width: 1px; + border-bottom-width: 1px; + border-left-width: 1px; + border-right-width: 1px; + border-top-color: #b2d5d6; + border-bottom-color: #b2d5d6; + border-left-color: #90b4bd; + border-right-color: #90b4bd; + letter-spacing: 3px; + text-align: left; + padding-top: 6px; + padding-bottom: 6px; + padding-left: 6px; + padding-right: 6px; + background-color: #b2d5d6; +} + +td { + vertical-align: top; + padding-bottom: 8px; + font-size: small; +} + +form.jsfcrud_list_form td, td td { + border-top-style: solid; + border-bottom-style: solid; + border-left-style: solid; + border-right-style: solid; + border-top-width: 1px; + border-bottom-width: 1px; + border-left-width: 1px; + border-right-width: 1px; + border-top-color: #b2d5d6; + border-bottom-color: #b2d5d6; + border-left-color: #b2d5d6; + border-right-color: #b2d5d6; + vertical-align: baseline; + padding-bottom: 0px; +} + +tr.jsfcrud_odd_row { + background-color: #fefeff; + color: #4e6a71; +} + + +tr.jsfcrud_even_row { + background-color: #eff5fa; + color: #4e6a71; +} + +#busyImage { + position: absolute; + left: 50%; + top: 50%; +} diff --git a/src/main/webapp/resources/js/jsfcrud.js b/src/main/webapp/resources/js/jsfcrud.js new file mode 100644 index 00000000000..fb1c8c63b01 --- /dev/null +++ b/src/main/webapp/resources/js/jsfcrud.js @@ -0,0 +1,8 @@ +function handleSubmit(args, dialog) { + var jqDialog = jQuery('#' + dialog); + if (args.validationFailed) { + jqDialog.effect('shake', {times: 3}, 100); + } else { + PF(dialog).hide(); + } +} diff --git a/src/main/webapp/trsaRegistry/Create.xhtml b/src/main/webapp/trsaRegistry/Create.xhtml new file mode 100644 index 00000000000..6b75b4ea05a --- /dev/null +++ b/src/main/webapp/trsaRegistry/Create.xhtml @@ -0,0 +1,79 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/webapp/trsaRegistry/Edit.xhtml b/src/main/webapp/trsaRegistry/Edit.xhtml new file mode 100644 index 00000000000..27f2b28dede --- /dev/null +++ b/src/main/webapp/trsaRegistry/Edit.xhtml @@ -0,0 +1,83 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/webapp/trsaRegistry/List.xhtml b/src/main/webapp/trsaRegistry/List.xhtml new file mode 100644 index 00000000000..b5abd0a1e86 --- /dev/null +++ b/src/main/webapp/trsaRegistry/List.xhtml @@ -0,0 +1,121 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/webapp/trsaRegistry/View.xhtml b/src/main/webapp/trsaRegistry/View.xhtml new file mode 100644 index 00000000000..e40bb6f8338 --- /dev/null +++ b/src/main/webapp/trsaRegistry/View.xhtml @@ -0,0 +1,61 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +