7812 download all api speedup

src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java

@@ -648,13 +648,8 @@ public RetrieveDatasetVersionResponse selectRequestedVersion(List<DatasetVersion
      */
     public RetrieveDatasetVersionResponse retrieveDatasetVersionById(Long datasetId, String version){
         msg("retrieveDatasetVersionById: " + datasetId + " " + version);
-        if (datasetId==null){
-            return null;
-        }        
 
-        String identifierClause = " AND ds.id = " + datasetId;
-
-        DatasetVersion ds = retrieveDatasetVersionByIdentiferClause(identifierClause, version);
+        DatasetVersion ds = getDatasetVersionById(datasetId, version);
 
         if (ds != null){
             return new RetrieveDatasetVersionResponse(ds, version);
@@ -665,6 +660,30 @@ public RetrieveDatasetVersionResponse retrieveDatasetVersionById(Long datasetId,
 
     } // end: retrieveDatasetVersionById
 
+    public DatasetVersion getDatasetVersionById(Long datasetId, String version){
+        msg("retrieveDatasetVersionById: " + datasetId + " " + version);
+        if (datasetId==null){
+            return null;
+        }        
+
+        String identifierClause = this.getIdClause(datasetId);
+
+        DatasetVersion ds = retrieveDatasetVersionByIdentiferClause(identifierClause, version);
+
+        return ds;
+
+
+    } // end: getDatasetVersionById
+
+    public DatasetVersion getLatestReleasedVersionFast(Long datasetId) {
+        String identifierClause = this.getIdClause(datasetId);
+        String latestVersionQuery = this.getLatestReleasedDatasetVersionQuery(identifierClause);
+        return this.getDatasetVersionByQuery(latestVersionQuery);
+    }
+
+    private String getIdClause(Long datasetId) {
+        return " AND ds.id = " + datasetId;
+    }
 
      /**
      * Find a DatasetVersion using the dataset versionId
@@ -688,7 +707,7 @@ public RetrieveDatasetVersionResponse retrieveDatasetVersionByVersionId(Long ver
         }
         return null;          
     } // end: retrieveDatasetVersionByVersionId
-
+    
     // This is an optimized, native query-based method for picking an image 
     // that can be used as the thumbnail for a given dataset/version. 
     // It is primarily designed to be used when thumbnails are requested

src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java

@@ -581,7 +581,7 @@ private Guestbook findDefaultGuestbook() {
         return guestbook;
 
     }
-
+    
     public String getUserName(User user) {
         if (user.isAuthenticated()) {
             AuthenticatedUser authUser = (AuthenticatedUser) user;
@@ -821,9 +821,6 @@ public GuestbookResponse initAPIGuestbookResponse(Dataset dataset, DataFile data
             guestbookResponse.setGuestbook(datasetGuestbook);            
         }
 
-       if(dataset.getLatestVersion() != null && dataset.getLatestVersion().isDraft()){
-            guestbookResponse.setWriteResponse(false);
-        }
         if (dataFile != null){
             guestbookResponse.setDataFile(dataFile);
         }        
@@ -894,7 +891,7 @@ public void save(GuestbookResponse guestbookResponse) {
         em.persist(guestbookResponse);
     }
 
-
+        
     public Long getCountGuestbookResponsesByDataFileId(Long dataFileId) {
         // datafile id is null, will return 0
         Query query = em.createNativeQuery("select count(o.id) from GuestbookResponse  o  where o.datafile_id  = " + dataFileId);

src/main/java/edu/harvard/iq/dataverse/api/Access.java

@@ -67,7 +67,6 @@
 import edu.harvard.iq.dataverse.util.FileUtil;
 import edu.harvard.iq.dataverse.util.StringUtil;
 import edu.harvard.iq.dataverse.util.SystemConfig;
-import edu.harvard.iq.dataverse.util.json.JsonPrinter;
 
 import java.util.logging.Logger;
 import javax.ejb.EJB;
@@ -122,8 +121,6 @@
 import javax.ws.rs.RedirectionException;
 import javax.ws.rs.core.MediaType;
 import static javax.ws.rs.core.Response.Status.FORBIDDEN;
-import org.glassfish.jersey.media.multipart.FormDataBodyPart;
-import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
 import org.glassfish.jersey.media.multipart.FormDataParam;
 
 /*
@@ -633,9 +630,38 @@ public Response postDownloadDatafiles(String fileIds, @QueryParam("gbrecs") bool
     @Produces({"application/zip"})
     public Response downloadAllFromLatest(@PathParam("id") String datasetIdOrPersistentId, @QueryParam("gbrecs") boolean gbrecs, @QueryParam("key") String apiTokenParam, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) throws WebApplicationException {
         try {
-            DataverseRequest req = createDataverseRequest(findUserOrDie());
-            final Dataset retrieved = execCommand(new GetDatasetCommand(req, findDatasetOrDie(datasetIdOrPersistentId)));
-            final DatasetVersion latest = execCommand(new GetLatestAccessibleDatasetVersionCommand(req, retrieved));
+            User user = findUserOrDie(); 
+            DataverseRequest req = createDataverseRequest(user);
+            final Dataset retrieved = findDatasetOrDie(datasetIdOrPersistentId);
+            if (!(user instanceof GuestUser)) {
+                // The reason we are only looking up a draft version for a NON-guest user
+                // is that we know that guest never has the Permission.ViewUnpublishedDataset. 
+                final DatasetVersion draft = versionService.getDatasetVersionById(retrieved.getId(), DatasetVersion.VersionState.DRAFT.toString());
+                if (draft != null && permissionService.requestOn(req, retrieved).has(Permission.ViewUnpublishedDataset)) {                    
+                    String fileIds = getFileIdsAsCommaSeparated(draft.getFileMetadatas());
+                    // We don't want downloads from Draft versions to be counted, 
+                    // so we are setting the gbrecs (aka "do not write guestbook response") 
+                    // variable accordingly:
+                    return downloadDatafiles(fileIds, true, apiTokenParam, uriInfo, headers, response);
+                }
+            }
+
+            // OK, it was not the draft. Let's see if we can serve a published version.
+
+            final DatasetVersion latest = versionService.getLatestReleasedVersionFast(retrieved.getId()); 
+
+            // Make sure to throw a clean error code if we have failed to locate an 
+            // accessible version:
+            // (A "Not Found" would be more appropriate here, I believe, than a "Bad Request". 
+            // But we've been using the latter for a while, and it's a popular API... 
+            // and this return code is expected by our tests - so I'm choosing it to keep 
+            // -- L.A.)
+
+            if (latest == null) {
+                return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.exception.dataset.not.found"));
+                //throw new NotFoundException();
+            }
+
             String fileIds = getFileIdsAsCommaSeparated(latest.getFileMetadatas());
             return downloadDatafiles(fileIds, gbrecs, apiTokenParam, uriInfo, headers, response);
         } catch (WrappedResponse wr) {
@@ -673,9 +699,19 @@ public Command<DatasetVersion> handleLatestPublished() {
                 }
             }));
             if (dsv == null) {
+                // (A "Not Found" would be more appropriate here, I believe, than a "Bad Request". 
+                // But we've been using the latter for a while, and it's a popular API... 
+                // and this return code is expected by our tests - so I'm choosing it to keep 
+                // -- L.A.)
                 return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.exception.version.not.found"));
             }
             String fileIds = getFileIdsAsCommaSeparated(dsv.getFileMetadatas());
+            // We don't want downloads from Draft versions to be counted, 
+            // so we are setting the gbrecs (aka "do not write guestbook response") 
+            // variable accordingly:
+            if (dsv.isDraft()) {
+                gbrecs = true;
+            }
             return downloadDatafiles(fileIds, gbrecs, apiTokenParam, uriInfo, headers, response);
         } catch (WrappedResponse wr) {
             return wr.getResponse();
@@ -701,7 +737,7 @@ public Response datafiles(@PathParam("fileIds") String fileIds, @QueryParam("gbr
         return downloadDatafiles(fileIds, gbrecs, apiTokenParam, uriInfo, headers, response);
     }
 
-    private Response downloadDatafiles(String rawFileIds, boolean gbrecs, String apiTokenParam, UriInfo uriInfo, HttpHeaders headers, HttpServletResponse response) throws WebApplicationException /* throws NotFoundException, ServiceUnavailableException, PermissionDeniedException, AuthorizationRequiredException*/ {
+    private Response downloadDatafiles(String rawFileIds, boolean donotwriteGBResponse, String apiTokenParam, UriInfo uriInfo, HttpHeaders headers, HttpServletResponse response) throws WebApplicationException /* throws NotFoundException, ServiceUnavailableException, PermissionDeniedException, AuthorizationRequiredException*/ {
         long setLimit = systemConfig.getZipDownloadLimit();
         if (!(setLimit > 0L)) {
             setLimit = DataFileZipper.DEFAULT_ZIPFILE_LIMIT;
@@ -745,7 +781,7 @@ private Response downloadDatafiles(String rawFileIds, boolean gbrecs, String api
         if (useCustomZipService) {
             URI redirect_uri = null; 
             try {
-                redirect_uri = handleCustomZipDownload(customZipServiceUrl, fileIds, apiToken, apiTokenUser, uriInfo, headers, gbrecs, true); 
+                redirect_uri = handleCustomZipDownload(customZipServiceUrl, fileIds, apiToken, apiTokenUser, uriInfo, headers, donotwriteGBResponse, true); 
             } catch (WebApplicationException wae) {
                 throw wae;
             }
@@ -789,7 +825,7 @@ public void write(OutputStream os) throws IOException,
 
                                     logger.fine("adding datafile (id=" + file.getId() + ") to the download list of the ZippedDownloadInstance.");
                                     //downloadInstance.addDataFile(file);
-                                    if (gbrecs != true && file.isReleased()){
+                                    if (donotwriteGBResponse != true && file.isReleased()){
                                         GuestbookResponse  gbr = guestbookResponseService.initAPIGuestbookResponse(file.getOwner(), file, session, apiTokenUser);
                                         guestbookResponseService.save(gbr);
                                         MakeDataCountEntry entry = new MakeDataCountEntry(uriInfo, headers, dvRequestService, file);                                        
@@ -1874,7 +1910,8 @@ private User findAPITokenUser(String apiToken) {
         return apiTokenUser;
     }
 
-    private URI handleCustomZipDownload(String customZipServiceUrl, String fileIds, String apiToken, User apiTokenUser, UriInfo uriInfo, HttpHeaders headers, boolean gbrecs, boolean orig) throws WebApplicationException {
+    private URI handleCustomZipDownload(String customZipServiceUrl, String fileIds, String apiToken, User apiTokenUser, UriInfo uriInfo, HttpHeaders headers, boolean donotwriteGBResponse, boolean orig) throws WebApplicationException {
+
         String zipServiceKey = null; 
         Timestamp timestamp = null; 
 
@@ -1901,7 +1938,7 @@ private URI handleCustomZipDownload(String customZipServiceUrl, String fileIds,
                     validFileCount++;
                     if (isAccessAuthorized(file, apiToken)) {
                         logger.fine("adding datafile (id=" + file.getId() + ") to the download list of the ZippedDownloadInstance.");
-                        if (gbrecs != true && file.isReleased()) {
+                        if (donotwriteGBResponse != true && file.isReleased()) {
                             GuestbookResponse gbr = guestbookResponseService.initAPIGuestbookResponse(file.getOwner(), file, session, apiTokenUser);
                             guestbookResponseService.save(gbr);
                             MakeDataCountEntry entry = new MakeDataCountEntry(uriInfo, headers, dvRequestService, file);

src/main/java/propertyFiles/Bundle.properties

@@ -2395,6 +2395,7 @@ access.api.requestList.noRequestsFound=There are no access requests for this fil
 access.api.exception.metadata.not.available.for.nontabular.file=This type of metadata is only available for tabular files.
 access.api.exception.metadata.restricted.no.permission=You do not have permission to download this file.
 access.api.exception.version.not.found=Could not find requested dataset version.
+access.api.exception.dataset.not.found=Could not find requested dataset.
 
 #permission
 permission.AddDataverse.label=AddDataverse

src/test/java/edu/harvard/iq/dataverse/api/DownloadFilesIT.java

@@ -20,6 +20,7 @@
 import static javax.ws.rs.core.Response.Status.FORBIDDEN;
 import static javax.ws.rs.core.Response.Status.OK;
 import static javax.ws.rs.core.Response.Status.UNAUTHORIZED;
+import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
 import static org.hamcrest.CoreMatchers.equalTo;
 import org.junit.Assert;
 import static org.junit.Assert.assertTrue;
@@ -92,10 +93,12 @@ public void downloadAllFilesByVersion() throws IOException {
         Assert.assertEquals(expectedFiles1, filenamesFound1);
 
         // A guest user can't download unpublished files.
+        // (a guest user cannot even see that the draft version actually exists;
+        // so they are going to get a "BAD REQUEST", not "UNAUTHORIZED":)
         Response downloadFiles2 = UtilIT.downloadFiles(datasetPid, null);
         downloadFiles2.prettyPrint();
         downloadFiles2.then().assertThat()
-                .statusCode(UNAUTHORIZED.getStatusCode())
+                .statusCode(BAD_REQUEST.getStatusCode())
                 .body("status", equalTo("ERROR"));
 
         UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken)