diff --git a/apps/backend/src/main/java/wildfire/visualization/backend/controller/DataController.java b/apps/backend/src/main/java/wildfire/visualization/backend/controller/DataController.java index 891d9b8f..1151bd64 100644 --- a/apps/backend/src/main/java/wildfire/visualization/backend/controller/DataController.java +++ b/apps/backend/src/main/java/wildfire/visualization/backend/controller/DataController.java @@ -27,6 +27,9 @@ public class DataController { private static final Logger logger = LoggerFactory.getLogger(DataController.class); + private static final String LOG_RECEIVED_REMOVE_ALL_ITEMS = "Received request to remove all items"; + private static final String LOG_SUCCESSFULLY_REMOVED_ALL_ITEMS = "successfully removed all items"; + private final DataService dataService; public DataController(DataService dataService) { @@ -35,8 +38,7 @@ public DataController(DataService dataService) { @GetMapping("/api/load-assets/{collectionId}") public ResponseEntity loadAssets( - @PathVariable String collectionId - ) { + @PathVariable String collectionId) { logger.info("Received request to load assets asynchronously for collection: {}", collectionId); try { // Run the processing task asynchronously @@ -68,7 +70,6 @@ public ResponseEntity>> getLoadedLayers() { return ResponseEntity.ok(dataService.getLoadedLayers()); } - /** * Parses a bounding box (BBOX) string from a request parameter. * The BBOX format is "minX,minY,maxX,maxY". @@ -250,9 +251,9 @@ public ResponseEntity>> getItem(@PathVariable("id") Str */ @DeleteMapping("/api/remove-all-items") public ResponseEntity removeAllItems() { - logger.info("Received request to remove all items"); + logger.info(LOG_RECEIVED_REMOVE_ALL_ITEMS); String result = dataService.removeAllItems(); - logger.debug("Successfully removed all items"); + logger.debug(LOG_SUCCESSFULLY_REMOVED_ALL_ITEMS); return ResponseEntity.ok(result); } @@ -266,9 +267,9 @@ public ResponseEntity removeAllItems() { */ @DeleteMapping("/api/remove-items-from-collection/{collectionId}") public ResponseEntity removeItemsFromCollection(@PathVariable("collectionId") String collectionId) { - logger.info("Received request to remove all items"); + logger.info(LOG_RECEIVED_REMOVE_ALL_ITEMS); String result = dataService.removeItemsFromCollection(collectionId); - logger.debug("Successfully removed all items"); + logger.debug(LOG_SUCCESSFULLY_REMOVED_ALL_ITEMS); return ResponseEntity.ok(result); } @@ -284,9 +285,9 @@ public ResponseEntity removeItemsFromCollection(@PathVariable("collectio @DeleteMapping("/api/remove-item/{id}/{collection}") public ResponseEntity removeItem(@PathVariable("id") String itemId, @PathVariable("collection") String collectionId) { - logger.info("Received request to remove all items"); + logger.info(LOG_RECEIVED_REMOVE_ALL_ITEMS); String result = dataService.removeItem(itemId, collectionId); - logger.debug("Successfully removed all items"); + logger.debug(LOG_SUCCESSFULLY_REMOVED_ALL_ITEMS); return ResponseEntity.ok(result); } @@ -332,17 +333,20 @@ public ResponseEntity resetItems() { } /** - * Fetches collections sorted by name, optionally filtering by a bounding box (BBOX) + * Fetches collections sorted by name, optionally filtering by a bounding box + * (BBOX) * with specified sort direction. * - * @param bboxStr The bounding box string in "minX,minY,maxX,maxY" format (optional). - * @param sortDirection The direction to sort collections ("asc" or "desc"). Default is "asc". + * @param bboxStr The bounding box string in "minX,minY,maxX,maxY" format + * (optional). + * @param sortDirection The direction to sort collections ("asc" or "desc"). + * Default is "asc". * @return A ResponseEntity containing a list of collections sorted by name. */ @GetMapping("/api/get-collections-by-name") public ResponseEntity>> getCollectionsByName( - @RequestParam(value = "bbox", required = false) String bboxStr, - @RequestParam(value = "sortDirection", defaultValue = "asc") String sortDirection) { + @RequestParam(value = "bbox", required = false) String bboxStr, + @RequestParam(value = "sortDirection", defaultValue = "asc") String sortDirection) { logger.info("Processing request to fetch collections sorted by name with direction: {}", sortDirection); double[] bbox = parseBbox(bboxStr); @@ -354,15 +358,15 @@ public ResponseEntity>> getCollectionsByName( * Fetches collections sorted by date, optionally filtering by a bounding box * (BBOX) with specified sort direction. * - * @param bboxStr The bounding box string in "minX,minY,maxX,maxY" format - * (optional). + * @param bboxStr The bounding box string in "minX,minY,maxX,maxY" format + * (optional). * @param sortDirection The direction to sort ("asc" or "desc"). * @return A ResponseEntity containing a list of collections sorted by date. */ @GetMapping("/api/get-collections-by-date") public ResponseEntity>> getCollectionsByDate( - @RequestParam(value = "bbox", required = false) String bboxStr, - @RequestParam(value = "sortDirection", defaultValue = "asc") String sortDirection) { + @RequestParam(value = "bbox", required = false) String bboxStr, + @RequestParam(value = "sortDirection", defaultValue = "asc") String sortDirection) { logger.info("Processing request to fetch collections sorted by date with direction: {}", sortDirection); double[] bbox = parseBbox(bboxStr); @@ -427,7 +431,7 @@ public ResponseEntity resetItemAssets() { return ResponseEntity.ok("Item asset layers reset successfully."); } else { return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) - .body("Failed to reset item asset layers."); + .body("Failed to reset item asset layers."); } } } diff --git a/apps/backend/src/main/java/wildfire/visualization/backend/repository/StacRepository.java b/apps/backend/src/main/java/wildfire/visualization/backend/repository/StacRepository.java index 199b3e3f..116fc303 100644 --- a/apps/backend/src/main/java/wildfire/visualization/backend/repository/StacRepository.java +++ b/apps/backend/src/main/java/wildfire/visualization/backend/repository/StacRepository.java @@ -23,6 +23,7 @@ public class StacRepository { private static final Logger logger = LoggerFactory.getLogger(StacRepository.class); private static final String FETCHING_ALL_COLLECTIONS = "Fetching all collections"; private static final String FETCHING_WITH_BBOX = "Fetching collections with bbox: "; + private static final String QUERY_RETURNED_RESULTS = "Query returned {} results"; private final JdbcTemplate jdbcTemplate; @@ -121,7 +122,7 @@ public List> queryCollection(String collectionId) { try { String sql = "SELECT * FROM pgstac.collections WHERE id = ?"; List> results = jdbcTemplate.queryForList(sql, collectionId); - logger.debug("Query returned {} results", results.size()); + logger.debug(QUERY_RETURNED_RESULTS, results.size()); return results; } catch (DataAccessException e) { logger.error("Error querying collection: {}", e.getMessage(), e); @@ -376,7 +377,8 @@ public List> getAllItems(String collectionId) { " }'::jsonb" + ")"; List> results = jdbcTemplate.queryForList(sql); - logger.debug("Query returned {} results", results.size()); + + logger.debug(QUERY_RETURNED_RESULTS, results.size()); return results; } catch (DataAccessException e) { logger.error("Error fetching items: {}", e.getMessage(), e); @@ -416,7 +418,7 @@ public List> getItem(String id) { try { String sql = "SELECT pgstac.get_item('" + id + "');"; List> results = jdbcTemplate.queryForList(sql); - logger.debug("Query returned {} results", results.size()); + logger.debug(QUERY_RETURNED_RESULTS, results.size()); return results; } catch (DataAccessException e) { logger.error("Error fetching item: {}", e.getMessage(), e); @@ -439,7 +441,7 @@ public List> getItem(String id, String collection) { try { String sql = "SELECT pgstac.get_item('" + id + "', '" + collection + "');"; List> results = jdbcTemplate.queryForList(sql); - logger.debug("Query returned {} results", results.size()); + logger.debug(QUERY_RETURNED_RESULTS, results.size()); return results; } catch (DataAccessException e) { logger.error("Error fetching item: {}", e.getMessage(), e); diff --git a/apps/backend/src/main/java/wildfire/visualization/backend/service/ConfigService.java b/apps/backend/src/main/java/wildfire/visualization/backend/service/ConfigService.java index 3b7b95a6..b2910521 100644 --- a/apps/backend/src/main/java/wildfire/visualization/backend/service/ConfigService.java +++ b/apps/backend/src/main/java/wildfire/visualization/backend/service/ConfigService.java @@ -33,11 +33,13 @@ public Map getConfig() { try { File configFile = new File(CONFIG_FILE_PATH); if (configFile.exists()) { + logger.info("Reading configuration from file: {}", CONFIG_FILE_PATH); return objectMapper.readValue(configFile, Map.class); } else { return Map.of(); } } catch (Exception e) { + logger.error("Failed to retrieve configuration from {}: {}", CONFIG_FILE_PATH, e.getMessage(), e); throw new ConfigException("Failed to retrieve configuration", e); } } @@ -50,8 +52,11 @@ public Map getConfig() { */ public void saveConfig(Map config) { try { + logger.info("Saving configuration to file: {}", CONFIG_FILE_PATH); objectMapper.writeValue(new File(CONFIG_FILE_PATH), config); + logger.debug("Configuration saved successfully"); } catch (Exception e) { + logger.error("Failed to save configuration to {}: {}", CONFIG_FILE_PATH, e.getMessage(), e); throw new ConfigException("Failed to save configuration", e); } } diff --git a/apps/backend/src/main/java/wildfire/visualization/backend/service/DataService.java b/apps/backend/src/main/java/wildfire/visualization/backend/service/DataService.java index 4c08a4b3..cf92400b 100644 --- a/apps/backend/src/main/java/wildfire/visualization/backend/service/DataService.java +++ b/apps/backend/src/main/java/wildfire/visualization/backend/service/DataService.java @@ -1,5 +1,6 @@ package wildfire.visualization.backend.service; +import java.io.IOException; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Arrays; @@ -37,6 +38,9 @@ public class DataService { private static final Logger logger = LoggerFactory.getLogger(DataService.class); + private static final String KEY_PROGRESS = "progress"; + private static final String LOG_NO_BBOX = "No bbox"; + @Autowired private StacRepository stacRepository; @@ -90,8 +94,8 @@ public void fetchAndSaveItems(String collectionId) { ResponseEntity> response = fetchData(collectionId); Map responseBody = response.getBody(); - if (responseBody != null && responseBody.containsKey("progress")) { - int progress = ((Number) responseBody.get("progress")).intValue(); + if (responseBody != null && responseBody.containsKey(KEY_PROGRESS)) { + ((Number) responseBody.get(KEY_PROGRESS)).intValue(); // Extract progress value from response body (not used) } else { logger.error("Progress not found in response body for collection: {}", collectionId); fetchProgress.get(collectionId).set(-1); // Set error state @@ -220,7 +224,7 @@ public void fetchAndSaveCollections(String endpointUrl) { */ public List> getCollections(double[] bbox) { logger.debug("Fetching collections from database with bbox: {}", - bbox != null ? Arrays.toString(bbox) : "No bbox"); + bbox != null ? Arrays.toString(bbox) : LOG_NO_BBOX); try { // Fetch collections from repository @@ -269,15 +273,16 @@ public String verifyCollections(String endpointUrl) { * This method queries collections ordered by their name (`id` field) and * restructures the response for client consumption. * - * @param bbox An optional bounding box filter (minX, minY, maxX, maxY). If - * null, no filter is applied. + * @param bbox An optional bounding box filter (minX, minY, maxX, + * maxY). If + * null, no filter is applied. * @param sortDirection The direction to sort ("asc" or "desc"). * @return A list of collections, each containing keys: `key`, `id`, and `bbox`. * @throws DataException If an error occurs while fetching collections. */ public List> getCollectionsByName(double[] bbox, String sortDirection) { logger.debug("Fetching collections from database sorted by name ({}) with bbox: {}", - sortDirection, bbox != null ? Arrays.toString(bbox) : "No bbox"); + sortDirection, bbox != null ? Arrays.toString(bbox) : LOG_NO_BBOX); try { // Fetch collections sorted by name (id) from repository @@ -287,13 +292,13 @@ public List> getCollectionsByName(double[] bbox, String sort // Transform collections into a structured format return collections.stream() - .map(collection -> Map.of( - "key", collection.get("key"), - "id", collection.get("id"), - "title", collection.get("title") == null ? "" : collection.get("title"), - "bbox", collection.getOrDefault("bbox", "[]") // Default empty bbox if null - )) - .collect(Collectors.toList()); + .map(collection -> Map.of( + "key", collection.get("key"), + "id", collection.get("id"), + "title", collection.get("title") == null ? "" : collection.get("title"), + "bbox", collection.getOrDefault("bbox", "[]") // Default empty bbox if null + )) + .collect(Collectors.toList()); } catch (Exception e) { logger.error("Error fetching collections by Name: {}", e.getMessage(), e); throw new DataException("Failed to fetch collections by name", e); @@ -308,15 +313,16 @@ public List> getCollectionsByName(double[] bbox, String sort * date and * restructures the response for client consumption. * - * @param bbox An optional bounding box filter (minX, minY, maxX, maxY). If - * null, no filter is applied. + * @param bbox An optional bounding box filter (minX, minY, maxX, + * maxY). If + * null, no filter is applied. * @param sortDirection The direction to sort ("asc" or "desc"). * @return A list of collections, each containing keys: `key`, `id`, and `bbox`. * @throws DataException If an error occurs while fetching collections. */ public List> getCollectionsByDate(double[] bbox, String sortDirection) { logger.debug("Fetching collections from database sorted by date ({}) with bbox: {}", - sortDirection, bbox != null ? Arrays.toString(bbox) : "No bbox"); + sortDirection, bbox != null ? Arrays.toString(bbox) : LOG_NO_BBOX); try { // Fetch collections sorted by date from repository @@ -326,13 +332,13 @@ public List> getCollectionsByDate(double[] bbox, String sort // Transform collections into a structured format return collections.stream() - .map(collection -> Map.of( - "key", collection.get("key"), - "id", collection.get("id"), - "title", collection.get("title") == null ? "" : collection.get("title"), - "bbox", collection.getOrDefault("bbox", "[]") // Default empty bbox if null - )) - .collect(Collectors.toList()); + .map(collection -> Map.of( + "key", collection.get("key"), + "id", collection.get("id"), + "title", collection.get("title") == null ? "" : collection.get("title"), + "bbox", collection.getOrDefault("bbox", "[]") // Default empty bbox if null + )) + .collect(Collectors.toList()); } catch (Exception e) { logger.error("Error fetching collections by Date: {}", e.getMessage(), e); throw new DataException("Failed to fetch collections by date", e); @@ -383,6 +389,144 @@ public void insertItem(String itemJson) { } } + /** + * Retrieves the configuration settings by invoking the config controller. + * If the configuration settings cannot be retrieved or are null, this method + * throws a {@link DataException}. + * + * @return a map containing the configuration settings. + * @throws DataException if the configuration settings cannot be retrieved. + */ + private Map getConfigOrThrow() { + ResponseEntity> responseEntity = configController.getConfig(); + Map config = responseEntity.getBody(); + if (config == null) { + throw new DataException("Failed to retrieve configuration settings"); + } + return config; + } + + /** + * Retrieves metadata for the specified collection ID from the STAC repository. + * If no metadata is found, a {@link DataException} is thrown. + * + * @param collectionId the ID of the collection for which metadata is to be + * retrieved + * @return a list of metadata maps, where each map contains key-value pairs + * representing metadata attributes + * @throws DataException if no metadata is found for the specified collection ID + */ + private List> getMetadataOrThrow(String collectionId) { + List> metadata = stacRepository.queryCollectionMetaData(collectionId); + if (metadata.isEmpty()) { + throw new DataException("No metadata found for collection: " + collectionId); + } + return metadata; + } + + /** + * Validates the provided start and end dates for a given collection. + * Ensures that neither the start nor the end date is null. + * + * @param start The start date to validate. Must not be null. + * @param end The end date to validate. Must not be null. + * @param collectionId The identifier of the collection being validated. + * @throws DataException If either the start or end date is null. + */ + private void validateDates(LocalDateTime start, LocalDateTime end, String collectionId) { + if (start == null || end == null) { + throw new DataException("Failed to extract temporal extent for " + collectionId); + } + } + + /** + * Processes a list of items by associating them with a collection ID, checking + * if they exist + * in the repository, and inserting them if they do not already exist. + * + * @param collectionId The ID of the collection to associate with the + * items. + * @param items A list of items represented as maps of key-value + * pairs. + * @param insertedItems A list to store the IDs of items that were + * successfully inserted. + * @param insertedTimestamps A list to store the extracted ISO timestamps of + * inserted items. + * @return The number of items successfully inserted into the repository. + * @throws JsonProcessingException If an error occurs while converting an item + * to JSON format. + */ + private int processItems(String collectionId, List> items, List insertedItems, + List insertedTimestamps) throws JsonProcessingException { + int count = 0; + for (Map item : items) { + item.put("collection_id", collectionId); + String id = (String) item.get("id"); + + if (!stacRepository.checkCollectionExists(id)) { + String itemJson = objectMapper.writeValueAsString(item); + stacRepository.insertItem(itemJson); + insertedItems.add(id); + insertedTimestamps.add(UtilHelper.extractTimestampISO(id)); + count++; + } + } + return count; + } + + /** + * Computes the progress percentage based on the provided parameters. + * + * @param itemCount The total number of items to be processed. If null or + * less than or equal to zero, + * the progress will be computed using the provided items, + * end, and start parameters. + * @param totalFetched The number of items that have been fetched or processed + * so far. + * @param items A list of maps representing the items being processed. + * Used to compute progress + * if itemCount is null or invalid. + * @param end The end time of the processing period. Used in + * conjunction with the start time + * to compute progress when itemCount is not provided. + * @param start The start time of the processing period. Used in + * conjunction with the end time + * to compute progress when itemCount is not provided. + * @return The progress percentage as an integer value between 0 and 100. + */ + private int computeProgress(Integer itemCount, int totalFetched, List> items, + LocalDateTime end, LocalDateTime start) { + if (itemCount != null && itemCount > 0) { + return (int) Math.floor(((double) totalFetched / itemCount) * 100); + } else { + return (int) Math.floor(UtilHelper.computeProgressFromItems(items, end, start)); + } + } + + /** + * Builds a response body by populating the provided map with the specified + * data. + * + * @param body The map to populate with response data. + * @param collectionId The identifier of the collection being processed. + * @param totalFetched The total number of items fetched so far. + * @param progress The progress percentage of the current operation. + * @param items The list of items that were inserted. + * @param timestamps The list of timestamps corresponding to the inserted + * items. + * @param nextPage The token or identifier for the next page of results, if + * applicable. + */ + private void buildResponseBody(Map body, String collectionId, int totalFetched, int progress, + List items, List timestamps, String nextPage) { + body.put("collectionId", collectionId); + body.put("totalFetched", totalFetched); + body.put(KEY_PROGRESS, progress); + body.put("insertedItems", items); + body.put("insertedTimestamps", timestamps); + body.put("nextPage", nextPage); + } + /** * Method responsible for fetching and saving items from a given collection * @@ -396,19 +540,10 @@ public ResponseEntity> fetchData(String collectionId) { int progress = 0; try { - ResponseEntity> responseEntity = configController.getConfig(); - Map config = responseEntity.getBody(); - if (config == null) { - throw new DataException("Failed to retrieve configuration settings"); - } - - assert config != null; + Map config = getConfigOrThrow(); // Fetch collection metadata from the database - List> collectionMetadata = stacRepository.queryCollectionMetaData(collectionId); - if (collectionMetadata.isEmpty()) { - throw new DataException("No metadata found for collection: " + collectionId); - } + List> collectionMetadata = getMetadataOrThrow(collectionId); // Extract item count from metadata (if available) Integer itemCount = (Integer) collectionMetadata.get(0).get("item_count"); // May be missing @@ -416,9 +551,8 @@ public ResponseEntity> fetchData(String collectionId) { // Extract start & end dates (for time-based progress if needed) LocalDateTime startDate = UtilHelper.extractTemporalStartFromDB(collectionMetadata); LocalDateTime endDate = UtilHelper.extractTemporalEndFromDB(collectionMetadata); - if (startDate == null || endDate == null) { - throw new DataException("Failed to extract temporal extent for " + collectionId); - } + + validateDates(startDate, endDate, collectionId); // Base items endpoint String endpointUrl = config.get("endpoint").toString() + "/" + collectionId + "/items"; @@ -437,27 +571,10 @@ public ResponseEntity> fetchData(String collectionId) { if (items == null || items.isEmpty()) break; - for (Map item : items) { - item.put("collection_id", collectionId); - String id = (String) item.get("id"); - - if (!stacRepository.checkCollectionExists(id)) { - String itemJson = objectMapper.writeValueAsString(item); - stacRepository.insertItem(itemJson); - insertedItems.add(id); - insertedTimestamps.add(UtilHelper.extractTimestampISO(id)); // Convert ID to timestamp - totalFetched++; - } - } + totalFetched += processItems(collectionId, items, insertedItems, insertedTimestamps); // Determine progress calculation method - if (itemCount != null && itemCount > 0) { - // Use `item_count` if available - progress = (int) Math.floor(((double) totalFetched / itemCount) * 100); - } else { - // Use timestamp-based progress if `item_count` is missing - progress = (int) Math.floor(UtilHelper.computeProgressFromItems(items, endDate, startDate)); - } + progress = computeProgress(itemCount, totalFetched, items, endDate, startDate); // Update progress in the database fetchProgress.put(collectionId, new AtomicInteger(progress)); @@ -469,12 +586,8 @@ public ResponseEntity> fetchData(String collectionId) { fetchProgress.put(collectionId, new AtomicInteger(100)); // Construct API response body - responseBody.put("collectionId", collectionId); - responseBody.put("totalFetched", totalFetched); - responseBody.put("progress", progress); - responseBody.put("insertedItems", insertedItems); - responseBody.put("insertedTimestamps", insertedTimestamps); - responseBody.put("nextPage", nextUrl); + buildResponseBody(responseBody, collectionId, totalFetched, progress, insertedItems, + insertedTimestamps, nextUrl); return ResponseEntity.ok(responseBody); @@ -496,6 +609,7 @@ public List fetchItemsTimestamps() { /** * This method fetches order list of item ids + * * @return returns the list of item ids sorted by timestamp */ public List getItemsIdsOrderedByTimestamp() { @@ -627,7 +741,8 @@ public String verifyInternetConnection(String endpointUrl) { /** * Method responsible for registering all assets for a specific item. *

- * This fetches the item from the database and registers its GeoTIFF assets one by one. + * This fetches the item from the database and registers its GeoTIFF assets one + * by one. * This is intended to be used when refreshing/re-registering existing assets. * * @param itemId ID of the item to process @@ -659,7 +774,7 @@ public boolean processItemAssetsRefresh(String itemId) { return false; } - for (Iterator it = assets.fieldNames(); it.hasNext(); ) { + for (Iterator it = assets.fieldNames(); it.hasNext();) { String assetKey = it.next(); boolean success = geoTIFFService.registerGeoTIFF(itemId, assetKey); @@ -679,10 +794,124 @@ public boolean processItemAssetsRefresh(String itemId) { } /** - * Method responsible for processing all assets for all items in a given collection. + * Extracts the content as a string from the given content object. The method + * handles + * different types of objects and converts them to a string representation. + * + * @param contentObj The content object to extract the string from. It can be + * of type + * {@code PGobject}, {@code String}, or any other object. + * @param itemId The ID of the item associated with the content. Used for + * logging purposes. + * @param collectionId The ID of the collection associated with the content. + * Used for logging purposes. + * @return The string representation of the content object if it is not null. If + * the content + * object is null, returns {@code null}. Logs warnings for unexpected + * types or null values. + */ + private String extractContentAsString(Object contentObj, String itemId, String collectionId) { + if (contentObj instanceof PGobject) { + return ((PGobject) contentObj).getValue(); + } else if (contentObj instanceof String) { + return (String) contentObj; + } else if (contentObj != null) { + logger.warn("Unexpected type for content column: {}", contentObj.getClass()); + return contentObj.toString(); + } else { + logger.warn("Null content column for item {} in collection {}", itemId, collectionId); + return null; + } + } + + /** + * Processes the assets associated with a specific item and collection. + * Iterates through the assets provided in the JSON node, validates their value + * ranges, + * and attempts to process each asset using the GeoTIFF service. + * + * @param itemId The ID of the item to which the assets belong. + * @param collId The ID of the collection to which the item belongs. + * @param assetsNode A JSON node containing the assets to be processed. Each + * asset is expected + * to have an "href" field for the resource location and a + * "value_range" field + * specifying the minimum and maximum values as an array. + * + * Logs warnings for invalid or missing value ranges, as well + * as for failed asset processing attempts. + */ + private void processAssetsForItem(String itemId, String collId, JsonNode assetsNode) { + Iterator fieldNames = assetsNode.fieldNames(); + while (fieldNames.hasNext()) { + String assetKey = fieldNames.next(); + JsonNode asset = assetsNode.get(assetKey); + String href = asset.get("href").asText(); + + JsonNode valueRange = asset.get("value_range"); + if (valueRange == null || !valueRange.isArray() || valueRange.size() < 2) { + logger.warn("Invalid or missing value_range for asset {} of item {}", assetKey, itemId); + continue; + } + + int min = valueRange.get(0).asInt(); + int max = valueRange.get(1).asInt(); + + boolean success = geoTIFFService.processGeoTIFF(itemId, collId, assetKey, href, min, max); + if (!success) { + logger.warn("Failed to register asset {} for item {}", assetKey, itemId); + } + } + } + + /** + * Processes each item in the provided list of results, extracting content, + * processing assets, and updating progress. + * + * @param results A list of maps where each map represents an item with its + * properties. + * @param collectionId The ID of the collection to which the items belong. + * @param progressKey A key used to track and update the progress of the + * processing. + * @throws IOException If an error occurs during content extraction or JSON + * parsing. + */ + private void processEachItem(List> results, String collectionId, String progressKey) + throws IOException { + int totalItems = results.size(); + int count = 0; + + for (Map row : results) { + String itemId = (String) row.get("id"); + String collId = (String) row.get("collection"); + + String contentStr = extractContentAsString(row.get("content"), itemId, collectionId); + if (contentStr == null) + continue; + + JsonNode assetsNode = objectMapper.readTree(contentStr).get("assets"); + if (assetsNode == null || assetsNode.isEmpty()) { + logger.info("No assets found for item {}", itemId); + continue; + } + + processAssetsForItem(itemId, collId, assetsNode); + + count++; + int progress = (int) (((double) count / totalItems) * 100); + fetchProgress.get(progressKey).set(progress); + logger.info("Processed item {}/{}: {}", count, totalItems, itemId); + } + } + + /** + * Method responsible for processing all assets for all items in a given + * collection. *

- * This fetches all items from the collection and processes their assets one by one. - * Layers are reset before processing begins, but data is preserved if unregistering fails. + * This fetches all items from the collection and processes their assets one by + * one. + * Layers are reset before processing begins, but data is preserved if + * unregistering fails. * * @param collectionId ID of the collection */ @@ -697,65 +926,7 @@ public void processItemAssets(String collectionId) { logger.warn("No items returned from getAllItems for collection {}", collectionId); } - int totalItems = results.size(); - logger.info("Processing {} items in collection: {}", totalItems, collectionId); - - int count = 0; - for (Map row : results) { - String itemId = (String) row.get("id"); - String collId = (String) row.get("collection"); - String contentStr = null; - Object contentObj = row.get("content"); - if (contentObj instanceof PGobject) { - contentStr = ((PGobject) contentObj).getValue(); - } else if (contentObj instanceof String) { - // If the driver already gave you a string, just cast it - contentStr = (String) contentObj; - } else if (contentObj != null) { - // Fallback: you can do contentObj.toString(), or throw an error - logger.warn("Unexpected type for content column: {}", contentObj.getClass()); - contentStr = contentObj.toString(); - } else { - // Handle null - logger.warn("Null content column for item in collection {}", collectionId); - continue; - } - - // Now parse that as JSON - JsonNode contentNode = objectMapper.readTree(contentStr); - JsonNode assetsNode = contentNode.get("assets"); - - if (assetsNode == null || assetsNode.isEmpty()) { - logger.info("No assets found for item {}", itemId); - continue; - } - - Iterator fieldNames = assetsNode.fieldNames(); - while (fieldNames.hasNext()) { - String assetKey = fieldNames.next(); - JsonNode asset = assetsNode.get(assetKey); - String href = asset.get("href").asText(); - - JsonNode valueRange = asset.get("value_range"); - if (valueRange == null || !valueRange.isArray() || valueRange.size() < 2) { - logger.warn("Invalid or missing value_range for asset {} of item {}", assetKey, itemId); - continue; - } - - int min = valueRange.get(0).asInt(); - int max = valueRange.get(1).asInt(); - - boolean success = geoTIFFService.processGeoTIFF(itemId, collId, assetKey, href, min, max); - if (!success) { - logger.warn("Failed to register asset {} for item {}", assetKey, itemId); - } - } - - count++; - int progress = (int) (((double) count / totalItems) * 100); - fetchProgress.get(collectionProgressKey).set(progress); - logger.info("Processed item {}/{}: {}", count, totalItems, itemId); - } + processEachItem(results, collectionId, collectionProgressKey); logger.info("Finished registering all assets in collection {}", collectionId); fetchProgress.get(collectionProgressKey).set(100); @@ -765,7 +936,6 @@ public void processItemAssets(String collectionId) { } } - /** * Overloaded method to reset item assets with full reset as default. */ @@ -774,11 +944,14 @@ public boolean itemAssetsReset() { } /** - * Method responsible for unregistering all registered asset layers from GeoServer. + * Method responsible for unregistering all registered asset layers from + * GeoServer. * Also updates their registration status in the database. - * If all layers are successfully unregistered, clears the entire Assets and Items tables. + * If all layers are successfully unregistered, clears the entire Assets and + * Items tables. * - * @param fullReset Whether to fully clear the Assets and Items tables after unregistering + * @param fullReset Whether to fully clear the Assets and Items tables after + * unregistering * @return true if the full reset succeeded, false otherwise */ public boolean itemAssetsReset(boolean fullReset) { @@ -814,8 +987,8 @@ public boolean itemAssetsReset(boolean fullReset) { */ public boolean tryItemAssetsResetOnce(boolean fullReset) { List> assetsToProcess = fullReset - ? stacRepository.getItemsWithAssets() - : stacRepository.getLoadedLayers(); + ? stacRepository.getItemsWithAssets() + : stacRepository.getLoadedLayers(); boolean allUnregisteredSuccessfully = true; diff --git a/apps/backend/src/main/java/wildfire/visualization/backend/service/GeoServerService.java b/apps/backend/src/main/java/wildfire/visualization/backend/service/GeoServerService.java index 6d4365af..6f36f0ac 100644 --- a/apps/backend/src/main/java/wildfire/visualization/backend/service/GeoServerService.java +++ b/apps/backend/src/main/java/wildfire/visualization/backend/service/GeoServerService.java @@ -32,14 +32,19 @@ public class GeoServerService { @Value("${geoserver.downloadDir}") private String geoserverDownloadDir; + @Value("${geoserver.workspacesPath}") + private String geoserverWorkspacesPath; + private final RestTemplate restTemplate = new RestTemplate(); private static final Logger logger = LoggerFactory.getLogger(GeoServerService.class); /** - * Creates and returns HTTP headers with basic authentication for GeoServer interaction. + * Creates and returns HTTP headers with basic authentication for GeoServer + * interaction. * - * @return HttpHeaders object with content type set to JSON and basic authentication included. + * @return HttpHeaders object with content type set to JSON and basic + * authentication included. */ private HttpHeaders createHeaders() { HttpHeaders headers = new HttpHeaders(); @@ -49,13 +54,14 @@ private HttpHeaders createHeaders() { } /** - * Registers a new coverage store in the GeoServer under the specified workspace. + * Registers a new coverage store in the GeoServer under the specified + * workspace. * * @param layerName the name of the coverage store to be registered * @return true if the registration was successful, false otherwise */ public boolean registerCoverageStore(String layerName) { - String url = geoserverUrl + "/rest/workspaces/" + workspace + "/coveragestores"; + String url = geoserverUrl + geoserverWorkspacesPath + workspace + "/coveragestores"; String body = """ { @@ -76,13 +82,14 @@ public boolean registerCoverageStore(String layerName) { } /** - * Registers a new coverage layer in the GeoServer under the specified workspace and coverage store. + * Registers a new coverage layer in the GeoServer under the specified workspace + * and coverage store. * * @param layerName the name of the layer to be registered * @return true if the registration was successful, false otherwise */ public boolean registerCoverageLayer(String layerName) { - String url = geoserverUrl + "/rest/workspaces/" + workspace + "/coveragestores/" + layerName + "/coverages"; + String url = geoserverUrl + geoserverWorkspacesPath + workspace + "/coveragestores/" + layerName + "/coverages"; String body = """ { @@ -109,9 +116,10 @@ public boolean registerCoverageLayer(String layerName) { * @return true if the layer was successfully unregistered, false otherwise */ public boolean unregisterLayer(String layerName) { - String url = geoserverUrl + "/rest/workspaces/" + workspace + "/coveragestores/" + layerName + "?purge=all&recurse=true"; + String url = geoserverUrl + geoserverWorkspacesPath + workspace + "/coveragestores/" + layerName + + "?purge=all&recurse=true"; - try{ + try { boolean apiSuccess = sendDeleteRequest(url); return apiSuccess; } catch (Exception e) { @@ -135,7 +143,8 @@ private boolean sendDeleteRequest(String url) { } /** - * Deletes the .tif file corresponding to a given layer or store from the local storage. + * Deletes the .tif file corresponding to a given layer or store from the local + * storage. * * @param layerName the name of the file to be deleted (without extension) */ diff --git a/apps/backend/src/main/resources/application.properties b/apps/backend/src/main/resources/application.properties index 614946e5..4a08a2bf 100644 --- a/apps/backend/src/main/resources/application.properties +++ b/apps/backend/src/main/resources/application.properties @@ -17,6 +17,7 @@ management.endpoints.web.exposure.include=health geoserver.url=${GEOSERVER_URL:http://localhost:8090/geoserver} geoserver.url.local=http://localhost:8090/geoserver geoserver.workspace=Default +geoserver.workspacesPath=/rest/workspaces/ geoserver.username=geoserver geoserver.password=password geoserver.dataDir=/var/geoserver/data_dir/tiffs diff --git a/apps/backend/src/test/java/wildfire/visualization/backend/controller/DataControllerTests.java b/apps/backend/src/test/java/wildfire/visualization/backend/controller/DataControllerTests.java index f9f1b503..253a8de2 100644 --- a/apps/backend/src/test/java/wildfire/visualization/backend/controller/DataControllerTests.java +++ b/apps/backend/src/test/java/wildfire/visualization/backend/controller/DataControllerTests.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.containsString; @@ -32,6 +33,7 @@ import static org.mockito.Mockito.*; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; +import static org.awaitility.Awaitility.await; @ExtendWith(MockitoExtension.class) class DataControllerTests { @@ -735,8 +737,7 @@ void loadAssets_Success() throws Exception { .andExpect(status().isOk()) .andExpect(content().string("Processing started in the background. Check logs for completion.")); - // Give async execution a brief moment (useful for debugging) - Thread.sleep(100); + await().atMost(2, TimeUnit.SECONDS).untilAsserted(() -> verify(dataService).processItemAssets(collectionId)); verify(dataService, times(1)).processItemAssets(collectionId); } @@ -755,7 +756,7 @@ void loadAssets_Failure_ShouldStillReturnSuccess() throws Exception { .andExpect(content().string("Processing started in the background. Check logs for completion.")); // Optional: Give async execution a brief moment - Thread.sleep(100); + await().atMost(2, TimeUnit.SECONDS).untilAsserted(() -> verify(dataService).processItemAssets(collectionId)); // Verify method was still called verify(dataService, times(1)).processItemAssets(collectionId); diff --git a/apps/backend/src/test/java/wildfire/visualization/backend/service/GeoServerServiceTests.java b/apps/backend/src/test/java/wildfire/visualization/backend/service/GeoServerServiceTests.java index 94c51df1..9bd45300 100644 --- a/apps/backend/src/test/java/wildfire/visualization/backend/service/GeoServerServiceTests.java +++ b/apps/backend/src/test/java/wildfire/visualization/backend/service/GeoServerServiceTests.java @@ -38,6 +38,7 @@ class GeoServerServiceTests { private final String password = "geoserver"; private final String geoserverDataDir = "/var/geoserver/data_dir/tiffs"; private final String layerName = "humidity"; + private final String geoserverWorkspacesPath = "/rest/workspaces/"; @BeforeEach void setUp() { @@ -47,131 +48,142 @@ void setUp() { setField(geoServerService, "geoserverPassword", password); setField(geoServerService, "geoserverDataDir", geoserverDataDir); setField(geoServerService, "restTemplate", restTemplate); + setField(geoServerService, "geoserverWorkspacesPath", geoserverWorkspacesPath); } @Test void testRegisterCoverageStore_Success() { - String expectedUrl = geoserverUrl + "/rest/workspaces/" + workspace + "/coveragestores"; + String expectedUrl = geoserverUrl + geoserverWorkspacesPath + workspace + "/coveragestores"; // Mock response ResponseEntity successResponse = new ResponseEntity<>("Created", HttpStatus.CREATED); when(restTemplate.exchange(eq(expectedUrl), eq(HttpMethod.POST), any(HttpEntity.class), eq(String.class))) - .thenReturn(successResponse); + .thenReturn(successResponse); // Execute boolean result = geoServerService.registerCoverageStore(layerName); // Verify - verify(restTemplate, times(1)).exchange(eq(expectedUrl), eq(HttpMethod.POST), any(HttpEntity.class), eq(String.class)); + verify(restTemplate, times(1)).exchange(eq(expectedUrl), eq(HttpMethod.POST), any(HttpEntity.class), + eq(String.class)); assertThat(result).isTrue(); } @Test void testRegisterCoverageStore_Failure() { - String expectedUrl = geoserverUrl + "/rest/workspaces/" + workspace + "/coveragestores"; + String expectedUrl = geoserverUrl + geoserverWorkspacesPath + workspace + "/coveragestores"; // Mock response ResponseEntity failureResponse = new ResponseEntity<>("Error", HttpStatus.BAD_REQUEST); when(restTemplate.exchange(eq(expectedUrl), eq(HttpMethod.POST), any(HttpEntity.class), eq(String.class))) - .thenReturn(failureResponse); + .thenReturn(failureResponse); // Execute boolean result = geoServerService.registerCoverageStore(layerName); // Verify - verify(restTemplate, times(1)).exchange(eq(expectedUrl), eq(HttpMethod.POST), any(HttpEntity.class), eq(String.class)); + verify(restTemplate, times(1)).exchange(eq(expectedUrl), eq(HttpMethod.POST), any(HttpEntity.class), + eq(String.class)); assertThat(result).isFalse(); } @Test void testRegisterCoverageLayer_Success() { - String expectedUrl = geoserverUrl + "/rest/workspaces/" + workspace + "/coveragestores/" + layerName + "/coverages"; + String expectedUrl = geoserverUrl + geoserverWorkspacesPath + workspace + "/coveragestores/" + layerName + + "/coverages"; // Mock response ResponseEntity successResponse = new ResponseEntity<>("Created", HttpStatus.CREATED); when(restTemplate.exchange(eq(expectedUrl), eq(HttpMethod.POST), any(HttpEntity.class), eq(String.class))) - .thenReturn(successResponse); + .thenReturn(successResponse); // Execute boolean result = geoServerService.registerCoverageLayer(layerName); // Verify - verify(restTemplate, times(1)).exchange(eq(expectedUrl), eq(HttpMethod.POST), any(HttpEntity.class), eq(String.class)); + verify(restTemplate, times(1)).exchange(eq(expectedUrl), eq(HttpMethod.POST), any(HttpEntity.class), + eq(String.class)); assertThat(result).isTrue(); } @Test void testRegisterCoverageLayer_Failure() { - String expectedUrl = geoserverUrl + "/rest/workspaces/" + workspace + "/coveragestores/" + layerName + "/coverages"; + String expectedUrl = geoserverUrl + geoserverWorkspacesPath + workspace + "/coveragestores/" + layerName + + "/coverages"; // Mock response ResponseEntity failureResponse = new ResponseEntity<>("Error", HttpStatus.BAD_REQUEST); when(restTemplate.exchange(eq(expectedUrl), eq(HttpMethod.POST), any(HttpEntity.class), eq(String.class))) - .thenReturn(failureResponse); + .thenReturn(failureResponse); // Execute boolean result = geoServerService.registerCoverageLayer(layerName); // Verify - verify(restTemplate, times(1)).exchange(eq(expectedUrl), eq(HttpMethod.POST), any(HttpEntity.class), eq(String.class)); + verify(restTemplate, times(1)).exchange(eq(expectedUrl), eq(HttpMethod.POST), any(HttpEntity.class), + eq(String.class)); assertThat(result).isFalse(); } @Test void testUnregisterLayer_Success() { - String expectedUrl = geoserverUrl + "/rest/workspaces/" + workspace + "/coveragestores/" + layerName + "?purge=all&recurse=true"; + String expectedUrl = geoserverUrl + geoserverWorkspacesPath + workspace + "/coveragestores/" + layerName + + "?purge=all&recurse=true"; // Mock response ResponseEntity successResponse = new ResponseEntity<>(HttpStatus.NO_CONTENT); when(restTemplate.exchange(eq(expectedUrl), eq(HttpMethod.DELETE), any(HttpEntity.class), eq(String.class))) - .thenReturn(successResponse); + .thenReturn(successResponse); // Execute boolean result = geoServerService.unregisterLayer(layerName); // Verify - verify(restTemplate, times(1)).exchange(eq(expectedUrl), eq(HttpMethod.DELETE), any(HttpEntity.class), eq(String.class)); + verify(restTemplate, times(1)).exchange(eq(expectedUrl), eq(HttpMethod.DELETE), any(HttpEntity.class), + eq(String.class)); assertThat(result).isTrue(); } @Test void testUnregisterLayer_Failure() { - String expectedUrl = geoserverUrl + "/rest/workspaces/" + workspace + "/coveragestores/" + layerName + "?purge=all&recurse=true"; + String expectedUrl = geoserverUrl + geoserverWorkspacesPath + workspace + "/coveragestores/" + layerName + + "?purge=all&recurse=true"; // Mock response ResponseEntity failureResponse = new ResponseEntity<>("Error", HttpStatus.INTERNAL_SERVER_ERROR); when(restTemplate.exchange(eq(expectedUrl), eq(HttpMethod.DELETE), any(HttpEntity.class), eq(String.class))) - .thenReturn(failureResponse); + .thenReturn(failureResponse); // Execute boolean result = geoServerService.unregisterLayer(layerName); // Verify - verify(restTemplate, times(1)).exchange(eq(expectedUrl), eq(HttpMethod.DELETE), any(HttpEntity.class), eq(String.class)); + verify(restTemplate, times(1)).exchange(eq(expectedUrl), eq(HttpMethod.DELETE), any(HttpEntity.class), + eq(String.class)); assertThat(result).isFalse(); } @Test void testUnregisterLayer_Failure_Exception() { - String expectedUrl = geoserverUrl + "/rest/workspaces/" + workspace + "/coveragestores/" + layerName + "?purge=all&recurse=true"; + String expectedUrl = geoserverUrl + geoserverWorkspacesPath + workspace + "/coveragestores/" + layerName + + "?purge=all&recurse=true"; // Simulate RestTemplate throwing an exception when(restTemplate.exchange( - eq(expectedUrl), - eq(HttpMethod.DELETE), - any(HttpEntity.class), - eq(String.class)) - ).thenThrow(new RuntimeException("Test exception")); + eq(expectedUrl), + eq(HttpMethod.DELETE), + any(HttpEntity.class), + eq(String.class))).thenThrow(new RuntimeException("Test exception")); // Execute boolean result = geoServerService.unregisterLayer(layerName); // Verify - verify(restTemplate, times(1)).exchange(eq(expectedUrl), eq(HttpMethod.DELETE), any(HttpEntity.class), eq(String.class)); + verify(restTemplate, times(1)).exchange(eq(expectedUrl), eq(HttpMethod.DELETE), any(HttpEntity.class), + eq(String.class)); assertThat(result).isFalse(); } - @Test void deleteTifFile_shouldDeleteFile_ifExists() throws Exception { // Arrange @@ -215,5 +227,4 @@ private void setField(Object target, String fieldName, Object value) { } } - } diff --git a/apps/backend/src/test/java/wildfire/visualization/backend/service/GeoTIFFServiceTests.java b/apps/backend/src/test/java/wildfire/visualization/backend/service/GeoTIFFServiceTests.java index 36b99901..203951eb 100644 --- a/apps/backend/src/test/java/wildfire/visualization/backend/service/GeoTIFFServiceTests.java +++ b/apps/backend/src/test/java/wildfire/visualization/backend/service/GeoTIFFServiceTests.java @@ -37,12 +37,6 @@ class GeoTIFFServiceTests { @InjectMocks private GeoTIFFService geoTIFFService; - private final String itemId = "wildfire_timestamp_2023_08_30_12_00_00"; - private final String collectionId = "montreal_2023"; - private final String assetName = "humidity"; - private final String tiffUrl = "https://example.com/humidity.tif"; // Fake URL - private final String localFilePath = "./geoserver_data/tiffs/humidity.tif"; - @BeforeEach void setUp() { // ✅ FIX: Use Reflection to set private fields correctly diff --git a/apps/geoserver_data/security/usergroup/default/users.xml b/apps/geoserver_data/security/usergroup/default/users.xml index b6bc52e6..83b1d7cd 100644 --- a/apps/geoserver_data/security/usergroup/default/users.xml +++ b/apps/geoserver_data/security/usergroup/default/users.xml @@ -1,7 +1,7 @@ - +