Upstream fixes

planetiler-core/src/main/java/com/onthegomap/planetiler/geo/GeometryType.java

@@ -13,7 +13,12 @@
 import vector_tile.VectorTileProto;
 
 public enum GeometryType {
-  UNKNOWN(VectorTileProto.Tile.GeomType.UNKNOWN, 0, "unknown"),
+  UNKNOWN(VectorTileProto.Tile.GeomType.UNKNOWN, 0, "unknown") {
+    @Override
+    public Expression featureTest() {
+      return Expression.TRUE;
+    }
+  },
   @JsonProperty("point")
   POINT(VectorTileProto.Tile.GeomType.POINT, 1, "point"),
   @JsonProperty("line")

planetiler-core/src/main/java/com/onthegomap/planetiler/util/Downloader.java

@@ -24,6 +24,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
+import java.util.OptionalLong;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
@@ -165,7 +166,7 @@ public void run() {
 
     for (var toDownload : toDownloadList) {
       try {
-        long size = toDownload.metadata.get(10, TimeUnit.SECONDS).size;
+        long size = toDownload.metadata.get(10, TimeUnit.SECONDS).size.orElse(0);
         loggers.addStorageRatePercentCounter(toDownload.id, size, toDownload::bytesDownloaded, true);
       } catch (InterruptedException e) {
         Thread.currentThread().interrupt();
@@ -183,22 +184,29 @@ CompletableFuture<Void> downloadIfNecessary(ResourceToDownload resourceToDownloa
     return CompletableFuture.runAsync(RunnableThatThrows.wrap(() -> {
       LogUtil.setStage("download", resourceToDownload.id);
       long existingSize = FileUtils.size(resourceToDownload.output);
-      var metadata = httpHeadFollowRedirects(resourceToDownload.url, 0);
-      Path tmpPath = resourceToDownload.tmpPath();
-      resourceToDownload.metadata.complete(metadata);
-      if (metadata.size == existingSize) {
-        LOGGER.info("Skipping {}: {} already up-to-date", resourceToDownload.id, resourceToDownload.output);
-        return;
+      try {
+        resourceToDownload.metadata.complete(httpHeadFollowRedirects(resourceToDownload.url, 0));
+      } catch (Exception e) {
+        resourceToDownload.metadata.completeExceptionally(e);
+        throw e;
       }
+      Path tmpPath = resourceToDownload.tmpPath();
       try {
+        var metadata = resourceToDownload.metadata.get();
+        if (metadata.size.orElse(-1) == existingSize) {
+          LOGGER.info("Skipping {}: {} already up-to-date", resourceToDownload.id, resourceToDownload.output);
+          return;
+        }
         String redirectInfo = metadata.canonicalUrl.equals(resourceToDownload.url) ? "" :
           " (redirected to " + metadata.canonicalUrl + ")";
         LOGGER.info("Downloading {}{} to {}", resourceToDownload.url, redirectInfo, resourceToDownload.output);
         FileUtils.delete(resourceToDownload.output);
         FileUtils.createParentDirectories(resourceToDownload.output);
         FileUtils.delete(tmpPath);
         FileUtils.deleteOnExit(tmpPath);
-        diskSpaceCheck.addDisk(tmpPath, metadata.size, resourceToDownload.id);
+        if (metadata.size.isPresent()) {
+          diskSpaceCheck.addDisk(tmpPath, metadata.size.getAsLong(), resourceToDownload.id);
+        }
         diskSpaceCheck.checkAgainstLimits(config.force(), false);
         httpDownload(resourceToDownload, tmpPath);
         Files.move(tmpPath, resourceToDownload.output);
@@ -225,7 +233,7 @@ ResourceMetadata httpHead(String url) throws IOException, InterruptedException {
       responseInfo -> {
         int status = responseInfo.statusCode();
         Optional<String> location = Optional.empty();
-        long contentLength = 0;
+        OptionalLong contentLength = OptionalLong.empty();
         HttpHeaders headers = responseInfo.headers();
         if (status >= 300 && status < 400) {
           location = responseInfo.headers().firstValue(LOCATION);
@@ -235,7 +243,7 @@ ResourceMetadata httpHead(String url) throws IOException, InterruptedException {
         } else if (responseInfo.statusCode() != 200) {
           throw new IllegalStateException("Bad response: " + responseInfo.statusCode());
         } else {
-          contentLength = headers.firstValueAsLong(CONTENT_LENGTH).orElseThrow();
+          contentLength = headers.firstValueAsLong(CONTENT_LENGTH);
         }
         boolean supportsRangeRequest = headers.allValues(ACCEPT_RANGES).contains("bytes");
         ResourceMetadata metadata = new ResourceMetadata(location, url, contentLength, supportsRangeRequest);
@@ -250,12 +258,13 @@ private void httpDownload(ResourceToDownload resource, Path tmpPath)
     record Range(long start, long end) {}
     List<Range> chunks = new ArrayList<>();
     boolean ranges = metadata.acceptRange && config.downloadThreads() > 1;
-    long chunkSize = ranges ? chunkSizeBytes : metadata.size;
-    for (long start = 0; start < metadata.size; start += chunkSize) {
-      long end = Math.min(start + chunkSize, metadata.size);
+    long fileSize = metadata.size.orElse(Long.MAX_VALUE);
+    long chunkSize = ranges ? chunkSizeBytes : fileSize;
+    for (long start = 0; start < fileSize; start += chunkSize) {
+      long end = Math.min(start + chunkSize, fileSize);
       chunks.add(new Range(start, end));
     }
-    FileUtils.setLength(tmpPath, metadata.size);
+    FileUtils.setLength(tmpPath, metadata.size.orElse(1));
     Semaphore perFileLimiter = new Semaphore(config.downloadThreads());
     Worker.joinFutures(chunks.stream().map(range -> CompletableFuture.runAsync(RunnableThatThrows.wrap(() -> {
       LogUtil.setStage("download", resource.id);
@@ -299,7 +308,7 @@ private HttpRequest.Builder newHttpRequest(String url) {
       .header(USER_AGENT, config.httpUserAgent());
   }
 
-  record ResourceMetadata(Optional<String> redirect, String canonicalUrl, long size, boolean acceptRange) {}
+  record ResourceMetadata(Optional<String> redirect, String canonicalUrl, OptionalLong size, boolean acceptRange) {}
 
   record ResourceToDownload(
     String id, String url, Path output, CompletableFuture<ResourceMetadata> metadata,

planetiler-core/src/main/java/com/onthegomap/planetiler/util/FileUtils.java

@@ -38,7 +38,7 @@ public class FileUtils {
   private static final Format FORMAT = Format.defaultInstance();
   // Prevent zip-bomb attack, see https://rules.sonarsource.com/java/RSPEC-5042
   private static final int ZIP_THRESHOLD_ENTRIES = 10_000;
-  private static final int ZIP_THRESHOLD_SIZE = 1_000_000_000;
+  private static final long ZIP_THRESHOLD_SIZE = 100_000_000_000L;
   private static final double ZIP_THRESHOLD_RATIO = 1_000;
 
   private static final Logger LOGGER = LoggerFactory.getLogger(FileUtils.class);
@@ -271,7 +271,7 @@ public static void unzipResource(String resource, Path dest) {
    */
   public static void safeCopy(InputStream inputStream, Path destPath) {
     try (var outputStream = Files.newOutputStream(destPath, StandardOpenOption.CREATE, WRITE)) {
-      int totalSize = 0;
+      long totalSize = 0;
 
       int nBytes;
       byte[] buffer = new byte[2048];
@@ -295,7 +295,7 @@ public static void safeCopy(InputStream inputStream, Path destPath) {
    * @throws UncheckedIOException if an IO exception occurs
    */
   public static void unzip(InputStream input, Path destDir) {
-    int totalSizeArchive = 0;
+    long totalSizeArchive = 0;
     int totalEntryArchive = 0;
     try (var zip = new ZipInputStream(input)) {
       ZipEntry entry;

planetiler-core/src/test/java/com/onthegomap/planetiler/geo/GeometryTypeTest.java

@@ -3,7 +3,6 @@
 import static java.util.Collections.emptyList;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import com.onthegomap.planetiler.TestUtils;
@@ -42,9 +41,9 @@ void testGeometryFactory() {
     assertFalse(GeometryType.POLYGON.featureTest().evaluate(point));
     assertTrue(GeometryType.POLYGON.featureTest().evaluate(poly));
 
-    assertThrows(Exception.class, () -> GeometryType.UNKNOWN.featureTest().evaluate(point));
-    assertThrows(Exception.class, () -> GeometryType.UNKNOWN.featureTest().evaluate(line));
-    assertThrows(Exception.class, () -> GeometryType.UNKNOWN.featureTest().evaluate(poly));
+    assertTrue(GeometryType.UNKNOWN.featureTest().evaluate(point));
+    assertTrue(GeometryType.UNKNOWN.featureTest().evaluate(line));
+    assertTrue(GeometryType.UNKNOWN.featureTest().evaluate(poly));
   }
 
   @ParameterizedTest

planetiler-core/src/test/java/com/onthegomap/planetiler/util/DownloaderTest.java

@@ -10,6 +10,7 @@
 import java.nio.file.Path;
 import java.util.Map;
 import java.util.Optional;
+import java.util.OptionalLong;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicLong;
@@ -25,7 +26,8 @@ class DownloaderTest {
   private final PlanetilerConfig config = PlanetilerConfig.defaults();
   private AtomicLong downloads = new AtomicLong(0);
 
-  private Downloader mockDownloader(Map<String, byte[]> resources, boolean supportsRange) {
+  private Downloader mockDownloader(Map<String, byte[]> resources, boolean supportsRange,
+    boolean supportsContentLength) {
     return new Downloader(config, 2L) {
 
       @Override
@@ -55,31 +57,38 @@ ResourceMetadata httpHead(String url) {
         if (parts.length > 1) {
           int redirectNum = Integer.parseInt(parts[1]);
           String next = redirectNum <= 1 ? parts[0] : (parts[0] + "#" + (redirectNum - 1));
-          return new ResourceMetadata(Optional.of(next), url, 0, supportsRange);
+          return new ResourceMetadata(Optional.of(next), url,
+            supportsContentLength ? OptionalLong.of(0) : OptionalLong.empty(), supportsRange);
         }
         byte[] bytes = resources.get(url);
-        return new ResourceMetadata(Optional.empty(), url, bytes.length, supportsRange);
+        return new ResourceMetadata(Optional.empty(), url,
+          supportsContentLength ? OptionalLong.of(bytes.length) : OptionalLong.empty(), supportsRange);
       }
     };
   }
 
   @ParameterizedTest
   @CsvSource({
-    "false,0",
-    "true,0",
-    "false,1",
-    "false,2",
-    "true,4",
+    "false,0,true",
+    "true,0,true",
+    "false,1,true",
+    "false,2,true",
+    "true,4,true",
+
+    "false,0,false",
+    "true,0,false",
+    "false,1,false",
+    "true,1,false",
   })
-  void testDownload(boolean range, int redirects) throws Exception {
+  void testDownload(boolean range, int redirects, boolean supportsContentLength) throws Exception {
     Path dest = path.resolve("out");
     String string = "0123456789";
     String url = "http://url";
     String initialUrl = url + (redirects > 0 ? "#" + redirects : "");
     Map<String, byte[]> resources = new ConcurrentHashMap<>();
 
     byte[] bytes = string.getBytes(StandardCharsets.UTF_8);
-    Downloader downloader = mockDownloader(resources, range);
+    Downloader downloader = mockDownloader(resources, range, supportsContentLength);
 
     // fails if no data
     var resource1 = new Downloader.ResourceToDownload("resource", initialUrl, dest);
@@ -96,13 +105,15 @@ void testDownload(boolean range, int redirects) throws Exception {
     assertEquals(10, resource2.bytesDownloaded());
 
     // does not re-request if size is the same
-    downloads.set(0);
-    var resource3 = new Downloader.ResourceToDownload("resource", initialUrl, dest);
-    downloader.downloadIfNecessary(resource3).get();
-    assertEquals(0, downloads.get());
-    assertEquals(string, Files.readString(dest));
-    assertEquals(FileUtils.size(path), FileUtils.size(dest));
-    assertEquals(0, resource3.bytesDownloaded());
+    if (supportsContentLength) {
+      downloads.set(0);
+      var resource3 = new Downloader.ResourceToDownload("resource", initialUrl, dest);
+      downloader.downloadIfNecessary(resource3).get();
+      assertEquals(0, downloads.get());
+      assertEquals(string, Files.readString(dest));
+      assertEquals(FileUtils.size(path), FileUtils.size(dest));
+      assertEquals(0, resource3.bytesDownloaded());
+    }
 
     // does re-download if size changes
     var resource4 = new Downloader.ResourceToDownload("resource", initialUrl, dest);
@@ -131,7 +142,7 @@ InputStream openStreamRange(String url, long start, long end) {
 
       @Override
       ResourceMetadata httpHead(String url) {
-        return new ResourceMetadata(Optional.empty(), url, Long.MAX_VALUE, true);
+        return new ResourceMetadata(Optional.empty(), url, OptionalLong.of(Long.MAX_VALUE), true);
       }
     };
 

planetiler-custommap/README.md

@@ -219,7 +219,10 @@ A feature is a defined set of objects that meet a specified filter criteria.
   of:
   - `point` `line` or `polygon` to pass the original feature through
   - `polygon_centroid` to match on polygons, and emit a point at the center
+  - `line_centroid` to match on lines, and emit a point at the center
+  - `centroid` to match any geometry, and emit a point at the center
   - `polygon_point_on_surface` to match on polygons, and emit an interior point
+  - `point_on_line` to match on lines, and emit a point somewhere along the line
   - `polygon_centroid_if_convex` to match on polygons, and if the polygon is convex emit the centroid, otherwise emit an
     interior point
 - `min_tile_cover_size` - Include objects of a certain geometry size, where 1.0 means "is

planetiler-custommap/planetiler.schema.json

@@ -354,8 +354,11 @@
             "line",
             "polygon",
             "polygon_centroid",
+            "line_centroid",
+            "centroid",
             "polygon_centroid_if_convex",
-            "polygon_point_on_surface"
+            "polygon_point_on_surface",
+            "point_on_line"
           ]
         },
         "source": {

planetiler-custommap/src/main/java/com/onthegomap/planetiler/custommap/configschema/FeatureGeometry.java

@@ -16,10 +16,16 @@ public enum FeatureGeometry {
   POLYGON(GeometryType.POLYGON, FeatureCollector::polygon),
   @JsonProperty("polygon_centroid")
   POLYGON_CENTROID(GeometryType.POLYGON, FeatureCollector::centroid),
+  @JsonProperty("line_centroid")
+  LINE_CENTROID(GeometryType.LINE, FeatureCollector::centroid),
+  @JsonProperty("centroid")
+  CENTROID(GeometryType.UNKNOWN, FeatureCollector::centroid),
   @JsonProperty("polygon_centroid_if_convex")
   POLYGON_CENTROID_IF_CONVEX(GeometryType.POLYGON, FeatureCollector::centroidIfConvex),
   @JsonProperty("polygon_point_on_surface")
-  POLYGON_POINT_ON_SURFACE(GeometryType.POLYGON, FeatureCollector::pointOnSurface);
+  POLYGON_POINT_ON_SURFACE(GeometryType.POLYGON, FeatureCollector::pointOnSurface),
+  @JsonProperty("point_on_line")
+  POINT_ON_LINE(GeometryType.LINE, FeatureCollector::pointOnSurface);
 
   public final GeometryType geometryType;
   public final BiFunction<FeatureCollector, String, FeatureCollector.Feature> geometryFactory;

planetiler-custommap/src/test/java/com/onthegomap/planetiler/custommap/ConfiguredFeatureTest.java

@@ -34,6 +34,7 @@
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.CsvSource;
 import org.junit.jupiter.params.provider.ValueSource;
+import org.locationtech.jts.geom.Puntal;
 
 class ConfiguredFeatureTest {
   private PlanetilerConfig planetilerConfig = PlanetilerConfig.defaults();
@@ -1222,4 +1223,59 @@ void testSchemaPostProcessWithMergePolygons() {
       )
     ), loadConfig(config).findFeatureLayer("testLayer").postProcess());
   }
+
+  @Test
+  void testCentroid() {
+    var config = """
+      sources:
+        osm:
+          type: osm
+          url: geofabrik:rhode-island
+          local_path: data/rhode-island.osm.pbf
+      layers:
+      - id: testLayer
+        features:
+        - source: osm
+          geometry: centroid
+      """;
+    this.planetilerConfig = PlanetilerConfig.from(Arguments.of(Map.of()));
+    testPolygon(config, Map.of(
+      "natural", "water"
+    ), feature -> {
+      assertInstanceOf(Puntal.class, feature.getGeometry());
+    }, 1);
+    testLinestring(config, Map.of(
+      "natural", "water"
+    ), feature -> {
+      assertInstanceOf(Puntal.class, feature.getGeometry());
+    }, 1);
+    testPoint(config, Map.of(
+      "natural", "water"
+    ), feature -> {
+      assertInstanceOf(Puntal.class, feature.getGeometry());
+    }, 1);
+  }
+
+  @ParameterizedTest
+  @ValueSource(strings = {"line_centroid", "point_on_line"})
+  void testLineCentroid(String type) {
+    var config = """
+      sources:
+        osm:
+          type: osm
+          url: geofabrik:rhode-island
+          local_path: data/rhode-island.osm.pbf
+      layers:
+      - id: testLayer
+        features:
+        - source: osm
+          geometry: %s
+      """.formatted(type);
+    this.planetilerConfig = PlanetilerConfig.from(Arguments.of(Map.of()));
+    testLinestring(config, Map.of(
+      "natural", "water"
+    ), feature -> {
+      assertInstanceOf(Puntal.class, feature.getGeometry());
+    }, 1);
+  }
 }

pom.xml

@@ -20,7 +20,7 @@
     <maven.compiler.source>21</maven.compiler.source>
     <maven.compiler.target>21</maven.compiler.target>
     <maven.source.excludeResources>true</maven.source.excludeResources>
-    <jackson.version>2.17.1</jackson.version>
+    <jackson.version>2.17.2</jackson.version>
     <junit.version>5.10.3</junit.version>
     <jts.version>1.19.0</jts.version>
     <sonar.host.url>https://sonarcloud.io</sonar.host.url>