Lua interop

CMakeLists.txt

@@ -81,10 +81,6 @@ ADD_CUSTOM_COMMAND(OUTPUT vector_tile.pb.cc vector_tile.pb.h
                    COMMAND ${PROTOBUF_PROTOC_EXECUTABLE}
 		   ARGS --cpp_out ${CMAKE_BINARY_DIR} -I ${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/include/vector_tile.proto)
 
-ADD_CUSTOM_COMMAND(OUTPUT osmformat.pb.cc osmformat.pb.h
-                   COMMAND ${PROTOBUF_PROTOC_EXECUTABLE}
-		   ARGS --cpp_out ${CMAKE_BINARY_DIR} -I ${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/include/osmformat.proto)
-
 file(GLOB tilemaker_src_files
 	src/attribute_store.cpp
 	src/coordinates.cpp
@@ -97,25 +93,30 @@ file(GLOB tilemaker_src_files
 	src/mbtiles.cpp
 	src/mmap_allocator.cpp
 	src/node_stores.cpp
+	src/options_parser.cpp
 	src/osm_lua_processing.cpp
 	src/osm_mem_tiles.cpp
 	src/osm_store.cpp
 	src/output_object.cpp
-	src/pbf_blocks.cpp
+	src/pbf_processor.cpp
+	src/pbf_reader.cpp
 	src/pmtiles.cpp
-	src/read_pbf.cpp
+	src/pooled_string.cpp
 	src/read_shp.cpp
+	src/sharded_node_store.cpp
+	src/sharded_way_store.cpp
 	src/shared_data.cpp
 	src/shp_mem_tiles.cpp
 	src/sorted_node_store.cpp
 	src/sorted_way_store.cpp
+	src/tag_map.cpp
 	src/tile_data.cpp
 	src/tilemaker.cpp
 	src/tile_worker.cpp
 	src/way_stores.cpp
 	src/write_geometry.cpp
   )
-add_executable(tilemaker vector_tile.pb.cc osmformat.pb.cc ${tilemaker_src_files})
+add_executable(tilemaker vector_tile.pb.cc ${tilemaker_src_files})
 target_include_directories(tilemaker PRIVATE include)
 target_include_directories(tilemaker PRIVATE ${CMAKE_BINARY_DIR}) # for generated files
 target_link_libraries(tilemaker

Makefile

@@ -93,7 +93,6 @@ INC := -I$(PLATFORM_PATH)/include -isystem ./include -I./src $(LUA_CFLAGS)
 all: tilemaker
 
 tilemaker: \
-	include/osmformat.pb.o \
 	include/vector_tile.pb.o \
 	src/attribute_store.o \
 	src/coordinates_geom.o \
@@ -106,36 +105,89 @@ tilemaker: \
 	src/mbtiles.o \
 	src/mmap_allocator.o \
 	src/node_stores.o \
+	src/options_parser.o \
 	src/osm_lua_processing.o \
 	src/osm_mem_tiles.o \
 	src/osm_store.o \
 	src/output_object.o \
-	src/pbf_blocks.o \
+	src/pbf_processor.o \
+	src/pbf_reader.o \
 	src/pmtiles.o \
-	src/read_pbf.o \
+	src/pooled_string.o \
 	src/read_shp.o \
+	src/sharded_node_store.o \
+	src/sharded_way_store.o \
 	src/shared_data.o \
 	src/shp_mem_tiles.o \
 	src/sorted_node_store.o \
 	src/sorted_way_store.o \
+	src/tag_map.o \
 	src/tile_data.o \
 	src/tilemaker.o \
 	src/tile_worker.o \
 	src/way_stores.o \
 	src/write_geometry.o
 	$(CXX) $(CXXFLAGS) -o tilemaker $^ $(INC) $(LIB) $(LDFLAGS)
 
-test: test_sorted_way_store
+test: \
+	test_append_vector \
+	test_attribute_store \
+	test_deque_map \
+	test_pbf_reader \
+	test_pooled_string \
+	test_sorted_node_store \
+	test_sorted_way_store
+
+test_append_vector: \
+	src/mmap_allocator.o \
+	test/append_vector.test.o
+	$(CXX) $(CXXFLAGS) -o test.append_vector $^ $(INC) $(LIB) $(LDFLAGS) && ./test.append_vector
+
+test_attribute_store: \
+	src/mmap_allocator.o \
+	src/attribute_store.o \
+	src/pooled_string.o \
+	test/attribute_store.test.o
+	$(CXX) $(CXXFLAGS) -o test.attribute_store $^ $(INC) $(LIB) $(LDFLAGS) && ./test.attribute_store
+
+test_deque_map: \
+	test/deque_map.test.o
+	$(CXX) $(CXXFLAGS) -o test.deque_map $^ $(INC) $(LIB) $(LDFLAGS) && ./test.deque_map
+
+test_options_parser: \
+	src/options_parser.o \
+	test/options_parser.test.o
+	$(CXX) $(CXXFLAGS) -o test.options_parser $^ $(INC) $(LIB) $(LDFLAGS) && ./test.options_parser
+
+test_pooled_string: \
+	src/mmap_allocator.o \
+	src/pooled_string.o \
+	test/pooled_string.test.o
+	$(CXX) $(CXXFLAGS) -o test.pooled_string $^ $(INC) $(LIB) $(LDFLAGS) && ./test.pooled_string
+
+test_sorted_node_store: \
+	src/external/streamvbyte_decode.o \
+	src/external/streamvbyte_encode.o \
+	src/external/streamvbyte_zigzag.o \
+	src/mmap_allocator.o \
+	src/sorted_node_store.o \
+	test/sorted_node_store.test.o
+	$(CXX) $(CXXFLAGS) -o test.sorted_node_store $^ $(INC) $(LIB) $(LDFLAGS) && ./test.sorted_node_store
 
 test_sorted_way_store: \
 	src/external/streamvbyte_decode.o \
 	src/external/streamvbyte_encode.o \
 	src/external/streamvbyte_zigzag.o \
 	src/mmap_allocator.o \
 	src/sorted_way_store.o \
-	src/sorted_way_store.test.o
+	test/sorted_way_store.test.o
 	$(CXX) $(CXXFLAGS) -o test.sorted_way_store $^ $(INC) $(LIB) $(LDFLAGS) && ./test.sorted_way_store
 
+test_pbf_reader: \
+	src/helpers.o \
+	src/pbf_reader.o \
+	test/pbf_reader.test.o
+	$(CXX) $(CXXFLAGS) -o test.pbf_reader $^ $(INC) $(LIB) $(LDFLAGS) && ./test.pbf_reader
 
 %.o: %.cpp
 	$(CXX) $(CXXFLAGS) -o $@ -c $< $(INC)
@@ -153,6 +205,6 @@ install:
 	install docs/man/tilemaker.1 ${DESTDIR}${MANPREFIX}/man1/
 
 clean:
-	rm -f tilemaker src/*.o src/external/*.o include/*.o include/*.pb.h
+	rm -f tilemaker src/*.o src/external/*.o include/*.o include/*.pb.h test/*.o
 
 .PHONY: install

docs/CONFIGURATION.md

@@ -107,13 +107,16 @@ For example:
 
 ### Lua processing
 
-Your Lua file needs to supply 5 things:
+Your Lua file needs to supply a few things:
 
 1. `node_keys`, a list of those OSM keys which indicate that a node should be processed
-2. `init_function(name)` (optional), a function to initialize Lua logic
-2. `node_function(node)`, a function to process an OSM node and add it to layers
-3. `way_function(way)`, a function to process an OSM way and add it to layers
-3. `exit_function` (optional), a function to finalize Lua logic (useful to show statistics)
+2. `node_function()`, a function to process an OSM node and add it to layers
+3. `way_function()`, a function to process an OSM way and add it to layers
+4. (optional) `init_function(name)`, a function to initialize Lua logic
+5. (optional) `exit_function`, a function to finalize Lua logic (useful to show statistics)
+6. (optional) `relation_scan_function`, a function to determine whether your Lua file wishes to process the given relation 
+7. (optional) `relation_function`, a function to process an OSM relation and add it to layers
+8. (optional) `attribute_function`, a function to remap attributes from shapefiles
 
 `node_keys` is a simple list (or in Lua parlance, a 'table') of OSM tag keys. If a node has one of those keys, it will be processed by `node_function`; if not, it'll be skipped. For example, if you wanted to show highway crossings and railway stations, it should be `{ "highway", "railway" }`. (This avoids the need to process the vast majority of nodes which contain no important tags at all.)
 
@@ -127,28 +130,30 @@ Note the order: you write to a layer first, then set attributes after.
 
 To do that, you use these methods:
 
-* `node:Find(key)` or `way:Find(key)`: get the value for a tag, or the empty string if not present. For example, `way:Find("railway")` might return "rail" for a railway, "siding" for a siding, or "" if it isn't a railway at all.
-* `node:Holds(key)` or `way:Holds(key)`: returns true if that key exists, false otherwise.
-* `node:Layer("layer_name", false)` or `way:Layer("layer_name", is_area)`: write this node/way to the named layer. This is how you put objects in your vector tile. is_area (true/false) specifies whether a way should be treated as an area, or just as a linestring.
-* `way:LayerAsCentroid("layer_name")`: write a single centroid point for this way to the named layer (useful for labels and POIs).
-* `node:Attribute(key,value,minzoom)` or `node:Attribute(key,value,minzoom)`: add an attribute to the most recently written layer. Argument `minzoom` is optional, use it if you do not want to write the attribute on lower zoom levels.
-* `node:AttributeNumeric(key,value,minzoom)`, `node:AttributeBoolean(key,value,minzoom)` (and `way:`...): for numeric/boolean columns.
-* `node:Id()` or `way:Id()`: get the OSM ID of the current object.
-* `node:ZOrder(number)` or `way:ZOrder(number)`: Set a numeric value (default 0, 1-byte signed integer) used to sort features within a layer. Use this feature to ensure a proper rendering order if the rendering engine itself does not support sorting. Sorting is not supported across layers merged with `write_to`. Features with different z-order are not merged if `combine_below` or `combine_polygons_below` is used.
-* `node:MinZoom(zoom)` or `way:MinZoom(zoom)`: set the minimum zoom level (0-15) at which this object will be written. Note that the JSON layer configuration minimum still applies (so `:MinZoom(5)` will have no effect if your layer only starts at z6).
-* `way:Length()` and `way:Area()`: return the length (metres)/area (square metres) of the current object. Requires recent Boost.
-* `way:Centroid()`: return the lat/lon of the centre of the current object as a two-element Lua table (element 1 is lat, 2 is lon).
+* `Find(key)`: get the value for a tag, or the empty string if not present. For example, `Find("railway")` might return "rail" for a railway, "siding" for a siding, or "" if it isn't a railway at all.
+* `Holds(key)`: returns true if that key exists, false otherwise.
+* `Layer("layer_name", is_area)`: write this node/way to the named layer. This is how you put objects in your vector tile. is_area (true/false) specifies whether a way should be treated as an area, or just as a linestring.
+* `LayerAsCentroid("layer_name")`: write a single centroid point for this way to the named layer (useful for labels and POIs).
+* `Attribute(key,value,minzoom)`: add an attribute to the most recently written layer. Argument `minzoom` is optional, use it if you do not want to write the attribute on lower zoom levels.
+* `AttributeNumeric(key,value,minzoom)`, `AttributeBoolean(key,value,minzoom)`: for numeric/boolean columns.
+* `Id()`: get the OSM ID of the current object.
+* `ZOrder(number)`: Set a numeric value (default 0, 1-byte signed integer) used to sort features within a layer. Use this feature to ensure a proper rendering order if the rendering engine itself does not support sorting. Sorting is not supported across layers merged with `write_to`. Features with different z-order are not merged if `combine_below` or `combine_polygons_below` is used.
+* `MinZoom(zoom)`: set the minimum zoom level (0-15) at which this object will be written. Note that the JSON layer configuration minimum still applies (so `:MinZoom(5)` will have no effect if your layer only starts at z6).
+* `Length()` and `Area()`: return the length (metres)/area (square metres) of the current object. Requires recent Boost.
+* `Centroid()`: return the lat/lon of the centre of the current object as a two-element Lua table (element 1 is lat, 2 is lon).
 
 The simplest possible function, to include roads/paths and nothing else, might look like this:
 
-    function way_function(way)
-      local highway = way:Find("highway")
+```lua
+    function way_function()
+      local highway = Find("highway")
       if highway~="" then
-        way:Layer("roads", false)
-        way:Attribute("name", way:Find("name"))
-        way:Attribute("type", highway)
+        Layer("roads", false)
+        Attribute("name", Find("name"))
+        Attribute("type", highway)
       end
     end
+```
 
 Take a look at the supplied process.lua for a simple example, or the more complex OpenMapTiles-compatible script in `resources/`. You can specify another filename with the `--process` option.
 
@@ -197,11 +202,11 @@ When processing OSM objects with your Lua script, you can perform simple spatial
 
 You can then find out whether a node is within one of these polygons using the `Intersects` method:
 
-    if node:Intersects("countries") then print("Looks like it's on land"); end
+    if Intersects("countries") then print("Looks like it's on land"); end
 
 Or you can find out what country(/ies) the node is within using `FindIntersecting`, which returns a table:
 
-    names = node:FindIntersecting("countries")
+    names = FindIntersecting("countries")
     print(table.concat(name,","))
 
 To enable these functions, set `index` to true in your shapefile layer definition. `index_column` is not needed for `Intersects` but required for `FindIntersecting`.

docs/RELATIONS.md

@@ -22,26 +22,30 @@ This is a two-stage process: first, when reading relations, indicate that these
 
 To define which relations should be accepted, add a `relation_scan_function`:
 
-    function relation_scan_function(relation)
-      if relation:Find("type")=="route" and relation:Find("route")=="bicycle" then
-        local network = relation:Find("network")
-        if network=="ncn" then relation:Accept() end
+```lua
+    function relation_scan_function()
+      if Find("type")=="route" and Find("route")=="bicycle" then
+        local network = Find("network")
+        if network=="ncn" then Accept() end
       end
     end
+```
 
-This function takes the relation as its sole argument. Examine the tags using `relation:Find(key)` as normal. (You can also use `relation:Holds(key)` and `relation:Id()`.) If you want to use this relation, call `relation:Accept()`.
+Examine the tags using `Find(key)` as normal. (You can also use `Holds(key)` and `Id()`.) If you want to use this relation, call `Accept()`.
 
 #### Stage 2: accessing relations from ways
 
-Now that you've accepted the relations, they will be available from `way_function`. They are accessed using an iterator (`way:NextRelation()`) which reads each relation for that way in turn, returning nil when there are no more relations available. Once you have accessed a relation with the iterator, you can read its tags with `way:FindInRelation(key)`. For example:
+Now that you've accepted the relations, they will be available from `way_function`. They are accessed using an iterator (`NextRelation()`) which reads each relation for that way in turn, returning nil when there are no more relations available. Once you have accessed a relation with the iterator, you can read its tags with `FindInRelation(key)`. For example:
 
+```lua
     while true do
-      local rel = way:NextRelation()
+      local rel = NextRelation()
       if not rel then break end
-      print ("Part of route "..way:FindInRelation("ref"))
+      print ("Part of route "..FindInRelation("ref"))
     end
+```
 
-(Should you need to re-read the relations, you can reset the iterator with `way:RestartRelations()`.)
+(Should you need to re-read the relations, you can reset the iterator with `RestartRelations()`.)
 
 
 ### Writing relation geometries
@@ -52,13 +56,15 @@ First, make sure that you have accepted the relations using `relation_scan_funct
 
 Then write a `relation_function`, which works in the same way as `way_function` would:
 
-    function relation_function(relation)
-      if relation:Find("type")=="route" and relation:Find("route")=="bicycle" then
-        relation:Layer("bike_routes", false)
-        relation:Attribute("class", relation:Find("network"))
-        relation:Attribute("ref", relation:Find("ref"))
+```lua
+    function relation_function()
+      if Find("type")=="route" and Find("route")=="bicycle" then
+        Layer("bike_routes", false)
+        Attribute("class", Find("network"))
+        Attribute("ref", Find("ref"))
       end
     end
+```
 
 
 ### Not supported