Only add ways used by relations to the store

include/osm_store.h

@@ -8,6 +8,7 @@
 #include <utility>
 #include <vector>
 #include <mutex>
+#include <unordered_set>
 
 class void_mmap_allocator
 {
@@ -189,6 +190,58 @@ class CompactNodeStore
 	std::shared_ptr<map_t> mLatpLons;
 };
 
+// list of ways used by relations
+// by noting these in advance, we don't need to store all ways in the store
+class UsedWays {
+
+private:
+	std::vector<bool> usedList;
+	mutable std::mutex mutex;
+
+public:
+	bool inited = false;
+
+	// Size the vector to a reasonable estimate, to avoid resizing on the fly
+	void reserve(bool compact, size_t numNodes) {
+		std::lock_guard<std::mutex> lock(mutex);
+		if (inited) return;
+		inited = true;
+		if (compact) {
+			// If we're running in compact mode, way count is roughly 1/9th of node count... say 1/8 to be safe
+			usedList.reserve(numNodes/8);
+		} else {
+			// Otherwise, we could have anything up to the current max node ID (approaching 2**30 in summer 2021)
+			// 2**31 is 0.25GB with a vector<bool>
+			usedList.reserve(pow(2,31));
+		}
+	}
+
+	// Mark a way as used
+	void insert(WayID wayid) {
+		std::lock_guard<std::mutex> lock(mutex);
+		if (wayid>usedList.size()) usedList.resize(wayid+1);
+		usedList[wayid] = true;
+	}
+
+	void insert_set(std::unordered_set<WayID> ids) {
+		std::lock_guard<std::mutex> lock(mutex);
+		for (WayID wayid : ids) {
+			if (wayid>usedList.size()) usedList.resize(wayid+1);
+			usedList[wayid] = true;
+		}
+	}
+
+	// See if a way is used
+	bool at(WayID wayid) const {
+		return (wayid>usedList.size()) ? false : usedList[wayid];
+	}
+
+	void clear() {
+		std::lock_guard<std::mutex> lock(mutex);
+		usedList.clear();
+	}
+};
+
 // way store
 class WayStore {
 
@@ -336,6 +389,7 @@ class OSMStore
 
 	WayStore ways;
 	RelationStore relations;
+	UsedWays used_ways;
 
 	generated osm_generated;
 	generated shp_generated;
@@ -382,8 +436,11 @@ class OSMStore
 			compact_nodes.insert_back(new_nodes);
 	}
 	void nodes_sort(unsigned int threadNum);
-
-	LatpLon nodes_at(NodeID i) const { 
+	std::size_t nodes_size() {
+		return use_compact_nodes ? compact_nodes.size() : nodes.size();
+	}
+
+  LatpLon nodes_at(NodeID i) const { 
 		return use_compact_nodes ? compact_nodes.at(i) : nodes.at(i);
 	}
 
@@ -397,6 +454,13 @@ class OSMStore
 	}
 	void relations_sort(unsigned int threadNum);
 
+	void mark_way_used(WayID i) { used_ways.insert(i); }
+	void mark_ways_used(std::unordered_set<WayID> ids) { used_ways.insert_set(ids); }
+	bool way_is_used(WayID i) { return used_ways.at(i); }
+	void ensure_used_ways_inited() {
+		if (!used_ways.inited) used_ways.reserve(use_compact_nodes, nodes_size());
+	}
+
 	generated &osm() { return osm_generated; }
 	generated const &osm() const { return osm_generated; }
 	generated &shp() { return shp_generated; }
@@ -477,6 +541,7 @@ class OSMStore
 		compact_nodes.clear();
 		ways.clear();
 		relations.clear();
+		used_ways.clear();
 	} 
 
 	void reportStoreSize(std::ostringstream &str);

include/read_pbf.h

@@ -22,7 +22,7 @@ class OsmLuaProcessing;
 class PbfReader
 {
 public:	
-	enum class ReadPhase { Nodes = 1, Ways = 2, Relations = 4, All = 7 };
+	enum class ReadPhase { Nodes = 1, Ways = 2, Relations = 4, RelationScan = 8, All = 15 };
 
 	PbfReader(OSMStore &osmStore);
 
@@ -38,7 +38,7 @@ class PbfReader
 	bool ReadNodes(OsmLuaProcessing &output, PrimitiveGroup &pg, PrimitiveBlock const &pb, const std::unordered_set<int> &nodeKeyPositions);
 
 	bool ReadWays(OsmLuaProcessing &output, PrimitiveGroup &pg, PrimitiveBlock const &pb);
-
+	bool ScanRelations(OsmLuaProcessing &output, PrimitiveGroup &pg, PrimitiveBlock const &pb);
 	bool ReadRelations(OsmLuaProcessing &output, PrimitiveGroup &pg, PrimitiveBlock const &pb);
 
 	/// Find a string in the dictionary

src/read_pbf.cpp

@@ -5,6 +5,7 @@
 #include <boost/interprocess/streams/bufferstream.hpp>
 #include <boost/asio/thread_pool.hpp>
 #include <boost/asio/post.hpp>
+#include <unordered_set>
 
 #include "osm_lua_processing.h"
 
@@ -91,9 +92,10 @@ bool PbfReader::ReadWays(OsmLuaProcessing &output, PrimitiveGroup &pg, Primitive
 					tags[pb.stringtable().s(keysPtr->Get(n))] = pb.stringtable().s(valsPtr->Get(n));
 				}
 
-				// Store the way's nodes in the global way store
-				ways.push_back(std::make_pair(static_cast<WayID>(pbfWay.id()), 
-					WayStore::nodeid_vector_t(nodeVec.begin(), nodeVec.end())));
+				// If we need it for later, store the way's nodes in the global way store
+				if (osmStore.way_is_used(wayId)) {
+					ways.push_back(std::make_pair(wayId, WayStore::nodeid_vector_t(nodeVec.begin(), nodeVec.end())));
+				}
 				output.setWay(static_cast<WayID>(pbfWay.id()), nodeVec, tags);
 
 			} catch (std::out_of_range &err) {
@@ -109,6 +111,30 @@ bool PbfReader::ReadWays(OsmLuaProcessing &output, PrimitiveGroup &pg, Primitive
 	return false;
 }
 
+bool PbfReader::ScanRelations(OsmLuaProcessing &output, PrimitiveGroup &pg, PrimitiveBlock const &pb) {
+	// Scan relations to see which ways we need to save
+	// as with ReadRelations, we currently just parse multipolygons
+	if (pg.relations_size()==0) return false;
+
+	int typeKey = findStringPosition(pb, "type");
+	int mpKey   = findStringPosition(pb, "multipolygon");
+
+	std::unordered_set<WayID> wayIDs;
+	for (int j=0; j<pg.relations_size(); j++) {
+		Relation pbfRelation = pg.relations(j);
+		if (find(pbfRelation.keys().begin(), pbfRelation.keys().end(), typeKey) == pbfRelation.keys().end()) { continue; }
+		if (find(pbfRelation.vals().begin(), pbfRelation.vals().end(), mpKey  ) == pbfRelation.vals().end()) { continue; }
+		int64_t lastID = 0;
+		for (int n=0; n < pbfRelation.memids_size(); n++) {
+			lastID += pbfRelation.memids(n);
+			if (pbfRelation.types(n) != Relation_MemberType_WAY) { continue; }
+			wayIDs.insert(static_cast<WayID>(lastID));
+		}
+	}
+	osmStore.mark_ways_used(wayIDs);
+	return true;
+}
+
 bool PbfReader::ReadRelations(OsmLuaProcessing &output, PrimitiveGroup &pg, PrimitiveBlock const &pb) {
 	// ----	Read relations
 	//		(just multipolygons for now; we should do routes in time)
@@ -207,6 +233,16 @@ bool PbfReader::ReadBlock(std::istream &infile, OsmLuaProcessing &output, std::p
 				continue;
 			}
 		}
+
+		if(phase == ReadPhase::RelationScan || phase == ReadPhase::All) {
+			osmStore.ensure_used_ways_inited();
+			bool done = ScanRelations(output, pg, pb);
+			if(done) { 
+				std::cout << "(Scanning for ways used in relations: " << (100*progress.first/progress.second) << "%)\r";
+				std::cout.flush();
+				continue;
+			}
+		}
 
 		if(phase == ReadPhase::Ways || phase == ReadPhase::All) {
 			bool done = ReadWays(output, pg, pb);
@@ -259,7 +295,7 @@ int PbfReader::ReadPbfFile(unordered_set<string> const &nodeKeys, unsigned int t
 
 	std::size_t total_blocks = blocks.size();
 
-	std::vector<ReadPhase> all_phases = { ReadPhase::Nodes, ReadPhase::Ways, ReadPhase::Relations };
+	std::vector<ReadPhase> all_phases = { ReadPhase::Nodes, ReadPhase::RelationScan, ReadPhase::Ways, ReadPhase::Relations };
 	for(auto phase: all_phases) {
 		// Launch the pool with threadNum threads
 		boost::asio::thread_pool pool(threadNum);

src/tilemaker.cpp

@@ -340,6 +340,7 @@ int main(int argc, char* argv[]) {
 				});	
 			if (ret != 0) return ret;
 		} 
+		osmStore.clear(); // no longer needed
 	}
 
 	// ----	Initialise SharedData