diff --git a/include/osm_store.h b/include/osm_store.h index a84b7900..e3f6b7dc 100644 --- a/include/osm_store.h +++ b/include/osm_store.h @@ -8,6 +8,7 @@ #include #include #include +#include class void_mmap_allocator { @@ -189,6 +190,58 @@ class CompactNodeStore std::shared_ptr mLatpLons; }; +// list of ways used by relations +// by noting these in advance, we don't need to store all ways in the store +class UsedWays { + +private: + std::vector usedList; + mutable std::mutex mutex; + +public: + bool inited = false; + + // Size the vector to a reasonable estimate, to avoid resizing on the fly + void reserve(bool compact, size_t numNodes) { + std::lock_guard lock(mutex); + if (inited) return; + inited = true; + if (compact) { + // If we're running in compact mode, way count is roughly 1/9th of node count... say 1/8 to be safe + usedList.reserve(numNodes/8); + } else { + // Otherwise, we could have anything up to the current max node ID (approaching 2**30 in summer 2021) + // 2**31 is 0.25GB with a vector + usedList.reserve(pow(2,31)); + } + } + + // Mark a way as used + void insert(WayID wayid) { + std::lock_guard lock(mutex); + if (wayid>usedList.size()) usedList.resize(wayid+1); + usedList[wayid] = true; + } + + void insert_set(std::unordered_set ids) { + std::lock_guard lock(mutex); + for (WayID wayid : ids) { + if (wayid>usedList.size()) usedList.resize(wayid+1); + usedList[wayid] = true; + } + } + + // See if a way is used + bool at(WayID wayid) const { + return (wayid>usedList.size()) ? false : usedList[wayid]; + } + + void clear() { + std::lock_guard lock(mutex); + usedList.clear(); + } +}; + // way store class WayStore { @@ -336,6 +389,7 @@ class OSMStore WayStore ways; RelationStore relations; + UsedWays used_ways; generated osm_generated; generated shp_generated; @@ -382,8 +436,11 @@ class OSMStore compact_nodes.insert_back(new_nodes); } void nodes_sort(unsigned int threadNum); - - LatpLon nodes_at(NodeID i) const { + std::size_t nodes_size() { + return use_compact_nodes ? compact_nodes.size() : nodes.size(); + } + + LatpLon nodes_at(NodeID i) const { return use_compact_nodes ? compact_nodes.at(i) : nodes.at(i); } @@ -397,6 +454,13 @@ class OSMStore } void relations_sort(unsigned int threadNum); + void mark_way_used(WayID i) { used_ways.insert(i); } + void mark_ways_used(std::unordered_set ids) { used_ways.insert_set(ids); } + bool way_is_used(WayID i) { return used_ways.at(i); } + void ensure_used_ways_inited() { + if (!used_ways.inited) used_ways.reserve(use_compact_nodes, nodes_size()); + } + generated &osm() { return osm_generated; } generated const &osm() const { return osm_generated; } generated &shp() { return shp_generated; } @@ -477,6 +541,7 @@ class OSMStore compact_nodes.clear(); ways.clear(); relations.clear(); + used_ways.clear(); } void reportStoreSize(std::ostringstream &str); diff --git a/include/read_pbf.h b/include/read_pbf.h index 326a188b..ae31d340 100644 --- a/include/read_pbf.h +++ b/include/read_pbf.h @@ -22,7 +22,7 @@ class OsmLuaProcessing; class PbfReader { public: - enum class ReadPhase { Nodes = 1, Ways = 2, Relations = 4, All = 7 }; + enum class ReadPhase { Nodes = 1, Ways = 2, Relations = 4, RelationScan = 8, All = 15 }; PbfReader(OSMStore &osmStore); @@ -38,7 +38,7 @@ class PbfReader bool ReadNodes(OsmLuaProcessing &output, PrimitiveGroup &pg, PrimitiveBlock const &pb, const std::unordered_set &nodeKeyPositions); bool ReadWays(OsmLuaProcessing &output, PrimitiveGroup &pg, PrimitiveBlock const &pb); - + bool ScanRelations(OsmLuaProcessing &output, PrimitiveGroup &pg, PrimitiveBlock const &pb); bool ReadRelations(OsmLuaProcessing &output, PrimitiveGroup &pg, PrimitiveBlock const &pb); /// Find a string in the dictionary diff --git a/src/read_pbf.cpp b/src/read_pbf.cpp index fa545f03..5ef048ee 100644 --- a/src/read_pbf.cpp +++ b/src/read_pbf.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include "osm_lua_processing.h" @@ -91,9 +92,10 @@ bool PbfReader::ReadWays(OsmLuaProcessing &output, PrimitiveGroup &pg, Primitive tags[pb.stringtable().s(keysPtr->Get(n))] = pb.stringtable().s(valsPtr->Get(n)); } - // Store the way's nodes in the global way store - ways.push_back(std::make_pair(static_cast(pbfWay.id()), - WayStore::nodeid_vector_t(nodeVec.begin(), nodeVec.end()))); + // If we need it for later, store the way's nodes in the global way store + if (osmStore.way_is_used(wayId)) { + ways.push_back(std::make_pair(wayId, WayStore::nodeid_vector_t(nodeVec.begin(), nodeVec.end()))); + } output.setWay(static_cast(pbfWay.id()), nodeVec, tags); } catch (std::out_of_range &err) { @@ -109,6 +111,30 @@ bool PbfReader::ReadWays(OsmLuaProcessing &output, PrimitiveGroup &pg, Primitive return false; } +bool PbfReader::ScanRelations(OsmLuaProcessing &output, PrimitiveGroup &pg, PrimitiveBlock const &pb) { + // Scan relations to see which ways we need to save + // as with ReadRelations, we currently just parse multipolygons + if (pg.relations_size()==0) return false; + + int typeKey = findStringPosition(pb, "type"); + int mpKey = findStringPosition(pb, "multipolygon"); + + std::unordered_set wayIDs; + for (int j=0; j(lastID)); + } + } + osmStore.mark_ways_used(wayIDs); + return true; +} + bool PbfReader::ReadRelations(OsmLuaProcessing &output, PrimitiveGroup &pg, PrimitiveBlock const &pb) { // ---- Read relations // (just multipolygons for now; we should do routes in time) @@ -207,6 +233,16 @@ bool PbfReader::ReadBlock(std::istream &infile, OsmLuaProcessing &output, std::p continue; } } + + if(phase == ReadPhase::RelationScan || phase == ReadPhase::All) { + osmStore.ensure_used_ways_inited(); + bool done = ScanRelations(output, pg, pb); + if(done) { + std::cout << "(Scanning for ways used in relations: " << (100*progress.first/progress.second) << "%)\r"; + std::cout.flush(); + continue; + } + } if(phase == ReadPhase::Ways || phase == ReadPhase::All) { bool done = ReadWays(output, pg, pb); @@ -259,7 +295,7 @@ int PbfReader::ReadPbfFile(unordered_set const &nodeKeys, unsigned int t std::size_t total_blocks = blocks.size(); - std::vector all_phases = { ReadPhase::Nodes, ReadPhase::Ways, ReadPhase::Relations }; + std::vector all_phases = { ReadPhase::Nodes, ReadPhase::RelationScan, ReadPhase::Ways, ReadPhase::Relations }; for(auto phase: all_phases) { // Launch the pool with threadNum threads boost::asio::thread_pool pool(threadNum); diff --git a/src/tilemaker.cpp b/src/tilemaker.cpp index 7fc4e122..7b95091f 100644 --- a/src/tilemaker.cpp +++ b/src/tilemaker.cpp @@ -340,6 +340,7 @@ int main(int argc, char* argv[]) { }); if (ret != 0) return ret; } + osmStore.clear(); // no longer needed } // ---- Initialise SharedData