diff --git a/Dockerfile b/Dockerfile index ca222e8..12be60c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,14 @@ # base image -FROM pelias/baseimage +FROM ubuntu:16.04 + +RUN apt-get update && apt-get install -y locales apt-utils iputils-ping curl wget git-core autoconf automake libtool pkg-config python + +RUN apt-get install -y software-properties-common && \ + apt-get update + +# install nodejs +ENV NODE_VERSION='10.14.0' +RUN git clone 'https://github.com/isaacs/nave.git' /code/nave && /code/nave/nave.sh 'usemain' "${NODE_VERSION}" && rm -rf ~/.nave /code/nave # install go 1.10 ENV GOPATH=/usr/src/.go @@ -7,6 +16,12 @@ RUN wget -qO- https://dl.google.com/go/go1.10.linux-amd64.tar.gz | tar -C /usr/l RUN mkdir -p "${GOPATH}" ENV PATH="${PATH}:/usr/local/go/bin:${GOPATH}/bin" +# Install valhalla (Requires Ubuntu 16.04) +RUN add-apt-repository -y ppa:valhalla-core/valhalla && \ + apt-get update + +RUN apt-get install -y valhalla-bin + # change working dir ENV WORKDIR /code/pelias/polylines WORKDIR ${WORKDIR} @@ -18,6 +33,12 @@ ENV PATH $PATH:$GOROOT/bin:$GOPATH/bin # get go dependencies RUN go get github.com/missinglink/pbf +RUN valhalla_build_config \ + --mjolnir-tile-dir '/data/valhalla/valhalla_tiles' \ + --mjolnir-tile-extract '/data/valhalla/valhalla_tiles.tar' \ + --mjolnir-timezone '/data/valhalla/valhalla_tiles/timezones.sqlite' \ + --mjolnir-admin '/data/valhalla/valhalla_tiles/admins.sqlite' > valhalla.json + # copy package.json first to prevent npm install being rerun when only code changes COPY ./package.json ${WORKDIR} RUN npm install diff --git a/docker_extract.sh b/docker_extract.sh index 1a0f3bb..b654573 100755 --- a/docker_extract.sh +++ b/docker_extract.sh @@ -3,6 +3,7 @@ set -euo pipefail # ensure data subdirectory exists mkdir -p /data/polylines/; +mkdir -p /data/valhalla/valhalla_tiles # enumerate a list of PBF files shopt -s nullglob @@ -17,23 +18,24 @@ fi # truncate polylines file echo '' > /data/polylines/extract.0sv; +# create empty sqlite db for valhalla +touch /data/valhalla/valhalla_tiles/timezones.sqlite +touch /data/valhalla/valhalla_tiles/admins.sqlite + # iterate over all PBF files in the osm directory for PBF_FILE in "${PBF_FILES[@]}"; do - # give a warning if the filesize is over 1GB - # the missinglink/pbf library is memory-bound and cannot safely handle very large extracts - find "${PBF_FILE}" -maxdepth 1 -size +1G | while read file; do - 2>&1 echo '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'; - 2>&1 echo "${PBF_FILE} is very large."; - 2>&1 echo 'You will likely experience memory issues working with large extracts like this.'; - 2>&1 echo 'We strongly recommend using Valhalla to produce extracts for large PBF extracts.'; - 2>&1 echo 'see: https://github.com/pelias/polylines#download!data'; - 2>&1 echo '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'; - done - # convert pbf file to 0sv (polylines) format, appending results to polyline file echo "converting ${PBF_FILE} to /data/polylines/extract.0sv"; - pbf streets "${PBF_FILE}" >> /data/polylines/extract.0sv; + + if [[ -n $(find "${PBF_FILE}" -maxdepth 1 -size +1G) ]]; then + # Use Valhalla to generate polylines + valhalla_build_tiles -c valhalla.json /data/openstreetmap/*.osm.pbf; + find /data/valhalla/valhalla_tiles | sort -n | tar cf /data/valhalla/valhalla_tiles.tar --no-recursion -T - + valhalla_export_edges --config valhalla.json >> /data/polylines/extract.0sv; + else + pbf streets "${PBF_FILE}" >> /data/polylines/extract.0sv; + fi done