diff --git a/Makefile b/Makefile index 5424c3a1858..d8bf04dcb61 100644 --- a/Makefile +++ b/Makefile @@ -184,12 +184,14 @@ LIBRARIES += glog gflags protobuf boost_system boost_filesystem m hdf5_hl hdf5 USE_LEVELDB ?= 1 USE_LMDB ?= 1 USE_OPENCV ?= 1 +LMDB_MAP_SIZE ?= 1099511627776 ifeq ($(USE_LEVELDB), 1) LIBRARIES += leveldb snappy endif ifeq ($(USE_LMDB), 1) LIBRARIES += lmdb + COMMON_FLAGS += -D_LMDB_MAP_SIZE=$(LMDB_MAP_SIZE) endif ifeq ($(USE_OPENCV), 1) LIBRARIES += opencv_core opencv_highgui opencv_imgproc diff --git a/Makefile.config.example b/Makefile.config.example index 8fd49c9c1a7..8477f4e1b6c 100644 --- a/Makefile.config.example +++ b/Makefile.config.example @@ -16,6 +16,7 @@ # You should not set this flag if you will be reading LMDBs with any # possibility of simultaneous read and write # ALLOW_LMDB_NOLOCK := 1 +# LMDB_MAP_SIZE := 4294967295 # 1TB is the default - uncomment this to avoid overflow on 32bit systems. # Uncomment if you're using OpenCV 3 # OPENCV_VERSION := 3 diff --git a/examples/mnist/convert_mnist_data.cpp b/examples/mnist/convert_mnist_data.cpp index 16d28093dd5..46a0515aeb4 100644 --- a/examples/mnist/convert_mnist_data.cpp +++ b/examples/mnist/convert_mnist_data.cpp @@ -23,6 +23,7 @@ #include #include "caffe/proto/caffe.pb.h" +#include "caffe/util/db_lmdb.hpp" #include "caffe/util/format.hpp" #if defined(USE_LEVELDB) && defined(USE_LMDB) @@ -93,7 +94,9 @@ void convert_dataset(const char* image_filename, const char* label_filename, CHECK_EQ(mkdir(db_path, 0744), 0) << "mkdir " << db_path << "failed"; CHECK_EQ(mdb_env_create(&mdb_env), MDB_SUCCESS) << "mdb_env_create failed"; - CHECK_EQ(mdb_env_set_mapsize(mdb_env, 1099511627776), MDB_SUCCESS) // 1TB + // Check that the db size is the user specified one, or the max of size_t. + CHECK_EQ(mdb_env_set_mapsize(mdb_env, BIGNUM_OR_MAX(_LMDB_MAP_SIZE)), \ + MDB_SUCCESS) << "mdb_env_set_mapsize failed"; CHECK_EQ(mdb_env_open(mdb_env, db_path, 0, 0664), MDB_SUCCESS) << "mdb_env_open failed"; diff --git a/include/caffe/util/db_lmdb.hpp b/include/caffe/util/db_lmdb.hpp index 4e1568ace50..1b0a686387f 100644 --- a/include/caffe/util/db_lmdb.hpp +++ b/include/caffe/util/db_lmdb.hpp @@ -2,12 +2,19 @@ #ifndef CAFFE_UTIL_DB_LMDB_HPP #define CAFFE_UTIL_DB_LMDB_HPP +#include #include #include "lmdb.h" #include "caffe/util/db.hpp" +#define BIGNUM_OR_MAX(A) ((A) > (std::numeric_limits::max()) ?\ + (std::numeric_limits::max()) : (A)) +#ifndef _LMDB_MAP_SIZE +#define _LMDB_MAP_SIZE 1099511627776 +#endif + namespace caffe { namespace db { inline void MDB_CHECK(int mdb_status) { diff --git a/src/caffe/util/db_lmdb.cpp b/src/caffe/util/db_lmdb.cpp index 0bc82b53e2b..817a05eecc2 100644 --- a/src/caffe/util/db_lmdb.cpp +++ b/src/caffe/util/db_lmdb.cpp @@ -7,7 +7,8 @@ namespace caffe { namespace db { -const size_t LMDB_MAP_SIZE = 1099511627776; // 1 TB +// User specified size or max value of size_t. +const size_t LMDB_MAP_SIZE = BIGNUM_OR_MAX(_LMDB_MAP_SIZE); void LMDB::Open(const string& source, Mode mode) { MDB_CHECK(mdb_env_create(&mdb_env_));