From fb1810d671cf5826c30d665b8d35a8acd42a7a6f Mon Sep 17 00:00:00 2001 From: Nicholas Fraser Date: Tue, 19 Apr 2016 19:17:10 +0100 Subject: [PATCH 1/4] Fixed integer overflow issue with LMDB on 32bit systems. --- examples/mnist/convert_mnist_data.cpp | 3 ++- include/caffe/util/db_lmdb.hpp | 4 ++++ src/caffe/util/db_lmdb.cpp | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/examples/mnist/convert_mnist_data.cpp b/examples/mnist/convert_mnist_data.cpp index 16d28093dd5..0ed5465f7c9 100644 --- a/examples/mnist/convert_mnist_data.cpp +++ b/examples/mnist/convert_mnist_data.cpp @@ -24,6 +24,7 @@ #include "caffe/proto/caffe.pb.h" #include "caffe/util/format.hpp" +#include "caffe/util/db_lmdb.hpp" #if defined(USE_LEVELDB) && defined(USE_LMDB) @@ -93,7 +94,7 @@ void convert_dataset(const char* image_filename, const char* label_filename, CHECK_EQ(mkdir(db_path, 0744), 0) << "mkdir " << db_path << "failed"; CHECK_EQ(mdb_env_create(&mdb_env), MDB_SUCCESS) << "mdb_env_create failed"; - CHECK_EQ(mdb_env_set_mapsize(mdb_env, 1099511627776), MDB_SUCCESS) // 1TB + CHECK_EQ(mdb_env_set_mapsize(mdb_env, BIGNUM_OR_MAX(1099511627776)), MDB_SUCCESS) // 1TB or the maximum size of size_t. << "mdb_env_set_mapsize failed"; CHECK_EQ(mdb_env_open(mdb_env, db_path, 0, 0664), MDB_SUCCESS) << "mdb_env_open failed"; diff --git a/include/caffe/util/db_lmdb.hpp b/include/caffe/util/db_lmdb.hpp index 4e1568ace50..c029fe36749 100644 --- a/include/caffe/util/db_lmdb.hpp +++ b/include/caffe/util/db_lmdb.hpp @@ -8,6 +8,10 @@ #include "caffe/util/db.hpp" +#include + +#define BIGNUM_OR_MAX(A) ((A) > (std::numeric_limits::max()) ? (std::numeric_limits::max()) : (A)) + namespace caffe { namespace db { inline void MDB_CHECK(int mdb_status) { diff --git a/src/caffe/util/db_lmdb.cpp b/src/caffe/util/db_lmdb.cpp index 0bc82b53e2b..70d7081ade4 100644 --- a/src/caffe/util/db_lmdb.cpp +++ b/src/caffe/util/db_lmdb.cpp @@ -7,7 +7,7 @@ namespace caffe { namespace db { -const size_t LMDB_MAP_SIZE = 1099511627776; // 1 TB +const size_t LMDB_MAP_SIZE = BIGNUM_OR_MAX(1099511627776); // 1 TB or maximum number for size_t type. void LMDB::Open(const string& source, Mode mode) { MDB_CHECK(mdb_env_create(&mdb_env_)); From 8aba4bd6e3f05f7ac1f06a6d34fe13fbdad3deb3 Mon Sep 17 00:00:00 2001 From: Nicholas Fraser Date: Wed, 20 Apr 2016 12:32:01 +0100 Subject: [PATCH 2/4] Added flag to specify LMDB_MAP_SIZE at compilation time. --- Makefile | 2 ++ Makefile.config.example | 1 + examples/mnist/convert_mnist_data.cpp | 2 +- include/caffe/util/db_lmdb.hpp | 3 +++ src/caffe/util/db_lmdb.cpp | 2 +- 5 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 5424c3a1858..00b6832ade1 100644 --- a/Makefile +++ b/Makefile @@ -184,12 +184,14 @@ LIBRARIES += glog gflags protobuf boost_system boost_filesystem m hdf5_hl hdf5 USE_LEVELDB ?= 1 USE_LMDB ?= 1 USE_OPENCV ?= 1 +LMDB_MAP_SIZE ?= 1099511627776 ifeq ($(USE_LEVELDB), 1) LIBRARIES += leveldb snappy endif ifeq ($(USE_LMDB), 1) LIBRARIES += lmdb + COMMON_FLAGS += -D_LMDB_MAP_SIZE=$(LMDB_MAP_SIZE) endif ifeq ($(USE_OPENCV), 1) LIBRARIES += opencv_core opencv_highgui opencv_imgproc diff --git a/Makefile.config.example b/Makefile.config.example index 8fd49c9c1a7..8477f4e1b6c 100644 --- a/Makefile.config.example +++ b/Makefile.config.example @@ -16,6 +16,7 @@ # You should not set this flag if you will be reading LMDBs with any # possibility of simultaneous read and write # ALLOW_LMDB_NOLOCK := 1 +# LMDB_MAP_SIZE := 4294967295 # 1TB is the default - uncomment this to avoid overflow on 32bit systems. # Uncomment if you're using OpenCV 3 # OPENCV_VERSION := 3 diff --git a/examples/mnist/convert_mnist_data.cpp b/examples/mnist/convert_mnist_data.cpp index 0ed5465f7c9..917e5bfc6d4 100644 --- a/examples/mnist/convert_mnist_data.cpp +++ b/examples/mnist/convert_mnist_data.cpp @@ -94,7 +94,7 @@ void convert_dataset(const char* image_filename, const char* label_filename, CHECK_EQ(mkdir(db_path, 0744), 0) << "mkdir " << db_path << "failed"; CHECK_EQ(mdb_env_create(&mdb_env), MDB_SUCCESS) << "mdb_env_create failed"; - CHECK_EQ(mdb_env_set_mapsize(mdb_env, BIGNUM_OR_MAX(1099511627776)), MDB_SUCCESS) // 1TB or the maximum size of size_t. + CHECK_EQ(mdb_env_set_mapsize(mdb_env, BIGNUM_OR_MAX(_LMDB_MAP_SIZE)), MDB_SUCCESS) // 1TB or the maximum size of size_t. << "mdb_env_set_mapsize failed"; CHECK_EQ(mdb_env_open(mdb_env, db_path, 0, 0664), MDB_SUCCESS) << "mdb_env_open failed"; diff --git a/include/caffe/util/db_lmdb.hpp b/include/caffe/util/db_lmdb.hpp index c029fe36749..80e916d968b 100644 --- a/include/caffe/util/db_lmdb.hpp +++ b/include/caffe/util/db_lmdb.hpp @@ -11,6 +11,9 @@ #include #define BIGNUM_OR_MAX(A) ((A) > (std::numeric_limits::max()) ? (std::numeric_limits::max()) : (A)) +#ifndef _LMDB_MAP_SIZE +#define _LMDB_MAP_SIZE 1099511627776 +#endif namespace caffe { namespace db { diff --git a/src/caffe/util/db_lmdb.cpp b/src/caffe/util/db_lmdb.cpp index 70d7081ade4..65fce5dd7d0 100644 --- a/src/caffe/util/db_lmdb.cpp +++ b/src/caffe/util/db_lmdb.cpp @@ -7,7 +7,7 @@ namespace caffe { namespace db { -const size_t LMDB_MAP_SIZE = BIGNUM_OR_MAX(1099511627776); // 1 TB or maximum number for size_t type. +const size_t LMDB_MAP_SIZE = BIGNUM_OR_MAX(_LMDB_MAP_SIZE); // 1 TB or maximum number for size_t type. void LMDB::Open(const string& source, Mode mode) { MDB_CHECK(mdb_env_create(&mdb_env_)); From 7fbf2f5ce03467e6f685c62be4e231931dda9b18 Mon Sep 17 00:00:00 2001 From: Nicholas Fraser Date: Wed, 20 Apr 2016 15:57:39 +0100 Subject: [PATCH 3/4] Replace spaces with tabs in Makefile to match original style. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 00b6832ade1..d8bf04dcb61 100644 --- a/Makefile +++ b/Makefile @@ -191,7 +191,7 @@ ifeq ($(USE_LEVELDB), 1) endif ifeq ($(USE_LMDB), 1) LIBRARIES += lmdb - COMMON_FLAGS += -D_LMDB_MAP_SIZE=$(LMDB_MAP_SIZE) + COMMON_FLAGS += -D_LMDB_MAP_SIZE=$(LMDB_MAP_SIZE) endif ifeq ($(USE_OPENCV), 1) LIBRARIES += opencv_core opencv_highgui opencv_imgproc From 097954219fcd8bc7b71428d36ce85878fdd570d6 Mon Sep 17 00:00:00 2001 From: Nicholas Fraser Date: Wed, 20 Apr 2016 17:43:37 +0100 Subject: [PATCH 4/4] Updated code format to pass 'make lint'. --- examples/mnist/convert_mnist_data.cpp | 6 ++++-- include/caffe/util/db_lmdb.hpp | 6 +++--- src/caffe/util/db_lmdb.cpp | 3 ++- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/examples/mnist/convert_mnist_data.cpp b/examples/mnist/convert_mnist_data.cpp index 917e5bfc6d4..46a0515aeb4 100644 --- a/examples/mnist/convert_mnist_data.cpp +++ b/examples/mnist/convert_mnist_data.cpp @@ -23,8 +23,8 @@ #include #include "caffe/proto/caffe.pb.h" -#include "caffe/util/format.hpp" #include "caffe/util/db_lmdb.hpp" +#include "caffe/util/format.hpp" #if defined(USE_LEVELDB) && defined(USE_LMDB) @@ -94,7 +94,9 @@ void convert_dataset(const char* image_filename, const char* label_filename, CHECK_EQ(mkdir(db_path, 0744), 0) << "mkdir " << db_path << "failed"; CHECK_EQ(mdb_env_create(&mdb_env), MDB_SUCCESS) << "mdb_env_create failed"; - CHECK_EQ(mdb_env_set_mapsize(mdb_env, BIGNUM_OR_MAX(_LMDB_MAP_SIZE)), MDB_SUCCESS) // 1TB or the maximum size of size_t. + // Check that the db size is the user specified one, or the max of size_t. + CHECK_EQ(mdb_env_set_mapsize(mdb_env, BIGNUM_OR_MAX(_LMDB_MAP_SIZE)), \ + MDB_SUCCESS) << "mdb_env_set_mapsize failed"; CHECK_EQ(mdb_env_open(mdb_env, db_path, 0, 0664), MDB_SUCCESS) << "mdb_env_open failed"; diff --git a/include/caffe/util/db_lmdb.hpp b/include/caffe/util/db_lmdb.hpp index 80e916d968b..1b0a686387f 100644 --- a/include/caffe/util/db_lmdb.hpp +++ b/include/caffe/util/db_lmdb.hpp @@ -2,15 +2,15 @@ #ifndef CAFFE_UTIL_DB_LMDB_HPP #define CAFFE_UTIL_DB_LMDB_HPP +#include #include #include "lmdb.h" #include "caffe/util/db.hpp" -#include - -#define BIGNUM_OR_MAX(A) ((A) > (std::numeric_limits::max()) ? (std::numeric_limits::max()) : (A)) +#define BIGNUM_OR_MAX(A) ((A) > (std::numeric_limits::max()) ?\ + (std::numeric_limits::max()) : (A)) #ifndef _LMDB_MAP_SIZE #define _LMDB_MAP_SIZE 1099511627776 #endif diff --git a/src/caffe/util/db_lmdb.cpp b/src/caffe/util/db_lmdb.cpp index 65fce5dd7d0..817a05eecc2 100644 --- a/src/caffe/util/db_lmdb.cpp +++ b/src/caffe/util/db_lmdb.cpp @@ -7,7 +7,8 @@ namespace caffe { namespace db { -const size_t LMDB_MAP_SIZE = BIGNUM_OR_MAX(_LMDB_MAP_SIZE); // 1 TB or maximum number for size_t type. +// User specified size or max value of size_t. +const size_t LMDB_MAP_SIZE = BIGNUM_OR_MAX(_LMDB_MAP_SIZE); void LMDB::Open(const string& source, Mode mode) { MDB_CHECK(mdb_env_create(&mdb_env_));