From c3cb945b458c8c526da85f2a9fa95344a8ce891a Mon Sep 17 00:00:00 2001 From: sh1r0 Date: Fri, 9 Oct 2015 00:31:05 +0800 Subject: [PATCH 1/7] Remove the 4D constraint of blobproto IO in python --- python/caffe/io.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/python/caffe/io.py b/python/caffe/io.py index 0cad7211291..40b7ac1ed78 100644 --- a/python/caffe/io.py +++ b/python/caffe/io.py @@ -21,22 +21,18 @@ def blobproto_to_array(blob, return_diff=False): unless return_diff is True, in which case we will return the diff. """ if return_diff: - return np.array(blob.diff).reshape( - blob.num, blob.channels, blob.height, blob.width) + return np.array(blob.diff).reshape(*blob.shape.dim) else: - return np.array(blob.data).reshape( - blob.num, blob.channels, blob.height, blob.width) + return np.array(blob.data).reshape(*blob.shape.dim) def array_to_blobproto(arr, diff=None): - """Converts a 4-dimensional array to blob proto. If diff is given, also + """Converts a N-dimensional array to blob proto. If diff is given, also convert the diff. You need to make sure that arr and diff have the same shape, and this function does not do sanity check. """ - if arr.ndim != 4: - raise ValueError('Incorrect array shape.') blob = caffe_pb2.BlobProto() - blob.num, blob.channels, blob.height, blob.width = arr.shape + blob.shape.dim.extend(arr.shape) blob.data.extend(arr.astype(float).flat) if diff is not None: blob.diff.extend(diff.astype(float).flat) From a0ac9057ad8ed91b3846aed475ae9ccaf7245bae Mon Sep 17 00:00:00 2001 From: Luke Yeager Date: Thu, 15 Oct 2015 11:03:09 -0700 Subject: [PATCH 2/7] Allow old-style shape in blobproto_to_array Fixes #3199 Bug introduced in #3170 --- python/caffe/io.py | 11 ++++++++-- python/caffe/test/test_io.py | 41 ++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 python/caffe/test/test_io.py diff --git a/python/caffe/io.py b/python/caffe/io.py index 40b7ac1ed78..11c84260f1a 100644 --- a/python/caffe/io.py +++ b/python/caffe/io.py @@ -20,11 +20,18 @@ def blobproto_to_array(blob, return_diff=False): Convert a blob proto to an array. In default, we will just return the data, unless return_diff is True, in which case we will return the diff. """ + # Read the data into an array if return_diff: - return np.array(blob.diff).reshape(*blob.shape.dim) + data = np.array(blob.diff) else: - return np.array(blob.data).reshape(*blob.shape.dim) + data = np.array(blob.data) + # Reshape the array + if blob.HasField('num') or blob.HasField('channels') or blob.HasField('height') or blob.HasField('width'): + # Use legacy 4D shape + return data.reshape(blob.num, blob.channels, blob.height, blob.width) + else: + return data.reshape(blob.shape.dim) def array_to_blobproto(arr, diff=None): """Converts a N-dimensional array to blob proto. If diff is given, also diff --git a/python/caffe/test/test_io.py b/python/caffe/test/test_io.py new file mode 100644 index 00000000000..8c86ef75fb2 --- /dev/null +++ b/python/caffe/test/test_io.py @@ -0,0 +1,41 @@ +import numpy as np +import unittest + +import caffe + +class TestBlobProtoToArray(unittest.TestCase): + + def test_old_format(self): + data = np.zeros((10,10)) + blob = caffe.proto.caffe_pb2.BlobProto() + blob.data.extend(list(data.flatten())) + shape = (1,1,10,10) + blob.num, blob.channels, blob.height, blob.width = shape + + arr = caffe.io.blobproto_to_array(blob) + self.assertEqual(arr.shape, shape) + + def test_new_format(self): + data = np.zeros((10,10)) + blob = caffe.proto.caffe_pb2.BlobProto() + blob.data.extend(list(data.flatten())) + blob.shape.dim.extend(list(data.shape)) + + arr = caffe.io.blobproto_to_array(blob) + self.assertEqual(arr.shape, data.shape) + + def test_no_shape(self): + data = np.zeros((10,10)) + blob = caffe.proto.caffe_pb2.BlobProto() + blob.data.extend(list(data.flatten())) + + with self.assertRaises(ValueError): + caffe.io.blobproto_to_array(blob) + + def test_scalar(self): + data = np.ones((1)) * 123 + blob = caffe.proto.caffe_pb2.BlobProto() + blob.data.extend(list(data.flatten())) + + arr = caffe.io.blobproto_to_array(blob) + self.assertEqual(arr, 123) From d2c063cfdc9444b93a20ca99645dbc8726bde118 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Wed, 14 Oct 2015 12:00:14 +0900 Subject: [PATCH 3/7] Fixed drawing problems with repeated convolution --- python/caffe/draw.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/caffe/draw.py b/python/caffe/draw.py index a002b60b59a..f8bf5722aba 100644 --- a/python/caffe/draw.py +++ b/python/caffe/draw.py @@ -82,11 +82,11 @@ def get_layer_label(layer, rankdir): separator, layer.type, separator, - layer.convolution_param.kernel_size, + layer.convolution_param.kernel_size[0] if len(layer.convolution_param.kernel_size._values) else 1, separator, - layer.convolution_param.stride, + layer.convolution_param.stride[0] if len(layer.convolution_param.stride._values) else 1, separator, - layer.convolution_param.pad) + layer.convolution_param.pad[0] if len(layer.convolution_param.pad._values) else 0) elif layer.type == 'Pooling': pooling_types_dict = get_pooling_types_dict() node_label = '"%s%s(%s %s)%skernel size: %d%sstride: %d%spad: %d"' %\ From 61379dcee0a1df248ceefc043de9a1e2cf89e691 Mon Sep 17 00:00:00 2001 From: Evan Shelhamer Date: Fri, 16 Oct 2015 17:32:27 -0700 Subject: [PATCH 4/7] clean up logging for Net init - condense conditions by `LOG_IF` - only log memory use once after all tops --- src/caffe/net.cpp | 182 +++++++++++++++++++--------------------------- 1 file changed, 76 insertions(+), 106 deletions(-) diff --git a/src/caffe/net.cpp b/src/caffe/net.cpp index ebb8b5d28c2..1ad93e6af5f 100644 --- a/src/caffe/net.cpp +++ b/src/caffe/net.cpp @@ -46,10 +46,9 @@ void Net::Init(const NetParameter& in_param) { // the current NetState. NetParameter filtered_param; FilterNet(in_param, &filtered_param); - if (Caffe::root_solver()) { - LOG(INFO) << "Initializing net from parameters: " << std::endl - << filtered_param.DebugString(); - } + LOG_IF(INFO, Caffe::root_solver()) + << "Initializing net from parameters: " << std::endl + << filtered_param.DebugString(); // Create a copy of filtered_param with splits added where necessary. NetParameter param; InsertSplits(filtered_param, ¶m); @@ -73,8 +72,6 @@ void Net::Init(const NetParameter& in_param) { const int layer_id = -1; // inputs have fake layer ID -1 AppendTop(param, layer_id, input_id, &available_blobs, &blob_name_to_idx); } - DLOG_IF(INFO, Caffe::root_solver()) - << "Memory required for data: " << memory_used_ * sizeof(Dtype); // For each layer, set up its input and output bottom_vecs_.resize(param.layer_size()); top_vecs_.resize(param.layer_size()); @@ -106,9 +103,8 @@ void Net::Init(const NetParameter& in_param) { layers_.push_back(LayerRegistry::CreateLayer(layer_param)); } layer_names_.push_back(layer_param.name()); - if (Caffe::root_solver()) { - LOG(INFO) << "Creating Layer " << layer_param.name(); - } + LOG_IF(INFO, Caffe::root_solver()) + << "Creating Layer " << layer_param.name(); bool need_backward = false; // Figure out this layer's input and output @@ -151,29 +147,23 @@ void Net::Init(const NetParameter& in_param) { } else { layers_[layer_id]->SetUp(bottom_vecs_[layer_id], top_vecs_[layer_id]); } - if (Caffe::root_solver()) { - LOG(INFO) << "Setting up " << layer_names_[layer_id]; - } + LOG_IF(INFO, Caffe::root_solver()) + << "Setting up " << layer_names_[layer_id]; for (int top_id = 0; top_id < top_vecs_[layer_id].size(); ++top_id) { if (blob_loss_weights_.size() <= top_id_vecs_[layer_id][top_id]) { blob_loss_weights_.resize(top_id_vecs_[layer_id][top_id] + 1, Dtype(0)); } blob_loss_weights_[top_id_vecs_[layer_id][top_id]] = layer->loss(top_id); - if (Caffe::root_solver()) { - LOG(INFO) << "Top shape: " - << top_vecs_[layer_id][top_id]->shape_string(); - } + LOG_IF(INFO, Caffe::root_solver()) + << "Top shape: " << top_vecs_[layer_id][top_id]->shape_string(); if (layer->loss(top_id)) { - if (Caffe::root_solver()) { - LOG(INFO) << " with loss weight " << layer->loss(top_id); - } + LOG_IF(INFO, Caffe::root_solver()) + << " with loss weight " << layer->loss(top_id); } memory_used_ += top_vecs_[layer_id][top_id]->count(); } - if (Caffe::root_solver()) { - DLOG(INFO) << "Memory required for data: " - << memory_used_ * sizeof(Dtype); - } + LOG_IF(INFO, Caffe::root_solver()) + << "Memory required for data: " << memory_used_ * sizeof(Dtype); const int param_size = layer_param.param_size(); const int num_param_blobs = layers_[layer_id]->blobs().size(); CHECK_LE(param_size, num_param_blobs) @@ -231,14 +221,12 @@ void Net::Init(const NetParameter& in_param) { } } if (!layer_contributes_loss) { layer_need_backward_[layer_id] = false; } - if (layer_need_backward_[layer_id]) { - if (Caffe::root_solver()) { + if (Caffe::root_solver()) { + if (layer_need_backward_[layer_id]) { LOG(INFO) << layer_names_[layer_id] << " needs backward computation."; - } - } else { - if (Caffe::root_solver()) { + } else { LOG(INFO) << layer_names_[layer_id] - << " does not need backward computation."; + << " does not need backward computation."; } } for (int bottom_id = 0; bottom_id < bottom_vecs_[layer_id].size(); @@ -279,9 +267,8 @@ void Net::Init(const NetParameter& in_param) { // In the end, all remaining blobs are considered output blobs. for (set::iterator it = available_blobs.begin(); it != available_blobs.end(); ++it) { - if (Caffe::root_solver()) { - LOG(INFO) << "This network produces output " << *it; - } + LOG_IF(INFO, Caffe::root_solver()) + << "This network produces output " << *it; net_output_blobs_.push_back(blobs_[blob_name_to_idx[*it]].get()); net_output_blob_indices_.push_back(blob_name_to_idx[*it]); } @@ -293,10 +280,7 @@ void Net::Init(const NetParameter& in_param) { } ShareWeights(); debug_info_ = param.debug_info(); - if (Caffe::root_solver()) { - LOG(INFO) << "Network initialization done."; - LOG(INFO) << "Memory required for data: " << memory_used_ * sizeof(Dtype); - } + LOG_IF(INFO, Caffe::root_solver()) << "Network initialization done."; } template @@ -335,33 +319,30 @@ bool Net::StateMeetsRule(const NetState& state, // Check whether the rule is broken due to phase. if (rule.has_phase()) { if (rule.phase() != state.phase()) { - if (Caffe::root_solver()) { - LOG(INFO) << "The NetState phase (" << state.phase() - << ") differed from the phase (" << rule.phase() - << ") specified by a rule in layer " << layer_name; - } + LOG_IF(INFO, Caffe::root_solver()) + << "The NetState phase (" << state.phase() + << ") differed from the phase (" << rule.phase() + << ") specified by a rule in layer " << layer_name; return false; } } // Check whether the rule is broken due to min level. if (rule.has_min_level()) { if (state.level() < rule.min_level()) { - if (Caffe::root_solver()) { - LOG(INFO) << "The NetState level (" << state.level() - << ") is above the min_level (" << rule.min_level() - << ") specified by a rule in layer " << layer_name; - } + LOG_IF(INFO, Caffe::root_solver()) + << "The NetState level (" << state.level() + << ") is above the min_level (" << rule.min_level() + << ") specified by a rule in layer " << layer_name; return false; } } // Check whether the rule is broken due to max level. if (rule.has_max_level()) { if (state.level() > rule.max_level()) { - if (Caffe::root_solver()) { - LOG(INFO) << "The NetState level (" << state.level() - << ") is above the max_level (" << rule.max_level() - << ") specified by a rule in layer " << layer_name; - } + LOG_IF(INFO, Caffe::root_solver()) + << "The NetState level (" << state.level() + << ") is above the max_level (" << rule.max_level() + << ") specified by a rule in layer " << layer_name; return false; } } @@ -374,10 +355,9 @@ bool Net::StateMeetsRule(const NetState& state, if (rule.stage(i) == state.stage(j)) { has_stage = true; } } if (!has_stage) { - if (Caffe::root_solver()) { - LOG(INFO) << "The NetState did not contain stage '" << rule.stage(i) - << "' specified by a rule in layer " << layer_name; - } + LOG_IF(INFO, Caffe::root_solver()) + << "The NetState did not contain stage '" << rule.stage(i) + << "' specified by a rule in layer " << layer_name; return false; } } @@ -390,10 +370,9 @@ bool Net::StateMeetsRule(const NetState& state, if (rule.not_stage(i) == state.stage(j)) { has_stage = true; } } if (has_stage) { - if (Caffe::root_solver()) { - LOG(INFO) << "The NetState contained a not_stage '" << rule.not_stage(i) - << "' specified by a rule in layer " << layer_name; - } + LOG_IF(INFO, Caffe::root_solver()) + << "The NetState contained a not_stage '" << rule.not_stage(i) + << "' specified by a rule in layer " << layer_name; return false; } } @@ -415,9 +394,8 @@ void Net::AppendTop(const NetParameter& param, const int layer_id, if (blob_name_to_idx && layer_param && layer_param->bottom_size() > top_id && blob_name == layer_param->bottom(top_id)) { // In-place computation - if (Caffe::root_solver()) { - LOG(INFO) << layer_param->name() << " -> " << blob_name << " (in-place)"; - } + LOG_IF(INFO, Caffe::root_solver()) + << layer_param->name() << " -> " << blob_name << " (in-place)"; top_vecs_[layer_id].push_back(blobs_[(*blob_name_to_idx)[blob_name]].get()); top_id_vecs_[layer_id].push_back((*blob_name_to_idx)[blob_name]); } else if (blob_name_to_idx && @@ -473,9 +451,8 @@ int Net::AppendBottom(const NetParameter& param, const int layer_id, << layer_param.name() << "', bottom index " << bottom_id << ")"; } const int blob_id = (*blob_name_to_idx)[blob_name]; - if (Caffe::root_solver()) { - LOG(INFO) << layer_names_[layer_id] << " <- " << blob_name; - } + LOG_IF(INFO, Caffe::root_solver()) + << layer_names_[layer_id] << " <- " << blob_name; bottom_vecs_[layer_id].push_back(blobs_[blob_id].get()); bottom_id_vecs_[layer_id].push_back(blob_id); available_blobs->erase(blob_name); @@ -672,10 +649,9 @@ void Net::InputDebugInfo(const int input_id) { const Blob& blob = *net_input_blobs_[input_id]; const string& blob_name = blob_names_[net_input_blob_indices_[input_id]]; const Dtype data_abs_val_mean = blob.asum_data() / blob.count(); - if (Caffe::root_solver()) { - LOG(INFO) << " [Forward] " - << "Input " << blob_name << " data: " << data_abs_val_mean; - } + LOG_IF(INFO, Caffe::root_solver()) + << " [Forward] " + << "Input " << blob_name << " data: " << data_abs_val_mean; } template @@ -684,12 +660,11 @@ void Net::ForwardDebugInfo(const int layer_id) { const Blob& blob = *top_vecs_[layer_id][top_id]; const string& blob_name = blob_names_[top_id_vecs_[layer_id][top_id]]; const Dtype data_abs_val_mean = blob.asum_data() / blob.count(); - if (Caffe::root_solver()) { - LOG(INFO) << " [Forward] " - << "Layer " << layer_names_[layer_id] - << ", top blob " << blob_name - << " data: " << data_abs_val_mean; - } + LOG_IF(INFO, Caffe::root_solver()) + << " [Forward] " + << "Layer " << layer_names_[layer_id] + << ", top blob " << blob_name + << " data: " << data_abs_val_mean; } for (int param_id = 0; param_id < layers_[layer_id]->blobs().size(); ++param_id) { @@ -697,12 +672,11 @@ void Net::ForwardDebugInfo(const int layer_id) { const int net_param_id = param_id_vecs_[layer_id][param_id]; const string& blob_name = param_display_names_[net_param_id]; const Dtype data_abs_val_mean = blob.asum_data() / blob.count(); - if (Caffe::root_solver()) { - LOG(INFO) << " [Forward] " - << "Layer " << layer_names_[layer_id] - << ", param blob " << blob_name - << " data: " << data_abs_val_mean; - } + LOG_IF(INFO, Caffe::root_solver()) + << " [Forward] " + << "Layer " << layer_names_[layer_id] + << ", param blob " << blob_name + << " data: " << data_abs_val_mean; } } @@ -714,24 +688,22 @@ void Net::BackwardDebugInfo(const int layer_id) { const Blob& blob = *bottom_vec[bottom_id]; const string& blob_name = blob_names_[bottom_id_vecs_[layer_id][bottom_id]]; const Dtype diff_abs_val_mean = blob.asum_diff() / blob.count(); - if (Caffe::root_solver()) { - LOG(INFO) << " [Backward] " - << "Layer " << layer_names_[layer_id] - << ", bottom blob " << blob_name - << " diff: " << diff_abs_val_mean; - } + LOG_IF(INFO, Caffe::root_solver()) + << " [Backward] " + << "Layer " << layer_names_[layer_id] + << ", bottom blob " << blob_name + << " diff: " << diff_abs_val_mean; } for (int param_id = 0; param_id < layers_[layer_id]->blobs().size(); ++param_id) { if (!layers_[layer_id]->param_propagate_down(param_id)) { continue; } const Blob& blob = *layers_[layer_id]->blobs()[param_id]; const Dtype diff_abs_val_mean = blob.asum_diff() / blob.count(); - if (Caffe::root_solver()) { - LOG(INFO) << " [Backward] " - << "Layer " << layer_names_[layer_id] - << ", param blob " << param_id - << " diff: " << diff_abs_val_mean; - } + LOG_IF(INFO, Caffe::root_solver()) + << " [Backward] " + << "Layer " << layer_names_[layer_id] + << ", param blob " << param_id + << " diff: " << diff_abs_val_mean; } } @@ -744,22 +716,20 @@ void Net::UpdateDebugInfo(const int param_id) { const Dtype diff_abs_val_mean = blob.asum_diff() / blob.count(); if (param_owner < 0) { const Dtype data_abs_val_mean = blob.asum_data() / blob.count(); - if (Caffe::root_solver()) { - LOG(INFO) << " [Update] Layer " << layer_name - << ", param " << param_display_name - << " data: " << data_abs_val_mean - << "; diff: " << diff_abs_val_mean; - } + LOG_IF(INFO, Caffe::root_solver()) + << " [Update] Layer " << layer_name + << ", param " << param_display_name + << " data: " << data_abs_val_mean + << "; diff: " << diff_abs_val_mean; } else { const string& owner_layer_name = layer_names_[param_layer_indices_[param_owner].first]; - if (Caffe::root_solver()) { - LOG(INFO) << " [Update] Layer " << layer_name - << ", param blob " << param_display_name - << " (owned by layer " << owner_layer_name << ", " << "param " - << param_display_names_[param_owners_[param_id]] << ")" - << " diff: " << diff_abs_val_mean; - } + LOG_IF(INFO, Caffe::root_solver()) + << " [Update] Layer " << layer_name + << ", param blob " << param_display_name + << " (owned by layer " << owner_layer_name << ", " << "param " + << param_display_names_[param_owners_[param_id]] << ")" + << " diff: " << diff_abs_val_mean; } } From b1e2caf9bbeffbc228bfa9fa9bdb1dc2a1e8cdd6 Mon Sep 17 00:00:00 2001 From: "T.E.A de Souza" Date: Mon, 19 Oct 2015 18:19:38 +0800 Subject: [PATCH 5/7] Qualify messages issued by CMake when CUDA is unavailable --- cmake/Dependencies.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake index d68d7bfba66..2005b9927b8 100644 --- a/cmake/Dependencies.cmake +++ b/cmake/Dependencies.cmake @@ -55,9 +55,9 @@ endif() include(cmake/Cuda.cmake) if(NOT HAVE_CUDA) if(CPU_ONLY) - message("-- CUDA is disabled. Building without it...") + message(STATUS "-- CUDA is disabled. Building without it...") else() - message("-- CUDA is not detected by cmake. Building without it...") + message(WARNING "-- CUDA is not detected by cmake. Building without it...") endif() # TODO: remove this not cross platform define in future. Use caffe_config.h instead. From 6f57970a139e60e11e5910eae547c723e1dd74b3 Mon Sep 17 00:00:00 2001 From: Luke Yeager Date: Fri, 25 Sep 2015 15:43:47 -0700 Subject: [PATCH 6/7] Re-ordering some lines in build files Enforcing a consistent ordering - OpenCV, LevelDB, LMDB This will allow me to add the ALLOW_LMDB_NOLOCK option just after the USE_LMDB option, while keeping the IO dependency options together. --- CMakeLists.txt | 4 ++-- Makefile.config.example | 2 +- cmake/Summary.cmake | 4 ++-- cmake/Templates/caffe_config.h.in | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a0ba9bd0a76..9a4ebae3ccb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,9 +29,9 @@ set(python_version "2" CACHE STRING "Specify which Python version to use") caffe_option(BUILD_matlab "Build Matlab wrapper" OFF IF UNIX OR APPLE) caffe_option(BUILD_docs "Build documentation" ON IF UNIX OR APPLE) caffe_option(BUILD_python_layer "Build the Caffe Python layer" ON) -caffe_option(USE_LMDB "Build with lmdb" ON) -caffe_option(USE_LEVELDB "Build with levelDB" ON) caffe_option(USE_OPENCV "Build with OpenCV support" ON) +caffe_option(USE_LEVELDB "Build with levelDB" ON) +caffe_option(USE_LMDB "Build with lmdb" ON) # ---[ Dependencies include(cmake/Dependencies.cmake) diff --git a/Makefile.config.example b/Makefile.config.example index a2540bb0c2b..bdd60ba2098 100644 --- a/Makefile.config.example +++ b/Makefile.config.example @@ -11,9 +11,9 @@ # CPU_ONLY := 1 # uncomment to disable IO dependencies and corresponding data layers +# USE_OPENCV := 0 # USE_LEVELDB := 0 # USE_LMDB := 0 -# USE_OPENCV := 0 # To customize your choice of compiler, uncomment and set the following. # N.B. the default for Linux is g++ and the default for OSX is clang++ diff --git a/cmake/Summary.cmake b/cmake/Summary.cmake index 0d8d080d11f..8b4b8ca244d 100644 --- a/cmake/Summary.cmake +++ b/cmake/Summary.cmake @@ -114,9 +114,9 @@ function(caffe_print_configuration_summary) caffe_status(" BUILD_matlab : ${BUILD_matlab}") caffe_status(" BUILD_docs : ${BUILD_docs}") caffe_status(" CPU_ONLY : ${CPU_ONLY}") - caffe_status(" USE_LMDB : ${USE_LMDB}") - caffe_status(" USE_LEVELDB : ${USE_LEVELDB}") caffe_status(" USE_OPENCV : ${USE_OPENCV}") + caffe_status(" USE_LEVELDB : ${USE_LEVELDB}") + caffe_status(" USE_LMDB : ${USE_LMDB}") caffe_status("") caffe_status("Dependencies:") caffe_status(" BLAS : " APPLE THEN "Yes (vecLib)" ELSE "Yes (${BLAS})") diff --git a/cmake/Templates/caffe_config.h.in b/cmake/Templates/caffe_config.h.in index 2a0185cdc39..6b7f7e505d8 100644 --- a/cmake/Templates/caffe_config.h.in +++ b/cmake/Templates/caffe_config.h.in @@ -37,5 +37,5 @@ /* IO libraries */ #cmakedefine USE_OPENCV -#cmakedefine USE_LMDB #cmakedefine USE_LEVELDB +#cmakedefine USE_LMDB From 749a63679779203c74bc5fce782897545061ba82 Mon Sep 17 00:00:00 2001 From: Luke Yeager Date: Fri, 25 Sep 2015 15:53:54 -0700 Subject: [PATCH 7/7] Add ALLOW_LMDB_NOLOCK build option This option lets you open LMDB files with the MDB_NOLOCK flag. You should not set this flag if you will be reading LMDBs with any possibility of simultaneous read and write. --- CMakeLists.txt | 1 + Makefile | 3 +++ Makefile.config.example | 5 +++++ cmake/ConfigGen.cmake | 3 +++ cmake/Dependencies.cmake | 3 +++ cmake/Summary.cmake | 1 + cmake/Templates/caffe_config.h.in | 1 + src/caffe/util/db_lmdb.cpp | 17 ++++++++++++++++- 8 files changed, 33 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9a4ebae3ccb..b1bb1a8b42a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,6 +32,7 @@ caffe_option(BUILD_python_layer "Build the Caffe Python layer" ON) caffe_option(USE_OPENCV "Build with OpenCV support" ON) caffe_option(USE_LEVELDB "Build with levelDB" ON) caffe_option(USE_LMDB "Build with lmdb" ON) +caffe_option(ALLOW_LMDB_NOLOCK "Allow MDB_NOLOCK when reading LMDB files (only if necessary)" OFF) # ---[ Dependencies include(cmake/Dependencies.cmake) diff --git a/Makefile b/Makefile index b99ca3d751b..a1fc925e40b 100644 --- a/Makefile +++ b/Makefile @@ -353,6 +353,9 @@ ifeq ($(USE_LEVELDB), 1) endif ifeq ($(USE_LMDB), 1) COMMON_FLAGS += -DUSE_LMDB +ifeq ($(ALLOW_LMDB_NOLOCK), 1) + COMMON_FLAGS += -DALLOW_LMDB_NOLOCK +endif endif # CPU-only configuration diff --git a/Makefile.config.example b/Makefile.config.example index bdd60ba2098..156a1828dc6 100644 --- a/Makefile.config.example +++ b/Makefile.config.example @@ -15,6 +15,11 @@ # USE_LEVELDB := 0 # USE_LMDB := 0 +# uncomment to allow MDB_NOLOCK when reading LMDB files (only if necessary) +# You should not set this flag if you will be reading LMDBs with any +# possibility of simultaneous read and write +# ALLOW_LMDB_NOLOCK := 1 + # To customize your choice of compiler, uncomment and set the following. # N.B. the default for Linux is g++ and the default for OSX is clang++ # CUSTOM_CXX := g++ diff --git a/cmake/ConfigGen.cmake b/cmake/ConfigGen.cmake index 373cc452ef1..f0bff4eba32 100644 --- a/cmake/ConfigGen.cmake +++ b/cmake/ConfigGen.cmake @@ -62,6 +62,9 @@ function(caffe_generate_export_configs) if(USE_LMDB) list(APPEND Caffe_DEFINITIONS -DUSE_LMDB) + if (ALLOW_LMDB_NOLOCK) + list(APPEND Caffe_DEFINITIONS -DALLOW_LMDB_NOLOCK) + endif() endif() if(USE_LEVELDB) diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake index 2005b9927b8..5651e2b086d 100644 --- a/cmake/Dependencies.cmake +++ b/cmake/Dependencies.cmake @@ -34,6 +34,9 @@ if(USE_LMDB) include_directories(SYSTEM ${LMDB_INCLUDE_DIR}) list(APPEND Caffe_LINKER_LIBS ${LMDB_LIBRARIES}) add_definitions(-DUSE_LMDB) + if(ALLOW_LMDB_NOLOCK) + add_definitions(-DALLOW_LMDB_NOLOCK) + endif() endif() # ---[ LevelDB diff --git a/cmake/Summary.cmake b/cmake/Summary.cmake index 8b4b8ca244d..f9e4a498063 100644 --- a/cmake/Summary.cmake +++ b/cmake/Summary.cmake @@ -117,6 +117,7 @@ function(caffe_print_configuration_summary) caffe_status(" USE_OPENCV : ${USE_OPENCV}") caffe_status(" USE_LEVELDB : ${USE_LEVELDB}") caffe_status(" USE_LMDB : ${USE_LMDB}") + caffe_status(" ALLOW_LMDB_NOLOCK : ${ALLOW_LMDB_NOLOCK}") caffe_status("") caffe_status("Dependencies:") caffe_status(" BLAS : " APPLE THEN "Yes (vecLib)" ELSE "Yes (${BLAS})") diff --git a/cmake/Templates/caffe_config.h.in b/cmake/Templates/caffe_config.h.in index 6b7f7e505d8..26526c2fe86 100644 --- a/cmake/Templates/caffe_config.h.in +++ b/cmake/Templates/caffe_config.h.in @@ -39,3 +39,4 @@ #cmakedefine USE_OPENCV #cmakedefine USE_LEVELDB #cmakedefine USE_LMDB +#cmakedefine ALLOW_LMDB_NOLOCK diff --git a/src/caffe/util/db_lmdb.cpp b/src/caffe/util/db_lmdb.cpp index 78dd880ac41..0bc82b53e2b 100644 --- a/src/caffe/util/db_lmdb.cpp +++ b/src/caffe/util/db_lmdb.cpp @@ -19,7 +19,22 @@ void LMDB::Open(const string& source, Mode mode) { if (mode == READ) { flags = MDB_RDONLY | MDB_NOTLS; } - MDB_CHECK(mdb_env_open(mdb_env_, source.c_str(), flags, 0664)); + int rc = mdb_env_open(mdb_env_, source.c_str(), flags, 0664); +#ifndef ALLOW_LMDB_NOLOCK + MDB_CHECK(rc); +#else + if (rc == EACCES) { + LOG(WARNING) << "Permission denied. Trying with MDB_NOLOCK ..."; + // Close and re-open environment handle + mdb_env_close(mdb_env_); + MDB_CHECK(mdb_env_create(&mdb_env_)); + // Try again with MDB_NOLOCK + flags |= MDB_NOLOCK; + MDB_CHECK(mdb_env_open(mdb_env_, source.c_str(), flags, 0664)); + } else { + MDB_CHECK(rc); + } +#endif LOG(INFO) << "Opened lmdb " << source; }