From e684f10d7bad16414be97a680e56c5a564cf3c4c Mon Sep 17 00:00:00 2001 From: Jeff Donahue Date: Mon, 21 Apr 2014 20:35:43 -0700 Subject: [PATCH 1/2] fix bug where DataLayerPrefetch creates its own Caffe singleton, causing the phase to always be set to TRAIN (always random crops) and RNG failures --- include/caffe/vision_layers.hpp | 1 + src/caffe/layers/data_layer.cpp | 4 +++- src/caffe/layers/data_layer.cu | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/include/caffe/vision_layers.hpp b/include/caffe/vision_layers.hpp index b40f4c2958b..f4abcf4fbd0 100644 --- a/include/caffe/vision_layers.hpp +++ b/include/caffe/vision_layers.hpp @@ -297,6 +297,7 @@ class DataLayer : public Layer { shared_ptr > prefetch_label_; Blob data_mean_; bool output_labels_; + Caffe::Phase phase_; }; template diff --git a/src/caffe/layers/data_layer.cpp b/src/caffe/layers/data_layer.cpp index 8340259535a..e4c34eb6ea7 100644 --- a/src/caffe/layers/data_layer.cpp +++ b/src/caffe/layers/data_layer.cpp @@ -53,7 +53,7 @@ void* DataLayerPrefetch(void* layer_pointer) { CHECK(data.size()) << "Image cropping only support uint8 data"; int h_off, w_off; // We only do random crop when we do training. - if (Caffe::phase() == Caffe::TRAIN) { + if (layer->phase_ == Caffe::TRAIN) { // NOLINT_NEXT_LINE(runtime/threadsafe_fn) h_off = rand() % (height - crop_size); // NOLINT_NEXT_LINE(runtime/threadsafe_fn) @@ -227,6 +227,7 @@ void DataLayer::SetUp(const vector*>& bottom, } data_mean_.cpu_data(); DLOG(INFO) << "Initializing prefetch"; + phase_ = Caffe::phase(); CHECK(!pthread_create(&thread_, NULL, DataLayerPrefetch, reinterpret_cast(this))) << "Pthread execution failed."; DLOG(INFO) << "Prefetch initialized."; @@ -245,6 +246,7 @@ Dtype DataLayer::Forward_cpu(const vector*>& bottom, (*top)[1]->mutable_cpu_data()); } // Start a new prefetch thread + phase_ = Caffe::phase(); CHECK(!pthread_create(&thread_, NULL, DataLayerPrefetch, reinterpret_cast(this))) << "Pthread execution failed."; return Dtype(0.); diff --git a/src/caffe/layers/data_layer.cu b/src/caffe/layers/data_layer.cu index 15ef016670b..3dd9f8b686e 100644 --- a/src/caffe/layers/data_layer.cu +++ b/src/caffe/layers/data_layer.cu @@ -30,6 +30,7 @@ Dtype DataLayer::Forward_gpu(const vector*>& bottom, cudaMemcpyHostToDevice)); } // Start a new prefetch thread + phase_ = Caffe::phase(); CHECK(!pthread_create(&thread_, NULL, DataLayerPrefetch, reinterpret_cast(this))) << "Pthread execution failed."; return Dtype(0.); From 70a53c0337612deb6bc850afbe9388d9a4da8add Mon Sep 17 00:00:00 2001 From: Jeff Donahue Date: Mon, 21 Apr 2014 20:43:39 -0700 Subject: [PATCH 2/2] do the same as prev commit for ImageDataLayer --- include/caffe/vision_layers.hpp | 1 + src/caffe/layers/image_data_layer.cpp | 4 +++- src/caffe/layers/image_data_layer.cu | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/include/caffe/vision_layers.hpp b/include/caffe/vision_layers.hpp index f4abcf4fbd0..6084c27d6f6 100644 --- a/include/caffe/vision_layers.hpp +++ b/include/caffe/vision_layers.hpp @@ -477,6 +477,7 @@ class ImageDataLayer : public Layer { shared_ptr > prefetch_data_; shared_ptr > prefetch_label_; Blob data_mean_; + Caffe::Phase phase_; }; template diff --git a/src/caffe/layers/image_data_layer.cpp b/src/caffe/layers/image_data_layer.cpp index 8c23cc41053..4182091d4a7 100644 --- a/src/caffe/layers/image_data_layer.cpp +++ b/src/caffe/layers/image_data_layer.cpp @@ -60,7 +60,7 @@ void* ImageDataLayerPrefetch(void* layer_pointer) { CHECK(data.size()) << "Image cropping only support uint8 data"; int h_off, w_off; // We only do random crop when we do training. - if (Caffe::phase() == Caffe::TRAIN) { + if (layer->phase_ == Caffe::TRAIN) { // NOLINT_NEXT_LINE(runtime/threadsafe_fn) h_off = rand() % (height - crop_size); // NOLINT_NEXT_LINE(runtime/threadsafe_fn) @@ -228,6 +228,7 @@ void ImageDataLayer::SetUp(const vector*>& bottom, prefetch_label_->mutable_cpu_data(); data_mean_.cpu_data(); DLOG(INFO) << "Initializing prefetch"; + phase_ = Caffe::phase(); CHECK(!pthread_create(&thread_, NULL, ImageDataLayerPrefetch, reinterpret_cast(this))) << "Pthread execution failed."; DLOG(INFO) << "Prefetch initialized."; @@ -244,6 +245,7 @@ Dtype ImageDataLayer::Forward_cpu(const vector*>& bottom, memcpy((*top)[1]->mutable_cpu_data(), prefetch_label_->cpu_data(), sizeof(Dtype) * prefetch_label_->count()); // Start a new prefetch thread + phase_ = Caffe::phase(); CHECK(!pthread_create(&thread_, NULL, ImageDataLayerPrefetch, reinterpret_cast(this))) << "Pthread execution failed."; return Dtype(0.); diff --git a/src/caffe/layers/image_data_layer.cu b/src/caffe/layers/image_data_layer.cu index 7b4952d7768..e95550fe1ff 100644 --- a/src/caffe/layers/image_data_layer.cu +++ b/src/caffe/layers/image_data_layer.cu @@ -34,6 +34,7 @@ Dtype ImageDataLayer::Forward_gpu(const vector*>& bottom, prefetch_label_->cpu_data(), sizeof(Dtype) * prefetch_label_->count(), cudaMemcpyHostToDevice)); // Start a new prefetch thread + phase_ = Caffe::phase(); CHECK(!pthread_create(&thread_, NULL, ImageDataLayerPrefetch, reinterpret_cast(this))) << "Pthread execution failed."; return Dtype(0.);