Add & test regularizer class hierarchy: L1, L2 & skeleton of MaxNorm

Makefile

@@ -86,27 +86,37 @@ CUDA_LIB_DIR := $(CUDA_DIR)/lib64 $(CUDA_DIR)/lib
 MKL_INCLUDE_DIR := $(MKL_DIR)/include
 MKL_LIB_DIR := $(MKL_DIR)/lib $(MKL_DIR)/lib/intel64
 
-INCLUDE_DIRS += ./src ./include $(CUDA_INCLUDE_DIR) $(MKL_INCLUDE_DIR)
-LIBRARY_DIRS += $(CUDA_LIB_DIR) $(MKL_LIB_DIR)
+INCLUDE_DIRS += ./src ./include $(CUDA_INCLUDE_DIR)
+LIBRARY_DIRS += $(CUDA_LIB_DIR)
 LIBRARIES := cudart cublas curand \
-	mkl_rt \
 	pthread \
-	glog protobuf leveldb \
-	snappy \
+	glog protobuf leveldb snappy \
 	boost_system \
 	hdf5_hl hdf5 \
 	opencv_core opencv_highgui opencv_imgproc
 PYTHON_LIBRARIES := boost_python python2.7
 WARNINGS := -Wall
 
-COMMON_FLAGS := -DNDEBUG -O2 $(foreach includedir,$(INCLUDE_DIRS),-I$(includedir))
+COMMON_FLAGS := -DNDEBUG -O2
+
+# MKL switch (default = non-MKL)
+USE_MKL ?= 0
+ifeq ($(USE_MKL), 1)
+  LIBRARIES += mkl_rt
+  COMMON_FLAGS += -DUSE_MKL
+  INCLUDE_DIRS += $(MKL_INCLUDE_DIR)
+  LIBRARY_DIRS += $(MKL_LIB_DIR)
+else
+  LIBRARIES += cblas atlas
+endif
+
+COMMON_FLAGS += $(foreach includedir,$(INCLUDE_DIRS),-I$(includedir))
 CXXFLAGS += -pthread -fPIC $(COMMON_FLAGS)
 NVCCFLAGS := -ccbin=$(CXX) -Xcompiler -fPIC $(COMMON_FLAGS)
 LDFLAGS += $(foreach librarydir,$(LIBRARY_DIRS),-L$(librarydir)) \
 		$(foreach library,$(LIBRARIES),-l$(library))
 PYTHON_LDFLAGS := $(LDFLAGS) $(foreach library,$(PYTHON_LIBRARIES),-l$(library))
 
-
 ##############################
 # Define build targets
 ##############################
@@ -210,6 +220,10 @@ $(BUILD_DIR)/src/gtest/%.o: src/gtest/%.cpp
 	$(CXX) $< $(CXXFLAGS) -c -o $@
 	@echo
 
+$(BUILD_DIR)/src/$(PROJECT)/%.cuo: src/$(PROJECT)/%.cu
+	$(CUDA_DIR)/bin/nvcc $(NVCCFLAGS) $(CUDA_ARCH) -c $< -o $@
+	@echo
+
 $(BUILD_DIR)/src/$(PROJECT)/layers/%.cuo: src/$(PROJECT)/layers/%.cu
 	$(CUDA_DIR)/bin/nvcc $(NVCCFLAGS) $(CUDA_ARCH) -c $< -o $@
 	@echo

Makefile.config.example

@@ -10,6 +10,8 @@ CUDA_ARCH := -gencode arch=compute_20,code=sm_20 \
     -gencode arch=compute_30,code=sm_30 \
     -gencode arch=compute_35,code=sm_35
 
+# MKL switch: set to 1 for MKL
+USE_MKL := 0
 # MKL directory contains include/ and lib/ directions that we need.
 MKL_DIR := /opt/intel/mkl
 

docs/feature_extraction.md

@@ -0,0 +1,67 @@
+---
+layout: default
+title: Caffe
+---
+
+Extracting Features
+===================
+
+In this tutorial, we will extract features using a pre-trained model.
+Follow instructions for [setting up caffe](installation.html) and for [getting](getting_pretrained_models.html) the pre-trained ImageNet model.
+If you need detailed information about the tools below, please consult their source code, in which additional documentation is usually provided.
+
+Select data to run on
+---------------------
+
+We'll make a temporary folder to store things into.
+
+    mkdir examples/_temp
+
+Generate a list of the files to process.
+We're going to use the images that ship with caffe.
+
+    find `pwd`/examples/images -type f -exec echo {} \; > examples/_temp/file_list.txt
+
+The `ImagesLayer` we'll use expects labels after each filenames, so let's add a 0 to the end of each line
+
+    sed "s/$/ 0/" examples/_temp/file_list.txt > examples/_temp/file_list.txt
+
+Define the Feature Extraction Network Architecture
+--------------------------------------------------
+
+In practice, subtracting the mean image from a dataset significantly improves classification accuracies.
+Download the mean image of the ILSVRC dataset.
+
+    data/ilsvrc12/get_ilsvrc_aux.sh
+
+We will use `data/ilsvrc212/imagenet_mean.binaryproto` in the network definition prototxt.
+
+Let's copy and modify the network definition.
+We'll be using the `ImagesLayer`, which will load and resize images for us.
+
+    cp examples/feature_extraction/imagenet_val.prototxt examples/_temp
+
+Edit `examples/_temp/imagenet_val.prototxt` to use correct path for your setup (replace `$CAFFE_DIR`)
+
+Extract Features
+----------------
+
+Now everything necessary is in place.
+
+    build/tools/extract_features.bin models/caffe_reference_imagenet_model examples/_temp/imagenet_val.prototxt fc7 examples/_temp/features 10
+
+The name of feature blob that you extract is `fc7`, which represents the highest level feature of the reference model.
+We can use any other layer, as well, such as `conv5` or `pool3`.
+
+The last parameter above is the number of data mini-batches.
+
+The features are stored to LevelDB `examples/_temp/features`, ready for access by some other code.
+
+If you'd like to use the Python wrapper for extracting features, check out the [layer visualization notebook](http://nbviewer.ipython.org/github/BVLC/caffe/blob/master/examples/filter_visualization.ipynb).
+
+Clean Up
+--------
+
+Let's remove the temporary directory now.
+
+    rm -r examples/_temp

docs/index.md

@@ -33,6 +33,7 @@ Even in CPU mode, computing predictions on an image takes only 20 ms when images
 * [LeNet / MNIST Demo](/mnist.html): end-to-end training and testing of LeNet on MNIST.
 * [CIFAR-10 Demo](/cifar10.html): training and testing on the CIFAR-10 data.
 * [Training ImageNet](/imagenet_training.html): end-to-end training of an ImageNet classifier.
+* [Feature extraction with C++](/feature_extraction.html): feature extraction using pre-trained model
 * [Running Pretrained ImageNet \[notebook\]][pretrained_imagenet]: run classification with the pretrained ImageNet model using the Python interface.
 * [Running Detection \[notebook\]][imagenet_detection]: run a pretrained model as a detector.
 * [Visualizing Features and Filters \[notebook\]][visualizing_filters]: trained filters and an example image, viewed layer-by-layer.

examples/feature_extraction/imagenet_val.prototxt

@@ -0,0 +1,247 @@
+name: "CaffeNet"
+layers {
+  layer {
+    name: "data"
+    type: "images"
+    source: "$CAFFE_DIR/examples/_temp/file_list.txt"
+    meanfile: "$CAFFE_DIR/data/ilsvrc12/imagenet_mean.binaryproto"
+    batchsize: 50
+    new_height: 256
+    new_width: 256
+    mirror: false
+    cropsize: 227
+  }
+  top: "data"
+  top: "label"
+}
+layers {
+  layer {
+    name: "conv1"
+    type: "conv"
+    num_output: 96
+    kernelsize: 11
+    stride: 4
+  }
+  bottom: "data"
+  top: "conv1"
+}
+layers {
+  layer {
+    name: "relu1"
+    type: "relu"
+  }
+  bottom: "conv1"
+  top: "conv1"
+}
+layers {
+  layer {
+    name: "pool1"
+    type: "pool"
+    pool: MAX
+    kernelsize: 3
+    stride: 2
+  }
+  bottom: "conv1"
+  top: "pool1"
+}
+layers {
+  layer {
+    name: "norm1"
+    type: "lrn"
+    local_size: 5
+    alpha: 0.0001
+    beta: 0.75
+  }
+  bottom: "pool1"
+  top: "norm1"
+}
+layers {
+  layer {
+    name: "conv2"
+    type: "conv"
+    num_output: 256
+    group: 2
+    kernelsize: 5
+    pad: 2
+  }
+  bottom: "norm1"
+  top: "conv2"
+}
+layers {
+  layer {
+    name: "relu2"
+    type: "relu"
+  }
+  bottom: "conv2"
+  top: "conv2"
+}
+layers {
+  layer {
+    name: "pool2"
+    type: "pool"
+    pool: MAX
+    kernelsize: 3
+    stride: 2
+  }
+  bottom: "conv2"
+  top: "pool2"
+}
+layers {
+  layer {
+    name: "norm2"
+    type: "lrn"
+    local_size: 5
+    alpha: 0.0001
+    beta: 0.75
+  }
+  bottom: "pool2"
+  top: "norm2"
+}
+layers {
+  layer {
+    name: "conv3"
+    type: "conv"
+    num_output: 384
+    kernelsize: 3
+    pad: 1
+  }
+  bottom: "norm2"
+  top: "conv3"
+}
+layers {
+  layer {
+    name: "relu3"
+    type: "relu"
+  }
+  bottom: "conv3"
+  top: "conv3"
+}
+layers {
+  layer {
+    name: "conv4"
+    type: "conv"
+    num_output: 384
+    group: 2
+    kernelsize: 3
+    pad: 1
+  }
+  bottom: "conv3"
+  top: "conv4"
+}
+layers {
+  layer {
+    name: "relu4"
+    type: "relu"
+  }
+  bottom: "conv4"
+  top: "conv4"
+}
+layers {
+  layer {
+    name: "conv5"
+    type: "conv"
+    num_output: 256
+    group: 2
+    kernelsize: 3
+    pad: 1
+  }
+  bottom: "conv4"
+  top: "conv5"
+}
+layers {
+  layer {
+    name: "relu5"
+    type: "relu"
+  }
+  bottom: "conv5"
+  top: "conv5"
+}
+layers {
+  layer {
+    name: "pool5"
+    type: "pool"
+    kernelsize: 3
+    pool: MAX
+    stride: 2
+  }
+  bottom: "conv5"
+  top: "pool5"
+}
+layers {
+  layer {
+    name: "fc6"
+    type: "innerproduct"
+    num_output: 4096
+  }
+  bottom: "pool5"
+  top: "fc6"
+}
+layers {
+  layer {
+    name: "relu6"
+    type: "relu"
+  }
+  bottom: "fc6"
+  top: "fc6"
+}
+layers {
+  layer {
+    name: "drop6"
+    type: "dropout"
+    dropout_ratio: 0.5
+  }
+  bottom: "fc6"
+  top: "fc6"
+}
+layers {
+  layer {
+    name: "fc7"
+    type: "innerproduct"
+    num_output: 4096
+  }
+  bottom: "fc6"
+  top: "fc7"
+}
+layers {
+  layer {
+    name: "relu7"
+    type: "relu"
+  }
+  bottom: "fc7"
+  top: "fc7"
+}
+layers {
+  layer {
+    name: "drop7"
+    type: "dropout"
+    dropout_ratio: 0.5
+  }
+  bottom: "fc7"
+  top: "fc7"
+}
+layers {
+  layer {
+    name: "fc8"
+    type: "innerproduct"
+    num_output: 1000
+  }
+  bottom: "fc7"
+  top: "fc8"
+}
+layers {
+  layer {
+    name: "prob"
+    type: "softmax"
+  }
+  bottom: "fc8"
+  top: "prob"
+}
+layers {
+  layer {
+    name: "accuracy"
+    type: "accuracy"
+  }
+  bottom: "prob"
+  bottom: "label"
+  top: "accuracy"
+}