From 88dfa9cfb72f167d058bb6a3d0194d63b12aba6a Mon Sep 17 00:00:00 2001 From: Bing Xu Date: Sun, 6 Sep 2015 23:34:39 -0600 Subject: [PATCH] mnist gpu --- example/mnist/mlp_gpu.py | 120 +++++++++++++++++++++++++++++++++ mshadow | 2 +- src/dag_engine/naive_engine.cc | 20 +++++- 3 files changed, 139 insertions(+), 3 deletions(-) create mode 100644 example/mnist/mlp_gpu.py diff --git a/example/mnist/mlp_gpu.py b/example/mnist/mlp_gpu.py new file mode 100644 index 000000000000..8fc44d0f5af4 --- /dev/null +++ b/example/mnist/mlp_gpu.py @@ -0,0 +1,120 @@ +# pylint: skip-file +import mxnet as mx +import numpy as np +import os, gzip +import pickle as pickle +import sys +import get_data + +def CalAcc(out, label): + pred = np.argmax(out, axis=1) + return np.sum(pred == label) * 1.0 / out.shape[0] + +# symbol net +batch_size = 100 +data = mx.symbol.Variable('data') +fc1 = mx.symbol.FullyConnected(data = data, name='fc1', num_hidden=128) +act1 = mx.symbol.Activation(data = fc1, name='relu1', act_type="relu") +fc2 = mx.symbol.FullyConnected(data = act1, name = 'fc2', num_hidden = 64) +act2 = mx.symbol.Activation(data = fc2, name='relu2', act_type="relu") +fc3 = mx.symbol.FullyConnected(data = act2, name='fc3', num_hidden=10) +softmax = mx.symbol.Softmax(data = fc3, name = 'sm') +args_list = softmax.list_arguments() +# infer shape +data_shape = (batch_size, 784) +arg_shapes, out_shapes, aux_shapes = softmax.infer_shape(data=data_shape) + +arg_narrays = [mx.narray.create(shape, ctx=mx.Context("gpu")) for shape in arg_shapes] +grad_narrays = [mx.narray.create(shape, ctx=mx.Context("gpu")) for shape in arg_shapes] + +inputs = dict(zip(args_list, arg_narrays)) + +name2shape = dict(zip(args_list, arg_shapes)) +pred = mx.narray.create(out_shapes[0]) + +np.random.seed(0) +# set random weight +for name, narray in inputs.items(): + if "weight" in name: + tmp = mx.narray.create(name2shape[name]) + tmp.numpy[:] = np.random.uniform(-0.07, 0.07, name2shape[name]) + tmp.copyto(narray) + if "bias" in name: + narray[:] = 0.0 + +# bind executer +# TODO(bing): think of a better bind interface +executor = softmax.bind(mx.Context('gpu'), arg_narrays, grad_narrays) +# update + +out_narray = executor.heads()[0] +grad_narray = mx.narray.create(out_narray.shape) + +epoch = 9 +lr = 0.1 +wd = 0.0004 + +def Update(grad, weight): + weight[:] -= lr * grad / batch_size + +block = list(zip(grad_narrays, arg_narrays)) + +#check data +get_data.GetMNIST_ubyte() + +train_dataiter = mx.io.MNISTIter( + image="data/train-images-idx3-ubyte", + label="data/train-labels-idx1-ubyte", + batch_size=batch_size, shuffle=True, flat=True, silent=False, seed=10) +val_dataiter = mx.io.MNISTIter( + image="data/t10k-images-idx3-ubyte", + label="data/t10k-labels-idx1-ubyte", + batch_size=batch_size, shuffle=True, flat=True, silent=False) + +tmp_label = mx.narray.create(name2shape["sm_label"]) + +def test_mlp(): + acc_train = 0. + acc_val = 0. + for i in range(epoch): + # train + print("Epoch %d" % i) + train_acc = 0.0 + val_acc = 0.0 + train_nbatch = 0 + val_nbatch = 0 + for data, label in train_dataiter: + data = data + tmp_label.numpy[:] = label.numpy.reshape(tmp_label.shape) + data.copyto(inputs["data"]) + tmp_label.copyto(inputs["sm_label"]) + executor.forward() + out_narray.copyto(pred) + train_acc += CalAcc(pred.numpy, label.numpy.flatten()) + train_nbatch += 1 + out_narray.copyto(grad_narray) + executor.backward([grad_narray]) + + for grad, weight in block: + Update(grad, weight) + + # evaluate + for data, label in val_dataiter: + data = data + label = label.numpy.flatten() + data.copyto(inputs["data"]) + executor.forward() + out_narray.copyto(pred) + val_acc += CalAcc(pred.numpy, label) + val_nbatch += 1 + acc_train = train_acc / train_nbatch + acc_val = val_acc / val_nbatch + print("Train Acc: ", train_acc / train_nbatch) + print("Valid Acc: ", val_acc / val_nbatch) + train_dataiter.reset() + val_dataiter.reset() + assert(acc_train > 0.98) + assert(acc_val > 0.97) + +if __name__ == "__main__": + test_mlp() diff --git a/mshadow b/mshadow index b983f9b92381..3053f8cdfea0 160000 --- a/mshadow +++ b/mshadow @@ -1 +1 @@ -Subproject commit b983f9b92381983e0917a20c530026ef1c8fc39a +Subproject commit 3053f8cdfea0274739282ced015ad458090760e8 diff --git a/src/dag_engine/naive_engine.cc b/src/dag_engine/naive_engine.cc index 41e88b4f9900..6ad780c9615c 100644 --- a/src/dag_engine/naive_engine.cc +++ b/src/dag_engine/naive_engine.cc @@ -7,6 +7,19 @@ namespace engine { // The Naive engine interface class NaiveEngine : public DAGEngine { public: + NaiveEngine() { + #if MXNET_USE_CUDA + stream_ = mshadow::NewStream(true, false); + ctx_.stream = stream_; + #endif + } + + ~NaiveEngine() { + #if MXNET_USE_CUDA + mshadow::DeleteStream(stream_); + #endif + } + Variable NewVar() override { return nullptr; } @@ -30,10 +43,11 @@ class NaiveEngine : public DAGEngine { std::vector const& use_vars, std::vector const& mutate_vars) override { if (exec_ctx.dev_mask == gpu::kDevMask) { - ctx_.stream = &stream_; #if MXNET_USE_CUDA mshadow::SetDevice(exec_ctx.dev_id); + ctx_.stream = stream_; exec_fun(ctx_); + stream_->Wait(); #else LOG(FATAL) << "GPU is not enabled"; #endif @@ -60,7 +74,9 @@ class NaiveEngine : public DAGEngine { private: RunContext ctx_; - mshadow::Stream stream_; + #if MXNET_USE_CUDA + mshadow::Stream *stream_; + #endif }; } // namespace engine