CUDA illegal memory access when training SSD

[dtmoodie]

For bugs or installation issues, please provide the following information.
The more information you provide, the more likely people will be able to help you.

Environment info

Operating System:
Ubuntu 16.04

Compiler:
GCC 5.4

Package used (Python/R/Scala/Julia):
Python

Or if installed from source:
6935195

Python version and distribution:
2.7.12 (default, Nov 19 2016, 06:48:10)
[GCC 5.4.0 20160609]

Error Message:

Please paste the full error message, including stack trace.

[13:13:08] /code/mxnet/dmlc-core/include/dmlc/logging.h:304: [13:13:08] /code/mxnet/mshadow/mshadow/./stream_gpu-inl.h:49: Check failed: e == cudaSuccess CUDA: an illegal memory access was encountered

Stack trace returned 8 entries:
[bt] (0) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZN4dmlc15LogMessageFatalD1Ev+0x3c) [0x7f35618a4bbc]
[bt] (1) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZN7mshadow6StreamINS_3gpuEE4WaitEv+0xd8) [0x7f35618bcb08]
[bt] (2) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(+0x60e328) [0x7f356192c328]
[bt] (3) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZN5mxnet6engine14ThreadedEngine15ExecuteOprBlockENS_10RunContextEPNS0_8OprBlockE+0x87) [0x7f356190e287]
[bt] (4) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZNSt17_Function_handlerIFvvEZZN5mxnet6engine23ThreadedEnginePerDevice13PushToExecuteEPNS2_8OprBlockEbENKUlvE1_clEvEUlvE_E9_M_invokeERKSt9_Any_data+0x78) [0x7f3561912a08]
[bt] (5) /usr/lib/x86_64-linux-gnu/libstdc++.so.6(+0xb8c80) [0x7f35dd244c80]
[bt] (6) /lib/x86_64-linux-gnu/libpthread.so.0(+0x76ba) [0x7f35e12a46ba]
[bt] (7) /lib/x86_64-linux-gnu/libc.so.6(clone+0x6d) [0x7f35e0fda82d]

Minimum reproducible example

The below example is a near copy and past of (https://github.com/zhreshold/mxnet-ssd/blob/master/train/train_net.py) with modifications to work on the mapillary dataset.

import mxnet as mx
import logging
import sys
import os
import importlib
import re
sys.path.append('/code/mxnet/example/ssd/')

from dataset.iterator import DetRecordIter
from train.metric import MultiBoxMetric
from evaluate.eval_metric import MApMetric, VOC07MApMetric
from config.config import cfg

data_width = 512
data_height = 512
mean_pixels = [123, 117, 104]
train_path='/data/mxnet/ssd/training_filelist.rec'
val_path = '/data/mxnet/ssd/validation_filelist.rec'
train_list = '/data/mxnet/ssd/training_filelist.lst'
val_list = './2017-06-04/validation_filelist.lst'
net = 'vgg16_ssd_512'
log_file = date + '/log.txt'
prefix = './model/vgg16_reduced'
batch_size = 128
label_pad_width = -1
nms_thresh = 0.45
nms_topk = 400
force_suppress=False
freeze_layer_pattern="^(conv1_|conv2_)."
ctx = [mx.gpu(int(i)) for i in range(8)]
pretrained='model/vgg16_reduced'
epoch=1
frequent = 20
begin_epoch=0
end_epoch=100
momentum = 0.9
learning_rate=0.004
lr_refactor_step=[20, 40, 60]
lr_refactor_ratio=0.1
ovp_thresh=0.5
use_difficult=False
num_example=20000
weight_decay=0.0005
iter_monitor=100
monitor_pattern='.'
voc07_metric=False
classes = ['human--person', 'human--rider--bicyclist', 'human--rider--motorcyclist',
'human--rider--other-rider', 'object--pothole', 'object--street-light', 'object--traffic-light',
'object--traffic-sign--back', 'object--traffic-sign--front', 'object--vehicle--bicycle',
'object--vehicle--boat', 'object--vehicle--bus', 'object--vehicle--car',
'object--vehicle--caravan', 'object--vehicle--motorcycle', 'object--vehicle--on-rails',
'object--vehicle--other-vehicle', 'object--vehicle--trailer', 'object--vehicle--truck',
'object--vehicle--wheeled-slow']
num_classes = len(classes)

def get_lr_scheduler(learning_rate, lr_refactor_step, lr_refactor_ratio,
num_example, batch_size, begin_epoch):
assert lr_refactor_ratio > 0
iter_refactor = lr_refactor_step
if lr_refactor_ratio >= 1:
return (learning_rate, None)
else:
lr = learning_rate
epoch_size = num_example // batch_size
for s in iter_refactor:
if begin_epoch >= s:
lr *= lr_refactor_ratio
if lr != learning_rate:
logging.getLogger().info("Adjusted learning rate to {} for epoch {}".format(lr, begin_epoch))
steps = [epoch_size * (x - begin_epoch) for x in iter_refactor if x > begin_epoch]
lr_scheduler = mx.lr_scheduler.MultiFactorScheduler(step=steps, factor=lr_refactor_ratio)
return (lr, lr_scheduler)

def convert_pretrained(name, args):
if 'vgg16_reduced' in name:
args['conv6_bias'] = args.pop('fc6_bias')
args['conv6_weight'] = args.pop('fc6_weight')
args['conv7_bias'] = args.pop('fc7_bias')
args['conv7_weight'] = args.pop('fc7_weight')
del args['fc8_weight']
del args['fc8_bias']
return args

logger = logging.getLogger()
logger.setLevel(logging.INFO)
if log_file:
fh = logging.FileHandler(log_file)
logger.addHandler(fh)

data_shape = (3, data_height, data_width)

prefix += '_' + str(data_shape[1])

if isinstance(mean_pixels, (int, float)):
mean_pixels = [mean_pixels, mean_pixels, mean_pixels]
assert len(mean_pixels) == 3, "must provide all RGB mean values"

train_iter = DetRecordIter(train_path, batch_size, data_shape, mean_pixels=mean_pixels,
label_pad_width=label_pad_width, path_imglist=train_list, **cfg.train)

if val_path:
val_iter = DetRecordIter(val_path, batch_size, data_shape, mean_pixels=mean_pixels,
label_pad_width=label_pad_width, path_imglist=val_list, **cfg.valid)
else:
val_iter = None

sys.path.append(os.path.join(cfg.ROOT_DIR, 'symbol'))
symbol_module = importlib.import_module("symbol_" + net)
net = symbol_module.get_symbol_train(num_classes, nms_thresh=nms_thresh,
force_suppress=False, nms_topk=nms_topk)

if freeze_layer_pattern.strip():
re_prog = re.compile(freeze_layer_pattern)
fixed_param_names = [name for name in net.list_arguments() if re_prog.match(name)]
else:
fixed_param_names = None

ctx_str = '('+ ','.join([str(c) for c in ctx]) + ')'

logger.info("Start training with {} from pretrained model {}"
.format(ctx_str, pretrained))
_, args, auxs = mx.model.load_checkpoint(pretrained, epoch)
args = convert_pretrained(pretrained, args)

mod = mx.mod.Module(net, label_names=('label',), logger=logger, context=ctx,
fixed_param_names=fixed_param_names)

batch_end_callback = mx.callback.Speedometer(train_iter.batch_size, frequent=frequent)
epoch_end_callback = mx.callback.do_checkpoint(prefix)
learning_rate, lr_scheduler = get_lr_scheduler(learning_rate, lr_refactor_step,
lr_refactor_ratio, num_example, batch_size, begin_epoch)
optimizer_params={'learning_rate':learning_rate,
'momentum':momentum,
'wd':weight_decay,
'lr_scheduler':lr_scheduler,
'clip_gradient':None,
'rescale_grad': 1.0}
monitor = mx.mon.Monitor(iter_monitor, pattern=monitor_pattern) if iter_monitor > 0 else None

class_names = classes
if voc07_metric:
valid_metric = VOC07MApMetric(ovp_thresh, use_difficult, class_names, pred_idx=3)
else:
valid_metric = MApMetric(ovp_thresh, use_difficult, class_names, pred_idx=3)

mod.fit(train_iter,
val_iter,
eval_metric=MultiBoxMetric(),
validation_metric=valid_metric,
batch_end_callback=batch_end_callback,
epoch_end_callback=epoch_end_callback,
optimizer='sgd',
optimizer_params=optimizer_params,
begin_epoch=begin_epoch,
num_epoch=end_epoch,
initializer=mx.init.Xavier(),
arg_params=args,
aux_params=auxs,
allow_missing=True,
monitor=monitor)

What I've tried:
Reducing batch size to to 16, and 8; reducing num GPUs to 4, and 1. Running with NaiveEngine gives:

[13:34:04] /code/mxnet/dmlc-core/include/dmlc/logging.h:304: [13:34:04] /code/mxnet/mshadow/mshadow/./stream_gpu-inl.h:49: Check failed: e == cudaSuccess CUDA: an illegal memory access was encountered

Stack trace returned 10 entries:
[bt] (0) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZN4dmlc15LogMessageFatalD1Ev+0x3c) [0x7efe81e1ebbc]
[bt] (1) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZN7mshadow6StreamINS_3gpuEE4WaitEv+0xd8) [0x7efe81e36b08]
[bt] (2) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(+0x60e328) [0x7efe81ea6328]
[bt] (3) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(ZNSt17_Function_handlerIFvN5mxnet10RunContextENS0_6engine18CallbackOnCompleteEEZNS2_11NaiveEngine4PushEPNS2_3OprENS0_7ContextEibEUlS1_S3_E_E9_M_invokeERKSt9_Any_dataOS1_OS3+0x50) [0x7efe81e637b0]
[bt] (4) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZN5mxnet6engine11NaiveEngine9PushAsyncESt8functionIFvNS_10RunContextENS0_18CallbackOnCompleteEEENS_7ContextERKSt6vectorIPNS0_3VarESaISA_EESE_NS_10FnPropertyEiPKc+0x606) [0x7efe81e6b966]
[bt] (5) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZN5mxnet6engine11NaiveEngine4PushEPNS0_3OprENS_7ContextEib+0x8f) [0x7efe81e6c9cf]
[bt] (6) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZN5mxnet4exec13GraphExecutor6RunOpsEbmm+0x21e) [0x7efe81ea748e]
[bt] (7) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(MXExecutorForward+0x11) [0x7efe81e23641]
[bt] (8) /usr/lib/x86_64-linux-gnu/libffi.so.6(ffi_call_unix64+0x4c) [0x7efee3795e40]
[bt] (9) /usr/lib/x86_64-linux-gnu/libffi.so.6(ffi_call+0x2eb) [0x7efee37958ab]

[piiswrong]

@zhreshold

[zhreshold]

@dtmoodie Can you send me the training record files if they are not so big?

[dtmoodie]

I'll make a smaller record and upload it somewhere.

[dtmoodie]

Try these out:
https://drive.google.com/open?id=0B2d_9zd_Bt2ZbkdDbk1vbTR4Yms

[zhreshold]

I have no problem training with your data.
There seems to be a problem with your code,

label_pad_width = -1

will estimate the maximum label width in record file, however, we have train.rec and val.rec, so the estimated widths might differ, causing the data shape inconsistent between training/validation, which will trigger the crash.
@dtmoodie

[dtmoodie]

Could you clarify the meaning of label_pad_width, I was under the impression that it pads the images such that all images are of label_pad_width size. Thus the input data and labels are of a consistent size within a batch.
I thought setting this to -1 was automatically determining the value at runtime.
I did this because the mapillary dataset has different size images, and it may be that the smaller dataset that I uploaded has a consistent size.

[zhreshold]

1. since each image has different number of objects, the label widths are different, this is fine within the record and list file.
2. During training, the data shape as well as label shape must be consistent across the training, thus the label_pad_width is used to pad every label to maximum width, use -1 here is still good because it will estimate the max width, for example, 200
3. However, the automatic estimation won't work for multiple record files, i.e. train.rec won't know how much to pad in val.rec, thus you have to set the label_pad_width manually. I know it is awkward, but synchronize the width between train_iterator and val_iterator will force them to reload again, which is not a good option neither.

There will be a update on the data loading module for object detection tasks, so please bare with the current API for now.

[dtmoodie]

Ah this makes more sense now. So width is not a pixel value but rather the number of objects?

[zhreshold]

yes

[dtmoodie]

@zhreshold are you building out of your repo or out of master? Currently it looks like MultiBoxMetric is broken in master due to changes to EvalMetric.

Apologies, I was out of date wrt PR 6675

[dtmoodie]

I've pulled dmlc/master and changed my training code to just be the following:

python train.py
--train-path=/data/mxnet/ssd/training_filelist.rec
--train-list=/data/mxnet/ssd/training_filelist.lst
--val-path=/data/mxnet/ssd/validation_filelist.rec
--val-list=/data/mxnet/ssd/validation_filelist.lst
--network=vgg16_ssd_512
--pretrained=/mnt/network_data/mapillary/model/vgg16_reduced
--gpus='0,1,2,3,4,5,6,7'
--num-class=$num_classes
--class-names='$class_str'
--label-width=10000
--monitor=100
--data-shape=512

And I've graduated from an illegal memory access to what appears to be a deadlock.
All of my GPUs are at 100% utilization as per nvidia-smi, but I haven't seen a change to the output in over 10min.
My current output is:
[18:00:23] /code/mxnet/src/io/iter_image_det_recordio.cc:262: ImageDetRecordIOParser: /data/mxnet/ssd/training_filelist.rec, use 6 threads for decoding..
[18:00:35] /code/mxnet/src/io/iter_image_det_recordio.cc:315: ImageDetRecordIOParser: /data/mxnet/ssd/training_filelist.rec, label padding width: 10000
[18:00:41] /code/mxnet/src/io/iter_image_det_recordio.cc:262: ImageDetRecordIOParser: /data/mxnet/ssd/validation_filelist.rec, use 4 threads for decoding..
[18:00:43] /code/mxnet/src/io/iter_image_det_recordio.cc:315: ImageDetRecordIOParser: /data/mxnet/ssd/validation_filelist.rec, label padding width: 10000
INFO:root:Start training with (gpu(0),gpu(1),gpu(2),gpu(3),gpu(4),gpu(5),gpu(6),gpu(7)) from pretrained model /mnt/network_data/mapillary/model/vgg16_reduced
[18:00:45] /code/mxnet/src/nnvm/legacy_json_util.cc:190: Loading symbol saved by previous version v0.8.0. Attempting to upgrade...
[18:00:45] /code/mxnet/src/nnvm/legacy_json_util.cc:198: Symbol successfully upgraded!
INFO:root:Freezed parameters: [conv1_1_weight,conv1_1_bias,conv1_2_weight,conv1_2_bias,conv2_1_weight,conv2_1_bias,conv2_2_weight,conv2_2_bias]
[18:00:55] /code/mxnet/src/operator/././cudnn_algoreg-inl.h:65: Running performance tests to find the best convolution algorithm, this can take a while... (setting env variable MXNET_CUDNN_AUTOTUNE_DEFAULT to 0 to disable)

Is it possible that it is stuck trying to determine the best convolution algorithm?

[zhreshold]

label-width 10000 is way too big for your dataset, thus taken too much memory and computation overhead

[dtmoodie]

The same thing happens with label-width=3000; system hang at conv performance tests. However if I set it to 2000 I get:
mxnet.base.MXNetError: [19:51:29] /code/mxnet/src/io/iter_image_det_recordio.cc:308: ImageDetRecordIOParser: label_pad_width: 2000 smaller than estimated width: 2288

What setting did you use that worked correctly on this dataset?

[dtmoodie]

With the following command:

python train.py
--train-path=/data/mxnet/ssd/training_filelist.rec
--train-list=/data/mxnet/ssd/training_filelist.lst
--val-path=/data/mxnet/ssd/validation_filelist.rec
--val-list=/data/mxnet/ssd/validation_filelist.lst
--network=vgg16_ssd_512
--pretrained=/mnt/network_data/mapillary/model/vgg16_reduced
--gpus='0,1,2,3,4,5,6,7'
--num-class=$num_classes
--class-names='$class_str'
--label-width=2500
--monitor=100
--data-shape=512

I get the following output:

[20:30:13] /code/mxnet/src/io/iter_image_det_recordio.cc:262: ImageDetRecordIOParser: /data/mxnet/ssd/training_filelist.rec, use 6 threads for decoding..
[20:30:25] /code/mxnet/src/io/iter_image_det_recordio.cc:315: ImageDetRecordIOParser: /data/mxnet/ssd/training_filelist.rec, label padding width: 2500
[20:30:31] /code/mxnet/src/io/iter_image_det_recordio.cc:262: ImageDetRecordIOParser: /data/mxnet/ssd/validation_filelist.rec, use 4 threads for decoding..
[20:30:33] /code/mxnet/src/io/iter_image_det_recordio.cc:315: ImageDetRecordIOParser: /data/mxnet/ssd/validation_filelist.rec, label padding width: 2500
INFO:root:Start training with (gpu(0),gpu(1),gpu(2),gpu(3),gpu(4),gpu(5),gpu(6),gpu(7)) from pretrained model /mnt/network_data/mapillary/model/vgg16_reduced
[20:30:35] /code/mxnet/src/nnvm/legacy_json_util.cc:190: Loading symbol saved by previous version v0.8.0. Attempting to upgrade...
[20:30:35] /code/mxnet/src/nnvm/legacy_json_util.cc:198: Symbol successfully upgraded!
INFO:root:Freezed parameters: [conv1_1_weight,conv1_1_bias,conv1_2_weight,conv1_2_bias,conv2_1_weight,conv2_1_bias,conv2_2_weight,conv2_2_bias]
[20:30:40] /code/mxnet/src/operator/././cudnn_algoreg-inl.h:65: Running performance tests to find the best convolution algorithm, this can take a while... (setting env variable MXNET_CUDNN_AUTOTUNE_DEFAULT to 0 to disable)
[20:31:26] /code/mxnet/dmlc-core/include/dmlc/logging.h:304: [20:31:26] /code/mxnet/mshadow/mshadow/./stream_gpu-inl.h:49: Check failed: e == cudaSuccess CUDA: an illegal memory access was encountered

Stack trace returned 8 entries:
[bt] (0) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZN4dmlc15LogMessageFatalD1Ev+0x3c) [0x7f17dde6c0fc]
[bt] (1) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZN7mshadow6StreamINS_3gpuEE4WaitEv+0xd8) [0x7f17dde89c38]
[bt] (2) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(+0x628078) [0x7f17ddefb078]
[bt] (3) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZN5mxnet6engine14ThreadedEngine15ExecuteOprBlockENS_10RunContextEPNS0_8OprBlockE+0x87) [0x7f17ddedc737]
[bt] (4) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZNSt17_Function_handlerIFvvEZZN5mxnet6engine23ThreadedEnginePerDevice13PushToExecuteEPNS2_8OprBlockEbENKUlvE1_clEvEUlvE_E9_M_invokeERKSt9_Any_data+0x78) [0x7f17ddee0eb8]
[bt] (5) /usr/lib/x86_64-linux-gnu/libstdc++.so.6(+0xb8c80) [0x7f1809ab5c80]
[bt] (6) /lib/x86_64-linux-gnu/libpthread.so.0(+0x76ba) [0x7f180f9e96ba]
[bt] (7) /lib/x86_64-linux-gnu/libc.so.6(clone+0x6d) [0x7f180f71f82d]

[20:31:26] /code/mxnet/dmlc-core/include/dmlc/logging.h:304: [20:31:26] /code/mxnet/mshadow/mshadow/./stream_gpu-inl.h:49: Check failed: e == cudaSuccess CUDA: an illegal memory access was encountered

Stack trace returned 8 entries:
[bt] (0) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZN4dmlc15LogMessageFatalD1Ev+0x3c) [0x7f17dde6c0fc]
[bt] (1) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZN7mshadow6StreamINS_3gpuEE4WaitEv+0xd8) [0x7f17dde89c38]
[bt] (2) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(+0x628078) [0x7f17ddefb078]
[bt] (3) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZN5mxnet6engine14ThreadedEngine15ExecuteOprBlockENS_10RunContextEPNS0_8OprBlockE+0x87) [0x7f17ddedc737]
[bt] (4) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZNSt17_Function_handlerIFvvEZZN5mxnet6engine23ThreadedEnginePerDevice13PushToExecuteEPNS2_8OprBlockEbENKUlvE1_clEvEUlvE_E9_M_invokeERKSt9_Any_data+0x78) [0x7f17ddee0eb8]
[bt] (5) /usr/lib/x86_64-linux-gnu/libstdc++.so.6(+0xb8c80) [0x7f1809ab5c80]
[bt] (6) /lib/x86_64-linux-gnu/libpthread.so.0(+0x76ba) [0x7f180f9e96ba]
[bt] (7) /lib/x86_64-linux-gnu/libc.so.6(clone+0x6d) [0x7f180f71f82d]

[20:31:26] /code/mxnet/dmlc-core/include/dmlc/logging.h:304: [20:31:26] /code/mxnet/mshadow/mshadow/./stream_gpu-inl.h:49: Check failed: e == cudaSuccess CUDA: an illegal memory access was encountered

Stack trace returned 8 entries:
[bt] (0) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZN4dmlc15LogMessageFatalD1Ev+0x3c) [0x7f17dde6c0fc]
[bt] (1) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZN7mshadow6StreamINS_3gpuEE4WaitEv+0xd8) [0x7f17dde89c38]
[bt] (2) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(+0x628078) [0x7f17ddefb078]
[bt] (3) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZN5mxnet6engine14ThreadedEngine15ExecuteOprBlockENS_10RunContextEPNS0_8OprBlockE+0x87) [0x7f17ddedc737]
[bt] (4) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZNSt17_Function_handlerIFvvEZZN5mxnet6engine23ThreadedEnginePerDevice13PushToExecuteEPNS2_8OprBlockEbENKUlvE1_clEvEUlvE_E9_M_invokeERKSt9_Any_data+0x78) [0x7f17ddee0eb8]
[bt] (5) /usr/lib/x86_64-linux-gnu/libstdc++.so.6(+0xb8c80) [0x7f1809ab5c80]
[bt] (6) /lib/x86_64-linux-gnu/libpthread.so.0(+0x76ba) [0x7f180f9e96ba]
[bt] (7) /lib/x86_64-linux-gnu/libc.so.6(clone+0x6d) [0x7f180f71f82d]

[20:31:26] /code/mxnet/dmlc-core/include/dmlc/logging.h:304: [20:31:26] /code/mxnet/mshadow/mshadow/./stream_gpu-inl.h:49: Check failed: e == cudaSuccess CUDA: an illegal memory access was encountered

Stack trace returned 8 entries:
[bt] (0) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZN4dmlc15LogMessageFatalD1Ev+0x3c) [0x7f17dde6c0fc]
[bt] (1) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZN7mshadow6StreamINS_3gpuEE4WaitEv+0xd8) [0x7f17dde89c38]
[bt] (2) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(+0x628078) [0x7f17ddefb078]
[bt] (3) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZN5mxnet6engine14ThreadedEngine15ExecuteOprBlockENS_10RunContextEPNS0_8OprBlockE+0x87) [0x7f17ddedc737]
[bt] (4) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZNSt17_Function_handlerIFvvEZZN5mxnet6engine23ThreadedEnginePerDevice13PushToExecuteEPNS2_8OprBlockEbENKUlvE1_clEvEUlvE_E9_M_invokeERKSt9_Any_data+0x78) [0x7f17ddee0eb8]
[bt] (5) /usr/lib/x86_64-linux-gnu/libstdc++.so.6(+0xb8c80) [0x7f1809ab5c80]
[bt] (6) /lib/x86_64-linux-gnu/libpthread.so.0(+0x76ba) [0x7f180f9e96ba]
[bt] (7) /lib/x86_64-linux-gnu/libc.so.6(clone+0x6d) [0x7f180f71f82d]

[20:31:26] /code/mxnet/dmlc-core/include/dmlc/logging.h:304: [20:31:26] /code/mxnet/src/engine/./threaded_engine.h:329: [20:31:26] /code/mxnet/mshadow/mshadow/./stream_gpu-inl.h:49: Check failed: e == cudaSuccess CUDA: an illegal memory access was encountered

Stack trace returned 8 entries:
[bt] (0) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZN4dmlc15LogMessageFatalD1Ev+0x3c) [0x7f17dde6c0fc]
[bt] (1) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZN7mshadow6StreamINS_3gpuEE4WaitEv+0xd8) [0x7f17dde89c38]
[bt] (2) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(+0x628078) [0x7f17ddefb078]
[bt] (3) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZN5mxnet6engine14ThreadedEngine15ExecuteOprBlockENS_10RunContextEPNS0_8OprBlockE+0x87) [0x7f17ddedc737]
[bt] (4) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZNSt17_Function_handlerIFvvEZZN5mxnet6engine23ThreadedEnginePerDevice13PushToExecuteEPNS2_8OprBlockEbENKUlvE1_clEvEUlvE_E9_M_invokeERKSt9_Any_data+0x78) [0x7f17ddee0eb8]
[bt] (5) /usr/lib/x86_64-linux-gnu/libstdc++.so.6(+0xb8c80) [0x7f1809ab5c80]
[bt] (6) /lib/x86_64-linux-gnu/libpthread.so.0(+0x76ba) [0x7f180f9e96ba]
[bt] (7) /lib/x86_64-linux-gnu/libc.so.6(clone+0x6d) [0x7f180f71f82d]

An fatal error occurred in asynchronous engine operation. If you do not know what caused this error, you can try set environment variable MXNET_ENGINE_TYPE to NaiveEngine and run with debugger (i.e. gdb). This will force all operations to be synchronous and backtrace will give you the series of calls that lead to this error. Remember to set MXNET_ENGINE_TYPE back to empty after debugging.

Stack trace returned 6 entries:
[bt] (0) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZN4dmlc15LogMessageFatalD1Ev+0x3c) [0x7f17dde6c0fc]
[bt] (1) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZN5mxnet6engine14ThreadedEngine15ExecuteOprBlockENS_10RunContextEPNS0_8OprBlockE+0x31a) [0x7f17ddedc9ca]
[bt] (2) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZNSt17_Function_handlerIFvvEZZN5mxnet6engine23ThreadedEnginePerDevice13PushToExecuteEPNS2_8OprBlockEbENKUlvE1_clEvEUlvE_E9_M_invokeERKSt9_Any_data+0x78) [0x7f17ddee0eb8]
[bt] (3) /usr/lib/x86_64-linux-gnu/libstdc++.so.6(+0xb8c80) [0x7f1809ab5c80]
[bt] (4) /lib/x86_64-linux-gnu/libpthread.so.0(+0x76ba) [0x7f180f9e96ba]
[bt] (5) /lib/x86_64-linux-gnu/libc.so.6(clone+0x6d) [0x7f180f71f82d]

terminate called after throwing an instance of 'dmlc::Error'
what(): [20:31:26] /code/mxnet/src/engine/./threaded_engine.h:329: [20:31:26] /code/mxnet/mshadow/mshadow/./stream_gpu-inl.h:49: Check failed: e == cudaSuccess CUDA: an illegal memory access was encountered

Stack trace returned 8 entries:
[bt] (0) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZN4dmlc15LogMessageFatalD1Ev+0x3c) [0x7f17dde6c0fc]
[bt] (1) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZN7mshadow6StreamINS_3gpuEE4WaitEv+0xd8) [0x7f17dde89c38]
[bt] (2) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(+0x628078) [0x7f17ddefb078]
[bt] (3) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZN5mxnet6engine14ThreadedEngine15ExecuteOprBlockENS_10RunContextEPNS0_8OprBlockE+0x87) [0x7f17ddedc737]
[bt] (4) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZNSt17_Function_handlerIFvvEZZN5mxnet6engine23ThreadedEnginePerDevice13PushToExecuteEPNS2_8OprBlockEbENKUlvE1_clEvEUlvE_E9_M_invokeERKSt9_Any_data+0x78) [0x7f17ddee0eb8]
[bt] (5) /usr/lib/x86_64-linux-gnu/libstdc++.so.6(+0xb8c80) [0x7f1809ab5c80]
[bt] (6) /lib/x86_64-linux-gnu/libpthread.so.0(+0x76ba) [0x7f180f9e96ba]
[bt] (7) /lib/x86_64-linux-gnu/libc.so.6(clone+0x6d) [0x7f180f71f82d]

An fatal error occurred in asynchronous engine operation. If you do not know what caused this error, you can try set environment variable MXNET_ENGINE_TYPE to NaiveEngine and run with debugger (i.e. gdb). This will force all operations to be synchronous and backtrace will give you the series of calls that lead to this error. Remember to set MXNET_ENGINE_TYPE back to empty after debugging.

Stack trace returned 6 entries:
[bt] (0) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZN4dmlc15LogMessageFatalD1Ev+0x3c) [0x7f17dde6c0fc]
[bt] (1) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZN5mxnet6engine14ThreadedEngine15ExecuteOprBlockENS_10RunContextEPNS0_8OprBlockE+0x31a) [0x7f17ddedc9ca]
[bt] (2) /usr/local/lib/python2.7/dist-packages/mxnet-0.10.1-py2.7.egg/mxnet/libmxnet.so(_ZNSt17_Function_handlerIFvvEZZN5mxnet6engine23ThreadedEnginePerDevice13PushToExecuteEPNS2_8OprBlockEbENKUlvE1_clEvEUlvE_E9_M_invokeERKSt9_Any_data+0x78) [0x7f17ddee0eb8]
[bt] (3) /usr/lib/x86_64-linux-gnu/libstdc++.so.6(+0xb8c80) [0x7f1809ab5c80]
[bt] (4) /lib/x86_64-linux-gnu/libpthread.so.0(+0x76ba) [0x7f180f9e96ba]
[bt] (5) /lib/x86_64-linux-gnu/libc.so.6(clone+0x6d) [0x7f180f71f82d]

Aborted (core dumped)

[dtmoodie]

This seems to be fixed by compiling with the makefile build instead of cmake and by making sure DMSHADOW_USE_PASCAL = 1
#6204