diff --git a/src/types/label.js b/src/types/label.js
index caee36ce2..957c6dee1 100644
--- a/src/types/label.js
+++ b/src/types/label.js
@@ -145,7 +145,7 @@ function drawCallout(ctx, element) {
   const {pointX, pointY, options} = element;
   const callout = options.callout;
   const calloutPosition = callout && callout.display && resolveCalloutPosition(element, callout);
-  if (!calloutPosition || element.inRange(pointX, pointY)) {
+  if (!calloutPosition || isPointInRange(element, callout, calloutPosition)) {
     return;
   }
 
@@ -256,3 +256,20 @@ function getLabelSize({x, y, width, height, options}) {
     height: height - padding.top - padding.bottom - options.borderWidth
   };
 }
+
+function isPointInRange(element, callout, position) {
+  const {pointX, pointY} = element;
+  const margin = callout.margin;
+  let x = pointX;
+  let y = pointY;
+  if (position === 'left') {
+    x += margin;
+  } else if (position === 'right') {
+    x -= margin;
+  } else if (position === 'top') {
+    y += margin;
+  } else if (position === 'bottom') {
+    y -= margin;
+  }
+  return element.inRange(x, y);
+}
diff --git a/test/fixtures/label/calloutNoShown.js b/test/fixtures/label/calloutNoShown.js
new file mode 100644
index 000000000..75830f4c9
--- /dev/null
+++ b/test/fixtures/label/calloutNoShown.js
@@ -0,0 +1,118 @@
+module.exports = {
+  tolerance: 0.0075,
+  config: {
+    type: 'scatter',
+    options: {
+      scales: {
+        x: {
+          min: 0,
+          max: 10
+        },
+        y: {
+          min: 0,
+          max: 10
+        }
+      },
+      plugins: {
+        legend: false,
+        annotation: {
+          annotations: {
+            test1: {
+              type: 'label',
+              xValue: 2.5,
+              yValue: 7.5,
+              backgroundColor: 'white',
+              borderColor: 'red',
+              borderWidth: 1,
+              content: ['callout is', 'not drawn'],
+              xAdjust: -10,
+              yAdjust: 26,
+              callout: {
+                display: true,
+                margin: 5,
+              }
+            },
+            test2: {
+              type: 'point',
+              xValue: 2.5,
+              yValue: 7.5,
+              radius: 2,
+              backgroundColor: 'red',
+              borderColor: 'red'
+            },
+            test3: {
+              type: 'label',
+              xValue: 2.5,
+              yValue: 2.5,
+              backgroundColor: 'white',
+              borderColor: 'red',
+              borderWidth: 1,
+              content: ['callout is', 'not drawn'],
+              xAdjust: -10,
+              yAdjust: -26,
+              callout: {
+                display: true,
+                margin: 5,
+              }
+            },
+            test4: {
+              type: 'point',
+              xValue: 2.5,
+              yValue: 2.5,
+              radius: 2,
+              backgroundColor: 'red',
+              borderColor: 'red'
+            },
+            test5: {
+              type: 'label',
+              xValue: 7.5,
+              yValue: 2.5,
+              backgroundColor: 'white',
+              borderColor: 'red',
+              borderWidth: 1,
+              content: ['callout is', 'not drawn'],
+              xAdjust: -44,
+              callout: {
+                display: true,
+                margin: 5,
+              }
+            },
+            test6: {
+              type: 'point',
+              xValue: 7.5,
+              yValue: 2.5,
+              radius: 2,
+              backgroundColor: 'red',
+              borderColor: 'red'
+            },
+            test7: {
+              type: 'label',
+              xValue: 7.5,
+              yValue: 7.5,
+              backgroundColor: 'white',
+              borderColor: 'red',
+              borderWidth: 1,
+              content: ['callout is', 'not drawn'],
+              xAdjust: 44,
+              callout: {
+                display: true,
+                margin: 5,
+              }
+            },
+            test8: {
+              type: 'point',
+              xValue: 7.5,
+              yValue: 7.5,
+              radius: 2,
+              backgroundColor: 'red',
+              borderColor: 'red'
+            }
+          }
+        }
+      }
+    }
+  },
+  options: {
+    spriteText: true
+  }
+};
diff --git a/test/fixtures/label/calloutNoShown.png b/test/fixtures/label/calloutNoShown.png
new file mode 100644
index 000000000..9ede2aa1c
Binary files /dev/null and b/test/fixtures/label/calloutNoShown.png differ