diff --git a/lib/node-repo.js b/lib/node-repo.js
index 23063960..e133bec0 100644
--- a/lib/node-repo.js
+++ b/lib/node-repo.js
@@ -137,6 +137,40 @@ function fetchLabelPages (options, startPageNum, cb) {
   })
 }
 
+function getBotPrLabels (options, cb) {
+  githubClient.issues.getEvents({
+    owner: options.owner,
+    repo: options.repo,
+    page: 1,
+    per_page: 100, // we probably won't hit this
+    issue_number: options.prId
+  }, (err, events) => {
+    if (err) {
+      return cb(err)
+    }
+
+    const ourLabels = []
+
+    for (const event of events) {
+      if (event.event === 'unlabeled') {
+        const index = ourLabels.indexOf(event.label.name)
+        if (index === -1) continue
+
+        ourLabels.splice(index, 1)
+      } else if (event.event === 'labeled') {
+        const index = ourLabels.indexOf(event.label.name)
+        if (index > -1) continue
+
+        if (event.actor.login === 'nodejs-github-bot') {
+          ourLabels.push(event.label.name)
+        }
+      }
+    }
+
+    cb(null, ourLabels)
+  })
+}
+
 function stringsInCommon (arr1, arr2) {
   const loweredArr2 = arr2.map((str) => str.toLowerCase())
   // we want the original string cases in arr1, therefore we don't lowercase them
@@ -144,8 +178,9 @@ function stringsInCommon (arr1, arr2) {
   return arr1.filter((str) => loweredArr2.indexOf(str.toLowerCase()) !== -1)
 }
 
+exports.getBotPrLabels = getBotPrLabels
 exports.removeLabelFromPR = removeLabelFromPR
-exports.updatePrWithLabels = updatePrWithLabels
+exports.fetchExistingThenUpdatePr = fetchExistingThenUpdatePr
 exports.resolveLabelsThenUpdatePr = deferredResolveLabelsThenUpdatePr
 
 // exposed for testability
diff --git a/scripts/attempt-backport.js b/scripts/attempt-backport.js
index f61a4108..5a0ccb89 100644
--- a/scripts/attempt-backport.js
+++ b/scripts/attempt-backport.js
@@ -4,8 +4,9 @@ const child_process = require('child_process')
 const debug = require('debug')('attempt-backport')
 const request = require('request')
 const node_repo = require('../lib/node-repo')
-const updatePrWithLabels = node_repo.updatePrWithLabels
-// const removeLabelFromPR = node_repo.removeLabelFromPR
+const fetchExistingThenUpdatePr = node_repo.fetchExistingThenUpdatePr
+const removeLabelFromPR = node_repo.removeLabelFromPR
+const getBotPrLabels = node_repo.getBotPrLabels
 
 const enabledRepos = ['node']
 const nodeVersions = [
@@ -129,7 +130,23 @@ function attemptBackport (options, version, isLTS, cb) {
     const _cb = cb
     setImmediate(() => {
       options.logger.debug(`backport to ${version} failed`)
-      if (!isLTS) updatePrWithLabels(options, [`dont-land-on-v${version}.x`])
+
+      if (!isLTS) {
+        fetchExistingThenUpdatePr(options, [`dont-land-on-v${version}.x`])
+      } else {
+        getBotPrLabels(options, (err, ourLabels) => {
+          if (err) {
+            options.logger.error(err, 'Error fetching existing bot labels')
+            return
+          }
+
+          const label = `lts-watch-v${version}.x`
+          if (!ourLabels.includes(label)) return
+
+          removeLabelFromPR(options, label)
+        })
+      }
+
       setImmediate(() => {
         inProgress = false
         _cb()
@@ -206,12 +223,22 @@ function attemptBackport (options, version, isLTS, cb) {
     const cp = wrapCP('git', ['am'], { stdio: 'pipe' }, function done () {
       // Success!
       if (isLTS) {
-        updatePrWithLabels(options, [`lts-watch-v${version}.x`])
-      }// else {
+        fetchExistingThenUpdatePr(options, [`lts-watch-v${version}.x`])
+      } else {
         // TODO(Fishrock123): Re-enable this, but do a check first
         // to make sure the label was set by the bot only.
-        // removeLabelFromPR(options, `dont-land-on-v${version}.x`)
-      // }
+        getBotPrLabels(options, (err, ourLabels) => {
+          if (err) {
+            options.logger.error(err, 'Error fetching existing bot labels')
+            return
+          }
+
+          const label = `dont-land-on-v${version}.x`
+          if (!ourLabels.includes(label)) return
+
+          removeLabelFromPR(options, label)
+        })
+      }
 
       setImmediate(() => {
         options.logger.debug(`backport to v${version} successful`)