diff --git a/ipykernel/debugger.py b/ipykernel/debugger.py
index 72fb25983..6a73ea2a4 100644
--- a/ipykernel/debugger.py
+++ b/ipykernel/debugger.py
@@ -1,5 +1,6 @@
 import os
 import re
+import threading
 
 import zmq
 from zmq.utils import jsonapi
@@ -284,7 +285,7 @@ def __init__(self, log, debugpy_stream, event_callback, shell_socket, session):
             self.static_debug_handlers[msg_type] = getattr(self, msg_type)
 
         self.breakpoint_list = {}
-        self.stopped_threads = []
+        self.stopped_threads = set()
 
         self.debugpy_initialized = False
         self._removed_cleanup = {}
@@ -297,12 +298,21 @@ def __init__(self, log, debugpy_stream, event_callback, shell_socket, session):
 
     def _handle_event(self, msg):
         if msg['event'] == 'stopped':
-            self.stopped_threads.append(msg['body']['threadId'])
+            self.stopped_threads.add(msg['body']['threadId'])
         elif msg['event'] == 'continued':
             try:
-                self.stopped_threads.remove(msg['body']['threadId'])
+                if msg['allThreadsContinued']:
+                    self.stopped_threads = set()
+                else:
+                    self.stopped_threads.remove(msg['body']['threadId'])
             except Exception:
-                pass
+                # Workaround for debugpy/pydev not setting the correct threadId
+                # after a next request. Does not work if a the code executed on
+                # the shell spawns additional threads
+                if len(self.stopped_threads) == 1:
+                    self.stopped_threads = set()
+                else:
+                    raise Exception('threadId from continued event not in stopped threads set')
         self.event_callback(msg)
 
     async def _forward_message(self, msg):
@@ -513,7 +523,7 @@ async def debugInfo(self, message):
                 'tmpFilePrefix': get_tmp_directory() + os.sep,
                 'tmpFileSuffix': '.py',
                 'breakpoints': breakpoint_list,
-                'stoppedThreads': self.stopped_threads,
+                'stoppedThreads': list(self.stopped_threads),
                 'richRendering': True,
                 'exceptionPaths': ['Python Exceptions']
             }
@@ -613,7 +623,7 @@ async def process_request(self, message):
         if message['command'] == 'disconnect':
             self.stop()
             self.breakpoint_list = {}
-            self.stopped_threads = []
+            self.stopped_threads = set()
             self.is_started = False
             self.log.info('The debugger has stopped')