diff --git a/Makefile b/Makefile
index 4fd031b697c..7a54a546d66 100644
--- a/Makefile
+++ b/Makefile
@@ -90,7 +90,7 @@ release-test:
 	python3 setup.py develop
 	python3 selftest.py
 	python3 -m pytest Tests
-	python3 setup.py install
+	python3 setup.py install --remove-dist-egg
 	python3 -m pytest -qq
 	check-manifest
 	pyroma .
diff --git a/setup.py b/setup.py
index 15a6ec282a3..d8106eee77b 100755
--- a/setup.py
+++ b/setup.py
@@ -857,8 +857,12 @@ def debug_build():
 with open("README.md") as f:
     long_description = f.read()
 
+remove_dist_egg = "--remove-dist-egg" in sys.argv
+if remove_dist_egg:
+    while "--remove-dist-egg" in sys.argv:
+        sys.argv.remove("--remove-dist-egg")
 try:
-    setup(
+    distribution = setup(
         name=NAME,
         version=PILLOW_VERSION,
         description="Python Imaging Library (Fork)",
@@ -921,3 +925,12 @@ def debug_build():
 """
     sys.stderr.write(msg)
     raise DependencyException(msg)
+
+if remove_dist_egg:
+    egg_path = distribution.get_command_obj("bdist_egg").egg_output
+    if egg_path:
+        os.remove(egg_path)
+        try:
+            os.rmdir(os.path.dirname(egg_path))
+        except OSError:
+            pass