diff --git a/setup.py b/setup.py
index dccfc935..81920fd6 100644
--- a/setup.py
+++ b/setup.py
@@ -128,10 +128,13 @@ class CustomInstall(install):
             message = "Cleaning previous installation directories"
         if to_remove is None:
             to_remove = []
-        to_remove.extend([os.path.join(self.install_lib, NAME),
-                          os.path.join(self.install_data, 'share', 'doc', NAME),
-                          os.path.join(self.install_lib, "%s-py%s.egg-info" % (self.config_vars['dist_fullname'], self.config_vars['py_version_short'])),
-                          ])
+        for path in [os.path.join(self.install_lib, NAME),
+                     os.path.join(self.install_data, 'share', 'doc', NAME),
+                     os.path.join(self.install_lib, "%s.egg-info" % self.config_vars['dist_fullname']),
+                     os.path.join(self.install_lib, "%s-py%s.egg-info" % (self.config_vars['dist_fullname'], self.config_vars['py_version_short'])),
+                     ]:
+            if os.path.isdir(path):
+                to_remove.append(path)
         for source, dest in self.sh_script_links:
             if os.path.islink(dest):
                 to_remove.append(dest)