diff --git a/britney.py b/britney.py
index 0d428f7..ef3a8f2 100755
--- a/britney.py
+++ b/britney.py
@@ -1531,15 +1531,15 @@ class Britney(object):
 
         # this list will contain the packages which are valid candidates;
         # if a package is going to be removed, it will have a "-" prefix
-        upgrade_me = []
-        upgrade_me_append = upgrade_me.append  # Every . in a loop slows it down
+        upgrade_me = set()
+        upgrade_me_add = upgrade_me.add  # Every . in a loop slows it down
 
         excuses = self.excuses = {}
 
         # for every source package in testing, check if it should be removed
         for pkg in testing:
             if should_remove_source(pkg):
-                upgrade_me_append("-" + pkg)
+                upgrade_me_add("-" + pkg)
 
         # for every source package in unstable check if it should be upgraded
         for pkg in unstable:
@@ -1549,11 +1549,11 @@ class Britney(object):
             if pkg in testing and not testing[pkg].is_fakesrc:
                 for arch in architectures:
                     if should_upgrade_srcarch(pkg, arch, 'unstable'):
-                        upgrade_me_append("%s/%s" % (pkg, arch))
+                        upgrade_me_add("%s/%s" % (pkg, arch))
 
             # check if the source package should be upgraded
             if should_upgrade_src(pkg, 'unstable'):
-                upgrade_me_append(pkg)
+                upgrade_me_add(pkg)
 
         # for every source package in *-proposed-updates, check if it should be upgraded
         for suite in ['pu', 'tpu']:
@@ -1563,11 +1563,11 @@ class Britney(object):
                 if pkg in testing:
                     for arch in architectures:
                         if should_upgrade_srcarch(pkg, arch, suite):
-                            upgrade_me_append("%s/%s_%s" % (pkg, arch, suite))
+                            upgrade_me_add("%s/%s_%s" % (pkg, arch, suite))
 
                 # check if the source package should be upgraded
                 if should_upgrade_src(pkg, suite):
-                    upgrade_me_append("%s_%s" % (pkg, suite))
+                    upgrade_me_add("%s_%s" % (pkg, suite))
 
         # process the `remove' hints, if the given package is not yet in upgrade_me
         for hint in self.hints['remove']:
@@ -1582,7 +1582,7 @@ class Britney(object):
                 continue
 
             # add the removal of the package to upgrade_me and build a new excuse
-            upgrade_me_append("-%s" % (src))
+            upgrade_me_add("-%s" % (src))
             excuse = Excuse("-%s" % (src))
             excuse.set_vers(tsrcv, None)
             excuse.addhtml("Removal request by %s" % (hint.user))
@@ -1595,7 +1595,7 @@ class Britney(object):
             excuses[excuse.name] = excuse
 
         # extract the not considered packages, which are in the excuses but not in upgrade_me
-        unconsidered = [ename for ename in excuses if ename not in upgrade_me]
+        unconsidered = {ename for ename in excuses if ename not in upgrade_me}
 
         # invalidate impossible excuses
         for e in excuses.values():
diff --git a/britney2/excuse.py b/britney2/excuse.py
index 4521864..17e4a40 100644
--- a/britney2/excuse.py
+++ b/britney2/excuse.py
@@ -74,7 +74,7 @@ class Excuse(object):
         self.forced = False
         self._policy_verdict = PolicyVerdict.REJECTED_PERMANENTLY
 
-        self.invalid_deps = []
+        self.invalid_deps = set()
         self.deps = {}
         self.sane_deps = []
         self.break_deps = []
@@ -137,7 +137,7 @@ class Excuse(object):
 
     def invalidate_dep(self, name):
         """Invalidate dependency"""
-        if name not in self.invalid_deps: self.invalid_deps.append(name)
+        self.invalid_deps.add(name)
 
     def setdaysold(self, daysold, mindays):
         """Set the number of days from the upload and the minimum number of days for the update"""
diff --git a/britney2/utils.py b/britney2/utils.py
index 99f83e9..d54ce5b 100644
--- a/britney2/utils.py
+++ b/britney2/utils.py
@@ -783,7 +783,7 @@ def invalidate_excuses(excuses, valid, invalid):
     """Invalidate impossible excuses
 
     This method invalidates the impossible excuses, which depend
-    on invalid excuses. The two parameters contains the list of
+    on invalid excuses. The two parameters contains the sets of
     `valid' and `invalid' excuses.
     """
 
@@ -794,7 +794,7 @@ def invalidate_excuses(excuses, valid, invalid):
             revdeps[d].append(exc.name)
 
     # loop on the invalid excuses
-    for i, ename in enumerate(invalid):
+    for ename in iter_except(invalid.pop, KeyError):
         # if there is no reverse dependency, skip the item
         if ename not in revdeps:
             continue
@@ -816,8 +816,8 @@ def invalidate_excuses(excuses, valid, invalid):
                 # otherwise, invalidate the dependency and mark as invalidated and
                 # remove the depending excuses
                 excuses[x].invalidate_dep(ename)
-                p = valid.index(x)
-                invalid.append(valid.pop(p))
+                valid.discard(x)
+                invalid.add(x)
                 excuses[x].addhtml("Invalidated by dependency")
                 excuses[x].addreason("depends")
                 if excuses[x].policy_verdict.value < rdep_verdict.value: