diff --git a/lib/list_lru.c b/lib/list_lru.c
index da9b837..4aa069b 100644
--- a/lib/list_lru.c
+++ b/lib/list_lru.c
@@ -195,12 +195,10 @@ list_lru_walk_node(
 	unsigned long isolated = 0;
 
 	spin_lock(&nlru->lock);
-restart:
 	list_for_each_safe(item, n, &nlru->list) {
+		bool first_pass = true;
 		enum lru_status ret;
-
-		if ((*nr_to_walk)-- < 0)
-			break;
+restart:
 
 		ret = isolate(item, &nlru->lock, cb_arg);
 		switch (ret) {
@@ -217,10 +215,17 @@ restart:
 		case LRU_SKIP:
 			break;
 		case LRU_RETRY:
+			if (!first_pass)
+				break;
+			first_pass = true;
 			goto restart;
 		default:
 			BUG();
 		}
+
+		if ((*nr_to_walk)-- == 0)
+			break;
+
 	}
 	spin_unlock(&nlru->lock);
 	return isolated;