diff --git a/models/perm/access/access.go b/models/perm/access/access.go
index 6433c4675c..068a122d01 100644
--- a/models/perm/access/access.go
+++ b/models/perm/access/access.go
@@ -105,8 +105,10 @@ func refreshAccesses(ctx context.Context, repo *repo_model.Repository, accessMap
 	}
 
 	newAccesses := make([]Access, 0, len(accessMap))
+	keysToDelete := []int64{}
 	for userID, ua := range accessMap {
 		if ua.Mode < minMode && !ua.User.IsRestricted {
+			keysToDelete = append(keysToDelete, userID)
 			continue
 		}
 
@@ -116,6 +118,9 @@ func refreshAccesses(ctx context.Context, repo *repo_model.Repository, accessMap
 			Mode:   ua.Mode,
 		})
 	}
+	for _, uid := range keysToDelete {
+		delete(accessMap, uid)
+	}
 
 	// Delete old accesses and insert new ones for repository.
 	if _, err = db.DeleteByBean(ctx, &Access{RepoID: repo.ID}); err != nil {