diff --git a/web_src/js/features/repo-diff-selection.ts b/web_src/js/features/repo-diff-selection.ts
index 041a5d0645..906bff3a02 100644
--- a/web_src/js/features/repo-diff-selection.ts
+++ b/web_src/js/features/repo-diff-selection.ts
@@ -216,7 +216,9 @@ function handleDiffLineNumberClick(cell: HTMLElement, e: MouseEvent) {
   };
 
   if (applyDiffLineSelection(container, range)) {
-    diffSelectionStart = {...info, container};
+    if (!e.shiftKey || !diffSelectionStart || diffSelectionStart.container !== container || diffSelectionStart.fragment !== info.fragment) {
+      diffSelectionStart = {...info, container};
+    }
     window.getSelection().removeAllRanges();
   }
 }