diff --git a/pkg/gui/controllers/commits_files_controller.go b/pkg/gui/controllers/commits_files_controller.go
index bd9306e5e..6e01d8d5e 100644
--- a/pkg/gui/controllers/commits_files_controller.go
+++ b/pkg/gui/controllers/commits_files_controller.go
@@ -519,7 +519,7 @@ func (self *CommitFilesController) toggleAllForPatch(_ *filetree.CommitFileNode)
 }
 
 func (self *CommitFilesController) enter(node *filetree.CommitFileNode) error {
-	return self.c.Helpers().CommitFiles.EnterCommitFile(node, types.OnFocusOpts{ClickedWindowName: "", ClickedViewLineIdx: -1, ClickedViewRealLineIdx: -1})
+	return self.c.Helpers().CommitFiles.EnterCommitFile(node, nil, types.OnFocusOpts{ClickedWindowName: "", ClickedViewLineIdx: -1, ClickedViewRealLineIdx: -1})
 }
 
 // NOTE: this is very similar to handleToggleFileTreeView, could be DRY'd with generics
@@ -576,7 +576,9 @@ func (self *CommitFilesController) GetOnClickFocusedMainView() func(mainViewName
 			}
 		}
 
-		return self.c.Helpers().CommitFiles.EnterCommitFile(node, types.OnFocusOpts{ClickedWindowName: "main", ClickedViewLineIdx: line, ClickedViewRealLineIdx: line})
+		// Entered from the commit files panel's own focused main view, so escape
+		// should just pop back to it; no special escape context needed.
+		return self.c.Helpers().CommitFiles.EnterCommitFile(node, nil, types.OnFocusOpts{ClickedWindowName: "main", ClickedViewLineIdx: line, ClickedViewRealLineIdx: line})
 	}
 }
 
diff --git a/pkg/gui/controllers/helpers/commit_files_helper.go b/pkg/gui/controllers/helpers/commit_files_helper.go
index bd553a605..e03e1f8ce 100644
--- a/pkg/gui/controllers/helpers/commit_files_helper.go
+++ b/pkg/gui/controllers/helpers/commit_files_helper.go
@@ -21,7 +21,11 @@ func NewCommitFilesHelper(c *HelperCommon, patchBuildingHelper *PatchBuildingHel
 	}
 }
 
-func (self *CommitFilesHelper) EnterCommitFile(node *filetree.CommitFileNode, opts types.OnFocusOpts) error {
+// escapeContext is the side panel that escaping the patch builder should return
+// to, for the case where we're entering it straight from a focused main view;
+// it's nil for the normal flow that goes through the commit files panel. See
+// PatchBuildingHelper.escapeContext.
+func (self *CommitFilesHelper) EnterCommitFile(node *filetree.CommitFileNode, escapeContext types.Context, opts types.OnFocusOpts) error {
 	if node.File == nil {
 		self.handleToggleCommitFileDirCollapsed(node)
 		return nil
@@ -48,6 +52,11 @@ func (self *CommitFilesHelper) EnterCommitFile(node *filetree.CommitFileNode, op
 				}
 			}
 
+			// Set on every entry (so it can't leak from a previous main-view
+			// entry into a subsequent normal one), right as we push the patch
+			// builder.
+			self.patchBuildingHelper.escapeContext = escapeContext
+
 			self.c.Context().Push(self.c.Contexts().CustomPatchBuilder, opts)
 			self.patchBuildingHelper.ShowHunkStagingHint()
 
diff --git a/pkg/gui/controllers/helpers/patch_building_helper.go b/pkg/gui/controllers/helpers/patch_building_helper.go
index dc611404f..592a17bde 100644
--- a/pkg/gui/controllers/helpers/patch_building_helper.go
+++ b/pkg/gui/controllers/helpers/patch_building_helper.go
@@ -10,6 +10,14 @@ import (
 
 type PatchBuildingHelper struct {
 	c *HelperCommon
+
+	// When patch building is entered straight from a focused main view (rather
+	// than from the commit files panel), this records the side panel to return
+	// to on escape, so that we skip the commit files panel we never really
+	// visited. It is nil for the normal flow, where escape just pops back to the
+	// commit files panel. Set on every entry into patch building (see
+	// CommitFilesHelper.EnterCommitFile) so it can't leak between flows.
+	escapeContext types.Context
 }
 
 func NewPatchBuildingHelper(
@@ -32,9 +40,17 @@ func (self *PatchBuildingHelper) ShowHunkStagingHint() {
 	}
 }
 
-// takes us from the patch building panel back to the commit files panel
+// takes us from the patch building panel back to the commit files panel, or
+// straight back to the side panel if we entered patch building from a focused
+// main view (see escapeContext)
 func (self *PatchBuildingHelper) Escape() {
-	self.c.Context().Pop()
+	if self.escapeContext != nil {
+		escapeContext := self.escapeContext
+		self.escapeContext = nil
+		self.c.Context().Push(escapeContext, types.OnFocusOpts{})
+	} else {
+		self.c.Context().Pop()
+	}
 }
 
 // kills the custom patch and returns us back to the commit files panel if needed
diff --git a/pkg/gui/controllers/switch_to_diff_files_controller.go b/pkg/gui/controllers/switch_to_diff_files_controller.go
index 2f45a1303..a7e18efb1 100644
--- a/pkg/gui/controllers/switch_to_diff_files_controller.go
+++ b/pkg/gui/controllers/switch_to_diff_files_controller.go
@@ -83,7 +83,10 @@ func (self *SwitchToDiffFilesController) GetOnClickFocusedMainView() func(mainVi
 		context.GetViewTrait().FocusPoint(
 			context.ModelIndexToViewIndex(idx), false)
 		node = context.GetSelected()
-		return self.c.Helpers().CommitFiles.EnterCommitFile(node, types.OnFocusOpts{ClickedWindowName: "main", ClickedViewLineIdx: line, ClickedViewRealLineIdx: line})
+		// We entered patch building straight from the focused main view, so
+		// escaping it should take us all the way back out to this side panel,
+		// skipping the commit files panel we never really visited.
+		return self.c.Helpers().CommitFiles.EnterCommitFile(node, self.context, types.OnFocusOpts{ClickedWindowName: "main", ClickedViewLineIdx: line, ClickedViewRealLineIdx: line})
 	}
 }