From e90daaf81287b4ab5d53166d69651d9aba75ec9e Mon Sep 17 00:00:00 2001 From: Stefan Haller Date: Fri, 10 Jul 2026 08:56:06 +0200 Subject: [PATCH 1/5] Unify the git command lock-retry loops RunWithOutput and RunWithOutputs each carried their own near-identical copy of the index.lock retry loop. Extract the loop into a single retryOnLockError helper so the retry policy lives in one place, ahead of changing that policy. Behavior is unchanged; the added tests characterize it (success and non-lock errors run once, a lock error in the output is retried). Co-Authored-By: Claude Opus 4.8 (1M context) --- pkg/commands/git_cmd_obj_runner.go | 42 +++++------ pkg/commands/git_cmd_obj_runner_test.go | 92 +++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 21 deletions(-) create mode 100644 pkg/commands/git_cmd_obj_runner_test.go diff --git a/pkg/commands/git_cmd_obj_runner.go b/pkg/commands/git_cmd_obj_runner.go index 668feef93..a72565b5c 100644 --- a/pkg/commands/git_cmd_obj_runner.go +++ b/pkg/commands/git_cmd_obj_runner.go @@ -33,11 +33,30 @@ func (self *gitCmdObjRunner) Run(cmdObj *oscommands.CmdObj) error { } func (self *gitCmdObjRunner) RunWithOutput(cmdObj *oscommands.CmdObj) (string, error) { + return self.retryOnLockError(func() (string, error) { + return self.innerRunner.RunWithOutput(cmdObj.Clone()) + }) +} + +func (self *gitCmdObjRunner) RunWithOutputs(cmdObj *oscommands.CmdObj) (string, string, error) { + var stdout, stderr string + _, err := self.retryOnLockError(func() (string, error) { + var runErr error + stdout, stderr, runErr = self.innerRunner.RunWithOutputs(cmdObj.Clone()) + return stdout + stderr, runErr + }) + return stdout, stderr, err +} + +// retryOnLockError runs the given function, retrying if it fails with a +// transient lock error (see isRetryableError). The string returned by run is +// the command output we inspect to classify the failure. We clone the command +// for each attempt (inside run) because an *exec.Cmd can only be run once. +func (self *gitCmdObjRunner) retryOnLockError(run func() (string, error)) (string, error) { var output string var err error for range RetryCount { - newCmdObj := cmdObj.Clone() - output, err = self.innerRunner.RunWithOutput(newCmdObj) + output, err = run() if err == nil || !isRetryableError(output) { return output, err @@ -51,25 +70,6 @@ func (self *gitCmdObjRunner) RunWithOutput(cmdObj *oscommands.CmdObj) (string, e return output, err } -func (self *gitCmdObjRunner) RunWithOutputs(cmdObj *oscommands.CmdObj) (string, string, error) { - var stdout, stderr string - var err error - for range RetryCount { - newCmdObj := cmdObj.Clone() - stdout, stderr, err = self.innerRunner.RunWithOutputs(newCmdObj) - - if err == nil || !isRetryableError(stdout+stderr) { - return stdout, stderr, err - } - - // if we have an error based on a lock, we should wait a bit and then retry - self.log.Warn("lock error prevented command from running. Retrying command after a small wait") - time.Sleep(WaitTime) - } - - return stdout, stderr, err -} - // Retry logic not implemented here, but these commands typically don't need to obtain a lock. func (self *gitCmdObjRunner) RunAndProcessLines(cmdObj *oscommands.CmdObj, onLine func(line string) (bool, error)) error { return self.innerRunner.RunAndProcessLines(cmdObj, onLine) diff --git a/pkg/commands/git_cmd_obj_runner_test.go b/pkg/commands/git_cmd_obj_runner_test.go new file mode 100644 index 000000000..e8c73727d --- /dev/null +++ b/pkg/commands/git_cmd_obj_runner_test.go @@ -0,0 +1,92 @@ +package commands + +import ( + "errors" + "testing" + + "github.com/jesseduffield/lazygit/pkg/commands/oscommands" + "github.com/jesseduffield/lazygit/pkg/utils" + "github.com/stretchr/testify/assert" +) + +type runnerResult struct { + output string + err error +} + +// scriptedRunner is an ICmdObjRunner stub that returns a preconfigured result +// for each successive call, letting us drive the retry loop deterministically. +// It counts calls so tests can assert whether a command was retried. +type scriptedRunner struct { + results []runnerResult + calls int +} + +func (self *scriptedRunner) next() (string, error) { + result := self.results[self.calls] + self.calls++ + return result.output, result.err +} + +func (self *scriptedRunner) Run(*oscommands.CmdObj) error { + _, err := self.next() + return err +} + +func (self *scriptedRunner) RunWithOutput(*oscommands.CmdObj) (string, error) { + return self.next() +} + +func (self *scriptedRunner) RunWithOutputs(*oscommands.CmdObj) (string, string, error) { + output, err := self.next() + return output, "", err +} + +func (self *scriptedRunner) RunAndProcessLines(*oscommands.CmdObj, func(string) (bool, error)) error { + panic("not implemented") +} + +func newTestRunner(inner *scriptedRunner) *gitCmdObjRunner { + return &gitCmdObjRunner{ + log: utils.NewDummyLog(), + innerRunner: inner, + } +} + +// dummyCmdObj returns a throwaway command; only its clonability matters, since +// the scriptedRunner ignores it and returns preconfigured results. +func dummyCmdObj() *oscommands.CmdObj { + return oscommands.NewDummyCmdObjBuilder(nil).New([]string{"git", "status"}) +} + +func TestRunWithOutputReturnsSuccessWithoutRetrying(t *testing.T) { + inner := &scriptedRunner{results: []runnerResult{{output: "done", err: nil}}} + + output, err := newTestRunner(inner).RunWithOutput(dummyCmdObj()) + + assert.NoError(t, err) + assert.Equal(t, "done", output) + assert.Equal(t, 1, inner.calls) +} + +func TestRunWithOutputDoesNotRetryNonLockError(t *testing.T) { + inner := &scriptedRunner{results: []runnerResult{{output: "boom", err: errors.New("boom")}}} + + _, err := newTestRunner(inner).RunWithOutput(dummyCmdObj()) + + assert.Error(t, err) + assert.Equal(t, 1, inner.calls) +} + +func TestRunWithOutputRetriesWhenLockErrorIsInOutput(t *testing.T) { + inner := &scriptedRunner{results: []runnerResult{ + {output: "fatal: Unable to create '/repo/.git/index.lock': File exists.", err: errors.New("exit status 128")}, + {output: "done", err: nil}, + }} + + output, err := newTestRunner(inner).RunWithOutput(dummyCmdObj()) + + assert.NoError(t, err) + assert.Equal(t, "done", output) + assert.Equal(t, 2, inner.calls) +} From 0902c5c05878b97c17611cec804a99562694372f Mon Sep 17 00:00:00 2001 From: Stefan Haller Date: Fri, 10 Jul 2026 08:56:26 +0200 Subject: [PATCH 2/5] Demonstrate that a lock error in a streamed command isn't retried The gpg helper runs commands like amend with StreamOutput, so their output isn't captured and a failed run returns an empty output string; the index.lock message is carried by the error instead. isRetryableError only inspects the output, so the retry loop never fires for these commands. In practice this means a `shift-A` amend issued while a foreground `git status` refresh briefly holds index.lock fails outright instead of retrying. Co-Authored-By: Claude Opus 4.8 (1M context) --- pkg/commands/git_cmd_obj_runner_test.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pkg/commands/git_cmd_obj_runner_test.go b/pkg/commands/git_cmd_obj_runner_test.go index e8c73727d..2ff2795dc 100644 --- a/pkg/commands/git_cmd_obj_runner_test.go +++ b/pkg/commands/git_cmd_obj_runner_test.go @@ -90,3 +90,22 @@ func TestRunWithOutputRetriesWhenLockErrorIsInOutput(t *testing.T) { assert.Equal(t, "done", output) assert.Equal(t, 2, inner.calls) } + +func TestRunWithOutputRetriesWhenLockErrorIsOnlyInError(t *testing.T) { + // A streamed command (e.g. an amend run through the gpg helper) doesn't + // capture its output, so a lock failure surfaces only in the returned error + // with an empty output string. The retry logic must still recognize it. + inner := &scriptedRunner{results: []runnerResult{ + {output: "", err: errors.New("fatal: Unable to create '/repo/.git/index.lock': File exists.")}, + {output: "", err: nil}, + }} + + _, err := newTestRunner(inner).RunWithOutput(dummyCmdObj()) + + /* EXPECTED: + assert.NoError(t, err) + assert.Equal(t, 2, inner.calls) + ACTUAL: */ + assert.Error(t, err) + assert.Equal(t, 1, inner.calls) +} From c1cd500fa776c54f61b4cf6dddf6de0d90c7aa1a Mon Sep 17 00:00:00 2001 From: Stefan Haller Date: Fri, 10 Jul 2026 08:56:56 +0200 Subject: [PATCH 3/5] Retry lock errors reported only through the command's error Have isRetryableError also inspect the returned error, not just the captured output. Streamed commands (amend, commit, and other operations run through the gpg helper) don't capture output, so their index.lock failures were slipping past the retry loop and surfacing to the user as a hard "Git command failed". Now they retry like every other command. Co-Authored-By: Claude Opus 4.8 (1M context) --- pkg/commands/git_cmd_obj_runner.go | 18 ++++++++++++------ pkg/commands/git_cmd_obj_runner_test.go | 4 ---- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/pkg/commands/git_cmd_obj_runner.go b/pkg/commands/git_cmd_obj_runner.go index a72565b5c..f22a0f7aa 100644 --- a/pkg/commands/git_cmd_obj_runner.go +++ b/pkg/commands/git_cmd_obj_runner.go @@ -20,11 +20,17 @@ type gitCmdObjRunner struct { innerRunner oscommands.ICmdObjRunner } -// isRetryableError returns true if the error output indicates a transient -// lock-related error that may succeed on retry -func isRetryableError(output string) bool { - return strings.Contains(output, ".git/index.lock") || - strings.Contains(output, "cannot lock ref") +// isRetryableError returns true if a failed command hit a transient +// lock-related condition that may succeed on retry. The lock message can reach +// us either in the command's captured output or, for streamed commands whose +// output we don't capture, only in the returned error, so we check both. +func isRetryableError(output string, err error) bool { + text := output + if err != nil { + text += "\n" + err.Error() + } + return strings.Contains(text, ".git/index.lock") || + strings.Contains(text, "cannot lock ref") } func (self *gitCmdObjRunner) Run(cmdObj *oscommands.CmdObj) error { @@ -58,7 +64,7 @@ func (self *gitCmdObjRunner) retryOnLockError(run func() (string, error)) (strin for range RetryCount { output, err = run() - if err == nil || !isRetryableError(output) { + if err == nil || !isRetryableError(output, err) { return output, err } diff --git a/pkg/commands/git_cmd_obj_runner_test.go b/pkg/commands/git_cmd_obj_runner_test.go index 2ff2795dc..903c6a90b 100644 --- a/pkg/commands/git_cmd_obj_runner_test.go +++ b/pkg/commands/git_cmd_obj_runner_test.go @@ -102,10 +102,6 @@ func TestRunWithOutputRetriesWhenLockErrorIsOnlyInError(t *testing.T) { _, err := newTestRunner(inner).RunWithOutput(dummyCmdObj()) - /* EXPECTED: assert.NoError(t, err) assert.Equal(t, 2, inner.calls) - ACTUAL: */ - assert.Error(t, err) - assert.Equal(t, 1, inner.calls) } From e3ecb77939e191281ef3a70509be1183acc2b41e Mon Sep 17 00:00:00 2001 From: Stefan Haller Date: Fri, 10 Jul 2026 08:57:26 +0200 Subject: [PATCH 4/5] Recognize index.lock contention in worktrees and submodules The retry check matched the literal ".git/index.lock", which only ever appears for the main worktree. A linked worktree's lock is at .git/worktrees//index.lock and a submodule's is under its own git dir, so contention there was never retried. Match the bare "index.lock" fragment instead, which covers all of them. Co-Authored-By: Claude Opus 4.8 (1M context) --- pkg/commands/git_cmd_obj_runner.go | 7 ++++++- pkg/commands/git_cmd_obj_runner_test.go | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/pkg/commands/git_cmd_obj_runner.go b/pkg/commands/git_cmd_obj_runner.go index f22a0f7aa..c4d2ec049 100644 --- a/pkg/commands/git_cmd_obj_runner.go +++ b/pkg/commands/git_cmd_obj_runner.go @@ -24,12 +24,17 @@ type gitCmdObjRunner struct { // lock-related condition that may succeed on retry. The lock message can reach // us either in the command's captured output or, for streamed commands whose // output we don't capture, only in the returned error, so we check both. +// +// We match the bare "index.lock" fragment rather than a fuller path or message +// so we catch the lock wherever git puts it: the main .git dir, a linked +// worktree's git dir (.git/worktrees//index.lock), or a submodule's git +// dir. func isRetryableError(output string, err error) bool { text := output if err != nil { text += "\n" + err.Error() } - return strings.Contains(text, ".git/index.lock") || + return strings.Contains(text, "index.lock") || strings.Contains(text, "cannot lock ref") } diff --git a/pkg/commands/git_cmd_obj_runner_test.go b/pkg/commands/git_cmd_obj_runner_test.go index 903c6a90b..2cde503f5 100644 --- a/pkg/commands/git_cmd_obj_runner_test.go +++ b/pkg/commands/git_cmd_obj_runner_test.go @@ -105,3 +105,18 @@ func TestRunWithOutputRetriesWhenLockErrorIsOnlyInError(t *testing.T) { assert.NoError(t, err) assert.Equal(t, 2, inner.calls) } + +func TestRunWithOutputRetriesLockErrorInLinkedWorktree(t *testing.T) { + // In a linked worktree the lock lives at .git/worktrees//index.lock + // rather than .git/index.lock, so only matching the bare "index.lock" + // fragment lets the retry fire there too. + inner := &scriptedRunner{results: []runnerResult{ + {output: "", err: errors.New("fatal: Unable to create '/repo/.git/worktrees/feature/index.lock': File exists.")}, + {output: "", err: nil}, + }} + + _, err := newTestRunner(inner).RunWithOutput(dummyCmdObj()) + + assert.NoError(t, err) + assert.Equal(t, 2, inner.calls) +} From 4052057eee1073f2afe7c5cc1e3027995167c642 Mon Sep 17 00:00:00 2001 From: Stefan Haller Date: Fri, 10 Jul 2026 08:58:22 +0200 Subject: [PATCH 5/5] Back off exponentially between lock-error retries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The retry budget was five fixed 50ms waits (250ms total). A foreground `git status` refresh can hold index.lock for longer than that on a large repo, so the retries could be exhausted before the lock clears. Wait 20ms before the first retry and double each time, giving seven attempts over a bit more than a second — enough to outlast a slow refresh while keeping the common case (a lock that clears almost immediately) fast. The initial delay is now a runner field so tests can zero it out instead of sleeping. Co-Authored-By: Claude Opus 4.8 (1M context) --- pkg/commands/git_cmd_obj_builder.go | 5 +++-- pkg/commands/git_cmd_obj_runner.go | 28 ++++++++++++++++++------- pkg/commands/git_cmd_obj_runner_test.go | 15 +++++++++++++ 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/pkg/commands/git_cmd_obj_builder.go b/pkg/commands/git_cmd_obj_builder.go index 6bc3b7d19..20d31c11c 100644 --- a/pkg/commands/git_cmd_obj_builder.go +++ b/pkg/commands/git_cmd_obj_builder.go @@ -25,8 +25,9 @@ func NewGitCmdObjBuilder(log *logrus.Entry, innerBuilder *oscommands.CmdObjBuild // the price of having a convenient interface where we can say .New(...).Run() is that our builder now depends on our runner, so when we want to wrap the default builder/runner in new functionality we need to jump through some hoops. We could avoid the use of a decorator function here by just exporting the runner field on the default builder but that would be misleading because we don't want anybody using that to run commands (i.e. we want there to be a single API used across the codebase) updatedBuilder := innerBuilder.CloneWithNewRunner(func(runner oscommands.ICmdObjRunner) oscommands.ICmdObjRunner { return &gitCmdObjRunner{ - log: log, - innerRunner: runner, + log: log, + innerRunner: runner, + initialRetryDelay: defaultInitialRetryDelay, } }) diff --git a/pkg/commands/git_cmd_obj_runner.go b/pkg/commands/git_cmd_obj_runner.go index c4d2ec049..8112b0f30 100644 --- a/pkg/commands/git_cmd_obj_runner.go +++ b/pkg/commands/git_cmd_obj_runner.go @@ -11,13 +11,24 @@ import ( // here we're wrapping the default command runner in some git-specific stuff e.g. retry logic if we get an error due to the presence of .git/index.lock const ( - WaitTime = 50 * time.Millisecond - RetryCount = 5 + // defaultInitialRetryDelay is how long we wait before the first retry of a + // command that failed with a transient lock error. We double it before each + // subsequent retry (see retryOnLockError), so across maxRetries attempts we + // wait for a bit over a second in total. That's long enough to outlast the + // brief window during which another git process holds a lock we need — + // typically our own foreground `git status` refresh, which takes index.lock + // to persist its refreshed stat-cache. + defaultInitialRetryDelay = 20 * time.Millisecond + maxRetries = 7 ) type gitCmdObjRunner struct { log *logrus.Entry innerRunner oscommands.ICmdObjRunner + // initialRetryDelay is the wait before the first lock-error retry. It's a + // field rather than the constant directly so tests can set it to zero and + // not actually sleep. + initialRetryDelay time.Duration } // isRetryableError returns true if a failed command hit a transient @@ -64,18 +75,21 @@ func (self *gitCmdObjRunner) RunWithOutputs(cmdObj *oscommands.CmdObj) (string, // the command output we inspect to classify the failure. We clone the command // for each attempt (inside run) because an *exec.Cmd can only be run once. func (self *gitCmdObjRunner) retryOnLockError(run func() (string, error)) (string, error) { + delay := self.initialRetryDelay var output string var err error - for range RetryCount { + for attempt := range maxRetries { output, err = run() if err == nil || !isRetryableError(output, err) { - return output, err + break } - // if we have an error based on a lock, we should wait a bit and then retry - self.log.Warn("lock error prevented command from running. Retrying command after a small wait") - time.Sleep(WaitTime) + if attempt < maxRetries-1 { + self.log.Warnf("lock error prevented command from running; retrying in %s", delay) + time.Sleep(delay) + delay *= 2 + } } return output, err diff --git a/pkg/commands/git_cmd_obj_runner_test.go b/pkg/commands/git_cmd_obj_runner_test.go index 2cde503f5..bf938da54 100644 --- a/pkg/commands/git_cmd_obj_runner_test.go +++ b/pkg/commands/git_cmd_obj_runner_test.go @@ -50,6 +50,8 @@ func newTestRunner(inner *scriptedRunner) *gitCmdObjRunner { return &gitCmdObjRunner{ log: utils.NewDummyLog(), innerRunner: inner, + // don't actually sleep between retries + initialRetryDelay: 0, } } @@ -106,6 +108,19 @@ func TestRunWithOutputRetriesWhenLockErrorIsOnlyInError(t *testing.T) { assert.Equal(t, 2, inner.calls) } +func TestRunWithOutputGivesUpAfterMaxRetries(t *testing.T) { + results := make([]runnerResult, maxRetries) + for i := range results { + results[i] = runnerResult{err: errors.New("fatal: Unable to create '/repo/.git/index.lock': File exists.")} + } + inner := &scriptedRunner{results: results} + + _, err := newTestRunner(inner).RunWithOutput(dummyCmdObj()) + + assert.Error(t, err) + assert.Equal(t, maxRetries, inner.calls) +} + func TestRunWithOutputRetriesLockErrorInLinkedWorktree(t *testing.T) { // In a linked worktree the lock lives at .git/worktrees//index.lock // rather than .git/index.lock, so only matching the bare "index.lock"