From 2ed887d149d95be4bc55756ecd08e44b7bb54d90 Mon Sep 17 00:00:00 2001 From: Stefan Kerkmann Date: Thu, 18 Sep 2025 17:52:47 +0200 Subject: [PATCH] Commits: add incremental git log limit The existing git log logic fetched the first 300 commits of a repo and displayed them in the local and sub-commit views. Once a user selected a commit beyond a threshold of 200 commits the whole repository was loaded. This is problematic with large repos e.g. the linux kernel with currently ~138k commits as lazygit slows down substantially with such a large number of commits in memory. This commit replaces the current all or only the first 300 commits logic with an incremental fetching approach: 1. The first 400 commits of repo are loaded by default. 2. If the user selects a commit beyond a threshold (current limit-100) the git log limit is increased by 400 if there are more commits available. If there are more commits available is currently checked by comparing the previous log limit with the real commit count in the model, if the commit count is less then the limit it is assumed that we reached the end of the commit log. Ideally it would be better to call `git rev-list --count xyz` in the right places and compare with this result, but this requires more changes. Adding a "paginated implementation" by utilizing `git log --skip=x --max-count=y` and appending commits to the model instead of replacing the whole collection would be nice, but this requires deeper changes to keep everything consistent. Signed-off-by: Stefan Kerkmann Co-authored-by: DeepSeek V4 Flash --- pkg/commands/git_commands/commit_loader.go | 30 ++++++++++++++- pkg/gui/context/local_commits_context.go | 16 ++++---- pkg/gui/context/sub_commits_context.go | 14 +++---- pkg/gui/controllers/helpers/refresh_helper.go | 24 +++++++++--- pkg/gui/controllers/helpers/refs_helper.go | 4 +- .../controllers/helpers/sub_commits_helper.go | 4 +- .../controllers/local_commits_controller.go | 37 +++++++++++++++---- pkg/gui/controllers/sub_commits_controller.go | 12 +++++- 8 files changed, 105 insertions(+), 36 deletions(-) diff --git a/pkg/commands/git_commands/commit_loader.go b/pkg/commands/git_commands/commit_loader.go index 381dd641e..76ddd395f 100644 --- a/pkg/commands/git_commands/commit_loader.go +++ b/pkg/commands/git_commands/commit_loader.go @@ -55,8 +55,29 @@ func NewCommitLoader( } } +const ( + GIT_LOG_FETCH_COUNT = 400 + GIT_LOG_FETCH_THRESHOLD = GIT_LOG_FETCH_COUNT / 4 +) + +type GitLogLimit struct { + Limit int +} + +func DefaultGitLogLimit() *GitLogLimit { + return &GitLogLimit{Limit: GIT_LOG_FETCH_COUNT} +} + +func (self *GitLogLimit) CanFetchMoreCommits(lineIdx, logCommitCount int) bool { + return lineIdx >= logCommitCount-GIT_LOG_FETCH_THRESHOLD && logCommitCount >= self.Limit +} + +func (self *GitLogLimit) Increase(realCommitCount int) { + self.Limit = realCommitCount + GIT_LOG_FETCH_COUNT +} + type GetCommitsOptions struct { - Limit bool + LogLimit *GitLogLimit FilterPath string FilterAuthor string IncludeRebaseCommits bool @@ -603,6 +624,11 @@ func (self *CommitLoader) getLogCmd(opts GetCommitsOptions) *oscommands.CmdObj { refSpec += "..." + opts.RefToShowDivergenceFrom } + var limitArg string + if opts.LogLimit != nil { + limitArg = fmt.Sprintf("--max-count=%d", opts.LogLimit.Limit) + } + cmdArgs := NewGitCmd("log"). Arg(refSpec). ArgIf(gitLogOrder != "default", "--"+gitLogOrder). @@ -611,7 +637,7 @@ func (self *CommitLoader) getLogCmd(opts GetCommitsOptions) *oscommands.CmdObj { Arg(prettyFormat). Arg("--abbrev=40"). ArgIf(opts.FilterAuthor != "", "--author="+opts.FilterAuthor). - ArgIf(opts.Limit, "-300"). + ArgIf(limitArg != "", limitArg). ArgIf(opts.FilterPath != "", "--follow", "--name-status"). Arg("--no-show-signature"). ArgIf(opts.RefToShowDivergenceFrom != "", "--left-right"). diff --git a/pkg/gui/context/local_commits_context.go b/pkg/gui/context/local_commits_context.go index 4a99259fd..efee9e567 100644 --- a/pkg/gui/context/local_commits_context.go +++ b/pkg/gui/context/local_commits_context.go @@ -8,6 +8,7 @@ import ( "sync/atomic" "time" + "github.com/jesseduffield/lazygit/pkg/commands/git_commands" "github.com/jesseduffield/lazygit/pkg/commands/models" "github.com/jesseduffield/lazygit/pkg/config" "github.com/jesseduffield/lazygit/pkg/gocui" @@ -217,11 +218,12 @@ func (self *LocalCommitsContext) ClearDropInsertionIndex() { type LocalCommitsViewModel struct { *ListViewModel[*models.Commit] - // If this is true we limit the amount of commits we load, for the sake of keeping things fast. + // If this is non-nil we limit the amount of commits we load, for the sake of keeping things fast. // If the user attempts to scroll past the end of the list, we will load more commits. + // A nil value means no limit; we load the whole log. // Atomic because a checkout or reset sets it from a worker goroutine while the // commits refresh reads it on the UI thread to decide how many commits to load. - limitCommits atomic.Bool + gitLogLimit atomic.Pointer[git_commands.GitLogLimit] // If this is true we'll use git log --all when fetching the commits. showWholeGitGraph bool @@ -232,7 +234,7 @@ func NewLocalCommitsViewModel(getModel func() []*models.Commit, c *ContextCommon ListViewModel: NewListViewModel(getModel), showWholeGitGraph: c.UserConfig().Git.Log.ShowWholeGraph, } - self.limitCommits.Store(true) + self.gitLogLimit.Store(git_commands.DefaultGitLogLimit()) return self } @@ -303,12 +305,12 @@ func (self *LocalCommitsContext) ModelSearchResults(searchStr string, caseSensit return searchModelCommits(caseSensitive, self.GetCommits(), self.ColumnPositions(), self.modelToViewIndexConverter(), searchStr) } -func (self *LocalCommitsViewModel) SetLimitCommits(value bool) { - self.limitCommits.Store(value) +func (self *LocalCommitsViewModel) SetGitLogLimit(limit *git_commands.GitLogLimit) { + self.gitLogLimit.Store(limit) } -func (self *LocalCommitsViewModel) GetLimitCommits() bool { - return self.limitCommits.Load() +func (self *LocalCommitsViewModel) GetGitLogLimit() *git_commands.GitLogLimit { + return self.gitLogLimit.Load() } func (self *LocalCommitsViewModel) SetShowWholeGitGraph(value bool) { diff --git a/pkg/gui/context/sub_commits_context.go b/pkg/gui/context/sub_commits_context.go index b0bcee30a..6bdd3ba36 100644 --- a/pkg/gui/context/sub_commits_context.go +++ b/pkg/gui/context/sub_commits_context.go @@ -33,8 +33,8 @@ func NewSubCommitsContext( ListViewModel: NewListViewModel( func() []*models.Commit { return c.Model().SubCommits }, ), - ref: nil, - limitCommits: true, + ref: nil, + gitLogLimit: git_commands.DefaultGitLogLimit(), } getDisplayStrings := func(startIdx int, endIdx int) [][]string { @@ -142,7 +142,7 @@ type SubCommitsViewModel struct { refToShowDivergenceFrom string *ListViewModel[*models.Commit] - limitCommits bool + gitLogLimit *git_commands.GitLogLimit showBranchHeads bool } @@ -199,12 +199,12 @@ func (self *SubCommitsContext) GetCommits() []*models.Commit { return self.getModel() } -func (self *SubCommitsContext) SetLimitCommits(value bool) { - self.limitCommits = value +func (self *SubCommitsContext) SetGitLogLimit(limit *git_commands.GitLogLimit) { + self.gitLogLimit = limit } -func (self *SubCommitsContext) GetLimitCommits() bool { - return self.limitCommits +func (self *SubCommitsContext) GetGitLogLimit() *git_commands.GitLogLimit { + return self.gitLogLimit } func (self *SubCommitsContext) GetDiffTerminals() []string { diff --git a/pkg/gui/controllers/helpers/refresh_helper.go b/pkg/gui/controllers/helpers/refresh_helper.go index 730ed9a24..4771816ca 100644 --- a/pkg/gui/controllers/helpers/refresh_helper.go +++ b/pkg/gui/controllers/helpers/refresh_helper.go @@ -759,7 +759,7 @@ func (self *RefreshHelper) refreshReflogAndBranches(capturedReflog capturedReflo // worker computes from an immutable snapshot rather than reading state the UI // thread concurrently mutates. type capturedCommitState struct { - limitCommits bool + gitLogLimit *git_commands.GitLogLimit showWholeGitGraph bool filterPath string filterAuthor string @@ -768,6 +768,18 @@ type capturedCommitState struct { parentIsLocalCommits bool } +// snapshotGitLogLimit returns a copy of the given git log limit, or nil if +// there is no limit. The limit object lives on the context, where the UI +// thread bumps it when the user scrolls close to the end of the commit list; +// the refresh's git work on a worker goroutine must read a copy taken at +// capture time rather than the live object. +func snapshotGitLogLimit(limit *git_commands.GitLogLimit) *git_commands.GitLogLimit { + if limit == nil { + return nil + } + return &git_commands.GitLogLimit{Limit: limit.Limit} +} + // captureCommitsState reads the commits refresh's model/context/mode inputs // into an immutable snapshot. It must run on the UI thread. // The selection is captured later, when applying the refresh, so user input @@ -776,7 +788,7 @@ func (self *RefreshHelper) captureCommitsState() capturedCommitState { parentCtx := self.c.Contexts().CommitFiles.GetParentContext() return capturedCommitState{ - limitCommits: self.c.Contexts().LocalCommits.GetLimitCommits(), + gitLogLimit: snapshotGitLogLimit(self.c.Contexts().LocalCommits.GetGitLogLimit()), showWholeGitGraph: self.c.Contexts().LocalCommits.GetShowWholeGitGraph(), filterPath: self.c.Modes().Filtering.GetPath(), filterAuthor: self.c.Modes().Filtering.GetAuthor(), @@ -847,7 +859,7 @@ func (self *RefreshHelper) refreshCommitsWithLimit(captured capturedCommitState, refName, bisectInfo := self.refForLog(env) commits, err := env.git.Loaders.CommitLoader.GetCommits( git_commands.GetCommitsOptions{ - Limit: captured.limitCommits, + LogLimit: captured.gitLogLimit, FilterPath: captured.filterPath, FilterAuthor: captured.filterAuthor, IncludeRebaseCommits: true, @@ -999,7 +1011,7 @@ func findNewConflictedCommit(previousCommits []*models.Commit, commits []*models // work is dispatched to a worker. type capturedSubCommitState struct { ref models.Ref - limitCommits bool + gitLogLimit *git_commands.GitLogLimit refToShowDivergenceFrom string filterPath string filterAuthor string @@ -1012,7 +1024,7 @@ type capturedSubCommitState struct { func (self *RefreshHelper) captureSubCommitState() capturedSubCommitState { return capturedSubCommitState{ ref: self.c.Contexts().SubCommits.GetRef(), - limitCommits: self.c.Contexts().SubCommits.GetLimitCommits(), + gitLogLimit: snapshotGitLogLimit(self.c.Contexts().SubCommits.GetGitLogLimit()), refToShowDivergenceFrom: self.c.Contexts().SubCommits.GetRefToShowDivergenceFrom(), filterPath: self.c.Modes().Filtering.GetPath(), filterAuthor: self.c.Modes().Filtering.GetAuthor(), @@ -1028,7 +1040,7 @@ func (self *RefreshHelper) refreshSubCommitsWithLimit(captured capturedSubCommit commits, err := env.git.Loaders.CommitLoader.GetCommits( git_commands.GetCommitsOptions{ - Limit: captured.limitCommits, + LogLimit: captured.gitLogLimit, FilterPath: captured.filterPath, FilterAuthor: captured.filterAuthor, IncludeRebaseCommits: false, diff --git a/pkg/gui/controllers/helpers/refs_helper.go b/pkg/gui/controllers/helpers/refs_helper.go index 0fcbeace3..33afd83bd 100644 --- a/pkg/gui/controllers/helpers/refs_helper.go +++ b/pkg/gui/controllers/helpers/refs_helper.go @@ -41,7 +41,7 @@ func (self *RefsHelper) CheckoutRef(ref string, options types.CheckoutRefOptions refresh := func() { // loading a heap of commits is slow so we limit them whenever doing a reset - self.c.Contexts().LocalCommits.SetLimitCommits(true) + self.c.Contexts().LocalCommits.SetGitLogLimit(git_commands.DefaultGitLogLimit()) scope := []types.RefreshableView{ types.COMMITS, @@ -204,7 +204,7 @@ func (self *RefsHelper) ResetToRef(ref string, strength string, envVars []string } // loading a heap of commits is slow so we limit them whenever doing a reset - self.c.Contexts().LocalCommits.SetLimitCommits(true) + self.c.Contexts().LocalCommits.SetGitLogLimit(git_commands.DefaultGitLogLimit()) self.c.RefreshFromWorker(types.RefreshOptions{ Scope: []types.RefreshableView{types.FILES, types.BRANCHES, types.REFLOG, types.COMMITS}, diff --git a/pkg/gui/controllers/helpers/sub_commits_helper.go b/pkg/gui/controllers/helpers/sub_commits_helper.go index 09f32d1a9..0ee29aee8 100644 --- a/pkg/gui/controllers/helpers/sub_commits_helper.go +++ b/pkg/gui/controllers/helpers/sub_commits_helper.go @@ -34,7 +34,7 @@ type ViewSubCommitsOpts struct { func (self *SubCommitsHelper) ViewSubCommits(opts ViewSubCommitsOpts) error { commits, err := self.c.Git().Loaders.CommitLoader.GetCommits( git_commands.GetCommitsOptions{ - Limit: true, + LogLimit: git_commands.DefaultGitLogLimit(), FilterPath: self.c.Modes().Filtering.GetPath(), FilterAuthor: self.c.Modes().Filtering.GetAuthor(), IncludeRebaseCommits: false, @@ -59,7 +59,7 @@ func (self *SubCommitsHelper) ViewSubCommits(opts ViewSubCommitsOpts) error { subCommitsContext.SetTitleRef(utils.TruncateWithEllipsis(opts.TitleRef, 50)) subCommitsContext.SetRef(opts.Ref) subCommitsContext.SetRefToShowDivergenceFrom(opts.RefToShowDivergenceFrom) - subCommitsContext.SetLimitCommits(true) + subCommitsContext.SetGitLogLimit(git_commands.DefaultGitLogLimit()) subCommitsContext.SetShowBranchHeads(opts.ShowBranchHeads) subCommitsContext.ClearSearchString() subCommitsContext.GetView().ClearSearch() diff --git a/pkg/gui/controllers/local_commits_controller.go b/pkg/gui/controllers/local_commits_controller.go index 1e1a01427..f77a035b8 100644 --- a/pkg/gui/controllers/local_commits_controller.go +++ b/pkg/gui/controllers/local_commits_controller.go @@ -18,9 +18,6 @@ import ( "github.com/stefanhaller/git-todo-parser/todo" ) -// after selecting the 200th commit, we'll load in all the rest -const COMMIT_THRESHOLD = 200 - // How long a commit move may take before the drop indicator switches to a // "moving commits here" spinner; quick moves stay free of flicker. const commitDragMovingIndicatorDelay = 200 * time.Millisecond @@ -1679,8 +1676,8 @@ func (self *LocalCommitsController) createTag(commit *models.Commit) error { func (self *LocalCommitsController) openSearch() error { // we usually lazyload these commits but now that we're searching we need to load them now - if self.context().GetLimitCommits() { - self.context().SetLimitCommits(false) + if self.context().GetGitLogLimit() != nil { + self.context().SetGitLogLimit(nil) self.c.Refresh(types.RefreshOptions{Scope: []types.RefreshableView{types.COMMITS}}) } @@ -1698,7 +1695,9 @@ func (self *LocalCommitsController) handleOpenLogMenu() error { self.context().SetShowWholeGitGraph(!self.context().GetShowWholeGitGraph()) if self.context().GetShowWholeGitGraph() { - self.context().SetLimitCommits(false) + self.context().SetGitLogLimit(nil) + } else { + self.context().SetGitLogLimit(git_commands.DefaultGitLogLimit()) } return self.c.WithWaitingStatus(self.c.Tr.LoadingCommits, func(gocui.Task) error { @@ -1799,8 +1798,30 @@ func (self *LocalCommitsController) handleOpenLogMenu() error { func (self *LocalCommitsController) GetOnFocus() func(types.OnFocusOpts) { return func(types.OnFocusOpts) { context := self.context() - if context.GetSelectedLineIdx() > COMMIT_THRESHOLD && context.GetLimitCommits() { - context.SetLimitCommits(false) + limit := context.GetGitLogLimit() + + if limit == nil { + return + } + + lineIdx := context.GetSelectedLineIdx() + logCommitCount := len(self.c.Model().Commits) + + if self.isRebasing() { + rebaseCommitCount := lo.CountBy(self.c.Model().Commits, func(c *models.Commit) bool { + return c.IsTODO() + }) + + if lineIdx < rebaseCommitCount { + lineIdx = 0 + } else { + lineIdx -= rebaseCommitCount + } + logCommitCount -= rebaseCommitCount + } + + if limit.CanFetchMoreCommits(lineIdx, logCommitCount) { + limit.Increase(logCommitCount) self.c.Refresh(types.RefreshOptions{Scope: []types.RefreshableView{types.COMMITS}}) } } diff --git a/pkg/gui/controllers/sub_commits_controller.go b/pkg/gui/controllers/sub_commits_controller.go index d3d0c0b98..7d0796b5d 100644 --- a/pkg/gui/controllers/sub_commits_controller.go +++ b/pkg/gui/controllers/sub_commits_controller.go @@ -64,8 +64,16 @@ func (self *SubCommitsController) GetOnRenderToMain() func() { func (self *SubCommitsController) GetOnFocus() func(types.OnFocusOpts) { return func(types.OnFocusOpts) { context := self.context() - if context.GetSelectedLineIdx() > COMMIT_THRESHOLD && context.GetLimitCommits() { - context.SetLimitCommits(false) + limit := context.GetGitLogLimit() + + if limit == nil { + return + } + + logCommitCount := len(self.c.Model().SubCommits) + + if limit.CanFetchMoreCommits(context.GetSelectedLineIdx(), logCommitCount) { + limit.Increase(logCommitCount) self.c.Refresh(types.RefreshOptions{Scope: []types.RefreshableView{types.SUB_COMMITS}}) } }