Commits: add incremental git log limit

The existing git log logic fetched the first 300 commits of a repo and
displayed them in the local and sub-commit views. Once a user selected a
commit beyond a threshold of 200 commits the whole repository was
loaded. This is problematic with large repos e.g. the linux kernel with
currently ~138k commits as lazygit slows down substantially with such a
large number of commits in memory.

This commit replaces the current all or only the first 300 commits logic
with an incremental fetching approach:

1. The first 400 commits of repo are loaded by default.
2. If the user selects a commit beyond a threshold (current limit-100)
   the git log limit is increased by 400 if there are more commits
   available. If there are more commits available is currently checked
   by comparing the previous log limit with the real commit count in the
   model, if the commit count is less then the limit it is assumed that
   we reached the end of the commit log. Ideally it would be better to
   call `git rev-list --count xyz` in the right places and compare with
   this result, but this requires more changes.

Adding a "paginated implementation" by utilizing `git log --skip=x
--max-count=y` and appending commits to the model instead of replacing
the whole collection would be nice, but this requires deeper changes to
keep everything consistent.

Signed-off-by: Stefan Kerkmann <s.kerkmann@pengutronix.de>

Co-authored-by: DeepSeek V4 Flash

pkg/commands/git_commands/commit_loader.go

@@ -55,8 +55,29 @@ func NewCommitLoader(
 	}
 }
 
+const (
+	GIT_LOG_FETCH_COUNT     = 400
+	GIT_LOG_FETCH_THRESHOLD = GIT_LOG_FETCH_COUNT / 4
+)
+
+type GitLogLimit struct {
+	Limit int
+}
+
+func DefaultGitLogLimit() *GitLogLimit {
+	return &GitLogLimit{Limit: GIT_LOG_FETCH_COUNT}
+}
+
+func (self *GitLogLimit) CanFetchMoreCommits(lineIdx, logCommitCount int) bool {
+	return lineIdx >= logCommitCount-GIT_LOG_FETCH_THRESHOLD && logCommitCount >= self.Limit
+}
+
+func (self *GitLogLimit) Increase(realCommitCount int) {
+	self.Limit = realCommitCount + GIT_LOG_FETCH_COUNT
+}
+
 type GetCommitsOptions struct {
-	Limit                bool
+	LogLimit             *GitLogLimit
 	FilterPath           string
 	FilterAuthor         string
 	IncludeRebaseCommits bool
@@ -603,6 +624,11 @@ func (self *CommitLoader) getLogCmd(opts GetCommitsOptions) *oscommands.CmdObj {
 		refSpec += "..." + opts.RefToShowDivergenceFrom
 	}
 
+	var limitArg string
+	if opts.LogLimit != nil {
+		limitArg = fmt.Sprintf("--max-count=%d", opts.LogLimit.Limit)
+	}
+
 	cmdArgs := NewGitCmd("log").
 		Arg(refSpec).
 		ArgIf(gitLogOrder != "default", "--"+gitLogOrder).
@@ -611,7 +637,7 @@ func (self *CommitLoader) getLogCmd(opts GetCommitsOptions) *oscommands.CmdObj {
 		Arg(prettyFormat).
 		Arg("--abbrev=40").
 		ArgIf(opts.FilterAuthor != "", "--author="+opts.FilterAuthor).
-		ArgIf(opts.Limit, "-300").
+		ArgIf(limitArg != "", limitArg).
 		ArgIf(opts.FilterPath != "", "--follow", "--name-status").
 		Arg("--no-show-signature").
 		ArgIf(opts.RefToShowDivergenceFrom != "", "--left-right").

pkg/gui/context/local_commits_context.go

@@ -8,6 +8,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/jesseduffield/lazygit/pkg/commands/git_commands"
 	"github.com/jesseduffield/lazygit/pkg/commands/models"
 	"github.com/jesseduffield/lazygit/pkg/config"
 	"github.com/jesseduffield/lazygit/pkg/gocui"
@@ -217,11 +218,12 @@ func (self *LocalCommitsContext) ClearDropInsertionIndex() {
 type LocalCommitsViewModel struct {
 	*ListViewModel[*models.Commit]
 
-	// If this is true we limit the amount of commits we load, for the sake of keeping things fast.
+	// If this is non-nil we limit the amount of commits we load, for the sake of keeping things fast.
 	// If the user attempts to scroll past the end of the list, we will load more commits.
+	// A nil value means no limit; we load the whole log.
 	// Atomic because a checkout or reset sets it from a worker goroutine while the
 	// commits refresh reads it on the UI thread to decide how many commits to load.
-	limitCommits atomic.Bool
+	gitLogLimit atomic.Pointer[git_commands.GitLogLimit]
 
 	// If this is true we'll use git log --all when fetching the commits.
 	showWholeGitGraph bool
@@ -232,7 +234,7 @@ func NewLocalCommitsViewModel(getModel func() []*models.Commit, c *ContextCommon
 		ListViewModel:     NewListViewModel(getModel),
 		showWholeGitGraph: c.UserConfig().Git.Log.ShowWholeGraph,
 	}
-	self.limitCommits.Store(true)
+	self.gitLogLimit.Store(git_commands.DefaultGitLogLimit())
 
 	return self
 }
@@ -303,12 +305,12 @@ func (self *LocalCommitsContext) ModelSearchResults(searchStr string, caseSensit
 	return searchModelCommits(caseSensitive, self.GetCommits(), self.ColumnPositions(), self.modelToViewIndexConverter(), searchStr)
 }
 
-func (self *LocalCommitsViewModel) SetLimitCommits(value bool) {
-	self.limitCommits.Store(value)
+func (self *LocalCommitsViewModel) SetGitLogLimit(limit *git_commands.GitLogLimit) {
+	self.gitLogLimit.Store(limit)
 }
 
-func (self *LocalCommitsViewModel) GetLimitCommits() bool {
-	return self.limitCommits.Load()
+func (self *LocalCommitsViewModel) GetGitLogLimit() *git_commands.GitLogLimit {
+	return self.gitLogLimit.Load()
 }
 
 func (self *LocalCommitsViewModel) SetShowWholeGitGraph(value bool) {

pkg/gui/context/sub_commits_context.go

@@ -33,8 +33,8 @@ func NewSubCommitsContext(
 		ListViewModel: NewListViewModel(
 			func() []*models.Commit { return c.Model().SubCommits },
 		),
-		ref:          nil,
-		limitCommits: true,
+		ref:         nil,
+		gitLogLimit: git_commands.DefaultGitLogLimit(),
 	}
 
 	getDisplayStrings := func(startIdx int, endIdx int) [][]string {
@@ -142,7 +142,7 @@ type SubCommitsViewModel struct {
 	refToShowDivergenceFrom string
 	*ListViewModel[*models.Commit]
 
-	limitCommits    bool
+	gitLogLimit     *git_commands.GitLogLimit
 	showBranchHeads bool
 }
 
@@ -199,12 +199,12 @@ func (self *SubCommitsContext) GetCommits() []*models.Commit {
 	return self.getModel()
 }
 
-func (self *SubCommitsContext) SetLimitCommits(value bool) {
-	self.limitCommits = value
+func (self *SubCommitsContext) SetGitLogLimit(limit *git_commands.GitLogLimit) {
+	self.gitLogLimit = limit
 }
 
-func (self *SubCommitsContext) GetLimitCommits() bool {
-	return self.limitCommits
+func (self *SubCommitsContext) GetGitLogLimit() *git_commands.GitLogLimit {
+	return self.gitLogLimit
 }
 
 func (self *SubCommitsContext) GetDiffTerminals() []string {

pkg/gui/controllers/helpers/refresh_helper.go

@@ -759,7 +759,7 @@ func (self *RefreshHelper) refreshReflogAndBranches(capturedReflog capturedReflo
 // worker computes from an immutable snapshot rather than reading state the UI
 // thread concurrently mutates.
 type capturedCommitState struct {
-	limitCommits         bool
+	gitLogLimit          *git_commands.GitLogLimit
 	showWholeGitGraph    bool
 	filterPath           string
 	filterAuthor         string
@@ -768,6 +768,18 @@ type capturedCommitState struct {
 	parentIsLocalCommits bool
 }
 
+// snapshotGitLogLimit returns a copy of the given git log limit, or nil if
+// there is no limit. The limit object lives on the context, where the UI
+// thread bumps it when the user scrolls close to the end of the commit list;
+// the refresh's git work on a worker goroutine must read a copy taken at
+// capture time rather than the live object.
+func snapshotGitLogLimit(limit *git_commands.GitLogLimit) *git_commands.GitLogLimit {
+	if limit == nil {
+		return nil
+	}
+	return &git_commands.GitLogLimit{Limit: limit.Limit}
+}
+
 // captureCommitsState reads the commits refresh's model/context/mode inputs
 // into an immutable snapshot. It must run on the UI thread.
 // The selection is captured later, when applying the refresh, so user input
@@ -776,7 +788,7 @@ func (self *RefreshHelper) captureCommitsState() capturedCommitState {
 	parentCtx := self.c.Contexts().CommitFiles.GetParentContext()
 
 	return capturedCommitState{
-		limitCommits:         self.c.Contexts().LocalCommits.GetLimitCommits(),
+		gitLogLimit:          snapshotGitLogLimit(self.c.Contexts().LocalCommits.GetGitLogLimit()),
 		showWholeGitGraph:    self.c.Contexts().LocalCommits.GetShowWholeGitGraph(),
 		filterPath:           self.c.Modes().Filtering.GetPath(),
 		filterAuthor:         self.c.Modes().Filtering.GetAuthor(),
@@ -847,7 +859,7 @@ func (self *RefreshHelper) refreshCommitsWithLimit(captured capturedCommitState,
 	refName, bisectInfo := self.refForLog(env)
 	commits, err := env.git.Loaders.CommitLoader.GetCommits(
 		git_commands.GetCommitsOptions{
-			Limit:                captured.limitCommits,
+			LogLimit:             captured.gitLogLimit,
 			FilterPath:           captured.filterPath,
 			FilterAuthor:         captured.filterAuthor,
 			IncludeRebaseCommits: true,
@@ -999,7 +1011,7 @@ func findNewConflictedCommit(previousCommits []*models.Commit, commits []*models
 // work is dispatched to a worker.
 type capturedSubCommitState struct {
 	ref                     models.Ref
-	limitCommits            bool
+	gitLogLimit             *git_commands.GitLogLimit
 	refToShowDivergenceFrom string
 	filterPath              string
 	filterAuthor            string
@@ -1012,7 +1024,7 @@ type capturedSubCommitState struct {
 func (self *RefreshHelper) captureSubCommitState() capturedSubCommitState {
 	return capturedSubCommitState{
 		ref:                     self.c.Contexts().SubCommits.GetRef(),
-		limitCommits:            self.c.Contexts().SubCommits.GetLimitCommits(),
+		gitLogLimit:             snapshotGitLogLimit(self.c.Contexts().SubCommits.GetGitLogLimit()),
 		refToShowDivergenceFrom: self.c.Contexts().SubCommits.GetRefToShowDivergenceFrom(),
 		filterPath:              self.c.Modes().Filtering.GetPath(),
 		filterAuthor:            self.c.Modes().Filtering.GetAuthor(),
@@ -1028,7 +1040,7 @@ func (self *RefreshHelper) refreshSubCommitsWithLimit(captured capturedSubCommit
 
 	commits, err := env.git.Loaders.CommitLoader.GetCommits(
 		git_commands.GetCommitsOptions{
-			Limit:                   captured.limitCommits,
+			LogLimit:                captured.gitLogLimit,
 			FilterPath:              captured.filterPath,
 			FilterAuthor:            captured.filterAuthor,
 			IncludeRebaseCommits:    false,

pkg/gui/controllers/helpers/refs_helper.go

@@ -41,7 +41,7 @@ func (self *RefsHelper) CheckoutRef(ref string, options types.CheckoutRefOptions
 
 	refresh := func() {
 		// loading a heap of commits is slow so we limit them whenever doing a reset
-		self.c.Contexts().LocalCommits.SetLimitCommits(true)
+		self.c.Contexts().LocalCommits.SetGitLogLimit(git_commands.DefaultGitLogLimit())
 
 		scope := []types.RefreshableView{
 			types.COMMITS,
@@ -204,7 +204,7 @@ func (self *RefsHelper) ResetToRef(ref string, strength string, envVars []string
 	}
 
 	// loading a heap of commits is slow so we limit them whenever doing a reset
-	self.c.Contexts().LocalCommits.SetLimitCommits(true)
+	self.c.Contexts().LocalCommits.SetGitLogLimit(git_commands.DefaultGitLogLimit())
 
 	self.c.RefreshFromWorker(types.RefreshOptions{
 		Scope:                 []types.RefreshableView{types.FILES, types.BRANCHES, types.REFLOG, types.COMMITS},

pkg/gui/controllers/helpers/sub_commits_helper.go

@@ -34,7 +34,7 @@ type ViewSubCommitsOpts struct {
 func (self *SubCommitsHelper) ViewSubCommits(opts ViewSubCommitsOpts) error {
 	commits, err := self.c.Git().Loaders.CommitLoader.GetCommits(
 		git_commands.GetCommitsOptions{
-			Limit:                   true,
+			LogLimit:                git_commands.DefaultGitLogLimit(),
 			FilterPath:              self.c.Modes().Filtering.GetPath(),
 			FilterAuthor:            self.c.Modes().Filtering.GetAuthor(),
 			IncludeRebaseCommits:    false,
@@ -59,7 +59,7 @@ func (self *SubCommitsHelper) ViewSubCommits(opts ViewSubCommitsOpts) error {
 	subCommitsContext.SetTitleRef(utils.TruncateWithEllipsis(opts.TitleRef, 50))
 	subCommitsContext.SetRef(opts.Ref)
 	subCommitsContext.SetRefToShowDivergenceFrom(opts.RefToShowDivergenceFrom)
-	subCommitsContext.SetLimitCommits(true)
+	subCommitsContext.SetGitLogLimit(git_commands.DefaultGitLogLimit())
 	subCommitsContext.SetShowBranchHeads(opts.ShowBranchHeads)
 	subCommitsContext.ClearSearchString()
 	subCommitsContext.GetView().ClearSearch()

pkg/gui/controllers/local_commits_controller.go

@@ -18,9 +18,6 @@ import (
 	"github.com/stefanhaller/git-todo-parser/todo"
 )
 
-// after selecting the 200th commit, we'll load in all the rest
-const COMMIT_THRESHOLD = 200
-
 // How long a commit move may take before the drop indicator switches to a
 // "moving commits here" spinner; quick moves stay free of flicker.
 const commitDragMovingIndicatorDelay = 200 * time.Millisecond
@@ -1679,8 +1676,8 @@ func (self *LocalCommitsController) createTag(commit *models.Commit) error {
 
 func (self *LocalCommitsController) openSearch() error {
 	// we usually lazyload these commits but now that we're searching we need to load them now
-	if self.context().GetLimitCommits() {
-		self.context().SetLimitCommits(false)
+	if self.context().GetGitLogLimit() != nil {
+		self.context().SetGitLogLimit(nil)
 		self.c.Refresh(types.RefreshOptions{Scope: []types.RefreshableView{types.COMMITS}})
 	}
 
@@ -1698,7 +1695,9 @@ func (self *LocalCommitsController) handleOpenLogMenu() error {
 					self.context().SetShowWholeGitGraph(!self.context().GetShowWholeGitGraph())
 
 					if self.context().GetShowWholeGitGraph() {
-						self.context().SetLimitCommits(false)
+						self.context().SetGitLogLimit(nil)
+					} else {
+						self.context().SetGitLogLimit(git_commands.DefaultGitLogLimit())
 					}
 
 					return self.c.WithWaitingStatus(self.c.Tr.LoadingCommits, func(gocui.Task) error {
@@ -1799,8 +1798,30 @@ func (self *LocalCommitsController) handleOpenLogMenu() error {
 func (self *LocalCommitsController) GetOnFocus() func(types.OnFocusOpts) {
 	return func(types.OnFocusOpts) {
 		context := self.context()
-		if context.GetSelectedLineIdx() > COMMIT_THRESHOLD && context.GetLimitCommits() {
-			context.SetLimitCommits(false)
+		limit := context.GetGitLogLimit()
+
+		if limit == nil {
+			return
+		}
+
+		lineIdx := context.GetSelectedLineIdx()
+		logCommitCount := len(self.c.Model().Commits)
+
+		if self.isRebasing() {
+			rebaseCommitCount := lo.CountBy(self.c.Model().Commits, func(c *models.Commit) bool {
+				return c.IsTODO()
+			})
+
+			if lineIdx < rebaseCommitCount {
+				lineIdx = 0
+			} else {
+				lineIdx -= rebaseCommitCount
+			}
+			logCommitCount -= rebaseCommitCount
+		}
+
+		if limit.CanFetchMoreCommits(lineIdx, logCommitCount) {
+			limit.Increase(logCommitCount)
 			self.c.Refresh(types.RefreshOptions{Scope: []types.RefreshableView{types.COMMITS}})
 		}
 	}

pkg/gui/controllers/sub_commits_controller.go

@@ -64,8 +64,16 @@ func (self *SubCommitsController) GetOnRenderToMain() func() {
 func (self *SubCommitsController) GetOnFocus() func(types.OnFocusOpts) {
 	return func(types.OnFocusOpts) {
 		context := self.context()
-		if context.GetSelectedLineIdx() > COMMIT_THRESHOLD && context.GetLimitCommits() {
-			context.SetLimitCommits(false)
+		limit := context.GetGitLogLimit()
+
+		if limit == nil {
+			return
+		}
+
+		logCommitCount := len(self.c.Model().SubCommits)
+
+		if limit.CanFetchMoreCommits(context.GetSelectedLineIdx(), logCommitCount) {
+			limit.Increase(logCommitCount)
 			self.c.Refresh(types.RefreshOptions{Scope: []types.RefreshableView{types.SUB_COMMITS}})
 		}
 	}