From 3af95d6d07c98ed5a11bf0b584e595978c1dff20 Mon Sep 17 00:00:00 2001 From: Takumi Sueda Date: Wed, 8 Mar 2023 11:54:40 +0900 Subject: [PATCH 1/4] Bump Go version to 1.20.x --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e5149c3..ea32114 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go: [ '1.16' ] + go: [ '~1.20.2' ] name: Go ${{ matrix.go }} test steps: - name: Checkout repository From 1e185a785f79e1c2deb6875e2b5f3e2c2f883ea0 Mon Sep 17 00:00:00 2001 From: Takumi Sueda Date: Wed, 8 Mar 2023 23:31:49 +0900 Subject: [PATCH 2/4] Add FuzzyLongestSort flag --- interface.go | 2 ++ peco.go | 1 + 2 files changed, 3 insertions(+) diff --git a/interface.go b/interface.go index ec2ea3e..02e6fec 100644 --- a/interface.go +++ b/interface.go @@ -104,6 +104,7 @@ type Peco struct { skipReadConfig bool styles StyleSet use256Color bool + fuzzyLongestSort bool // Source is where we buffer input. It gets reused when a new query is // executed. @@ -299,6 +300,7 @@ type Config struct { QueryExecutionDelay int StickySelection bool MaxScanBufferSize int + FuzzyLongestSort bool // If this is true, then the prefix for single key jump mode // is displayed by default. diff --git a/peco.go b/peco.go index 966d2be..1ef92d3 100644 --- a/peco.go +++ b/peco.go @@ -568,6 +568,7 @@ func (p *Peco) ApplyConfig(opts CLIOptions) error { if len(p.initialFilter) <= 0 { p.initialFilter = opts.OptInitialMatcher } + p.fuzzyLongestSort = p.config.FuzzyLongestSort if err := p.populateFilters(); err != nil { return errors.Wrap(err, "failed to populate filters") From 8c3265f15964d38b24b680913fcd5c9357e47b5d Mon Sep 17 00:00:00 2001 From: Takumi Sueda Date: Mon, 6 Mar 2023 23:18:12 +0900 Subject: [PATCH 3/4] Add FuzzyLongest filter --- filter/filter_test.go | 254 +++++++++++++++++++++++++++++++++++++++++- filter/fuzzy.go | 202 +++++++++++++++++++++++++++------ filter/interface.go | 1 + peco.go | 2 +- 4 files changed, 425 insertions(+), 34 deletions(-) diff --git a/filter/filter_test.go b/filter/filter_test.go index d301b3d..8e4ea2f 100644 --- a/filter/filter_test.go +++ b/filter/filter_test.go @@ -15,11 +15,27 @@ type indexer interface { Indices() [][]int } -// TestFuzzy tests a fuzzy filter against various inputs +// TestFuzzy tests the Fuzzy filter against various inputs. +// +// testFuzzy: simple substring match +// testFuzzyLongest: sorted longest substring match +// testFuzzyMatch: match position without sorting +// testFuzzyLongestMatch: match position with sorting func TestFuzzy(t *testing.T) { octx, ocancel := context.WithCancel(context.Background()) defer ocancel() + testFuzzy(octx, t, NewFuzzy(false)) + testFuzzyLongest(octx, t, NewFuzzy(true)) + testFuzzyMatch(octx, t, NewFuzzy(false)) +} + +// testFuzzy tests if given filter matches/rejects the query. +// This test checks the following functionalities: +// - Fuzzy substring match +// - Case-insensitive match +// - Multi-byte rune match +func testFuzzy(octx context.Context, t *testing.T, filter Filter) { testValues := []struct { input string query string @@ -38,7 +54,6 @@ func TestFuzzy(t *testing.T) { {"🚴đŸģ abcd efgh", "🚴đŸģe", true}, // unicode {"This is a test to Test the fuzzy filteR", "TTR", true}, } - filter := NewFuzzy() for i, v := range testValues { t.Run(fmt.Sprintf(`"%s" against "%s", expect "%t"`, v.input, v.query, v.selected), func(t *testing.T) { ctx, cancel := context.WithTimeout(filter.NewContext(octx, v.query), 10*time.Second) @@ -70,3 +85,238 @@ func TestFuzzy(t *testing.T) { }) } } + +// testFuzzyLongest tests if given filter matches/rejects the query. +// This test check the following functionalities: +// - Longest substring match +// - Ordering the match result by precedence (substring length > match index > string length) +func testFuzzyLongest(octx context.Context, t *testing.T, filter Filter) { + testValues := []struct { + name string + query string + input []string + expect []string + }{ + { + name: "The longer the matched string, the higher it ranks", + query: "abcd", + input: []string{ + "abc-d", + "ab-cd", + "abcd", + "a-bcd", + }, + expect: []string{ + "abcd", + "abc-d", + "a-bcd", + "ab-cd", + }, + }, + { + name: "The earlier it matches, the higher it ranks", + query: "abcd", + input: []string{ + "___abcd", + "_abcd", + "abcd", + "__abcd", + }, + expect: []string{ + "abcd", + "_abcd", + "__abcd", + "___abcd", + }, + }, + { + name: "The shorter the original string, the higher it ranks", + query: "abcd", + input: []string{ + "abcdef", + "abcdefg", + "abcd", + "abcde", + }, + expect: []string{ + "abcd", + "abcde", + "abcdef", + "abcdefg", + }, + }, + { + name: "Mixed precedence", + query: "abcd", + input: []string{ + "abc-d", + "ab-cd", + "abcd", + "ab_abcd", + "a-bcd", + "___abcd", + "_abcd", + "abcd", + "__abcd", + "abcdef", + "abcdefg", + "abcd", + "abcde", + }, + expect: []string{ + "abcd", + "abcd", + "abcd", + "abcde", + "abcdef", + "abcdefg", + "_abcd", + "__abcd", + "ab_abcd", // ab_abcd shall be above ___abcd because matched lines are stable-sorted + "___abcd", + "abc-d", + "a-bcd", + "ab-cd", + }, + }, + } + + for i, v := range testValues { + t.Run(v.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(filter.NewContext(octx, v.query), 10*time.Second) + defer cancel() + + var lines []line.Line + for _, raw := range v.input { + lines = append(lines, line.NewRaw(uint64(i), raw, false)) + } + + var actual []string + lc := make(chan interface{}) + ec := make(chan error) + go func() { + ec <- filter.Apply(ctx, lines, lc) + }() + + OUTER: + for { + select { + case l := <-lc: + if !assert.Implements(t, (*line.Line)(nil), l, "result is a line") { + return + } + actual = append(actual, l.(line.Line).DisplayString()) + case err := <-ec: + if !assert.NoError(t, err, `filter.Apply should succeed`) { + return + } + break OUTER + case <-ctx.Done(): + t.Fatalf("unexpected timeout") + } + } + + if !assert.Equal(t, v.expect, actual, "result is ordered in expected order") { + return + } + }) + } +} + +// testFuzzyMatch tests if non-sorted & sorted Fuzzy filter returns the expected result +func testFuzzyMatch(octx context.Context, t *testing.T, filter Filter) { + testValues := []struct { + name string + sort bool + query string + input string + expect [][]int + }{ + { + name: "Fuzzy: exact match", + sort: false, + query: "asdf", + input: "___asdf", + // ^^^^ + expect: [][]int{ + {3, 4}, + {4, 5}, + {5, 6}, + {6, 7}, + }, + }, + { + name: "Fuzzy: scattered match", + sort: false, + query: "asdf", + input: "as_asdf", + // ^^ ^^ + expect: [][]int{ + {0, 1}, + {1, 2}, + {5, 6}, + {6, 7}, + }, + }, + { + name: "FuzzyLongest: exact match", + sort: true, + query: "asdf", + input: "___asdf", + // ^^^^ + expect: [][]int{ + {3, 4}, + {4, 5}, + {5, 6}, + {6, 7}, + }, + }, + { + name: "FuzzyLongest: scattered match", + sort: true, + query: "asdf", + input: "as_asdf", + // ^^^^ + expect: [][]int{ + {3, 4}, + {4, 5}, + {5, 6}, + {6, 7}, + }, + }, + } + + for i, v := range testValues { + t.Run(v.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(filter.NewContext(octx, v.query), 10*time.Second) + defer cancel() + + filter := NewFuzzy(v.sort) + lc := make(chan interface{}) + ec := make(chan error) + go func() { + ec <- filter.Apply(ctx, []line.Line{line.NewRaw(uint64(i), v.input, false)}, lc) + }() + + OUTER: + for { + select { + case l := <-lc: + if !assert.Implements(t, (*indexer)(nil), l, "result is an indexer") { + return + } + if !assert.Equal(t, v.expect, l.(indexer).Indices(), "result has expected indices") { + return + } + case err := <-ec: + if !assert.NoError(t, err, `filter.Apply should succeed`) { + return + } + break OUTER + case <-ctx.Done(): + t.Fatalf("unexpected timeout") + } + } + }) + } +} diff --git a/filter/fuzzy.go b/filter/fuzzy.go index e2c659a..9cb2ef2 100644 --- a/filter/fuzzy.go +++ b/filter/fuzzy.go @@ -2,7 +2,11 @@ package filter import ( "context" + "fmt" + "math" + "sort" "strings" + "unicode" "unicode/utf8" "github.com/peco/peco/internal/util" @@ -12,9 +16,17 @@ import ( // NewFuzzy builds a fuzzy-finder type of filter. // In effect, this uses a smart case filter, and for q query -// like "ABC" it matches the equivalent of "A(.*)B(.*)C(.*)" -func NewFuzzy() *Fuzzy { - return &Fuzzy{} +// like "ABC" it matches the equivalent of "A(.*)B(.*)C(.*)". +// +// With sortLongest = true, Fuzzy filter outputs the result +// sorted in the following precedence: +// 1. Longer match +// 2. Earlier match +// 3. Shorter line length +func NewFuzzy(sortLongest bool) *Fuzzy { + return &Fuzzy{ + sortLongest: sortLongest, + } } func (ff Fuzzy) BufSize() int { @@ -32,38 +44,166 @@ func (ff Fuzzy) String() string { func (ff *Fuzzy) Apply(ctx context.Context, lines []line.Line, out pipeline.ChanOutput) error { originalQuery := ctx.Value(queryKey).(string) hasUpper := util.ContainsUpper(originalQuery) + matched := []fuzzyMatchedItem{} -OUTER: +LINE: for _, l := range lines { - base := 0 - matches := [][]int{} - txt := l.DisplayString() - query := originalQuery - for len(query) > 0 { - r, n := utf8.DecodeRuneInString(query) - query = query[n:] - if r == utf8.RuneError { - // "Silently" ignore - continue OUTER + // Find the first valid rune of the query + firstRune := utf8.RuneError + for _, r := range originalQuery { + if r != utf8.RuneError { + firstRune = r + break } - - var i int - if hasUpper { // explicit match - i = strings.IndexRune(txt, r) - } else { - i = strings.IndexFunc(txt, util.CaseInsensitiveIndexFunc(r)) - } - if i == -1 { - continue OUTER - } - - // otherwise we have a match, but the next match must match against - // something AFTER the current match - txt = txt[i+n:] - matches = append(matches, []int{base + i, base + i + n}) - base = base + i + n } - out.Send(line.NewMatched(l, matches)) + if firstRune == utf8.RuneError { + return fmt.Errorf("the query has no valid character") + } + + // Find the index of the first valid rune in the input line + txt := l.DisplayString() + firstRuneOffsets := []int{} + accum := 0 + r := rune(0) + n := 0 + for len(txt) > 0 { + txt, r, n = popRune(txt) + found := false + if hasUpper { + found = r == firstRune + } else { + found = unicode.ToUpper(r) == unicode.ToUpper(firstRune) + } + if found { + firstRuneOffsets = append(firstRuneOffsets, accum) + + if !ff.sortLongest { + // Old behavior only sees the first match + break + } + } + accum += n + } + if len(firstRuneOffsets) == 0 { + continue LINE + } + + // Find all candidate matches + candidates := []fuzzyMatchedItem{} + + OUTER: + for _, offset := range firstRuneOffsets { + query := originalQuery + txt = l.DisplayString()[offset:] + base := offset + matches := [][]int{} + + for len(query) > 0 { + query, r, n = popRune(query) + if r == utf8.RuneError { + // "Silently" ignore + continue OUTER + } + + var i int + if hasUpper { + i = strings.IndexRune(txt, r) + } else { + i = strings.IndexFunc(txt, util.CaseInsensitiveIndexFunc(r)) + } + if i == -1 { + continue OUTER + } + + txt = txt[i+n:] + matches = append(matches, []int{base + i, base + i + n}) + base = base + i + n + } + + candidates = append(candidates, newFuzzyMatchedItem(l, matches)) + } + + if len(candidates) == 0 { + continue + } + + if ff.sortLongest { + // Sort the candidate matches of a line and pick the best one + sort.SliceStable(candidates, less(candidates)) + } + matched = append(matched, candidates[0]) } + + if ff.sortLongest { + // Sort all matched lines + sort.SliceStable(matched, less(matched)) + } + + for i := range matched { + out.Send(line.NewMatched(matched[i].line, matched[i].matches)) + } + return nil } + +func popRune(s string) (string, rune, int) { + r, n := utf8.DecodeRuneInString(s) + return s[n:], r, n +} + +func less(s []fuzzyMatchedItem) func(i, j int) bool { + return func(i, j int) bool { + if s[i].longest != s[j].longest { + // Longer match is better + return s[i].longest > s[j].longest + } else if s[i].earliest != s[j].earliest { + // Earlier match is better + return s[i].earliest < s[j].earliest + } else { + // Shorter line is better + return s[i].Len() < s[j].Len() + } + } +} + +type fuzzyMatchedItem struct { + line line.Line + matches [][]int + longest int + earliest int +} + +func newFuzzyMatchedItem(line line.Line, matches [][]int) fuzzyMatchedItem { + longest := 0 + count := 0 + lastEnd := 0 + earliest := math.MaxInt + + for i := range matches { + length := matches[i][1] - matches[i][0] + if matches[i][0] == lastEnd { + count += length + } else { + count = length + } + if count > longest { + longest = count + } + lastEnd = matches[i][1] + + if matches[i][0] < earliest { + earliest = matches[i][0] + } + } + + return fuzzyMatchedItem{ + line: line, + matches: matches, + longest: longest, + earliest: earliest, + } +} + +func (f fuzzyMatchedItem) Len() int { + return len(f.line.DisplayString()) +} diff --git a/filter/interface.go b/filter/interface.go index b256615..75f0916 100644 --- a/filter/interface.go +++ b/filter/interface.go @@ -48,6 +48,7 @@ type regexpQuery struct { } type Fuzzy struct { + sortLongest bool } type Regexp struct { diff --git a/peco.go b/peco.go index 1ef92d3..bb05919 100644 --- a/peco.go +++ b/peco.go @@ -624,7 +624,7 @@ func (p *Peco) populateFilters() error { p.filters.Add(filter.NewCaseSensitive()) p.filters.Add(filter.NewSmartCase()) p.filters.Add(filter.NewRegexp()) - p.filters.Add(filter.NewFuzzy()) + p.filters.Add(filter.NewFuzzy(p.fuzzyLongestSort)) for name, c := range p.config.CustomFilter { f := filter.NewExternalCmd(name, c.Cmd, c.Args, c.BufferThreshold, p.idgen, p.enableSep) From 06a80bf8a9782e483b1faa79c654d5abaaa53429 Mon Sep 17 00:00:00 2001 From: Takumi Sueda Date: Wed, 8 Mar 2023 23:46:19 +0900 Subject: [PATCH 4/4] Describe FuzzyLongestSort flag in the document --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a14137c..f1f711e 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ The SmartCase filter uses case-*insensitive* matching when all of the queries ar The Regexp filter allows you to use any valid regular expression to match lines. -The Fuzzy filter allows you to find matches using partial patterns. For example, when searching for `ALongString`, you can enable the Fuzzy filter and search `ALS` to find it. The Fuzzy filter uses smart case search like the SmartCase filter. +The Fuzzy filter allows you to find matches using partial patterns. For example, when searching for `ALongString`, you can enable the Fuzzy filter and search `ALS` to find it. The Fuzzy filter uses smart case search like the SmartCase filter. With the `FuzzyLongestSort` flag enabled in the configuration file, it does a smarter match. It sorts the matched lines by the following precedence: 1. longer substring, 2. earlier (left positioned) substring, and 3. shorter line. ![Executed `ps aux | peco`, then typed `google`, which matches the Chrome.app under IgnoreCase filter type. When you change it to Regexp filter, this is no longer the case. But you can type `(?i)google` instead to toggle case-insensitive mode](http://peco.github.io/images/peco-demo-matcher.gif) @@ -286,6 +286,12 @@ You can change the query line's prompt, which is `QUERY>` by default. Specifies the filter name to start peco with. You should specify the name of the filter, such as `IgnoreCase`, `CaseSensitive`, `SmartCase`, `Regexp` and `Fuzzy`. +### FuzzyLongestSort + +Enables the longest substring match and sorts the output. It affects only the Fuzzy filter. + +Default value for FuzzyLongestSort is false. + ### StickySelection ```json