Merge pull request #539 from puhitaku/puhitaku/fuzzylongest

Add FuzzyLongest filter

.github/workflows/ci.yml

@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go: [ '1.16' ]
+        go: [ '~1.20.2' ]
     name: Go ${{ matrix.go }} test
     steps:
       - name: Checkout repository

README.md

@@ -65,7 +65,7 @@ The SmartCase filter uses case-*insensitive* matching when all of the queries ar
 
 The Regexp filter allows you to use any valid regular expression to match lines.
 
-The Fuzzy filter allows you to find matches using partial patterns. For example, when searching for `ALongString`, you can enable the Fuzzy filter and search `ALS` to find it. The Fuzzy filter uses smart case search like the SmartCase filter.
+The Fuzzy filter allows you to find matches using partial patterns. For example, when searching for `ALongString`, you can enable the Fuzzy filter and search `ALS` to find it. The Fuzzy filter uses smart case search like the SmartCase filter. With the `FuzzyLongestSort` flag enabled in the configuration file, it does a smarter match. It sorts the matched lines by the following precedence: 1. longer substring, 2. earlier (left positioned) substring, and 3. shorter line.
 
 ![Executed `ps aux | peco`, then typed `google`, which matches the Chrome.app under IgnoreCase filter type. When you change it to Regexp filter, this is no longer the case. But you can type `(?i)google` instead to toggle case-insensitive mode](http://peco.github.io/images/peco-demo-matcher.gif)
 
@@ -286,6 +286,12 @@ You can change the query line's prompt, which is `QUERY>` by default.
 
 Specifies the filter name to start peco with. You should specify the name of the filter, such as `IgnoreCase`, `CaseSensitive`, `SmartCase`, `Regexp` and `Fuzzy`.
 
+### FuzzyLongestSort
+
+Enables the longest substring match and sorts the output. It affects only the Fuzzy filter.
+
+Default value for FuzzyLongestSort is false.
+
 ### StickySelection
 
 ```json

filter/filter_test.go

@@ -15,11 +15,27 @@ type indexer interface {
 	Indices() [][]int
 }
 
-// TestFuzzy tests a fuzzy filter against various inputs
+// TestFuzzy tests the Fuzzy filter against various inputs.
+//
+//	testFuzzy: simple substring match
+//	testFuzzyLongest: sorted longest substring match
+//	testFuzzyMatch: match position without sorting
+//	testFuzzyLongestMatch: match position with sorting
 func TestFuzzy(t *testing.T) {
 	octx, ocancel := context.WithCancel(context.Background())
 	defer ocancel()
 
+	testFuzzy(octx, t, NewFuzzy(false))
+	testFuzzyLongest(octx, t, NewFuzzy(true))
+	testFuzzyMatch(octx, t, NewFuzzy(false))
+}
+
+// testFuzzy tests if given filter matches/rejects the query.
+// This test checks the following functionalities:
+//   - Fuzzy substring match
+//   - Case-insensitive match
+//   - Multi-byte rune match
+func testFuzzy(octx context.Context, t *testing.T, filter Filter) {
 	testValues := []struct {
 		input    string
 		query    string
@@ -38,7 +54,6 @@ func TestFuzzy(t *testing.T) {
 		{"🚴🏻 abcd efgh", "🚴🏻e", true},                            // unicode
 		{"This is a test to Test the fuzzy filteR", "TTR", true},
 	}
-	filter := NewFuzzy()
 	for i, v := range testValues {
 		t.Run(fmt.Sprintf(`"%s" against "%s", expect "%t"`, v.input, v.query, v.selected), func(t *testing.T) {
 			ctx, cancel := context.WithTimeout(filter.NewContext(octx, v.query), 10*time.Second)
@@ -70,3 +85,238 @@ func TestFuzzy(t *testing.T) {
 		})
 	}
 }
+
+// testFuzzyLongest tests if given filter matches/rejects the query.
+// This test check the following functionalities:
+//   - Longest substring match
+//   - Ordering the match result by precedence (substring length > match index > string length)
+func testFuzzyLongest(octx context.Context, t *testing.T, filter Filter) {
+	testValues := []struct {
+		name   string
+		query  string
+		input  []string
+		expect []string
+	}{
+		{
+			name:  "The longer the matched string, the higher it ranks",
+			query: "abcd",
+			input: []string{
+				"abc-d",
+				"ab-cd",
+				"abcd",
+				"a-bcd",
+			},
+			expect: []string{
+				"abcd",
+				"abc-d",
+				"a-bcd",
+				"ab-cd",
+			},
+		},
+		{
+			name:  "The earlier it matches, the higher it ranks",
+			query: "abcd",
+			input: []string{
+				"___abcd",
+				"_abcd",
+				"abcd",
+				"__abcd",
+			},
+			expect: []string{
+				"abcd",
+				"_abcd",
+				"__abcd",
+				"___abcd",
+			},
+		},
+		{
+			name:  "The shorter the original string, the higher it ranks",
+			query: "abcd",
+			input: []string{
+				"abcdef",
+				"abcdefg",
+				"abcd",
+				"abcde",
+			},
+			expect: []string{
+				"abcd",
+				"abcde",
+				"abcdef",
+				"abcdefg",
+			},
+		},
+		{
+			name:  "Mixed precedence",
+			query: "abcd",
+			input: []string{
+				"abc-d",
+				"ab-cd",
+				"abcd",
+				"ab_abcd",
+				"a-bcd",
+				"___abcd",
+				"_abcd",
+				"abcd",
+				"__abcd",
+				"abcdef",
+				"abcdefg",
+				"abcd",
+				"abcde",
+			},
+			expect: []string{
+				"abcd",
+				"abcd",
+				"abcd",
+				"abcde",
+				"abcdef",
+				"abcdefg",
+				"_abcd",
+				"__abcd",
+				"ab_abcd", // ab_abcd shall be above ___abcd because matched lines are stable-sorted
+				"___abcd",
+				"abc-d",
+				"a-bcd",
+				"ab-cd",
+			},
+		},
+	}
+
+	for i, v := range testValues {
+		t.Run(v.name, func(t *testing.T) {
+			ctx, cancel := context.WithTimeout(filter.NewContext(octx, v.query), 10*time.Second)
+			defer cancel()
+
+			var lines []line.Line
+			for _, raw := range v.input {
+				lines = append(lines, line.NewRaw(uint64(i), raw, false))
+			}
+
+			var actual []string
+			lc := make(chan interface{})
+			ec := make(chan error)
+			go func() {
+				ec <- filter.Apply(ctx, lines, lc)
+			}()
+
+		OUTER:
+			for {
+				select {
+				case l := <-lc:
+					if !assert.Implements(t, (*line.Line)(nil), l, "result is a line") {
+						return
+					}
+					actual = append(actual, l.(line.Line).DisplayString())
+				case err := <-ec:
+					if !assert.NoError(t, err, `filter.Apply should succeed`) {
+						return
+					}
+					break OUTER
+				case <-ctx.Done():
+					t.Fatalf("unexpected timeout")
+				}
+			}
+
+			if !assert.Equal(t, v.expect, actual, "result is ordered in expected order") {
+				return
+			}
+		})
+	}
+}
+
+// testFuzzyMatch tests if non-sorted & sorted Fuzzy filter returns the expected result
+func testFuzzyMatch(octx context.Context, t *testing.T, filter Filter) {
+	testValues := []struct {
+		name   string
+		sort   bool
+		query  string
+		input  string
+		expect [][]int
+	}{
+		{
+			name:  "Fuzzy: exact match",
+			sort:  false,
+			query: "asdf",
+			input: "___asdf",
+			//         ^^^^
+			expect: [][]int{
+				{3, 4},
+				{4, 5},
+				{5, 6},
+				{6, 7},
+			},
+		},
+		{
+			name:  "Fuzzy: scattered match",
+			sort:  false,
+			query: "asdf",
+			input: "as_asdf",
+			//      ^^   ^^
+			expect: [][]int{
+				{0, 1},
+				{1, 2},
+				{5, 6},
+				{6, 7},
+			},
+		},
+		{
+			name:  "FuzzyLongest: exact match",
+			sort:  true,
+			query: "asdf",
+			input: "___asdf",
+			//         ^^^^
+			expect: [][]int{
+				{3, 4},
+				{4, 5},
+				{5, 6},
+				{6, 7},
+			},
+		},
+		{
+			name:  "FuzzyLongest: scattered match",
+			sort:  true,
+			query: "asdf",
+			input: "as_asdf",
+			//         ^^^^
+			expect: [][]int{
+				{3, 4},
+				{4, 5},
+				{5, 6},
+				{6, 7},
+			},
+		},
+	}
+
+	for i, v := range testValues {
+		t.Run(v.name, func(t *testing.T) {
+			ctx, cancel := context.WithTimeout(filter.NewContext(octx, v.query), 10*time.Second)
+			defer cancel()
+
+			filter := NewFuzzy(v.sort)
+			lc := make(chan interface{})
+			ec := make(chan error)
+			go func() {
+				ec <- filter.Apply(ctx, []line.Line{line.NewRaw(uint64(i), v.input, false)}, lc)
+			}()
+
+		OUTER:
+			for {
+				select {
+				case l := <-lc:
+					if !assert.Implements(t, (*indexer)(nil), l, "result is an indexer") {
+						return
+					}
+					if !assert.Equal(t, v.expect, l.(indexer).Indices(), "result has expected indices") {
+						return
+					}
+				case err := <-ec:
+					if !assert.NoError(t, err, `filter.Apply should succeed`) {
+						return
+					}
+					break OUTER
+				case <-ctx.Done():
+					t.Fatalf("unexpected timeout")
+				}
+			}
+		})
+	}
+}

filter/fuzzy.go

@@ -2,7 +2,11 @@ package filter
 
 import (
 	"context"
+	"fmt"
+	"math"
+	"sort"
 	"strings"
+	"unicode"
 	"unicode/utf8"
 
 	"github.com/peco/peco/internal/util"
@@ -12,9 +16,17 @@ import (
 
 // NewFuzzy builds a fuzzy-finder type of filter.
 // In effect, this uses a smart case filter, and for q query
-// like "ABC" it matches the equivalent of "A(.*)B(.*)C(.*)"
-func NewFuzzy() *Fuzzy {
-	return &Fuzzy{}
+// like "ABC" it matches the equivalent of "A(.*)B(.*)C(.*)".
+//
+// With sortLongest = true, Fuzzy filter outputs the result
+// sorted in the following precedence:
+//  1. Longer match
+//  2. Earlier match
+//  3. Shorter line length
+func NewFuzzy(sortLongest bool) *Fuzzy {
+	return &Fuzzy{
+		sortLongest: sortLongest,
+	}
 }
 
 func (ff Fuzzy) BufSize() int {
@@ -32,38 +44,166 @@ func (ff Fuzzy) String() string {
 func (ff *Fuzzy) Apply(ctx context.Context, lines []line.Line, out pipeline.ChanOutput) error {
 	originalQuery := ctx.Value(queryKey).(string)
 	hasUpper := util.ContainsUpper(originalQuery)
+	matched := []fuzzyMatchedItem{}
 
-OUTER:
+LINE:
 	for _, l := range lines {
-		base := 0
-		matches := [][]int{}
-		txt := l.DisplayString()
-		query := originalQuery
-		for len(query) > 0 {
-			r, n := utf8.DecodeRuneInString(query)
-			query = query[n:]
-			if r == utf8.RuneError {
-				// "Silently" ignore
-				continue OUTER
+		// Find the first valid rune of the query
+		firstRune := utf8.RuneError
+		for _, r := range originalQuery {
+			if r != utf8.RuneError {
+				firstRune = r
+				break
 			}
-
-			var i int
-			if hasUpper { // explicit match
-				i = strings.IndexRune(txt, r)
-			} else {
-				i = strings.IndexFunc(txt, util.CaseInsensitiveIndexFunc(r))
-			}
-			if i == -1 {
-				continue OUTER
-			}
-
-			// otherwise we have a match, but the next match must match against
-			// something AFTER the current match
-			txt = txt[i+n:]
-			matches = append(matches, []int{base + i, base + i + n})
-			base = base + i + n
 		}
-		out.Send(line.NewMatched(l, matches))
+		if firstRune == utf8.RuneError {
+			return fmt.Errorf("the query has no valid character")
+		}
+
+		// Find the index of the first valid rune in the input line
+		txt := l.DisplayString()
+		firstRuneOffsets := []int{}
+		accum := 0
+		r := rune(0)
+		n := 0
+		for len(txt) > 0 {
+			txt, r, n = popRune(txt)
+			found := false
+			if hasUpper {
+				found = r == firstRune
+			} else {
+				found = unicode.ToUpper(r) == unicode.ToUpper(firstRune)
+			}
+			if found {
+				firstRuneOffsets = append(firstRuneOffsets, accum)
+
+				if !ff.sortLongest {
+					// Old behavior only sees the first match
+					break
+				}
+			}
+			accum += n
+		}
+		if len(firstRuneOffsets) == 0 {
+			continue LINE
+		}
+
+		// Find all candidate matches
+		candidates := []fuzzyMatchedItem{}
+
+	OUTER:
+		for _, offset := range firstRuneOffsets {
+			query := originalQuery
+			txt = l.DisplayString()[offset:]
+			base := offset
+			matches := [][]int{}
+
+			for len(query) > 0 {
+				query, r, n = popRune(query)
+				if r == utf8.RuneError {
+					// "Silently" ignore
+					continue OUTER
+				}
+
+				var i int
+				if hasUpper {
+					i = strings.IndexRune(txt, r)
+				} else {
+					i = strings.IndexFunc(txt, util.CaseInsensitiveIndexFunc(r))
+				}
+				if i == -1 {
+					continue OUTER
+				}
+
+				txt = txt[i+n:]
+				matches = append(matches, []int{base + i, base + i + n})
+				base = base + i + n
+			}
+
+			candidates = append(candidates, newFuzzyMatchedItem(l, matches))
+		}
+
+		if len(candidates) == 0 {
+			continue
+		}
+
+		if ff.sortLongest {
+			// Sort the candidate matches of a line and pick the best one
+			sort.SliceStable(candidates, less(candidates))
+		}
+		matched = append(matched, candidates[0])
 	}
+
+	if ff.sortLongest {
+		// Sort all matched lines
+		sort.SliceStable(matched, less(matched))
+	}
+
+	for i := range matched {
+		out.Send(line.NewMatched(matched[i].line, matched[i].matches))
+	}
+
 	return nil
 }
+
+func popRune(s string) (string, rune, int) {
+	r, n := utf8.DecodeRuneInString(s)
+	return s[n:], r, n
+}
+
+func less(s []fuzzyMatchedItem) func(i, j int) bool {
+	return func(i, j int) bool {
+		if s[i].longest != s[j].longest {
+			// Longer match is better
+			return s[i].longest > s[j].longest
+		} else if s[i].earliest != s[j].earliest {
+			// Earlier match is better
+			return s[i].earliest < s[j].earliest
+		} else {
+			// Shorter line is better
+			return s[i].Len() < s[j].Len()
+		}
+	}
+}
+
+type fuzzyMatchedItem struct {
+	line     line.Line
+	matches  [][]int
+	longest  int
+	earliest int
+}
+
+func newFuzzyMatchedItem(line line.Line, matches [][]int) fuzzyMatchedItem {
+	longest := 0
+	count := 0
+	lastEnd := 0
+	earliest := math.MaxInt
+
+	for i := range matches {
+		length := matches[i][1] - matches[i][0]
+		if matches[i][0] == lastEnd {
+			count += length
+		} else {
+			count = length
+		}
+		if count > longest {
+			longest = count
+		}
+		lastEnd = matches[i][1]
+
+		if matches[i][0] < earliest {
+			earliest = matches[i][0]
+		}
+	}
+
+	return fuzzyMatchedItem{
+		line:     line,
+		matches:  matches,
+		longest:  longest,
+		earliest: earliest,
+	}
+}
+
+func (f fuzzyMatchedItem) Len() int {
+	return len(f.line.DisplayString())
+}

filter/interface.go

@@ -48,6 +48,7 @@ type regexpQuery struct {
 }
 
 type Fuzzy struct {
+	sortLongest bool
 }
 
 type Regexp struct {

interface.go

@@ -104,6 +104,7 @@ type Peco struct {
 	skipReadConfig          bool
 	styles                  StyleSet
 	use256Color             bool
+	fuzzyLongestSort        bool
 
 	// Source is where we buffer input. It gets reused when a new query is
 	// executed.
@@ -299,6 +300,7 @@ type Config struct {
 	QueryExecutionDelay int
 	StickySelection     bool
 	MaxScanBufferSize   int
+	FuzzyLongestSort    bool
 
 	// If this is true, then the prefix for single key jump mode
 	// is displayed by default.

peco.go

@@ -568,6 +568,7 @@ func (p *Peco) ApplyConfig(opts CLIOptions) error {
 	if len(p.initialFilter) <= 0 {
 		p.initialFilter = opts.OptInitialMatcher
 	}
+	p.fuzzyLongestSort = p.config.FuzzyLongestSort
 
 	if err := p.populateFilters(); err != nil {
 		return errors.Wrap(err, "failed to populate filters")
@@ -623,7 +624,7 @@ func (p *Peco) populateFilters() error {
 	p.filters.Add(filter.NewCaseSensitive())
 	p.filters.Add(filter.NewSmartCase())
 	p.filters.Add(filter.NewRegexp())
-	p.filters.Add(filter.NewFuzzy())
+	p.filters.Add(filter.NewFuzzy(p.fuzzyLongestSort))
 
 	for name, c := range p.config.CustomFilter {
 		f := filter.NewExternalCmd(name, c.Cmd, c.Args, c.BufferThreshold, p.idgen, p.enableSep)