diff --git a/Changes b/Changes index 1e5b5c7..e82f510 100644 --- a/Changes +++ b/Changes @@ -2,6 +2,8 @@ Changes ======= v0.4.6 - Not yet released + Features: + * A new fuzzy filter has now been added. See README for details (#369, #370) Bugs/Fixes * A very subtle timing issue that causes the search to be reset has been resolved (#364) diff --git a/README.md b/README.md index 4c30cdc..16adc65 100644 --- a/README.md +++ b/README.md @@ -58,13 +58,15 @@ Not only can you select multiple lines one by one, you can select a range of lin ## Select Filters -Different types of filters are available. Default is case-insensitive filter, so lines with any case will match. You can toggle between IgnoreCase, CaseSensitive, SmartCase and RegExp filters. +Different types of filters are available. Default is case-insensitive filter, so lines with any case will match. You can toggle between IgnoreCase, CaseSensitive, SmartCase RegExp and Fuzzy filters. The SmartCase filter uses case-*insensitive* matching when all of the queries are lower case, and case-*sensitive* matching otherwise. The RegExp filter allows you to use any valid regular expression to match lines -![Executed `ps aux | peco`, then typed `google`, which matches the Chrome.app under IgnoreCase filter type. Whenyou change it to Regexp filter, this is no longer the case. But you can type `(?i)google` instead to toggle case-insensitive mode](http://peco.github.io/images/peco-demo-matcher.gif) +The Fuzzy filter allows you to find matches using partial patterns. For example, when searching for `ALongString`, you can enable the Fuzzy filter and search `ALS` to find it. The Fuzzy filter uses smart case search like the SmartCase filter. + +![Executed `ps aux | peco`, then typed `google`, which matches the Chrome.app under IgnoreCase filter type. When you change it to Regexp filter, this is no longer the case. But you can type `(?i)google` instead to toggle case-insensitive mode](http://peco.github.io/images/peco-demo-matcher.gif) ## Selectable Layout @@ -158,9 +160,9 @@ Changes how peco interprets incoming data. When this flag is set, you may insert Specifies the initial line position upon start up. E.g. If you want to start out with the second line selected, set it to "1" (because the index is 0 based) -### --initial-filter `IgnoreCase|CaseSensitive|SmartCase|Regexp` +### --initial-filter `IgnoreCase|CaseSensitive|SmartCase|Regexp|Fuzzy` -Specifies the initial filter to use upon start up. You should specify the name of the filter like `IgnoreCase`, `CaseSensitive`, `SmartCase` and `Regexp`. Default is `IgnoreCase`. +Specifies the initial filter to use upon start up. You should specify the name of the filter like `IgnoreCase`, `CaseSensitive`, `SmartCase`, `Regexp` and `Fuzzy`. Default is `IgnoreCase`. ### --prompt @@ -219,7 +221,7 @@ You can change the query line's prompt, which is `QUERY>` by default. ### InitialFilter -Specifies the filter name to start peco with. You should specify the name of the filter, such as `IgnoreCase`, `CaseSensitive`, `SmartCase` and `Regexp` +Specifies the filter name to start peco with. You should specify the name of the filter, such as `IgnoreCase`, `CaseSensitive`, `SmartCase`, `Regexp` and `Fuzzy`. ### StickySelection @@ -473,7 +475,7 @@ For now, styles of following 5 items can be customized in `config.json`. This is an experimental feature. Please note that some details of this specification may change -By default `peco` comes with `IgnoreCase`, `CaseSensitive`, `SmartCase` and `Regexp` filters, but since v0.1.3, it is possible to create your own custom filter. +By default `peco` comes with `IgnoreCase`, `CaseSensitive`, `SmartCase`, `Regexp` and `Fuzzy` filters, but since v0.1.3, it is possible to create your own custom filter. The filter will be executed via `Command.Run()` as an external process, and it will be passed the query values in the command line, and the original unaltered buffer is passed via `os.Stdin`. Your filter must perform the matching, and print out to `os.Stdout` matched lines. You filter MAY be called multiple times if the buffer given to peco is big enough. See `BufferThreshold` below. diff --git a/filter.go b/filter.go index 7a13da5..a94f1ff 100644 --- a/filter.go +++ b/filter.go @@ -6,8 +6,10 @@ import ( "os/exec" "regexp" "sort" + "strings" "sync" "time" + "unicode/utf8" "github.com/lestrrat/go-pdebug" "github.com/peco/peco/hub" @@ -126,7 +128,7 @@ func (f *Filter) Work(ctx context.Context, q hub.Payload) { g := pdebug.Marker("Periodic draw request for '%s'", query) defer g.End() } - t := time.NewTicker(5*time.Millisecond) + t := time.NewTicker(5 * time.Millisecond) defer t.Stop() defer state.Hub().SendStatusMsg("") defer state.Hub().SendDraw(&DrawOptions{RunningQuery: true}) @@ -215,7 +217,7 @@ var filterBufPool = sync.Pool{ }, } -func releaseRegexpFilterBuf(l []Line) { +func releaseFilterLineBuf(l []Line) { if l == nil { return } @@ -223,42 +225,55 @@ func releaseRegexpFilterBuf(l []Line) { filterBufPool.Put(l) } -func getRegexpFilterBuf() []Line { +func getFilterLineBuf() []Line { l := filterBufPool.Get().([]Line) return l } +type filter interface { + filter(Line) (Line, error) +} + +// This flusher is run in a separate goroutine so that the filter can +// run separately from accepting incoming messages +func flusher(f filter, incoming chan []Line, done chan struct{}, out pipeline.OutputChannel) { + if pdebug.Enabled { + g := pdebug.Marker("flusher goroutine") + defer g.End() + } + + defer close(done) + defer out.SendEndMark("end of filter") + for buf := range incoming { + for _, in := range buf { + if l, err := f.filter(in); err == nil { + out.Send(l) + } + } + releaseFilterLineBuf(buf) + } +} + func (rf *RegexpFilter) Accept(ctx context.Context, in chan interface{}, out pipeline.OutputChannel) { if pdebug.Enabled { g := pdebug.Marker("RegexpFilter.Accept") defer g.End() } + filterAcceptAndFilter(ctx, rf, in, out) +} + +func filterAcceptAndFilter(ctx context.Context, f filter, in chan interface{}, out pipeline.OutputChannel) { flush := make(chan []Line) flushDone := make(chan struct{}) - go func() { - if pdebug.Enabled { - g := pdebug.Marker("RegexpFilter.Accept flusher goroutine") - defer g.End() - } - defer close(flushDone) - defer out.SendEndMark("end of RegexpFilter") - for buf := range flush { - for _, in := range buf { - if l, err := rf.filter(in); err == nil { - out.Send(l) - } - } - releaseRegexpFilterBuf(buf) - } - }() + go flusher(f, flush, flushDone, out) - buf := getRegexpFilterBuf() - defer func() { releaseRegexpFilterBuf(buf) }() + buf := getFilterLineBuf() + defer releaseFilterLineBuf(buf) defer func() { <-flushDone }() // Wait till the flush goroutine is done defer close(flush) // Kill the flush goroutine - flushTicker := time.NewTicker(50*time.Millisecond) + flushTicker := time.NewTicker(50 * time.Millisecond) defer flushTicker.Stop() start := time.Now() @@ -267,7 +282,7 @@ func (rf *RegexpFilter) Accept(ctx context.Context, in chan interface{}, out pip select { case <-ctx.Done(): if pdebug.Enabled { - pdebug.Printf("RegexpFilter received done") + pdebug.Printf("filter received done") } return case v := <-in: @@ -275,7 +290,7 @@ func (rf *RegexpFilter) Accept(ctx context.Context, in chan interface{}, out pip case error: if pipeline.IsEndMark(v.(error)) { if pdebug.Enabled { - pdebug.Printf("RegexpFilter received end mark (read %d lines, %s since starting accept loop)", lines+len(buf), time.Since(start).String()) + pdebug.Printf("filter received end mark (read %d lines, %s since starting accept loop)", lines+len(buf), time.Since(start).String()) } if len(buf) > 0 { flush <- buf @@ -295,11 +310,11 @@ func (rf *RegexpFilter) Accept(ctx context.Context, in chan interface{}, out pip select { case <-flushTicker.C: flush <- buf - buf = getRegexpFilterBuf() + buf = getFilterLineBuf() default: if len(buf) >= cap(buf) { flush <- buf - buf = getRegexpFilterBuf() + buf = getFilterLineBuf() } } } @@ -420,6 +435,78 @@ func NewSmartCaseFilter() *RegexpFilter { return rf } +// NewFuzzyFilter builds a fuzzy-finder type of filter. +// In effect, this uses a smart case filter, and for q query +// like "ABC" it matches the equivalent of "A(.*)B(.*)C(.*)" +func NewFuzzyFilter() *FuzzyFilter { + return &FuzzyFilter{} +} + +func (ff FuzzyFilter) Clone() LineFilter { + return &FuzzyFilter{ + query: ff.query, + } +} + +func (ff *FuzzyFilter) SetQuery(q string) { + ff.mutex.Lock() + defer ff.mutex.Unlock() + + ff.query = q +} + +func (ff FuzzyFilter) String() string { + return "Fuzzy" +} + +func (ff *FuzzyFilter) Accept(ctx context.Context, in chan interface{}, out pipeline.OutputChannel) { + if pdebug.Enabled { + g := pdebug.Marker("FuzzyFilter.Accept") + defer g.End() + } + + filterAcceptAndFilter(ctx, ff, in, out) +} + +func (ff *FuzzyFilter) filter(l Line) (Line, error) { + query := "" + ff.mutex.Lock() + query = ff.query + ff.mutex.Unlock() + + base := 0 + txt := l.DisplayString() + matches := [][]int{} + + hasUpper := util.ContainsUpper(query) + + for len(query) > 0 { + r, n := utf8.DecodeRuneInString(query) + if r == utf8.RuneError { + // "Silently" ignore (just return a no match) + return nil, errors.New("failed to decode input string") + } + query = query[n:] + + var i int + if hasUpper { // explicit match + i = strings.IndexRune(txt, r) + } else { + i = strings.IndexFunc(txt, util.CaseInsensitiveIndexFunc(r)) + } + if i == -1 { + return nil, errors.New("filter did not match against given line") + } + + // otherwise we have a match, but the next match must match against + // something AFTER the current match + txt = txt[i+n:] + matches = append(matches, []int{base + i, base + i + n}) + base = base + i + n + } + return NewMatchedLine(l, matches), nil +} + func NewExternalCmdFilter(name string, cmd string, args []string, threshold int, idgen lineIDGenerator, enableSep bool) *ExternalCmdFilter { if len(args) == 0 { args = []string{"$QUERY"} diff --git a/filter_test.go b/filter_test.go new file mode 100644 index 0000000..4ab2b4c --- /dev/null +++ b/filter_test.go @@ -0,0 +1,56 @@ +package peco + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestFuzzyFilter tests a fuzzy filter against various inputs +func TestFuzzyFilter(t *testing.T) { + testValues := []struct { + input string + query string + selected bool + }{ + {"this is a test to test the fuzzy Filter", "tf", true}, // normal selection + {"this is a test to test the fuzzy Filter", "wp", false}, // incorrect selection + {"THIS IS A TEST TO TEST THE FUZZY FILTER", "tu", true}, // case insensitivity + {"this is a Test to test the fuzzy filter", "Tu", true}, // case sensitivity + {"this is a Test to test the fUzzy filter", "TU", true}, // case sensitivity + {"this is a test to test the fuzzy filter", "Tu", false}, // case sensitivity + {"this is a test to Test the fuzzy filter", "TU", false}, // case sensitivity + {"日本語は難しいです", "難", true}, // kanji + {"あ、日本語は難しいですよ", "あい", true}, // hiragana + {"パソコンは遅いですネ", "ソネ", true}, // katakana + {"🚴🏻 abcd efgh", "🚴🏻e", true}, // unicode + } + filter := NewFuzzyFilter() + for i, v := range testValues { + t.Run(fmt.Sprintf(`"%s" against "%s", expect "%t"`, v.input, v.query, v.selected), func(t *testing.T) { + filter.SetQuery(v.query) + l := NewRawLine(uint64(i), v.input, false) + res, err := filter.filter(l) + + if !v.selected { + if !assert.Error(t, err, "filter should fail") { + return + } + if !assert.Nil(t, res, "return value should be nil") { + return + } + return + } + + if !assert.NoError(t, err, "filtering failed") { + return + } + + if !assert.NotNil(t, res, "return value should NOT be nil") { + return + } + t.Logf("%#v", res.Indices()) + }) + } +} diff --git a/interface.go b/interface.go index c70db8a..069c0d9 100644 --- a/interface.go +++ b/interface.go @@ -324,7 +324,7 @@ type FilteredBuffer struct { } // Config holds all the data that can be configured in the -// external configuran file +// external configuration file type Config struct { Action map[string][]string `json:"Action"` // Keymap used to be directly responsible for dispatching @@ -513,6 +513,11 @@ type LineFilter interface { String() string } +type FuzzyFilter struct { + mutex sync.Mutex + query string +} + type RegexpFilter struct { compiledQuery []*regexp.Regexp flags regexpFlags diff --git a/internal/util/util.go b/internal/util/util.go index 97a24a5..a54c380 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -9,6 +9,13 @@ type fder interface { Fd() uintptr } +func CaseInsensitiveIndexFunc(r rune) func(rune) bool { + lr := unicode.ToUpper(r) + return func(v rune) bool { + return lr == unicode.ToUpper(v) + } +} + func ContainsUpper(query string) bool { for _, c := range query { if unicode.IsUpper(c) { diff --git a/peco.go b/peco.go index 0fcb49c..095028a 100644 --- a/peco.go +++ b/peco.go @@ -566,6 +566,7 @@ func (p *Peco) populateFilters() error { p.filters.Add(NewCaseSensitiveFilter()) p.filters.Add(NewSmartCaseFilter()) p.filters.Add(NewRegexpFilter()) + p.filters.Add(NewFuzzyFilter()) for name, c := range p.config.CustomFilter { f := NewExternalCmdFilter(name, c.Cmd, c.Args, c.BufferThreshold, p.idgen, p.enableSep) diff --git a/peco_test.go b/peco_test.go index c486055..bd7f186 100644 --- a/peco_test.go +++ b/peco_test.go @@ -197,6 +197,17 @@ func TestGHIssue331(t *testing.T) { } } +func TestConfigFuzzyFilter(t *testing.T) { + var opts CLIOptions + p := newPeco() + + // Ensure that it's possible to enable the Fuzzy filter + opts.OptInitialFilter = "Fuzzy" + if !assert.NoError(t, p.ApplyConfig(opts), "p.ApplyConfig should succeed") { + return + } +} + func TestApplyConfig(t *testing.T) { // XXX We should add all the possible configurations that needs to be // propagated to Peco from config