Merge pull request #370 from peco/topic/pr-369

Cleanup for PR 369

Changes

@@ -2,6 +2,8 @@ Changes
 =======
 
 v0.4.6 - Not yet released
+  Features:
+  * A new fuzzy filter has now been added. See README for details (#369, #370)
   Bugs/Fixes
   * A very subtle timing issue that causes the search to be reset
     has been resolved (#364)

README.md

@@ -58,13 +58,15 @@ Not only can you select multiple lines one by one, you can select a range of lin
 
 ## Select Filters
 
-Different types of filters are available. Default is case-insensitive filter, so lines with any case will match. You can toggle between IgnoreCase, CaseSensitive, SmartCase and RegExp filters. 
+Different types of filters are available. Default is case-insensitive filter, so lines with any case will match. You can toggle between IgnoreCase, CaseSensitive, SmartCase RegExp and Fuzzy filters. 
 
 The SmartCase filter uses case-*insensitive* matching when all of the queries are lower case, and case-*sensitive* matching otherwise.
 
 The RegExp filter allows you to use any valid regular expression to match lines
 
-![Executed `ps aux | peco`, then typed `google`, which matches the Chrome.app under IgnoreCase filter type. Whenyou change it to Regexp filter, this is no longer the case. But you can type `(?i)google` instead to toggle case-insensitive mode](http://peco.github.io/images/peco-demo-matcher.gif)
+The Fuzzy filter allows you to find matches using partial patterns. For example, when searching for `ALongString`, you can enable the Fuzzy filter and search `ALS` to find it. The Fuzzy filter uses smart case search like the SmartCase filter.
+
+![Executed `ps aux | peco`, then typed `google`, which matches the Chrome.app under IgnoreCase filter type. When you change it to Regexp filter, this is no longer the case. But you can type `(?i)google` instead to toggle case-insensitive mode](http://peco.github.io/images/peco-demo-matcher.gif)
 
 ## Selectable Layout
 
@@ -158,9 +160,9 @@ Changes how peco interprets incoming data. When this flag is set, you may insert
 
 Specifies the initial line position upon start up. E.g. If you want to start out with the second line selected, set it to "1" (because the index is 0 based)
 
-### --initial-filter `IgnoreCase|CaseSensitive|SmartCase|Regexp`
+### --initial-filter `IgnoreCase|CaseSensitive|SmartCase|Regexp|Fuzzy`
 
-Specifies the initial filter to use upon start up. You should specify the name of the filter like `IgnoreCase`, `CaseSensitive`, `SmartCase` and `Regexp`. Default is `IgnoreCase`.
+Specifies the initial filter to use upon start up. You should specify the name of the filter like `IgnoreCase`, `CaseSensitive`, `SmartCase`, `Regexp` and `Fuzzy`. Default is `IgnoreCase`.
 
 ### --prompt
 
@@ -219,7 +221,7 @@ You can change the query line's prompt, which is `QUERY>` by default.
 
 ### InitialFilter
 
-Specifies the filter name to start peco with. You should specify the name of the filter, such as `IgnoreCase`, `CaseSensitive`, `SmartCase` and `Regexp`
+Specifies the filter name to start peco with. You should specify the name of the filter, such as `IgnoreCase`, `CaseSensitive`, `SmartCase`, `Regexp` and `Fuzzy`.
 
 ### StickySelection
 
@@ -473,7 +475,7 @@ For now, styles of following 5 items can be customized in `config.json`.
 
 This is an experimental feature. Please note that some details of this specification may change
 
-By default `peco` comes with `IgnoreCase`, `CaseSensitive`, `SmartCase` and `Regexp` filters, but since v0.1.3, it is possible to create your own custom filter.
+By default `peco` comes with `IgnoreCase`, `CaseSensitive`, `SmartCase`, `Regexp` and `Fuzzy` filters, but since v0.1.3, it is possible to create your own custom filter.
 
 The filter will be executed via  `Command.Run()` as an external process, and it will be passed the query values in the command line, and the original unaltered buffer is passed via `os.Stdin`. Your filter must perform the matching, and print out to `os.Stdout` matched lines. You filter MAY be called multiple times if the buffer
 given to peco is big enough. See `BufferThreshold` below.

filter.go

@@ -6,8 +6,10 @@ import (
 	"os/exec"
 	"regexp"
 	"sort"
+	"strings"
 	"sync"
 	"time"
+	"unicode/utf8"
 
 	"github.com/lestrrat/go-pdebug"
 	"github.com/peco/peco/hub"
@@ -126,7 +128,7 @@ func (f *Filter) Work(ctx context.Context, q hub.Payload) {
 			g := pdebug.Marker("Periodic draw request for '%s'", query)
 			defer g.End()
 		}
-		t := time.NewTicker(5*time.Millisecond)
+		t := time.NewTicker(5 * time.Millisecond)
 		defer t.Stop()
 		defer state.Hub().SendStatusMsg("")
 		defer state.Hub().SendDraw(&DrawOptions{RunningQuery: true})
@@ -215,7 +217,7 @@ var filterBufPool = sync.Pool{
 	},
 }
 
-func releaseRegexpFilterBuf(l []Line) {
+func releaseFilterLineBuf(l []Line) {
 	if l == nil {
 		return
 	}
@@ -223,42 +225,55 @@ func releaseRegexpFilterBuf(l []Line) {
 	filterBufPool.Put(l)
 }
 
-func getRegexpFilterBuf() []Line {
+func getFilterLineBuf() []Line {
 	l := filterBufPool.Get().([]Line)
 	return l
 }
 
+type filter interface {
+	filter(Line) (Line, error)
+}
+
+// This flusher is run in a separate goroutine so that the filter can
+// run separately from accepting incoming messages
+func flusher(f filter, incoming chan []Line, done chan struct{}, out pipeline.OutputChannel) {
+	if pdebug.Enabled {
+		g := pdebug.Marker("flusher goroutine")
+		defer g.End()
+	}
+
+	defer close(done)
+	defer out.SendEndMark("end of filter")
+	for buf := range incoming {
+		for _, in := range buf {
+			if l, err := f.filter(in); err == nil {
+				out.Send(l)
+			}
+		}
+		releaseFilterLineBuf(buf)
+	}
+}
+
 func (rf *RegexpFilter) Accept(ctx context.Context, in chan interface{}, out pipeline.OutputChannel) {
 	if pdebug.Enabled {
 		g := pdebug.Marker("RegexpFilter.Accept")
 		defer g.End()
 	}
 
+	filterAcceptAndFilter(ctx, rf, in, out)
+}
+
+func filterAcceptAndFilter(ctx context.Context, f filter, in chan interface{}, out pipeline.OutputChannel) {
 	flush := make(chan []Line)
 	flushDone := make(chan struct{})
-	go func() {
-		if pdebug.Enabled {
-			g := pdebug.Marker("RegexpFilter.Accept flusher goroutine")
-			defer g.End()
-		}
-		defer close(flushDone)
-		defer out.SendEndMark("end of RegexpFilter")
-		for buf := range flush {
-			for _, in := range buf {
-				if l, err := rf.filter(in); err == nil {
-					out.Send(l)
-				}
-			}
-			releaseRegexpFilterBuf(buf)
-		}
-	}()
+	go flusher(f, flush, flushDone, out)
 
-	buf := getRegexpFilterBuf()
-	defer func() { releaseRegexpFilterBuf(buf) }()
+	buf := getFilterLineBuf()
+	defer releaseFilterLineBuf(buf)
 	defer func() { <-flushDone }() // Wait till the flush goroutine is done
 	defer close(flush)             // Kill the flush goroutine
 
-	flushTicker := time.NewTicker(50*time.Millisecond)
+	flushTicker := time.NewTicker(50 * time.Millisecond)
 	defer flushTicker.Stop()
 
 	start := time.Now()
@@ -267,7 +282,7 @@ func (rf *RegexpFilter) Accept(ctx context.Context, in chan interface{}, out pip
 		select {
 		case <-ctx.Done():
 			if pdebug.Enabled {
-				pdebug.Printf("RegexpFilter received done")
+				pdebug.Printf("filter received done")
 			}
 			return
 		case v := <-in:
@@ -275,7 +290,7 @@ func (rf *RegexpFilter) Accept(ctx context.Context, in chan interface{}, out pip
 			case error:
 				if pipeline.IsEndMark(v.(error)) {
 					if pdebug.Enabled {
-						pdebug.Printf("RegexpFilter received end mark (read %d lines, %s since starting accept loop)", lines+len(buf), time.Since(start).String())
+						pdebug.Printf("filter received end mark (read %d lines, %s since starting accept loop)", lines+len(buf), time.Since(start).String())
 					}
 					if len(buf) > 0 {
 						flush <- buf
@@ -295,11 +310,11 @@ func (rf *RegexpFilter) Accept(ctx context.Context, in chan interface{}, out pip
 				select {
 				case <-flushTicker.C:
 					flush <- buf
-					buf = getRegexpFilterBuf()
+					buf = getFilterLineBuf()
 				default:
 					if len(buf) >= cap(buf) {
 						flush <- buf
-						buf = getRegexpFilterBuf()
+						buf = getFilterLineBuf()
 					}
 				}
 			}
@@ -420,6 +435,78 @@ func NewSmartCaseFilter() *RegexpFilter {
 	return rf
 }
 
+// NewFuzzyFilter builds a fuzzy-finder type of filter.
+// In effect, this uses a smart case filter, and for q query 
+// like "ABC" it matches the equivalent of "A(.*)B(.*)C(.*)"
+func NewFuzzyFilter() *FuzzyFilter {
+	return &FuzzyFilter{}
+}
+
+func (ff FuzzyFilter) Clone() LineFilter {
+	return &FuzzyFilter{
+		query: ff.query,
+	}
+}
+
+func (ff *FuzzyFilter) SetQuery(q string) {
+	ff.mutex.Lock()
+	defer ff.mutex.Unlock()
+
+	ff.query = q
+}
+
+func (ff FuzzyFilter) String() string {
+	return "Fuzzy"
+}
+
+func (ff *FuzzyFilter) Accept(ctx context.Context, in chan interface{}, out pipeline.OutputChannel) {
+	if pdebug.Enabled {
+		g := pdebug.Marker("FuzzyFilter.Accept")
+		defer g.End()
+	}
+
+	filterAcceptAndFilter(ctx, ff, in, out)
+}
+
+func (ff *FuzzyFilter) filter(l Line) (Line, error) {
+	query := ""
+	ff.mutex.Lock()
+	query = ff.query
+	ff.mutex.Unlock()
+
+	base := 0
+	txt := l.DisplayString()
+	matches := [][]int{}
+
+	hasUpper := util.ContainsUpper(query)
+
+	for len(query) > 0 {
+		r, n := utf8.DecodeRuneInString(query)
+		if r == utf8.RuneError {
+			// "Silently" ignore (just return a no match)
+			return nil, errors.New("failed to decode input string")
+		}
+		query = query[n:]
+
+		var i int
+		if hasUpper { // explicit match
+			i = strings.IndexRune(txt, r)
+		} else {
+			i = strings.IndexFunc(txt, util.CaseInsensitiveIndexFunc(r))
+		}
+		if i == -1 {
+			return nil, errors.New("filter did not match against given line")
+		}
+
+		// otherwise we have a match, but the next match must match against
+		// something AFTER the current match
+		txt = txt[i+n:]
+		matches = append(matches, []int{base + i, base + i + n})
+		base = base + i + n
+	}
+	return NewMatchedLine(l, matches), nil
+}
+
 func NewExternalCmdFilter(name string, cmd string, args []string, threshold int, idgen lineIDGenerator, enableSep bool) *ExternalCmdFilter {
 	if len(args) == 0 {
 		args = []string{"$QUERY"}

filter_test.go

@@ -0,0 +1,56 @@
+package peco
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// TestFuzzyFilter tests a fuzzy filter against various inputs
+func TestFuzzyFilter(t *testing.T) {
+	testValues := []struct {
+		input    string
+		query    string
+		selected bool
+	}{
+		{"this is a test to test the fuzzy Filter", "tf", true},  // normal selection
+		{"this is a test to test the fuzzy Filter", "wp", false}, // incorrect selection
+		{"THIS IS A TEST TO TEST THE FUZZY FILTER", "tu", true},  // case insensitivity
+		{"this is a Test to test the fuzzy filter", "Tu", true},  // case sensitivity
+		{"this is a Test to test the fUzzy filter", "TU", true},  // case sensitivity
+		{"this is a test to test the fuzzy filter", "Tu", false}, // case sensitivity
+		{"this is a test to Test the fuzzy filter", "TU", false}, // case sensitivity
+		{"日本語は難しいです", "難", true}, // kanji
+		{"あ、日本語は難しいですよ", "あい", true}, // hiragana
+		{"パソコンは遅いですネ", "ソネ", true}, // katakana
+		{"🚴🏻 abcd efgh", "🚴🏻e", true}, // unicode
+	}
+	filter := NewFuzzyFilter()
+	for i, v := range testValues {
+		t.Run(fmt.Sprintf(`"%s" against "%s", expect "%t"`, v.input, v.query, v.selected), func(t *testing.T) {
+			filter.SetQuery(v.query)
+			l := NewRawLine(uint64(i), v.input, false)
+			res, err := filter.filter(l)
+
+			if !v.selected {
+				if !assert.Error(t, err, "filter should fail") {
+					return
+				}
+				if !assert.Nil(t, res, "return value should be nil") {
+					return
+				}
+				return
+			}
+
+			if !assert.NoError(t, err, "filtering failed") {
+				return
+			}
+
+			if !assert.NotNil(t, res, "return value should NOT be nil") {
+				return
+			}
+			t.Logf("%#v", res.Indices())
+		})
+	}
+}

interface.go

@@ -324,7 +324,7 @@ type FilteredBuffer struct {
 }
 
 // Config holds all the data that can be configured in the
-// external configuran file
+// external configuration file
 type Config struct {
 	Action map[string][]string `json:"Action"`
 	// Keymap used to be directly responsible for dispatching
@@ -513,6 +513,11 @@ type LineFilter interface {
 	String() string
 }
 
+type FuzzyFilter struct {
+	mutex sync.Mutex
+	query string
+}
+
 type RegexpFilter struct {
 	compiledQuery []*regexp.Regexp
 	flags         regexpFlags

internal/util/util.go

@@ -9,6 +9,13 @@ type fder interface {
 	Fd() uintptr
 }
 
+func CaseInsensitiveIndexFunc(r rune) func(rune) bool {
+	lr := unicode.ToUpper(r)
+	return func(v rune) bool {
+		return lr == unicode.ToUpper(v)
+	}
+}
+
 func ContainsUpper(query string) bool {
 	for _, c := range query {
 		if unicode.IsUpper(c) {

peco.go

@@ -566,6 +566,7 @@ func (p *Peco) populateFilters() error {
 	p.filters.Add(NewCaseSensitiveFilter())
 	p.filters.Add(NewSmartCaseFilter())
 	p.filters.Add(NewRegexpFilter())
+	p.filters.Add(NewFuzzyFilter())
 
 	for name, c := range p.config.CustomFilter {
 		f := NewExternalCmdFilter(name, c.Cmd, c.Args, c.BufferThreshold, p.idgen, p.enableSep)

peco_test.go

@@ -197,6 +197,17 @@ func TestGHIssue331(t *testing.T) {
 	}
 }
 
+func TestConfigFuzzyFilter(t *testing.T) {
+	var opts CLIOptions
+	p := newPeco()
+
+	// Ensure that it's possible to enable the Fuzzy filter
+	opts.OptInitialFilter = "Fuzzy"
+	if !assert.NoError(t, p.ApplyConfig(opts), "p.ApplyConfig should succeed") {
+		return
+	}
+}
+
 func TestApplyConfig(t *testing.T) {
 	// XXX We should add all the possible configurations that needs to be
 	// propagated to Peco from config