feat: typo resistant matcher using frizbee from blink.cmp (#891)

* feat: typo resistant matcher using frizbee from blink.cmp

* chore: generate completions & manpage

* fix: back to stable rustc using fork

* chore: update lockfile

* chore: feature gate

* ci: update actions

* chore: generate completions & manpage

* ci: use rustup directly

* ci: fix feature name

---------

Co-authored-by: Skim bot <skim-bot@skim-rs.github.io>

.githooks/pre-commit

@@ -2,4 +2,6 @@ set -xeuo pipefail
 
 cargo fmt --check --all
 cargo clippy --all
-cargo check --no-default-features
+cargo +stable check --no-default-features
+cargo +stable check --no-default-features --features cli
+cargo check --no-default-features --features nightly-frizbee

.github/workflows/pr.yml

@@ -30,16 +30,13 @@ jobs:
           app-id: ${{ vars.SKIM_RS_BOT_APP_ID }}
           private-key: ${{ secrets.SKIM_RS_BOT_PRIVATE_KEY }}
       - name: Checkout Git repo
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         with:
           fetch-depth: 0
           ref: ${{github.event.pull_request.head.ref}}
           repository: ${{github.event.pull_request.head.repo.full_name}}
           token: ${{ steps.app-token.outputs.token }}
-      - name: Install correct toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
+      - run: rustup toolchain install
       - name: Cache
         uses: Swatinem/rust-cache@v2
       - name: Generate files

.github/workflows/publish.yml

@@ -12,14 +12,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 1
-      - name: Install correct toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          profile: default
+        uses: actions/checkout@v6
+      - run: rustup toolchain install
       - name: Cache
         uses: Swatinem/rust-cache@v2
       - name: Login

.github/workflows/test.yml

@@ -25,11 +25,9 @@ jobs:
         include:
         - build: linux
           os: ubuntu-latest
-          rust: stable
           target: x86_64-unknown-linux-musl
         - build: macos
           os: macos-latest
-          rust: stable
           target: x86_64-apple-darwin
     steps:
       - name: "[linux] Install dependencies"
@@ -50,11 +48,7 @@ jobs:
         uses: actions/checkout@v2
         with:
           fetch-depth: 1
-      - name: Install correct toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: ${{ matrix.rust }}
-          target: ${{ matrix.target }}
+      - run: rustup toolchain install
       - uses: taiki-e/install-action@v2
         with:
           tool: nextest@0.9
@@ -73,12 +67,7 @@ jobs:
         uses: actions/checkout@v2
         with:
           fetch-depth: 1
-      - name: Install correct toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          profile: minimal
-          components: clippy
+      - run: rustup toolchain install
       - name: Cache
         uses: Swatinem/rust-cache@v2
       - name: Clippy
@@ -89,12 +78,7 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2
-      - name: Install Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          profile: minimal
-          components: rustfmt
+      - run: rustup toolchain install
       - name: Check formatting
         run: |
           cargo fmt --all -- --check
@@ -103,15 +87,16 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
-      - name: Install Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          profile: minimal
-          components: rustfmt
+        uses: actions/checkout@v6
+      - run: rustup toolchain install
       - name: Cache
         uses: Swatinem/rust-cache@v2
-      - name: Build without cli feature
+      - name: Build without any feature
         run: |
-          cargo build --no-default-features
+          cargo +stable build --no-default-features
+      - name: Build with cli feature
+        run: |
+          cargo +stable build --no-default-features --features cli
+      - name: Build with frizbee feature
+        run: |
+          cargo +nightly build --no-default-features --features nightly-frizbee

Cargo.lock

@@ -757,6 +757,16 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
 
+[[package]]
+name = "frizbee"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3365720de81dac18e889afa72f5907aa061c975548da68e2400c056ebc94aec"
+dependencies = [
+ "multiversion",
+ "rayon",
+]
+
 [[package]]
 name = "futures"
 version = "0.3.31"
@@ -1143,6 +1153,28 @@ dependencies = [
  "windows-sys 0.61.2",
 ]
 
+[[package]]
+name = "multiversion"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7edb7f0ff51249dfda9ab96b5823695e15a052dc15074c9dbf3d118afaf2c201"
+dependencies = [
+ "multiversion-macros",
+ "target-features",
+]
+
+[[package]]
+name = "multiversion-macros"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b093064383341eb3271f42e381cb8f10a01459478446953953c75d24bd339fc0"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.114",
+ "target-features",
+]
+
 [[package]]
 name = "nix"
 version = "0.29.0"
@@ -1848,6 +1880,7 @@ dependencies = [
  "defer-drop",
  "derive_builder",
  "env_logger",
+ "frizbee",
  "futures",
  "indexmap",
  "interprocess",
@@ -1949,6 +1982,12 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "target-features"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c1bbb9f3c5c463a01705937a24fdabc5047929ac764b2d5b9cf681c1f5041ed5"
+
 [[package]]
 name = "tempfile"
 version = "3.24.0"

Cargo.toml

@@ -63,7 +63,11 @@ clap_complete_nushell = "4.5.10"
 interprocess = { version = "2.2.3", features = ["tokio"] }
 serde = { version = "1.0.228", features = ["derive"] }
 ron = "0.12.0"
+frizbee = { version = "0.6.0", optional = true }
 
 [features]
-default = ["cli"]
+default = ["cli", "nightly-frizbee"]
+# Everyting needed to use skim as a cli (argument parsing, shell integrations...)
 cli = ["dep:clap", "dep:clap_complete", "dep:shlex", "dep:env_logger", "dep:clap_mangen"]
+# The frizbee matcher, which requires cargo nightly
+nightly-frizbee = ["dep:frizbee"]

README.md

@@ -495,9 +495,13 @@ First, add skim into your `Cargo.toml`:
 
 ```toml
 [dependencies]
-skim = "*"
+skim = { version = "<version>", default-features = false, features = [..] }
 ```
 
+_Note on features_:
+    - the `cli` feature is required to use skim as a cli, it *should* not be needed when using it as a library.
+    - the `nightly-frizbee` feature adds the frizbee algorithm, but requires cargo nigthly.
+
 Then try to run this simple example:
 
 ```rust

bench.sh

@@ -2,24 +2,62 @@
 
 # Benchmark script to measure ingestion + matching rate in skim interactive mode
 # This measures how fast skim can ingest items and display matched results
+#
+# Usage: bench.sh [BINARY_PATH] [NUM_ITEMS] [QUERY] [-- EXTRA_ARGS...]
+#
+# Examples:
+#   ./bench.sh                                    # Use defaults
+#   ./bench.sh ./target/release/sk 500000 foo    # Custom binary, items, query
+#   ./bench.sh -- --algo=skim                    # Pass extra args to binary
+#   ./bench.sh ./target/release/sk 1000000 test -- --no-sort --exact
 
 set -e
 export SHELL="/bin/sh"
 unset HISTFILE
 
 # Parse arguments
-BINARY_PATH=${1:-"./target/release/sk"}
-NUM_ITEMS=${2:-1000000}
-QUERY=${3:-"test"}
+# Arguments before -- are: BINARY_PATH NUM_ITEMS QUERY
+# Arguments after -- are passed directly to the binary
+BINARY_PATH="./target/release/sk"
+NUM_ITEMS=1000000
+QUERY="test"
+EXTRA_ARGS=""
+
+# Separate arguments before and after --
+BEFORE_SEP=()
+AFTER_SEP=()
+FOUND_SEP=0
+
+for arg in "$@"; do
+    if [ "$arg" = "--" ]; then
+        FOUND_SEP=1
+    elif [ $FOUND_SEP -eq 0 ]; then
+        BEFORE_SEP+=("$arg")
+    else
+        AFTER_SEP+=("$arg")
+    fi
+done
+
+# Parse positional arguments before --
+[ ${#BEFORE_SEP[@]} -ge 1 ] && BINARY_PATH="${BEFORE_SEP[0]}"
+[ ${#BEFORE_SEP[@]} -ge 2 ] && NUM_ITEMS="${BEFORE_SEP[1]}"
+[ ${#BEFORE_SEP[@]} -ge 3 ] && QUERY="${BEFORE_SEP[2]}"
+
+# Join extra args for passing to binary
+if [ ${#AFTER_SEP[@]} -gt 0 ]; then
+    EXTRA_ARGS="${AFTER_SEP[*]}"
+fi
 
 echo "=== Skim Ingestion + Matching Benchmark ==="
 echo "Binary: $BINARY_PATH | Items: $NUM_ITEMS | Query: '$QUERY'"
+[ -n "$EXTRA_ARGS" ] && echo "Extra args: $EXTRA_ARGS"
 
 # Generate test data
 TMP_FILE=$(mktemp)
 STATUS_FILE=$(mktemp)
 SESSION_NAME="skim_bench_$$"
-trap "rm -f $TMP_FILE $STATUS_FILE; tmux kill-session -t $SESSION_NAME 2>/dev/null || true" EXIT
+# trap "rm -f $TMP_FILE $STATUS_FILE; tmux kill-session -t $SESSION_NAME 2>/dev/null || true" EXIT
+trap "rm -f $STATUS_FILE; tmux kill-session -t $SESSION_NAME 2>/dev/null || true" EXIT
 
 # Generate random path-like strings with 2-10 words separated by slashes
 awk -v num="$NUM_ITEMS" 'BEGIN {
@@ -30,7 +68,7 @@ awk -v num="$NUM_ITEMS" 'BEGIN {
     words[16]="src"; words[17]="test"; words[18]="config"; words[19]="data"; words[20]="logs"
     words[21]="cache"; words[22]="backup"; words[23]="docs"; words[24]="images"; words[25]="videos"
     words[26]="audio"; words[27]="downloads"; words[28]="uploads"; words[29]="temp"; words[30]="shared"
-    
+
     for (i = 1; i <= num; i++) {
         depth = int(rand() * 9) + 2  # 2-10 depth
         path = ""
@@ -48,7 +86,7 @@ tmux new-session -s "$SESSION_NAME" -d
 
 # Prepare to capture the start time as close to data ingestion as possible
 # Run skim with the query already set, and measure until matcher completes
-tmux send-keys -t "$SESSION_NAME" "cat $TMP_FILE | $BINARY_PATH --query '$QUERY'" Enter
+tmux send-keys -t "$SESSION_NAME" "cat $TMP_FILE | $BINARY_PATH --query '$QUERY' $EXTRA_ARGS" Enter
 
 # Record start time
 START=$(date +%s%N)
@@ -119,7 +157,7 @@ while [ $ELAPSED -lt $MAX_WAIT ]; do
         if [ -n "$STATUS_LINE" ]; then
             MATCHED_COUNT=$(echo "$STATUS_LINE" | cut -d'/' -f1)
             TOTAL_INGESTED=$(echo "$STATUS_LINE" | cut -d'/' -f2)
-            
+
             # Check if ingestion is complete
             if [ "$TOTAL_INGESTED" = "$NUM_ITEMS" ]; then
                 COMPLETED=1
@@ -167,4 +205,4 @@ echo "Items/second: ${RATE}"
 if [ -n "$PEAK_MEM" ] && [ "$PEAK_MEM" -gt 0 ]; then
     echo "Peak memory usage: $((PEAK_MEM / 1024)) MB"
     echo "Peak CPU usage: ${PEAK_CPU}%"
-fi
+fi

man/man1/sk.1

@@ -161,12 +161,14 @@ skim_v1: Original skim fuzzy matching algorithm (v1)
 skim_v2: Improved skim fuzzy matching algorithm (v2, default)
 .IP \(bu 2
 clangd: Clangd fuzzy matching algorithm
+.IP \(bu 2
+frizbee: Frizbee matching algorithm, typo resistant Will fallback to SkimV2 if the feature is not enabled
 .RE
 .TP
 \fB\-\-case\fR \fI<CASE>\fR [default: smart]
 Case sensitivity
 
-Determines whether or not to ignore case while matching
+Determines whether or not to ignore case while matching Note: this is not used for the Frizbee matcher, it uses a penalty system to favor case\-sensitivity without enforcing it
 .br
 
 .br

rust-toolchain.toml

@@ -1,4 +1,4 @@
 [toolchain]
-channel = "stable"
+channel = "nightly"
 profile = "default"
-components = ["rust-analyzer"]
+components = ["rust-analyzer"]

shell/completion.bash

@@ -62,7 +62,7 @@ _sk() {
                     return 0
                     ;;
                 --algo)
-                    COMPREPLY=($(compgen -W "skim_v1 skim_v2 clangd" -- "${cur}"))
+                    COMPREPLY=($(compgen -W "skim_v1 skim_v2 clangd frizbee" -- "${cur}"))
                     return 0
                     ;;
                 --case)

shell/completion.fish

@@ -14,7 +14,8 @@ complete -c sk -l with-nth -d 'Fields to be transformed' -r
 complete -c sk -s d -l delimiter -d 'Delimiter between fields' -r
 complete -c sk -l algo -d 'Fuzzy matching algorithm' -r -f -a "skim_v1\t'Original skim fuzzy matching algorithm (v1)'
 skim_v2\t'Improved skim fuzzy matching algorithm (v2, default)'
-clangd\t'Clangd fuzzy matching algorithm'"
+clangd\t'Clangd fuzzy matching algorithm'
+frizbee\t'Frizbee matching algorithm, typo resistant Will fallback to SkimV2 if the feature is not enabled'"
 complete -c sk -l case -d 'Case sensitivity' -r -f -a "respect\t'Case-sensitive matching'
 ignore\t'Case-insensitive matching'
 smart\t'Smart case: case-insensitive unless query contains uppercase'"

shell/completion.nu

@@ -5,7 +5,7 @@ module completions {
   }
 
   def "nu-complete sk algorithm" [] {
-    [ "skim_v1" "skim_v2" "clangd" ]
+    [ "skim_v1" "skim_v2" "clangd" "frizbee" ]
   }
 
   def "nu-complete sk case" [] {

shell/completion.zsh

@@ -25,7 +25,8 @@ _sk() {
 '--delimiter=[Delimiter between fields]:DELIMITER:_default' \
 '--algo=[Fuzzy matching algorithm]:ALGORITHM:((skim_v1\:"Original skim fuzzy matching algorithm (v1)"
 skim_v2\:"Improved skim fuzzy matching algorithm (v2, default)"
-clangd\:"Clangd fuzzy matching algorithm"))' \
+clangd\:"Clangd fuzzy matching algorithm"
+frizbee\:"Frizbee matching algorithm, typo resistant Will fallback to SkimV2 if the feature is not enabled"))' \
 '--case=[Case sensitivity]:CASE:((respect\:"Case-sensitive matching"
 ignore\:"Case-insensitive matching"
 smart\:"Smart case\: case-insensitive unless query contains uppercase"))' \

src/engine/fuzzy.rs

@@ -2,6 +2,8 @@ use std::cmp::min;
 use std::fmt::{Display, Error, Formatter};
 use std::sync::Arc;
 
+#[cfg(feature = "nightly-frizbee")]
+use crate::fuzzy_matcher::frizbee::FrizbeeMatcher;
 use crate::fuzzy_matcher::{FuzzyMatcher, IndexType, ScoreType, clangd::ClangdMatcher, skim::SkimMatcherV2};
 
 use crate::item::RankBuilder;
@@ -10,7 +12,7 @@ use crate::{MatchRange, MatchResult, SkimItem};
 
 //------------------------------------------------------------------------------
 /// Fuzzy matching algorithm to use
-#[derive(Debug, Copy, Clone, Default)]
+#[derive(Debug, Copy, Clone, Default, PartialEq)]
 #[cfg_attr(feature = "cli", derive(clap::ValueEnum))]
 #[cfg_attr(feature = "cli", clap(rename_all = "snake_case"))]
 pub enum FuzzyAlgorithm {
@@ -21,6 +23,9 @@ pub enum FuzzyAlgorithm {
     SkimV2,
     /// Clangd fuzzy matching algorithm
     Clangd,
+    /// Frizbee matching algorithm, typo resistant
+    /// Will fallback to SkimV2 if the feature is not enabled
+    Frizbee,
 }
 
 const BYTES_1M: usize = 1024 * 1024 * 1024;
@@ -59,7 +64,14 @@ impl FuzzyEngineBuilder {
     #[allow(deprecated)]
     pub fn build(self) -> FuzzyEngine {
         use crate::fuzzy_matcher::skim::SkimMatcher;
-        let matcher: Box<dyn FuzzyMatcher> = match self.algorithm {
+        #[allow(unused_mut)]
+        let mut algorithm = self.algorithm;
+        #[cfg(not(feature = "nightly-frizbee"))]
+        if algorithm == FuzzyAlgorithm::Frizbee {
+            warn!("Frizbee algorithm not enabled, using SkimV2");
+            algorithm = FuzzyAlgorithm::SkimV2;
+        }
+        let matcher: Box<dyn FuzzyMatcher> = match algorithm {
             FuzzyAlgorithm::SkimV1 => {
                 debug!("Initialized SkimV1 algorithm");
                 Box::new(SkimMatcher::default())
@@ -84,6 +96,12 @@ impl FuzzyEngineBuilder {
                 debug!("Initialized Clangd algorithm");
                 Box::new(matcher)
             }
+            FuzzyAlgorithm::Frizbee => {
+                #[cfg(not(feature = "nightly-frizbee"))]
+                unreachable!();
+                #[cfg(feature = "nightly-frizbee")]
+                Box::new(FrizbeeMatcher::default())
+            }
         };
 
         FuzzyEngine {
@@ -141,9 +159,7 @@ impl MatchEngine for FuzzyEngine {
             }
         }
 
-        matched_result.as_ref()?;
-
-        let (score, matched_range) = matched_result.unwrap();
+        let (score, matched_range) = matched_result?;
 
         trace!("matched range {matched_range:?}");
         let begin = *matched_range.first().unwrap_or(&0);

src/fuzzy_matcher/frizbee.rs

@@ -0,0 +1,31 @@
+//! Matcher using https://crates.io/crates/frizbee
+use frizbee::{Config, Scoring, match_indices};
+
+use crate::fuzzy_matcher::{FuzzyMatcher, IndexType, ScoreType};
+
+/// Matcher using frizbee,
+/// the same one that `blink.cmp` uses in neovim
+/// credits to @saghen
+pub struct FrizbeeMatcher {
+    config: Config,
+}
+
+impl Default for FrizbeeMatcher {
+    fn default() -> Self {
+        Self {
+            config: Config {
+                prefilter: true,
+                max_typos: Some(2),
+                sort: false,
+                scoring: Scoring::default(),
+            },
+        }
+    }
+}
+
+impl FuzzyMatcher for FrizbeeMatcher {
+    fn fuzzy_indices(&self, choice: &str, pattern: &str) -> Option<(ScoreType, Vec<IndexType>)> {
+        let res = match_indices(pattern, choice, &self.config);
+        res.map(|indices| (indices.score.into(), indices.indices.to_vec()))
+    }
+}

src/fuzzy_matcher/mod.rs

@@ -5,6 +5,8 @@
 
 /// Clangd fuzzy matching algorithm
 pub mod clangd;
+#[cfg(feature = "nightly-frizbee")]
+pub mod frizbee;
 /// Skim fuzzy matching algorithm
 pub mod skim;
 mod util;

src/fuzzy_matcher/skim.rs

@@ -40,9 +40,8 @@ const PENALTY_MAX_LEADING: ScoreType = -18;
 const PENALTY_UNMATCHED: ScoreType = -2;
 
 #[deprecated(since = "0.3.5", note = "Please use SkimMatcherV2 instead")]
-#[derive(Debug)]
 /// Legacy fuzzy matcher (V1) - deprecated, use SkimMatcherV2 instead
-#[derive(Default)]
+#[derive(Default, Debug)]
 pub struct SkimMatcher {}
 
 /// The V1 matcher is based on ForrestTheWoods's post

src/options.rs

@@ -223,6 +223,8 @@ pub struct SkimOptions {
     /// Case sensitivity
     ///
     /// Determines whether or not to ignore case while matching
+    /// Note: this is not used for the Frizbee matcher, it uses a penalty system to favor
+    /// case-sensitivity without enforcing it
     #[cfg_attr(
         feature = "cli",
         arg(long, default_value = "smart", value_enum, help_heading = "Search")

src/tui/item_list.rs

@@ -587,7 +587,10 @@ impl SkimWidget for ItemList {
 
                 // Apply pre-selection only when new items arrive and only if we haven't reached target
                 // This runs once per item batch, not on every render
-                if this.multi_select && this.selector.is_some() && this.selection.len() < this.pre_select_target {
+                if this.multi_select
+                    && let Some(selector) = &this.selector
+                    && this.selection.len() < this.pre_select_target
+                {
                     debug!(
                         "Applying pre-selection to {} items (currently {} selected, target {})",
                         this.items.len(),
@@ -598,7 +601,7 @@ impl SkimWidget for ItemList {
                         if this.selection.len() >= this.pre_select_target {
                             break;
                         }
-                        let should_select = this.selector.as_ref().unwrap().should_select(index, item.item.as_ref());
+                        let should_select = selector.should_select(index, item.item.as_ref());
                         if should_select {
                             debug!("Pre-selecting item[{}]: '{}'", index, item.item.text());
                             this.selection.insert(item.clone());