mirror of
https://github.com/lotabout/skim.git
synced 2026-09-10 07:16:23 -04:00
chore: use python for bench script for comparison
This commit is contained in:
Loric ANDRE
parent
5b0323a9fb
commit
95f09a455a
|
|
@ -677,13 +677,13 @@ export TERMINFO=/data/data/com.termux/files/usr/share/terminfo
|
|||
|
||||
## Shell script
|
||||
|
||||
The `bench.sh` script is available to benchmark the code against other versions or fzf using `tmux` and querying the output. This is by no means a precise or foolproof way of running benchmarks, but it has the added benefit of allowing us to benchmark against `fzf` and of giving us resource metrics.
|
||||
The `bench.py` script is available to benchmark the code against other versions or fzf using `tmux` and querying the output. This is by no means a precise or foolproof way of running benchmarks, but it has the added benefit of allowing us to benchmark against `fzf` and of giving us resource metrics.
|
||||
|
||||
You can use it directly using `./bench.sh <binary> -n <number of items> -r <number of runs>`, or generate the data using `./bench.sh -g <output file> -n <number of items>`, then `./bench.sh <binary> -f <file> -r <number of runs>`
|
||||
You can use it directly using `./bench.py <binary> -n <number of items> -r <number of runs>`, or generate the data using `./bench.py -g <output file> -n <number of items>`, then `./bench.py <binary> -f <file> -r <number of runs>`
|
||||
|
||||
### Criterion benchmarks
|
||||
|
||||
Criterion benchmarks are available to measure skim's performance more precisely.
|
||||
To run them, you need to generate input data using `./bench.sh -g benches/fixtures/10M.txt -n 10000000 && ./bench.sh -g benches/fixtures/1M.txt -n 1000000`, then run `cargo bench -j 1`.
|
||||
To run them, you need to generate input data using `./bench.py -g benches/fixtures/10M.txt -n 10000000 && ./bench.py -g benches/fixtures/1M.txt -n 1000000`, then run `cargo bench -j 1`.
|
||||
|
||||
These will run for several minutes.
|
||||
|
|
|
|||
752
bench.py
Executable file
752
bench.py
Executable file
|
|
@ -0,0 +1,752 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Benchmark script to measure ingestion + matching rate in skim interactive mode.
|
||||
This measures how fast skim can ingest items and display matched results.
|
||||
|
||||
Usage: bench.py [BINARY_PATH ...] [-n|--num-items NUM] [-q|--query QUERY]
|
||||
[-r|--runs RUNS] [-f|--file FILE] [-g|--generate-file FILE]
|
||||
[-j|--json] [-- EXTRA_ARGS...]
|
||||
|
||||
Arguments:
|
||||
BINARY_PATH ... One or more paths to binaries (default: ./target/release/sk)
|
||||
When multiple are given they run in round-robin and the
|
||||
first is used as the baseline for +/- comparisons.
|
||||
-n, --num-items NUM Number of items to generate (default: 1000000)
|
||||
-q, --query QUERY Query string to search (default: "test")
|
||||
-r, --runs RUNS Number of benchmark runs per binary (default: 1)
|
||||
-f, --file FILE Use existing file as input instead of generating
|
||||
-g, --generate-file FILE Generate test data to file and exit
|
||||
-j, --json Output results as JSON
|
||||
-- Pass remaining arguments to the binary
|
||||
|
||||
Examples:
|
||||
./bench.py # Use defaults
|
||||
./bench.py ./target/release/sk -n 500000 -q foo
|
||||
./bench.py ./old/sk ./new/sk -r 5 # Compare two binaries
|
||||
./bench.py -r 5 # Run 5 times and show average
|
||||
./bench.py -f input.txt -q search # Use existing file
|
||||
./bench.py -g testdata.txt -n 2000000 # Generate file and exit
|
||||
"""
|
||||
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import random
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Defaults
|
||||
# ---------------------------------------------------------------------------
|
||||
DEFAULT_BINARY = "./target/release/sk"
|
||||
DEFAULT_NUM_ITEMS = 1_000_000
|
||||
DEFAULT_QUERY = "test"
|
||||
DEFAULT_RUNS = 1
|
||||
|
||||
# Stability / timeout tuning (mirrors bench.sh values)
|
||||
REQUIRED_STABLE_S = 5.0 # seconds the matched count must be unchanged
|
||||
MAX_WAIT_S = 60.0 # hard timeout per run
|
||||
CHECK_INTERVAL_S = 0.05 # polling interval
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test-data generation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
WORDS = [
|
||||
"home",
|
||||
"usr",
|
||||
"etc",
|
||||
"var",
|
||||
"opt",
|
||||
"tmp",
|
||||
"dev",
|
||||
"proc",
|
||||
"sys",
|
||||
"lib",
|
||||
"bin",
|
||||
"sbin",
|
||||
"boot",
|
||||
"mnt",
|
||||
"media",
|
||||
"src",
|
||||
"test",
|
||||
"config",
|
||||
"data",
|
||||
"logs",
|
||||
"cache",
|
||||
"backup",
|
||||
"docs",
|
||||
"images",
|
||||
"videos",
|
||||
"audio",
|
||||
"downloads",
|
||||
"uploads",
|
||||
"temp",
|
||||
"shared",
|
||||
]
|
||||
|
||||
|
||||
def generate_test_data(output_file: str, num_items: int) -> None:
|
||||
rng = random.Random()
|
||||
with open(output_file, "w") as fh:
|
||||
for i in range(1, num_items + 1):
|
||||
depth = rng.randint(2, 10)
|
||||
parts = [rng.choice(WORDS) for _ in range(depth)]
|
||||
fh.write("/".join(parts) + f"_{i}\n")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Argument parsing (no argparse – stdlib only, but argparse IS stdlib…
|
||||
# however, to keep the spirit of "no dependencies" we use manual parsing
|
||||
# since argparse is a stdlib module, not a third-party dep. We use it.)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def parse_args(argv):
|
||||
"""Return (binaries, opts, extra_args)."""
|
||||
import argparse
|
||||
|
||||
# Split off everything after "--"
|
||||
extra_args = []
|
||||
if "--" in argv:
|
||||
sep_idx = argv.index("--")
|
||||
extra_args = argv[sep_idx + 1 :]
|
||||
argv = argv[:sep_idx]
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Skim benchmark script",
|
||||
add_help=True,
|
||||
)
|
||||
parser.add_argument(
|
||||
"binaries",
|
||||
nargs="*",
|
||||
metavar="BINARY_PATH",
|
||||
help="Path(s) to binary (default: ./target/release/sk)",
|
||||
)
|
||||
parser.add_argument("-n", "--num-items", type=int, default=DEFAULT_NUM_ITEMS)
|
||||
parser.add_argument("-q", "--query", default=DEFAULT_QUERY)
|
||||
parser.add_argument("-r", "--runs", type=int, default=DEFAULT_RUNS)
|
||||
parser.add_argument("-f", "--file", default="")
|
||||
parser.add_argument("-g", "--generate-file", default="")
|
||||
parser.add_argument("-j", "--json", action="store_true")
|
||||
|
||||
opts = parser.parse_args(argv)
|
||||
|
||||
if not opts.binaries:
|
||||
opts.binaries = [DEFAULT_BINARY]
|
||||
|
||||
if opts.file and opts.generate_file:
|
||||
parser.error("Cannot use both --file and --generate-file")
|
||||
|
||||
return opts.binaries, opts, extra_args
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Resource monitor (background thread)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
import threading
|
||||
|
||||
|
||||
class ResourceMonitor(threading.Thread):
|
||||
"""Sample CPU and RSS of *pid* every 50 ms until the process exits."""
|
||||
|
||||
def __init__(self, pid: int):
|
||||
super().__init__(daemon=True)
|
||||
self.pid = pid
|
||||
self.peak_mem_kb: int = 0 # RSS in kB
|
||||
self.peak_cpu: float = 0.0 # %CPU
|
||||
|
||||
def run(self):
|
||||
while True:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["ps", "-p", str(self.pid), "-o", "rss=,%cpu="],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
line = result.stdout.strip()
|
||||
if not line:
|
||||
break
|
||||
parts = line.split()
|
||||
if len(parts) >= 2:
|
||||
try:
|
||||
mem = int(parts[0])
|
||||
cpu = float(parts[1])
|
||||
if mem > self.peak_mem_kb:
|
||||
self.peak_mem_kb = mem
|
||||
if cpu > self.peak_cpu:
|
||||
self.peak_cpu = cpu
|
||||
except ValueError:
|
||||
pass
|
||||
except Exception:
|
||||
break
|
||||
time.sleep(0.05)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Single run
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _find_sk_pid(pane_pid: int, binary_path: str) -> int:
|
||||
"""Try for up to 5 s to find the sk child PID under *pane_pid*."""
|
||||
for _ in range(50):
|
||||
time.sleep(0.1)
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["pgrep", "-P", str(pane_pid), "-f", binary_path],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
pids = result.stdout.strip().splitlines()
|
||||
if pids:
|
||||
return int(pids[0])
|
||||
except Exception:
|
||||
pass
|
||||
return 0
|
||||
|
||||
|
||||
def run_once(
|
||||
binary_path: str,
|
||||
query: str,
|
||||
tmp_file: str,
|
||||
num_items: int,
|
||||
extra_args: list,
|
||||
run_index: int,
|
||||
session_suffix: str,
|
||||
) -> dict:
|
||||
"""
|
||||
Execute one benchmark run against *binary_path*.
|
||||
Returns a dict with keys: elapsed_s, rate, matched, peak_mem_kb, peak_cpu,
|
||||
completed.
|
||||
"""
|
||||
session_name = f"skim_bench_{os.getpid()}_{session_suffix}_{run_index}"
|
||||
status_fd, status_file = tempfile.mkstemp(prefix="skim_bench_status_")
|
||||
os.close(status_fd)
|
||||
|
||||
env = os.environ.copy()
|
||||
env["SHELL"] = "/bin/sh"
|
||||
env.pop("HISTFILE", None)
|
||||
env.pop("FZF_DEFAULT_OPTS", None)
|
||||
env.pop("SKIM_DEFAULT_OPTIONS", None)
|
||||
|
||||
try:
|
||||
# Create tmux session
|
||||
subprocess.run(
|
||||
["tmux", "new-session", "-s", session_name, "-d"],
|
||||
check=True,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
)
|
||||
|
||||
# Clear env vars inside the session
|
||||
for cmd in [
|
||||
"unset HISTFILE",
|
||||
"unset FZF_DEFAULT_OPTS",
|
||||
"unset SKIM_DEFAULT_OPTIONS",
|
||||
]:
|
||||
subprocess.run(
|
||||
["tmux", "send-keys", "-t", session_name, cmd, "Enter"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
time.sleep(0.1)
|
||||
|
||||
# Build the command string
|
||||
extra_str = " ".join(extra_args)
|
||||
cmd_str = f"cat {tmp_file} | {binary_path} --query '{query}' {extra_str}"
|
||||
subprocess.run(
|
||||
["tmux", "send-keys", "-t", session_name, cmd_str, "Enter"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
|
||||
start_ns = time.perf_counter_ns()
|
||||
|
||||
# Locate sk PID for resource monitoring
|
||||
pane_pid = 0
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["tmux", "list-panes", "-t", session_name, "-F", "#{pane_pid}"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
pane_pid = int(r.stdout.strip().splitlines()[0])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
sk_pid = 0
|
||||
monitor = None
|
||||
if pane_pid:
|
||||
sk_pid = _find_sk_pid(pane_pid, binary_path)
|
||||
if sk_pid:
|
||||
monitor = ResourceMonitor(sk_pid)
|
||||
monitor.start()
|
||||
|
||||
# Poll for matcher completion
|
||||
completed = False
|
||||
matched_count = 0
|
||||
prev_matched_count = -1
|
||||
stable_start: float = 0.0
|
||||
end_ns = 0
|
||||
loop_start = time.monotonic()
|
||||
|
||||
while True:
|
||||
time.sleep(CHECK_INTERVAL_S)
|
||||
|
||||
now = time.monotonic()
|
||||
if now - loop_start >= MAX_WAIT_S:
|
||||
break
|
||||
|
||||
# Early exit if sk process is gone
|
||||
if sk_pid:
|
||||
try:
|
||||
os.kill(sk_pid, 0)
|
||||
except ProcessLookupError:
|
||||
break
|
||||
|
||||
# Capture tmux pane
|
||||
try:
|
||||
subprocess.run(
|
||||
[
|
||||
"tmux",
|
||||
"capture-pane",
|
||||
"-b",
|
||||
f"status-{session_name}",
|
||||
"-t",
|
||||
session_name,
|
||||
],
|
||||
capture_output=True,
|
||||
)
|
||||
subprocess.run(
|
||||
[
|
||||
"tmux",
|
||||
"save-buffer",
|
||||
"-b",
|
||||
f"status-{session_name}",
|
||||
status_file,
|
||||
],
|
||||
capture_output=True,
|
||||
)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# Parse "matched/total" from status line
|
||||
try:
|
||||
with open(status_file) as fh:
|
||||
content = fh.read()
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
import re
|
||||
|
||||
m = re.search(r"(\d+)/(\d+)", content)
|
||||
if not m:
|
||||
continue
|
||||
|
||||
mc = int(m.group(1))
|
||||
total = int(m.group(2))
|
||||
|
||||
if total == num_items:
|
||||
if mc != prev_matched_count:
|
||||
prev_matched_count = mc
|
||||
matched_count = mc
|
||||
stable_start = time.monotonic()
|
||||
end_ns = time.perf_counter_ns()
|
||||
elif stable_start > 0:
|
||||
if time.monotonic() - stable_start >= REQUIRED_STABLE_S:
|
||||
completed = True
|
||||
break
|
||||
|
||||
if end_ns == 0:
|
||||
end_ns = time.perf_counter_ns()
|
||||
|
||||
# Exit skim
|
||||
subprocess.run(
|
||||
["tmux", "send-keys", "-t", session_name, "Escape"],
|
||||
capture_output=True,
|
||||
)
|
||||
time.sleep(0.1)
|
||||
|
||||
# Wait for monitor
|
||||
if monitor is not None:
|
||||
monitor.join(timeout=2.0)
|
||||
|
||||
elapsed_s = (end_ns - start_ns) / 1e9
|
||||
rate = num_items / elapsed_s if elapsed_s > 0 else 0
|
||||
|
||||
peak_mem_kb = monitor.peak_mem_kb if monitor and monitor.peak_mem_kb else 0
|
||||
peak_cpu = monitor.peak_cpu if monitor and monitor.peak_cpu else 0.0
|
||||
|
||||
return {
|
||||
"elapsed_s": elapsed_s,
|
||||
"rate": rate,
|
||||
"matched": matched_count,
|
||||
"peak_mem_kb": peak_mem_kb if peak_mem_kb else None,
|
||||
"peak_cpu": peak_cpu if peak_cpu else None,
|
||||
"completed": completed,
|
||||
}
|
||||
|
||||
finally:
|
||||
subprocess.run(
|
||||
["tmux", "kill-session", "-t", session_name],
|
||||
capture_output=True,
|
||||
)
|
||||
try:
|
||||
os.unlink(status_file)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Aggregate statistics
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _avg(values):
|
||||
vals = [v for v in values if v is not None]
|
||||
return sum(vals) / len(vals) if vals else None
|
||||
|
||||
|
||||
def _min(values):
|
||||
vals = [v for v in values if v is not None]
|
||||
return min(vals) if vals else None
|
||||
|
||||
|
||||
def _max(values):
|
||||
vals = [v for v in values if v is not None]
|
||||
return max(vals) if vals else None
|
||||
|
||||
|
||||
def aggregate(results: list) -> dict:
|
||||
times = [r["elapsed_s"] for r in results]
|
||||
rates = [r["rate"] for r in results]
|
||||
matched = [r["matched"] for r in results]
|
||||
mems = [r["peak_mem_kb"] for r in results]
|
||||
cpus = [r["peak_cpu"] for r in results]
|
||||
completed = sum(1 for r in results if r["completed"])
|
||||
|
||||
return {
|
||||
"completed": completed,
|
||||
"runs": len(results),
|
||||
"avg_time": _avg(times),
|
||||
"min_time": _min(times),
|
||||
"max_time": _max(times),
|
||||
"avg_rate": _avg(rates),
|
||||
"min_rate": _min(rates),
|
||||
"max_rate": _max(rates),
|
||||
"avg_matched": _avg(matched),
|
||||
"min_matched": _min(matched),
|
||||
"max_matched": _max(matched),
|
||||
"avg_mem": _avg(mems),
|
||||
"min_mem": _min(mems),
|
||||
"max_mem": _max(mems),
|
||||
"avg_cpu": _avg(cpus),
|
||||
"min_cpu": _min(cpus),
|
||||
"max_cpu": _max(cpus),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Formatting helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _pct(baseline, value):
|
||||
"""Return a +/-XX.X% string comparing *value* to *baseline*."""
|
||||
if baseline is None or value is None or baseline == 0:
|
||||
return ""
|
||||
diff = (value - baseline) / abs(baseline) * 100
|
||||
sign = "+" if diff >= 0 else ""
|
||||
return f"{sign}{diff:.1f}%"
|
||||
|
||||
|
||||
def _fmt_mem(kb):
|
||||
if kb is None:
|
||||
return None
|
||||
return kb / 1024 # MB
|
||||
|
||||
|
||||
def _fmt_optional(value, fmt):
|
||||
if value is None:
|
||||
return "N/A"
|
||||
return fmt.format(value)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Output
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def print_human(
|
||||
binary_label: str,
|
||||
agg: dict,
|
||||
num_items: int,
|
||||
baseline: dict | None = None,
|
||||
is_baseline: bool = False,
|
||||
):
|
||||
tag = " [baseline]" if is_baseline else ""
|
||||
print(f"\n=== Results: {binary_label}{tag} ===")
|
||||
print(f"Completed runs: {agg['completed']} / {agg['runs']}")
|
||||
|
||||
def cmp(key, baseline_key=None):
|
||||
"""Format comparison vs baseline for a given aggregate key."""
|
||||
bk = baseline_key or key
|
||||
if baseline is None or is_baseline:
|
||||
return ""
|
||||
return " " + _pct(baseline.get(bk), agg.get(key))
|
||||
|
||||
# Items matched
|
||||
avg_m = _fmt_optional(agg["avg_matched"], "{:.0f}")
|
||||
min_m = _fmt_optional(agg["min_matched"], "{:.0f}")
|
||||
max_m = _fmt_optional(agg["max_matched"], "{:.0f}")
|
||||
print(
|
||||
f"Average items matched: {avg_m} / {num_items}"
|
||||
f" (min: {min_m}, max: {max_m})"
|
||||
f"{cmp('avg_matched')}"
|
||||
)
|
||||
|
||||
# Time
|
||||
avg_t = _fmt_optional(agg["avg_time"], "{:.3f}s")
|
||||
min_t = _fmt_optional(agg["min_time"], "{:.3f}s")
|
||||
max_t = _fmt_optional(agg["max_time"], "{:.3f}s")
|
||||
# Lower time is better, so flip sign for display
|
||||
time_cmp = ""
|
||||
if (
|
||||
baseline
|
||||
and not is_baseline
|
||||
and baseline.get("avg_time")
|
||||
and agg.get("avg_time")
|
||||
):
|
||||
diff = (
|
||||
(agg["avg_time"] - baseline["avg_time"]) / abs(baseline["avg_time"]) * 100
|
||||
)
|
||||
sign = "+" if diff >= 0 else ""
|
||||
time_cmp = f" {sign}{diff:.1f}%"
|
||||
print(f"Average time: {avg_t} (min: {min_t}, max: {max_t}){time_cmp}")
|
||||
|
||||
# Rate
|
||||
avg_r = _fmt_optional(agg["avg_rate"], "{:.0f}")
|
||||
min_r = _fmt_optional(agg["min_rate"], "{:.0f}")
|
||||
max_r = _fmt_optional(agg["max_rate"], "{:.0f}")
|
||||
print(
|
||||
f"Average items/second: {avg_r} (min: {min_r}, max: {max_r}){cmp('avg_rate')}"
|
||||
)
|
||||
|
||||
# Memory
|
||||
if agg["avg_mem"] is not None:
|
||||
avg_mb = _fmt_mem(agg["avg_mem"])
|
||||
min_mb = _fmt_mem(agg["min_mem"])
|
||||
max_mb = _fmt_mem(agg["max_mem"])
|
||||
print(
|
||||
f"Average peak memory usage: {avg_mb:.1f} MB"
|
||||
f" (min: {min_mb:.1f} MB, max: {max_mb:.1f} MB)"
|
||||
f"{cmp('avg_mem')}"
|
||||
)
|
||||
|
||||
# CPU
|
||||
if agg["avg_cpu"] is not None:
|
||||
avg_c = _fmt_optional(agg["avg_cpu"], "{:.1f}%")
|
||||
min_c = _fmt_optional(agg["min_cpu"], "{:.1f}%")
|
||||
max_c = _fmt_optional(agg["max_cpu"], "{:.1f}%")
|
||||
print(
|
||||
f"Average peak CPU usage: {avg_c}"
|
||||
f" (min: {min_c}, max: {max_c})"
|
||||
f"{cmp('avg_cpu')}"
|
||||
)
|
||||
|
||||
|
||||
def print_json_multi(binaries: list, aggregates: list, num_items: int, runs: int):
|
||||
output = []
|
||||
for binary, agg in zip(binaries, aggregates):
|
||||
entry = {
|
||||
"binary": binary,
|
||||
"num_items": num_items,
|
||||
"runs": runs,
|
||||
"completed_runs": agg["completed"],
|
||||
"items_matched": {
|
||||
"avg": agg["avg_matched"],
|
||||
"min": agg["min_matched"],
|
||||
"max": agg["max_matched"],
|
||||
},
|
||||
"time_s": {
|
||||
"avg": agg["avg_time"],
|
||||
"min": agg["min_time"],
|
||||
"max": agg["max_time"],
|
||||
},
|
||||
"items_per_second": {
|
||||
"avg": agg["avg_rate"],
|
||||
"min": agg["min_rate"],
|
||||
"max": agg["max_rate"],
|
||||
},
|
||||
"peak_memory_kb": {
|
||||
"avg": agg["avg_mem"],
|
||||
"min": agg["min_mem"],
|
||||
"max": agg["max_mem"],
|
||||
},
|
||||
"peak_cpu": {
|
||||
"avg": agg["avg_cpu"],
|
||||
"min": agg["min_cpu"],
|
||||
"max": agg["max_cpu"],
|
||||
},
|
||||
}
|
||||
output.append(entry)
|
||||
print(json.dumps(output if len(output) > 1 else output[0]))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def main():
|
||||
import re # ensure import at top of main scope for run_once
|
||||
|
||||
binaries, opts, extra_args = parse_args(sys.argv[1:])
|
||||
|
||||
num_items = opts.num_items
|
||||
query = opts.query
|
||||
runs = opts.runs
|
||||
input_file = opts.file
|
||||
generate_file = opts.generate_file
|
||||
as_json = opts.json
|
||||
|
||||
# ---- generate-file mode ------------------------------------------------
|
||||
if generate_file:
|
||||
print(f"Generating {num_items} items to {generate_file}...", file=sys.stderr)
|
||||
generate_test_data(generate_file, num_items)
|
||||
print(f"Generated {num_items} items successfully", file=sys.stderr)
|
||||
return
|
||||
|
||||
# ---- prepare input data ------------------------------------------------
|
||||
cleanup_input = False
|
||||
if input_file:
|
||||
if not os.path.isfile(input_file):
|
||||
print(f"Error: Input file '{input_file}' not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
tmp_file = input_file
|
||||
with open(input_file) as fh:
|
||||
num_items = sum(1 for _ in fh)
|
||||
print(f"Using input file with {num_items} items", file=sys.stderr)
|
||||
else:
|
||||
fd, tmp_file = tempfile.mkstemp(prefix="skim_bench_input_")
|
||||
os.close(fd)
|
||||
cleanup_input = True
|
||||
print("Generating test data...", file=sys.stderr)
|
||||
generate_test_data(tmp_file, num_items)
|
||||
|
||||
try:
|
||||
# ---- header ---------------------------------------------------------
|
||||
binary_list = ", ".join(binaries)
|
||||
print(f"=== Skim Ingestion + Matching Benchmark ===", file=sys.stderr)
|
||||
print(
|
||||
f"Binaries: {binary_list} | Items: {num_items} | "
|
||||
f"Query: '{query}' | Runs: {runs} (per binary)",
|
||||
file=sys.stderr,
|
||||
)
|
||||
if input_file:
|
||||
print(f"Input file: {input_file}", file=sys.stderr)
|
||||
if extra_args:
|
||||
print(f"Extra args: {' '.join(extra_args)}", file=sys.stderr)
|
||||
|
||||
# ---- run benchmark in round-robin -----------------------------------
|
||||
# all_results[i] = list of per-run dicts for binaries[i]
|
||||
all_results = [[] for _ in binaries]
|
||||
|
||||
for run_num in range(1, runs + 1):
|
||||
for bi, binary in enumerate(binaries):
|
||||
label = f"[{os.path.basename(binary)}]"
|
||||
if runs > 1 or len(binaries) > 1:
|
||||
print(
|
||||
f"\n=== Run {run_num}/{runs} — binary {bi + 1}/{len(binaries)}: {binary} ===",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
result = run_once(
|
||||
binary_path=binary,
|
||||
query=query,
|
||||
tmp_file=tmp_file,
|
||||
num_items=num_items,
|
||||
extra_args=extra_args,
|
||||
run_index=run_num,
|
||||
session_suffix=f"b{bi}",
|
||||
)
|
||||
all_results[bi].append(result)
|
||||
|
||||
if runs > 1 or len(binaries) > 1:
|
||||
status = "COMPLETED" if result["completed"] else "TIMEOUT"
|
||||
print(f"Status: {status}", file=sys.stderr)
|
||||
print(
|
||||
f"Items matched: {result['matched']} / {num_items}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
print(f"Total time: {result['elapsed_s']:.3f}s", file=sys.stderr)
|
||||
print(f"Items/second: {result['rate']:.0f}", file=sys.stderr)
|
||||
if result["peak_mem_kb"]:
|
||||
print(
|
||||
f"Peak memory usage: {result['peak_mem_kb'] / 1024:.1f} MB",
|
||||
file=sys.stderr,
|
||||
)
|
||||
if result["peak_cpu"]:
|
||||
print(
|
||||
f"Peak CPU usage: {result['peak_cpu']:.1f}%",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
# ---- aggregate ------------------------------------------------------
|
||||
aggregates = [aggregate(all_results[i]) for i in range(len(binaries))]
|
||||
|
||||
# ---- output ---------------------------------------------------------
|
||||
if as_json:
|
||||
print_json_multi(binaries, aggregates, num_items, runs)
|
||||
else:
|
||||
baseline_agg = aggregates[0]
|
||||
for i, (binary, agg) in enumerate(zip(binaries, aggregates)):
|
||||
print_human(
|
||||
binary_label=binary,
|
||||
agg=agg,
|
||||
num_items=num_items,
|
||||
baseline=baseline_agg if len(binaries) > 1 else None,
|
||||
is_baseline=(i == 0),
|
||||
)
|
||||
|
||||
# Summary comparison table when multiple binaries
|
||||
if len(binaries) > 1:
|
||||
print(f"\n=== Comparison Summary (vs baseline: {binaries[0]}) ===")
|
||||
header = f"{'Binary':<40} {'Avg time':>12} {'Δ time':>10} {'Avg rate':>14} {'Δ rate':>10}"
|
||||
print(header)
|
||||
print("-" * len(header))
|
||||
for i, (binary, agg) in enumerate(zip(binaries, aggregates)):
|
||||
t = (
|
||||
f"{agg['avg_time']:.3f}s"
|
||||
if agg["avg_time"] is not None
|
||||
else "N/A"
|
||||
)
|
||||
r = (
|
||||
f"{agg['avg_rate']:.0f}"
|
||||
if agg["avg_rate"] is not None
|
||||
else "N/A"
|
||||
)
|
||||
if i == 0:
|
||||
dt = "baseline"
|
||||
dr = "baseline"
|
||||
else:
|
||||
dt = _pct(baseline_agg["avg_time"], agg["avg_time"])
|
||||
dr = _pct(baseline_agg["avg_rate"], agg["avg_rate"])
|
||||
name = os.path.basename(binary) if len(binary) > 40 else binary
|
||||
print(f"{name:<40} {t:>12} {dt:>10} {r:>14} {dr:>10}")
|
||||
|
||||
finally:
|
||||
if cleanup_input:
|
||||
try:
|
||||
os.unlink(tmp_file)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
510
bench.sh
510
bench.sh
|
|
@ -1,510 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Benchmark script to measure ingestion + matching rate in skim interactive mode
|
||||
# This measures how fast skim can ingest items and display matched results
|
||||
#
|
||||
# Usage: bench.sh [BINARY_PATH] [-n|--num-items NUM] [-q|--query QUERY] [-r|--runs RUNS]
|
||||
# [-f|--file FILE] [-g|--generate-file FILE] [-- EXTRA_ARGS...]
|
||||
#
|
||||
# Arguments:
|
||||
# BINARY_PATH Path to binary (default: ./target/release/sk)
|
||||
# -n, --num-items NUM Number of items to generate (default: 1000000)
|
||||
# -q, --query QUERY Query string to search (default: "test")
|
||||
# -r, --runs RUNS Number of benchmark runs to average (default: 1)
|
||||
# -f, --file FILE Use existing file as input instead of generating
|
||||
# -g, --generate-file FILE Generate test data to file and exit
|
||||
# -- Pass remaining arguments to the binary
|
||||
#
|
||||
# Examples:
|
||||
# ./bench.sh # Use defaults
|
||||
# ./bench.sh ./target/release/sk -n 500000 -q foo
|
||||
# ./bench.sh -n 1000000 -q test -- --no-sort --exact
|
||||
# ./bench.sh -r 5 # Run 5 times and show average
|
||||
# ./bench.sh -f input.txt -q search # Use existing file
|
||||
# ./bench.sh -g testdata.txt -n 2000000 # Generate file and exit
|
||||
|
||||
set -e
|
||||
|
||||
# Send all non-final output to stderr. Save original stdout on fd 3 so we can
|
||||
# restore it later for the final results which should go to stdout.
|
||||
exec 3>&1 1>&2
|
||||
export SHELL="/bin/sh"
|
||||
unset HISTFILE
|
||||
|
||||
# Default values
|
||||
BINARY_PATH="./target/release/sk"
|
||||
NUM_ITEMS=1000000
|
||||
QUERY="test"
|
||||
RUNS=1
|
||||
INPUT_FILE=""
|
||||
GENERATE_FILE=""
|
||||
EXTRA_ARGS=""
|
||||
JSON=0
|
||||
|
||||
# Print unified JSON result. Expects the aggregate variables to be set:
|
||||
# AVG_MATCHED, MIN_MATCHED, MAX_MATCHED,
|
||||
# AVG_TIME, MIN_TIME, MAX_TIME,
|
||||
# AVG_RATE, MIN_RATE, MAX_RATE,
|
||||
# AVG_MEM, MIN_MEM, MAX_MEM (use string "null" when not measured)
|
||||
# AVG_CPU, MIN_CPU, MAX_CPU (use string "null" when not measured)
|
||||
print_json() {
|
||||
printf '{'
|
||||
printf '"num_items":%s,' "$NUM_ITEMS"
|
||||
printf '"runs":%s,' "$RUNS"
|
||||
printf '"completed_runs":%s,' "$COMPLETED_COUNT"
|
||||
printf '"items_matched":{"avg":%s,"min":%s,"max":%s},' "$AVG_MATCHED" "$MIN_MATCHED" "$MAX_MATCHED"
|
||||
printf '"time_s":{"avg":%s,"min":%s,"max":%s},' "$AVG_TIME" "$MIN_TIME" "$MAX_TIME"
|
||||
printf '"items_per_second":{"avg":%s,"min":%s,"max":%s},' "$AVG_RATE" "$MIN_RATE" "$MAX_RATE"
|
||||
|
||||
if [ "$AVG_MEM" = "null" ] || [ "$MIN_MEM" = "null" ] || [ "$MAX_MEM" = "null" ]; then
|
||||
printf '"peak_memory_kb":{"avg":null,"min":null,"max":null},'
|
||||
else
|
||||
printf '"peak_memory_kb":{"avg":%s,"min":%s,"max":%s},' "$AVG_MEM" "$MIN_MEM" "$MAX_MEM"
|
||||
fi
|
||||
|
||||
if [ "$AVG_CPU" = "null" ] || [ "$MIN_CPU" = "null" ] || [ "$MAX_CPU" = "null" ]; then
|
||||
printf '"peak_cpu":{"avg":null,"min":null,"max":null}'
|
||||
else
|
||||
printf '"peak_cpu":{"avg":%s,"min":%s,"max":%s}' "$AVG_CPU" "$MIN_CPU" "$MAX_CPU"
|
||||
fi
|
||||
|
||||
printf '}\n'
|
||||
}
|
||||
|
||||
# Parse arguments
|
||||
ARGS=()
|
||||
FOUND_SEP=0
|
||||
|
||||
for arg in "$@"; do
|
||||
if [ "$arg" = "--" ]; then
|
||||
FOUND_SEP=1
|
||||
elif [ $FOUND_SEP -eq 0 ]; then
|
||||
ARGS+=("$arg")
|
||||
else
|
||||
EXTRA_ARGS="$EXTRA_ARGS $arg"
|
||||
fi
|
||||
done
|
||||
|
||||
# Parse named arguments
|
||||
i=0
|
||||
while [ $i -lt ${#ARGS[@]} ]; do
|
||||
arg="${ARGS[$i]}"
|
||||
case "$arg" in
|
||||
-n | --num-items)
|
||||
i=$((i + 1))
|
||||
NUM_ITEMS="${ARGS[$i]}"
|
||||
;;
|
||||
-q | --query)
|
||||
i=$((i + 1))
|
||||
QUERY="${ARGS[$i]}"
|
||||
;;
|
||||
-r | --runs)
|
||||
i=$((i + 1))
|
||||
RUNS="${ARGS[$i]}"
|
||||
;;
|
||||
-f | --file)
|
||||
i=$((i + 1))
|
||||
INPUT_FILE="${ARGS[$i]}"
|
||||
;;
|
||||
-g | --generate-file)
|
||||
i=$((i + 1))
|
||||
GENERATE_FILE="${ARGS[$i]}"
|
||||
;;
|
||||
-j | --json)
|
||||
JSON=1
|
||||
;;
|
||||
-*)
|
||||
echo "Unknown option: $arg" >&2
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
# First non-option argument is binary path
|
||||
BINARY_PATH="$arg"
|
||||
;;
|
||||
esac
|
||||
i=$((i + 1))
|
||||
done
|
||||
|
||||
# Trim leading space from EXTRA_ARGS
|
||||
EXTRA_ARGS=$(echo "$EXTRA_ARGS" | sed 's/^ *//')
|
||||
|
||||
# Validate conflicting options
|
||||
if [ -n "$INPUT_FILE" ] && [ -n "$GENERATE_FILE" ]; then
|
||||
echo "Error: Cannot use both --file and --generate-file" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Function to generate test data
|
||||
generate_test_data() {
|
||||
local output_file="$1"
|
||||
local num_items="$2"
|
||||
|
||||
awk -v num="$num_items" 'BEGIN {
|
||||
srand()
|
||||
words[1]="home"; words[2]="usr"; words[3]="etc"; words[4]="var"; words[5]="opt"
|
||||
words[6]="tmp"; words[7]="dev"; words[8]="proc"; words[9]="sys"; words[10]="lib"
|
||||
words[11]="bin"; words[12]="sbin"; words[13]="boot"; words[14]="mnt"; words[15]="media"
|
||||
words[16]="src"; words[17]="test"; words[18]="config"; words[19]="data"; words[20]="logs"
|
||||
words[21]="cache"; words[22]="backup"; words[23]="docs"; words[24]="images"; words[25]="videos"
|
||||
words[26]="audio"; words[27]="downloads"; words[28]="uploads"; words[29]="temp"; words[30]="shared"
|
||||
|
||||
for (i = 1; i <= num; i++) {
|
||||
depth = int(rand() * 9) + 2 # 2-10 depth
|
||||
path = ""
|
||||
for (j = 1; j <= depth; j++) {
|
||||
word_idx = int(rand() * 30) + 1
|
||||
path = path words[word_idx]
|
||||
if (j < depth) path = path "/"
|
||||
}
|
||||
print path "_" i
|
||||
}
|
||||
}' >"$output_file"
|
||||
}
|
||||
|
||||
# Handle --generate-file mode
|
||||
if [ -n "$GENERATE_FILE" ]; then
|
||||
echo "Generating $NUM_ITEMS items to $GENERATE_FILE..."
|
||||
generate_test_data "$GENERATE_FILE" "$NUM_ITEMS"
|
||||
echo "Generated $NUM_ITEMS items successfully"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "=== Skim Ingestion + Matching Benchmark ==="
|
||||
echo "Binary: $BINARY_PATH | Items: $NUM_ITEMS | Query: '$QUERY' | Runs: $RUNS"
|
||||
[ -n "$INPUT_FILE" ] && echo "Input file: $INPUT_FILE"
|
||||
[ -n "$EXTRA_ARGS" ] && echo "Extra args: $EXTRA_ARGS"
|
||||
|
||||
# Arrays to store results from multiple runs
|
||||
ELAPSED_TIMES=()
|
||||
RATES=()
|
||||
PEAK_MEMS=()
|
||||
PEAK_CPUS=()
|
||||
MATCHED_COUNTS=()
|
||||
COMPLETED_COUNT=0
|
||||
|
||||
# Prepare test data file
|
||||
STATUS_FILE=$(mktemp)
|
||||
CLEANUP_INPUT=0
|
||||
|
||||
if [ -n "$INPUT_FILE" ]; then
|
||||
# Use provided input file
|
||||
if [ ! -f "$INPUT_FILE" ]; then
|
||||
echo "Error: Input file '$INPUT_FILE' not found" >&2
|
||||
exit 1
|
||||
fi
|
||||
TMP_FILE="$INPUT_FILE"
|
||||
# Count lines in the file to determine NUM_ITEMS
|
||||
NUM_ITEMS=$(wc -l <"$INPUT_FILE")
|
||||
echo "Using input file with $NUM_ITEMS items"
|
||||
else
|
||||
# Generate test data to temporary file
|
||||
TMP_FILE=$(mktemp)
|
||||
CLEANUP_INPUT=1
|
||||
echo "Generating test data..."
|
||||
generate_test_data "$TMP_FILE" "$NUM_ITEMS"
|
||||
fi
|
||||
|
||||
trap "rm -f $STATUS_FILE; [ $CLEANUP_INPUT -eq 1 ] && rm -f $TMP_FILE || true" EXIT
|
||||
|
||||
# Run benchmark multiple times
|
||||
for RUN in $(seq 1 $RUNS); do
|
||||
if [ $RUNS -gt 1 ]; then
|
||||
echo ""
|
||||
echo "=== Run $RUN/$RUNS ==="
|
||||
fi
|
||||
|
||||
SESSION_NAME="skim_bench_$$_$RUN"
|
||||
|
||||
# Create a new tmux session in the background
|
||||
tmux new-session -s "$SESSION_NAME" -d
|
||||
|
||||
# Unset HISTFILE in the tmux session to prevent command from appearing in shell history
|
||||
tmux send-keys -t "$SESSION_NAME" "unset HISTFILE" Enter
|
||||
tmux send-keys -t "$SESSION_NAME" "unset FZF_DEFAULT_OPTS" Enter
|
||||
tmux send-keys -t "$SESSION_NAME" "unset SKIM_DEFAULT_OPTIONS" Enter
|
||||
sleep 0.1
|
||||
|
||||
# Prepare to capture the start time as close to data ingestion as possible
|
||||
# Run skim with the query already set, and measure until matcher completes
|
||||
tmux send-keys -t "$SESSION_NAME" "cat $TMP_FILE | $BINARY_PATH --query '$QUERY' $EXTRA_ARGS" Enter
|
||||
|
||||
# Record start time
|
||||
START=$(date +%s%N)
|
||||
|
||||
# Find skim PID for resource monitoring.
|
||||
# We combine -P (parent = tmux pane shell) with -f (full cmdline contains
|
||||
# BINARY_PATH) so that transient children like direnv that run before sk
|
||||
# are ignored. Using -P avoids the self-match problem of plain pgrep -f
|
||||
# (where the pgrep invocation's own argv would contain BINARY_PATH).
|
||||
TMUX_PANE_PID=$(tmux list-panes -t "$SESSION_NAME" -F '#{pane_pid}' 2>/dev/null || echo "")
|
||||
SK_PID=""
|
||||
MONITOR_LOG=""
|
||||
if [ -n "$TMUX_PANE_PID" ]; then
|
||||
for i in $(seq 1 50); do
|
||||
sleep 0.1
|
||||
SK_PID=$(pgrep -P "$TMUX_PANE_PID" -f "$BINARY_PATH" 2>/dev/null | head -1)
|
||||
if [ -n "$SK_PID" ]; then
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [ -n "$SK_PID" ]; then
|
||||
# Start background monitoring of CPU and RAM
|
||||
MONITOR_LOG="/tmp/skim-monitor-$SK_PID.log"
|
||||
rm -f "$MONITOR_LOG"
|
||||
(
|
||||
PEAK_MEM=0
|
||||
PEAK_CPU=0
|
||||
while kill -0 "$SK_PID" 2>/dev/null; do
|
||||
MEM=$(ps -p "$SK_PID" -o rss= 2>/dev/null | tr -d ' ')
|
||||
CPU=$(ps -p "$SK_PID" -o %cpu= 2>/dev/null | tr -d ' ')
|
||||
if [ -n "$MEM" ] && [ "$MEM" -gt "$PEAK_MEM" ]; then
|
||||
PEAK_MEM=$MEM
|
||||
fi
|
||||
if [ -n "$CPU" ]; then
|
||||
CPU_INT=$(echo "$CPU" | cut -d. -f1)
|
||||
PEAK_CPU_INT=$(echo "$PEAK_CPU" | cut -d. -f1)
|
||||
if [ "$CPU_INT" -gt "$PEAK_CPU_INT" ]; then
|
||||
PEAK_CPU=$CPU
|
||||
fi
|
||||
fi
|
||||
echo "$MEM $CPU" >>"$MONITOR_LOG"
|
||||
sleep 0.05
|
||||
done
|
||||
echo "PEAK:$PEAK_MEM:$PEAK_CPU" >>"$MONITOR_LOG"
|
||||
) &
|
||||
MONITOR_PID=$!
|
||||
else
|
||||
MONITOR_PID=""
|
||||
fi
|
||||
|
||||
# Monitor for matcher completion by checking the tmux status line.
|
||||
# We consider matching done when:
|
||||
# (a) total ingested == NUM_ITEMS, AND
|
||||
# (b) the matched count has been stable for REQUIRED_STABLE_DURATION_NS.
|
||||
# We also bail out early if skim has exited (tmux pane gone / SK_PID dead).
|
||||
COMPLETED=0
|
||||
MATCHED_COUNT=0
|
||||
TOTAL_INGESTED=0
|
||||
PREV_MATCHED_COUNT=-1
|
||||
STABLE_START_TIME=0
|
||||
REQUIRED_STABLE_DURATION_NS=5000000000 # 5 seconds in nanoseconds
|
||||
MAX_WAIT_NS=$((60 * 1000000000)) # 60-second hard timeout
|
||||
CHECK_INTERVAL=0.05 # 50 ms between checks
|
||||
END=0
|
||||
LOOP_START=$(date +%s%N)
|
||||
|
||||
while true; do
|
||||
sleep $CHECK_INTERVAL
|
||||
|
||||
# Hard timeout: give up after MAX_WAIT_NS regardless
|
||||
NOW=$(date +%s%N)
|
||||
if [ $((NOW - LOOP_START)) -ge $MAX_WAIT_NS ]; then
|
||||
break
|
||||
fi
|
||||
|
||||
# Early-exit: if the skim process has exited, stop waiting.
|
||||
# We only break on a dead PID — the pane-child fallback is removed
|
||||
# because transient pre-sk processes (e.g. direnv) would trigger it
|
||||
# falsely before sk has even launched.
|
||||
if [ -n "$SK_PID" ] && ! kill -0 "$SK_PID" 2>/dev/null; then
|
||||
break
|
||||
fi
|
||||
|
||||
# Capture and check status using bench.sh's method
|
||||
tmux capture-pane -b "status-$SESSION_NAME" -t "$SESSION_NAME" 2>/dev/null || true
|
||||
tmux save-buffer -b "status-$SESSION_NAME" "$STATUS_FILE" 2>/dev/null || true
|
||||
|
||||
if [ -f "$STATUS_FILE" ]; then
|
||||
# Skim status line format is typically: " > query matched/total"
|
||||
# The first number is matched items, second is total ingested items
|
||||
STATUS_LINE=$(grep -oE '[0-9]+/[0-9]+' "$STATUS_FILE" 2>/dev/null | head -1 || echo "")
|
||||
if [ -n "$STATUS_LINE" ]; then
|
||||
MATCHED_COUNT=$(echo "$STATUS_LINE" | cut -d'/' -f1)
|
||||
TOTAL_INGESTED=$(echo "$STATUS_LINE" | cut -d'/' -f2)
|
||||
|
||||
# Check if ingestion is complete
|
||||
if [ "$TOTAL_INGESTED" = "$NUM_ITEMS" ]; then
|
||||
if [ "$MATCHED_COUNT" != "$PREV_MATCHED_COUNT" ]; then
|
||||
# Count changed: reset stability timer and record candidate end time
|
||||
PREV_MATCHED_COUNT=$MATCHED_COUNT
|
||||
STABLE_START_TIME=$(date +%s%N)
|
||||
END=$STABLE_START_TIME
|
||||
elif [ $STABLE_START_TIME -gt 0 ]; then
|
||||
# Count unchanged: check if stable long enough
|
||||
CURRENT_TIME=$(date +%s%N)
|
||||
if [ $((CURRENT_TIME - STABLE_START_TIME)) -ge $REQUIRED_STABLE_DURATION_NS ]; then
|
||||
COMPLETED=1
|
||||
break
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# If we didn't capture an end time, set it now
|
||||
if [ $END -eq 0 ]; then
|
||||
END=$(date +%s%N)
|
||||
fi
|
||||
|
||||
# Exit skim
|
||||
tmux send-keys -t "$SESSION_NAME" Escape
|
||||
sleep 0.1
|
||||
|
||||
# Wait for monitor to finish if it was started
|
||||
if [ -n "$MONITOR_PID" ]; then
|
||||
wait "$MONITOR_PID" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# Clean up session
|
||||
tmux kill-session -t "$SESSION_NAME" 2>/dev/null || true
|
||||
|
||||
ELAPSED_NS=$((END - START))
|
||||
ELAPSED_SEC=$(awk "BEGIN {printf \"%.3f\", $ELAPSED_NS / 1000000000}")
|
||||
RATE=$(awk "BEGIN {printf \"%.0f\", $NUM_ITEMS / $ELAPSED_SEC}")
|
||||
|
||||
# Extract peak CPU and RAM usage.
|
||||
# Use empty string as sentinel for "not measured" so that averaging logic
|
||||
# can skip these runs rather than treating 0 as a valid sample.
|
||||
PEAK_MEM=""
|
||||
PEAK_CPU=""
|
||||
if [ -n "$MONITOR_PID" ] && [ -n "$MONITOR_LOG" ] && [ -f "$MONITOR_LOG" ]; then
|
||||
PEAK_LINE=$(grep "^PEAK:" "$MONITOR_LOG" 2>/dev/null || echo "")
|
||||
if [ -n "$PEAK_LINE" ]; then
|
||||
PEAK_MEM=$(echo "$PEAK_LINE" | cut -d: -f2)
|
||||
PEAK_CPU=$(echo "$PEAK_LINE" | cut -d: -f3)
|
||||
# Treat 0 as "not measured" (monitor never sampled anything meaningful)
|
||||
[ "$PEAK_MEM" = "0" ] && PEAK_MEM=""
|
||||
[ "$PEAK_CPU" = "0" ] && PEAK_CPU=""
|
||||
fi
|
||||
rm -f "$MONITOR_LOG"
|
||||
fi
|
||||
|
||||
# Store results
|
||||
ELAPSED_TIMES+=("$ELAPSED_SEC")
|
||||
RATES+=("$RATE")
|
||||
MATCHED_COUNTS+=("$MATCHED_COUNT")
|
||||
PEAK_MEMS+=("$PEAK_MEM")
|
||||
PEAK_CPUS+=("$PEAK_CPU")
|
||||
|
||||
if [ $COMPLETED -eq 1 ]; then
|
||||
COMPLETED_COUNT=$((COMPLETED_COUNT + 1))
|
||||
fi
|
||||
|
||||
# Print individual run results
|
||||
if [ $RUNS -gt 1 ]; then
|
||||
echo "Status: $(if [ $COMPLETED -eq 1 ]; then echo 'COMPLETED'; else echo 'TIMEOUT'; fi)"
|
||||
echo "Items matched: $MATCHED_COUNT / $NUM_ITEMS"
|
||||
echo "Total time: ${ELAPSED_SEC}s"
|
||||
echo "Items/second: ${RATE}"
|
||||
if [ -n "$PEAK_MEM" ]; then
|
||||
echo "Peak memory usage: $((PEAK_MEM / 1024)) MB"
|
||||
fi
|
||||
if [ -n "$PEAK_CPU" ]; then
|
||||
echo "Peak CPU usage: ${PEAK_CPU}%"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Completed runs: $COMPLETED_COUNT / $RUNS"
|
||||
|
||||
# Calculate averages
|
||||
AVG_TIME=$(awk -v times="${ELAPSED_TIMES[*]}" 'BEGIN {
|
||||
n = split(times, arr, " ")
|
||||
sum = 0
|
||||
for (i = 1; i <= n; i++) sum += arr[i]
|
||||
printf "%.3f", sum / n
|
||||
}')
|
||||
|
||||
AVG_RATE=$(awk -v rates="${RATES[*]}" 'BEGIN {
|
||||
n = split(rates, arr, " ")
|
||||
sum = 0
|
||||
for (i = 1; i <= n; i++) sum += arr[i]
|
||||
printf "%.0f", sum / n
|
||||
}')
|
||||
|
||||
AVG_MATCHED=$(awk -v counts="${MATCHED_COUNTS[*]}" 'BEGIN {
|
||||
n = split(counts, arr, " ")
|
||||
sum = 0
|
||||
for (i = 1; i <= n; i++) sum += arr[i]
|
||||
printf "%.0f", sum / n
|
||||
}')
|
||||
|
||||
AVG_MEM=$(awk -v mems="${PEAK_MEMS[*]}" 'BEGIN {
|
||||
n = split(mems, arr, " ")
|
||||
sum = 0
|
||||
count = 0
|
||||
for (i = 1; i <= n; i++) {
|
||||
if (arr[i] != "" && arr[i] + 0 > 0) {
|
||||
sum += arr[i]
|
||||
count++
|
||||
}
|
||||
}
|
||||
if (count > 0) printf "%.0f", sum / count
|
||||
else print ""
|
||||
}')
|
||||
|
||||
AVG_CPU=$(awk -v cpus="${PEAK_CPUS[*]}" 'BEGIN {
|
||||
n = split(cpus, arr, " ")
|
||||
sum = 0
|
||||
count = 0
|
||||
for (i = 1; i <= n; i++) {
|
||||
if (arr[i] != "" && arr[i] + 0 > 0) {
|
||||
sum += arr[i]
|
||||
count++
|
||||
}
|
||||
}
|
||||
if (count > 0) printf "%.1f", sum / count
|
||||
else print ""
|
||||
}')
|
||||
|
||||
# Calculate min/max for several metrics so we can show them alongside averages
|
||||
read -r MIN_TIME MAX_TIME <<<"$(awk -v times="${ELAPSED_TIMES[*]}" 'BEGIN { n=split(times,a," "); min=a[1]; max=a[1]; for(i=1;i<=n;i++){ if(a[i]<min) min=a[i]; if(a[i]>max) max=a[i]; } printf "%.3f %.3f", min, max }')"
|
||||
read -r MIN_RATE MAX_RATE <<<"$(awk -v rates="${RATES[*]}" 'BEGIN { n=split(rates,a," "); min=a[1]; max=a[1]; for(i=1;i<=n;i++){ if(a[i]<min) min=a[i]; if(a[i]>max) max=a[i]; } printf "%.0f %.0f", min, max }')"
|
||||
read -r MIN_MATCHED MAX_MATCHED <<<"$(awk -v counts="${MATCHED_COUNTS[*]}" 'BEGIN { n=split(counts,a," "); min=a[1]; max=a[1]; for(i=1;i<=n;i++){ if(a[i]<min) min=a[i]; if(a[i]>max) max=a[i]; } printf "%.0f %.0f", min, max }')"
|
||||
|
||||
# For memory and CPU, ignore empty/zero entries (meaning not measured).
|
||||
# Output is empty string when no run was measured, so that downstream null
|
||||
# checks work correctly.
|
||||
read -r MIN_MEM MAX_MEM <<<"$(awk -v mems="${PEAK_MEMS[*]}" 'BEGIN { n=split(mems,a," "); min=1e18; max=0; found=0; for(i=1;i<=n;i++){ if(a[i] != "" && a[i]+0 > 0){ if(a[i]<min) min=a[i]; if(a[i]>max) max=a[i]; found=1 } } if(found) printf "%.0f %.0f", min, max; else print "" }')"
|
||||
read -r MIN_CPU MAX_CPU <<<"$(awk -v cpus="${PEAK_CPUS[*]}" 'BEGIN { n=split(cpus,a," "); min=1e18; max=0; found=0; for(i=1;i<=n;i++){ if(a[i] != "" && a[i]+0 > 0){ if(a[i]<min) min=a[i]; if(a[i]>max) max=a[i]; found=1 } } if(found) printf "%.1f %.1f", min, max; else print "" }')"
|
||||
|
||||
# Restore stdout for final results and display them on stdout
|
||||
echo ""
|
||||
exec 1>&3 3>&-
|
||||
|
||||
# If JSON output requested, emit a single-line JSON object and exit
|
||||
if [ "$JSON" -eq 1 ]; then
|
||||
# Ensure numeric defaults
|
||||
AVG_MEM=${AVG_MEM:-"null"}
|
||||
MIN_MEM=${MIN_MEM:-"null"}
|
||||
MAX_MEM=${MAX_MEM:-"null"}
|
||||
AVG_CPU=${AVG_CPU:-"null"}
|
||||
MIN_CPU=${MIN_CPU:-"null"}
|
||||
MAX_CPU=${MAX_CPU:-"null"}
|
||||
printf '{'
|
||||
printf '"num_items":%s,' "$NUM_ITEMS"
|
||||
printf '"runs":%s,' "$RUNS"
|
||||
printf '"completed_runs":%s,' "$COMPLETED_COUNT"
|
||||
printf '"items_matched":{"avg":%s,"min":%s,"max":%s},' "$AVG_MATCHED" "$MIN_MATCHED" "$MAX_MATCHED"
|
||||
printf '"time_s":{"avg":%s,"min":%s,"max":%s},' "$AVG_TIME" "$MIN_TIME" "$MAX_TIME"
|
||||
printf '"items_per_second":{"avg":%s,"min":%s,"max":%s},' "$AVG_RATE" "$MIN_RATE" "$MAX_RATE"
|
||||
printf '"peak_memory_kb":{"avg":%s,"min":%s,"max":%s},' "$AVG_MEM" "$MIN_MEM" "$MAX_MEM"
|
||||
printf '"peak_cpu":{"avg":%s,"min":%s,"max":%s}' "$AVG_CPU" "$MIN_CPU" "$MAX_CPU"
|
||||
printf '}\n'
|
||||
exit 0
|
||||
else
|
||||
echo "=== Results ==="
|
||||
|
||||
echo "Average items matched: $AVG_MATCHED / $NUM_ITEMS (min: $MIN_MATCHED, max: $MAX_MATCHED)"
|
||||
echo "Average time: ${AVG_TIME}s (min: ${MIN_TIME}s, max: ${MAX_TIME}s)"
|
||||
echo "Average items/second: ${AVG_RATE} (min: ${MIN_RATE}, max: ${MAX_RATE})"
|
||||
if [ -n "$AVG_MEM" ]; then
|
||||
echo "Average peak memory usage: $((AVG_MEM / 1024)) MB (min: $((MIN_MEM / 1024)) MB, max: $((MAX_MEM / 1024)) MB)"
|
||||
fi
|
||||
if [ -n "$AVG_CPU" ]; then
|
||||
echo "Average peak CPU usage: ${AVG_CPU}% (min: ${MIN_CPU}%, max: ${MAX_CPU}%)"
|
||||
fi
|
||||
fi
|
||||
|
|
@ -31,13 +31,6 @@ fn prepare(file: &str, opt_builder: &mut SkimOptionsBuilder) -> (SkimOptions, Sk
|
|||
}
|
||||
|
||||
fn criterion_benchmark_10m(c: &mut Criterion) {
|
||||
c.bench_function("filter_10M_default", |b| {
|
||||
b.iter_batched(
|
||||
|| prepare("10M.txt", SkimOptionsBuilder::default().filter("test")),
|
||||
|(opts, rx)| Skim::run_with(opts, Some(rx)),
|
||||
criterion::BatchSize::SmallInput,
|
||||
);
|
||||
});
|
||||
c.bench_function("filter_10M_regex", |b| {
|
||||
b.iter_batched(
|
||||
|| prepare("10M.txt", SkimOptionsBuilder::default().filter("test").regex(true)),
|
||||
|
|
@ -152,14 +145,6 @@ fn criterion_benchmark_10m(c: &mut Criterion) {
|
|||
}
|
||||
|
||||
fn criterion_benchmark_1m(c: &mut Criterion) {
|
||||
c.bench_function("filter_1M_default", |b| {
|
||||
b.iter_batched(
|
||||
|| prepare("1M.txt", SkimOptionsBuilder::default().filter("test")),
|
||||
|(opts, rx)| Skim::run_with(opts, Some(rx)),
|
||||
criterion::BatchSize::SmallInput,
|
||||
);
|
||||
});
|
||||
|
||||
c.bench_function("filter_1M_regex", |b| {
|
||||
b.iter_batched(
|
||||
|| prepare("1M.txt", SkimOptionsBuilder::default().filter("test").regex(true)),
|
||||
|
|
|
|||
|
|
@ -1,239 +0,0 @@
|
|||
# /// script
|
||||
# requires-python = ">=3.13"
|
||||
# dependencies = [
|
||||
# "requests>=2.32.5",
|
||||
# ]
|
||||
# ///
|
||||
import json
|
||||
|
||||
REPO = "skim-rs/skim"
|
||||
|
||||
from requests import get
|
||||
import os
|
||||
import tempfile
|
||||
import shutil
|
||||
import subprocess
|
||||
import tarfile
|
||||
import zipfile
|
||||
import stat
|
||||
|
||||
|
||||
def _extract_release(rel):
|
||||
return {
|
||||
"date": rel["created_at"],
|
||||
"version": rel["tag_name"],
|
||||
"download_url": next(
|
||||
x["browser_download_url"]
|
||||
for x in rel["assets"]
|
||||
if ("linux" in x["name"] and "x86" in x["name"])
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def get_releases():
|
||||
releases = []
|
||||
page_offset = 1
|
||||
while True:
|
||||
page = get(
|
||||
f"https://api.github.com/repos/{REPO}/releases?per_page=100&page={page_offset}",
|
||||
headers={
|
||||
"Accept": "application/vnd.github+json",
|
||||
"X-GitHub-Api-Version": "2022-11-28",
|
||||
"Authorization": f"Bearer {os.environ['GH_API_KEY']}",
|
||||
},
|
||||
).json()
|
||||
if len(page) > 0:
|
||||
releases += page
|
||||
page_offset += 1
|
||||
else:
|
||||
break
|
||||
return list(map(_extract_release, releases))
|
||||
|
||||
|
||||
def download(rel):
|
||||
# Download the release binary and unpack it in a temporary file
|
||||
url = rel["download_url"]
|
||||
dst_dir = tempfile.mkdtemp(prefix="skim-release-")
|
||||
try:
|
||||
resp = get(url, stream=True)
|
||||
resp.raise_for_status()
|
||||
|
||||
# Try to determine filename
|
||||
fname = None
|
||||
cd = resp.headers.get("content-disposition")
|
||||
if cd and "filename=" in cd:
|
||||
fname = cd.split("filename=")[-1].strip('"')
|
||||
if not fname:
|
||||
fname = os.path.basename(url.split("?")[0]) or "asset"
|
||||
|
||||
file_path = os.path.join(dst_dir, fname)
|
||||
with open(file_path, "wb") as f:
|
||||
for chunk in resp.iter_content(chunk_size=8192):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
|
||||
# If archive, extract
|
||||
lower = file_path.lower()
|
||||
extracted_root = dst_dir
|
||||
if lower.endswith(".zip"):
|
||||
with zipfile.ZipFile(file_path, "r") as z:
|
||||
z.extractall(dst_dir)
|
||||
elif lower.endswith((".tar.gz", ".tgz", ".tar.xz", ".tar")):
|
||||
with tarfile.open(file_path, "r:*") as t:
|
||||
t.extractall(dst_dir)
|
||||
else:
|
||||
# Assume it's a raw binary; mark executable and return
|
||||
st = os.stat(file_path)
|
||||
os.chmod(file_path, st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
|
||||
return file_path, dst_dir
|
||||
|
||||
# Find an executable in extracted tree. Prefer name containing 'sk' or 'skim'
|
||||
candidates = []
|
||||
for root, _, files in os.walk(extracted_root):
|
||||
for fn in files:
|
||||
p = os.path.join(root, fn)
|
||||
try:
|
||||
if os.path.isfile(p) and os.access(p, os.X_OK):
|
||||
candidates.append(p)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# Try to pick best candidate
|
||||
def score(pth: str) -> int:
|
||||
n = os.path.basename(pth).lower()
|
||||
if "sk" == n or n == "sk" or n.startswith("sk"):
|
||||
return 100
|
||||
if "skim" in n:
|
||||
return 90
|
||||
return 10
|
||||
|
||||
if not candidates:
|
||||
# If nothing marked executable, try to make some files executable and pick largest
|
||||
files_all = [
|
||||
os.path.join(root, f)
|
||||
for root, _, fs in os.walk(extracted_root)
|
||||
for f in fs
|
||||
]
|
||||
if not files_all:
|
||||
raise RuntimeError(
|
||||
f"no files found in archive for release {rel.get('version')}"
|
||||
)
|
||||
# pick largest file
|
||||
files_all = sorted(
|
||||
files_all, key=lambda p: os.path.getsize(p), reverse=True
|
||||
)
|
||||
p = files_all[0]
|
||||
st = os.stat(p)
|
||||
os.chmod(p, st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
|
||||
return p, dst_dir
|
||||
|
||||
candidates.sort(key=score, reverse=True)
|
||||
chosen = candidates[0]
|
||||
# ensure executable bit set
|
||||
st = os.stat(chosen)
|
||||
os.chmod(chosen, st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
|
||||
return chosen, dst_dir
|
||||
except Exception:
|
||||
# cleanup on error
|
||||
try:
|
||||
shutil.rmtree(dst_dir)
|
||||
except Exception:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def bench(path):
|
||||
# Run bench.sh on the binary at `path` with 3 runs in json mode and return the json output as a dict
|
||||
repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
||||
bench_sh = os.path.join(repo_root, "bench.sh")
|
||||
fixture_path = os.path.join(repo_root, "benches", "fixtures", "10M.txt")
|
||||
if not os.path.isfile(bench_sh):
|
||||
raise RuntimeError("bench.sh not found in repo root")
|
||||
|
||||
cmd = [bench_sh, path, "-r", "10", "-j", "-f", fixture_path]
|
||||
# Ensure bench.sh is executable
|
||||
st = os.stat(bench_sh)
|
||||
os.chmod(bench_sh, st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
|
||||
|
||||
proc = subprocess.run(cmd, cwd=repo_root, capture_output=True, text=True)
|
||||
# bench.sh prints the JSON to stdout when -j passed
|
||||
out = proc.stdout.strip()
|
||||
if proc.returncode != 0:
|
||||
# include stderr for debugging
|
||||
raise RuntimeError(
|
||||
f"bench.sh failed: {proc.returncode}\nstdout:\n{out}\nstderr:\n{proc.stderr}"
|
||||
)
|
||||
|
||||
try:
|
||||
return json.loads(out)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"failed to parse bench output as json: {e}\noutput:\n{out}")
|
||||
|
||||
|
||||
def run_benches(releases):
|
||||
# Download and bench each release and save the results to a json file
|
||||
results = []
|
||||
for i, rel in enumerate(releases, start=1):
|
||||
print(f"[{i}/{len(releases)}] processing {rel.get('version')}")
|
||||
try:
|
||||
bin_path, tmpdir = download(rel)
|
||||
except Exception as e:
|
||||
print(f"failed to download {rel.get('version')}: {e}")
|
||||
results.append(
|
||||
{
|
||||
"version": rel.get("version"),
|
||||
"date": rel.get("date"),
|
||||
"download_url": rel.get("download_url"),
|
||||
"error": f"download: {e}",
|
||||
}
|
||||
)
|
||||
# persist intermediate results
|
||||
with open(out_path, "w") as f:
|
||||
json.dump(results, f, indent=2)
|
||||
continue
|
||||
|
||||
try:
|
||||
bench_out = bench(bin_path)
|
||||
except Exception as e:
|
||||
print(f"bench failed for {rel.get('version')}: {e}")
|
||||
bench_out = {"error": str(e)}
|
||||
finally:
|
||||
# cleanup downloaded/extracted files
|
||||
try:
|
||||
shutil.rmtree(tmpdir)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
record = {
|
||||
"version": rel.get("version"),
|
||||
"date": rel.get("date"),
|
||||
"download_url": rel.get("download_url"),
|
||||
"bench": bench_out,
|
||||
}
|
||||
results.append(record)
|
||||
|
||||
out_path = "/tmp/bench-history.json"
|
||||
with open(out_path, "w") as f:
|
||||
json.dump(results, f, indent=2)
|
||||
return results
|
||||
|
||||
|
||||
def main() -> None:
|
||||
releases = [r for r in get_releases() if r["date"].startswith("2026")]
|
||||
# with open("/tmp/releases.json", "r") as f:
|
||||
# releases = json.load(f)
|
||||
|
||||
print(f"fetched {len(releases)} releases")
|
||||
|
||||
out_path = "/tmp/bench-history.json"
|
||||
|
||||
results = run_benches(releases)
|
||||
with open(out_path, "w") as f:
|
||||
json.dump(results, f, indent=2)
|
||||
|
||||
# with open(out_path, "r") as f:
|
||||
# results = json.load(f)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -1,355 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# /// script
|
||||
# requires-python = ">=3.11"
|
||||
# dependencies = [
|
||||
# "matplotlib>=3.10.8",
|
||||
# ]
|
||||
# ///
|
||||
"""
|
||||
Plot benchmark history (time, throughput, CPU, RAM) with fzf baseline.
|
||||
|
||||
Usage:
|
||||
./scripts/plot-bench.py [--history PATH] [--baseline PATH] [--out PATH]
|
||||
[--since YYYY-MM-DD]
|
||||
|
||||
Defaults:
|
||||
history: ./scripts/data/bench-history.json
|
||||
baseline: ./scripts/data/fzf-baseline.json
|
||||
out: ./scripts/data/bench-plots.png
|
||||
since: 2026-01-01
|
||||
|
||||
The script expects `history` to be a JSON array where each element is an
|
||||
object produced by scripts/bench-history.py (with keys: version, date, bench).
|
||||
Each bench entry should follow the bench.sh JSON layout (time_s, peak_cpu,
|
||||
peak_memory_kb, items_per_second) with avg/min/max values.
|
||||
|
||||
The plot shows min/avg/max bands and a horizontal baseline from the fzf
|
||||
benchmark file. Row 1: time (s) and throughput (items/s). Row 2: CPU % and
|
||||
memory (MB).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from typing import List, Dict, Any, Optional, Tuple
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
import matplotlib.dates as mdates
|
||||
|
||||
|
||||
def load_json(path: str) -> Any:
|
||||
with open(path, "r") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def safe_get(d: Dict[str, Any], *keys, default=None):
|
||||
cur = d
|
||||
for k in keys:
|
||||
if cur is None:
|
||||
return default
|
||||
cur = cur.get(k)
|
||||
return cur if cur is not None else default
|
||||
|
||||
|
||||
def _to_float_or_none(v):
|
||||
if v is None:
|
||||
return None
|
||||
try:
|
||||
return float(v)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def parse_history(
|
||||
history: List[Dict[str, Any]],
|
||||
since: Optional[datetime] = None,
|
||||
) -> Tuple[List[str], List[datetime], Dict[str, Dict[str, List[Optional[float]]]]]:
|
||||
"""Sort history by date, optionally filter to entries >= since, return plotting data."""
|
||||
|
||||
def parse_date(r):
|
||||
try:
|
||||
dt = datetime.fromisoformat(r.get("date").replace("Z", "+00:00"))
|
||||
return dt
|
||||
except Exception:
|
||||
return datetime.min.replace(tzinfo=timezone.utc)
|
||||
|
||||
history_sorted = sorted(history, key=parse_date)
|
||||
|
||||
if since is not None:
|
||||
if since.tzinfo is None:
|
||||
since = since.replace(tzinfo=timezone.utc)
|
||||
history_sorted = [r for r in history_sorted if parse_date(r) >= since]
|
||||
|
||||
versions = [r.get("version") or f"#{i}" for i, r in enumerate(history_sorted)]
|
||||
dates = [parse_date(r) for r in history_sorted]
|
||||
|
||||
metrics: Dict[str, Dict[str, List[Optional[float]]]] = {
|
||||
"time": {"avg": [], "min": [], "max": []},
|
||||
"throughput": {"avg": [], "min": [], "max": []},
|
||||
"cpu": {"avg": [], "min": [], "max": []},
|
||||
"mem": {"avg": [], "min": [], "max": []},
|
||||
}
|
||||
|
||||
for r in history_sorted:
|
||||
b = r.get("bench") or {}
|
||||
|
||||
metrics["time"]["avg"].append(_to_float_or_none(safe_get(b, "time_s", "avg")))
|
||||
metrics["time"]["min"].append(_to_float_or_none(safe_get(b, "time_s", "min")))
|
||||
metrics["time"]["max"].append(_to_float_or_none(safe_get(b, "time_s", "max")))
|
||||
|
||||
metrics["throughput"]["avg"].append(
|
||||
_to_float_or_none(safe_get(b, "items_per_second", "avg"))
|
||||
)
|
||||
metrics["throughput"]["min"].append(
|
||||
_to_float_or_none(safe_get(b, "items_per_second", "min"))
|
||||
)
|
||||
metrics["throughput"]["max"].append(
|
||||
_to_float_or_none(safe_get(b, "items_per_second", "max"))
|
||||
)
|
||||
|
||||
metrics["cpu"]["avg"].append(
|
||||
_to_float_or_none(safe_get(b, "peak_cpu", "avg") or safe_get(b, "peak_cpu"))
|
||||
)
|
||||
metrics["cpu"]["min"].append(_to_float_or_none(safe_get(b, "peak_cpu", "min")))
|
||||
metrics["cpu"]["max"].append(_to_float_or_none(safe_get(b, "peak_cpu", "max")))
|
||||
|
||||
metrics["mem"]["avg"].append(
|
||||
_to_float_or_none(safe_get(b, "peak_memory_kb", "avg"))
|
||||
)
|
||||
metrics["mem"]["min"].append(
|
||||
_to_float_or_none(safe_get(b, "peak_memory_kb", "min"))
|
||||
)
|
||||
metrics["mem"]["max"].append(
|
||||
_to_float_or_none(safe_get(b, "peak_memory_kb", "max"))
|
||||
)
|
||||
|
||||
return versions, dates, metrics
|
||||
|
||||
|
||||
def plot_band(ax, x_nums, y_min, y_avg, y_max, label: str, color: str):
|
||||
"""Plot an avg line with a min/max shaded band, skipping None values."""
|
||||
import math
|
||||
|
||||
y_min_f = [math.nan if v is None else float(v) for v in y_min]
|
||||
y_avg_f = [math.nan if v is None else float(v) for v in y_avg]
|
||||
y_max_f = [math.nan if v is None else float(v) for v in y_max]
|
||||
|
||||
ax.plot(
|
||||
x_nums, y_avg_f, label=label + " (avg)", color=color, marker="o", markersize=4
|
||||
)
|
||||
ax.fill_between(
|
||||
x_nums, y_min_f, y_max_f, color=color, alpha=0.2, label=label + " (min-max)"
|
||||
)
|
||||
|
||||
|
||||
def _is_minor_release(version: str) -> bool:
|
||||
"""Return True iff version is a minor release (patch == 0, or -pre1 suffix).
|
||||
|
||||
Examples:
|
||||
v1.2.0 -> True
|
||||
v1.2.1 -> False
|
||||
v1.0.0-pre3 -> False
|
||||
#3 -> False
|
||||
"""
|
||||
import re
|
||||
|
||||
if version == "HEAD":
|
||||
return True
|
||||
|
||||
m = re.fullmatch(r"v?(\d+)\.(\d+)\.(\d+)(.*)", version)
|
||||
if not m:
|
||||
return False
|
||||
return int(m.group(3)) == 0 and (m.group(4) == "" or m.group(4) == "-pre1")
|
||||
|
||||
|
||||
def apply_date_xaxis(ax, x_nums, versions):
|
||||
"""Configure the x-axis: tick at every data point, label only minor releases."""
|
||||
ax.set_xticks(x_nums)
|
||||
labels = [v if _is_minor_release(v) else "" for v in versions]
|
||||
ax.set_xticklabels(labels, rotation=45, ha="right", fontsize=8)
|
||||
# Keep minor tick marks visible for unlabelled points without a label
|
||||
ax.tick_params(axis="x", which="major", length=4)
|
||||
# Draw a longer tick for labelled (minor release) positions
|
||||
for tick, label in zip(ax.xaxis.get_major_ticks(), labels):
|
||||
if label:
|
||||
tick.tick1line.set_markersize(8)
|
||||
|
||||
|
||||
def prepare_baseline(b: Dict[str, Any]) -> Dict[str, Tuple]:
|
||||
return {
|
||||
"time": (
|
||||
_to_float_or_none(safe_get(b, "time_s", "avg")),
|
||||
_to_float_or_none(safe_get(b, "time_s", "min")),
|
||||
_to_float_or_none(safe_get(b, "time_s", "max")),
|
||||
),
|
||||
"throughput": (
|
||||
_to_float_or_none(safe_get(b, "items_per_second", "avg")),
|
||||
_to_float_or_none(safe_get(b, "items_per_second", "min")),
|
||||
_to_float_or_none(safe_get(b, "items_per_second", "max")),
|
||||
),
|
||||
"cpu": (
|
||||
_to_float_or_none(safe_get(b, "peak_cpu", "avg")),
|
||||
_to_float_or_none(safe_get(b, "peak_cpu", "min")),
|
||||
_to_float_or_none(safe_get(b, "peak_cpu", "max")),
|
||||
),
|
||||
"mem": (
|
||||
_to_float_or_none(safe_get(b, "peak_memory_kb", "avg")),
|
||||
_to_float_or_none(safe_get(b, "peak_memory_kb", "min")),
|
||||
_to_float_or_none(safe_get(b, "peak_memory_kb", "max")),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def add_baseline_hline(ax, baseline_tuple, scale=1.0):
|
||||
"""Draw a horizontal dashed line for the fzf baseline if available."""
|
||||
if baseline_tuple is None:
|
||||
return
|
||||
b_avg, b_min, b_max = baseline_tuple
|
||||
if b_avg is None:
|
||||
return
|
||||
ax.axhline(
|
||||
b_avg * scale, color="k", linestyle="--", linewidth=1, label="fzf baseline"
|
||||
)
|
||||
if b_min is not None and b_max is not None:
|
||||
# light band for baseline min/max — we use axhspan via dummy x range
|
||||
ax.axhspan(b_min * scale, b_max * scale, color="k", alpha=0.08)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--history", default="./scripts/data/bench-history.json")
|
||||
p.add_argument("--baseline", default="./scripts/data/fzf-baseline.json")
|
||||
p.add_argument("--out", default="./scripts/data/bench-plots.png")
|
||||
p.add_argument(
|
||||
"--since",
|
||||
default="2026-01-01",
|
||||
help="Only show data on or after this date (YYYY-MM-DD). Pass '' to disable.",
|
||||
)
|
||||
args = p.parse_args(argv)
|
||||
|
||||
since: Optional[datetime] = None
|
||||
if args.since:
|
||||
since = datetime.strptime(args.since, "%Y-%m-%d").replace(tzinfo=timezone.utc)
|
||||
|
||||
history_path = args.history
|
||||
baseline_path = args.baseline
|
||||
out_path = args.out
|
||||
|
||||
if not os.path.isfile(history_path):
|
||||
raise SystemExit(f"history file not found: {history_path}")
|
||||
history = load_json(history_path)
|
||||
|
||||
baseline = None
|
||||
if os.path.isfile(baseline_path):
|
||||
baseline = prepare_baseline(load_json(baseline_path))
|
||||
else:
|
||||
print(
|
||||
f"warning: baseline file not found: {baseline_path} — proceeding without baseline"
|
||||
)
|
||||
|
||||
versions, dates, metrics = parse_history(history, since=since)
|
||||
if not versions:
|
||||
raise SystemExit("No data points after the --since filter.")
|
||||
|
||||
x_nums = mdates.date2num(dates)
|
||||
|
||||
# MB conversion for memory
|
||||
mem_avg_mb = [None if v is None else v / 1024.0 for v in metrics["mem"]["avg"]]
|
||||
mem_min_mb = [None if v is None else v / 1024.0 for v in metrics["mem"]["min"]]
|
||||
mem_max_mb = [None if v is None else v / 1024.0 for v in metrics["mem"]["max"]]
|
||||
|
||||
# Throughput in millions of items/s for readability
|
||||
tp_scale = 1e6
|
||||
tp_avg = [None if v is None else v / tp_scale for v in metrics["throughput"]["avg"]]
|
||||
tp_min = [None if v is None else v / tp_scale for v in metrics["throughput"]["min"]]
|
||||
tp_max = [None if v is None else v / tp_scale for v in metrics["throughput"]["max"]]
|
||||
|
||||
n = len(versions)
|
||||
fig_w = max(14, n * 0.55)
|
||||
fig, axes = plt.subplots(2, 2, figsize=(fig_w, 9), sharex=True)
|
||||
ax_time, ax_tp, ax_cpu, ax_mem = axes[0, 0], axes[0, 1], axes[1, 0], axes[1, 1]
|
||||
|
||||
# --- Row 1, col 0: Time ---
|
||||
plot_band(
|
||||
ax_time,
|
||||
x_nums,
|
||||
metrics["time"]["min"],
|
||||
metrics["time"]["avg"],
|
||||
metrics["time"]["max"],
|
||||
"Time",
|
||||
"C0",
|
||||
)
|
||||
add_baseline_hline(ax_time, baseline.get("time") if baseline else None)
|
||||
ax_time.set_ylabel("Time (s)")
|
||||
ax_time.legend(fontsize=8)
|
||||
ax_time.set_ylim(bottom=0)
|
||||
ax_time.set_title("Execution time (lower is better)")
|
||||
|
||||
# --- Row 1, col 1: Throughput ---
|
||||
plot_band(ax_tp, x_nums, tp_min, tp_avg, tp_max, "Throughput", "C1")
|
||||
if (
|
||||
baseline
|
||||
and baseline.get("throughput")
|
||||
and baseline["throughput"][0] is not None
|
||||
):
|
||||
add_baseline_hline(
|
||||
ax_tp,
|
||||
tuple(None if v is None else v / tp_scale for v in baseline["throughput"]),
|
||||
)
|
||||
ax_tp.set_ylabel("Throughput (M items/s)")
|
||||
ax_tp.legend(fontsize=8)
|
||||
ax_tp.set_ylim(bottom=0)
|
||||
ax_tp.set_title("Throughput (higher is better)")
|
||||
|
||||
# --- Row 2, col 0: CPU ---
|
||||
plot_band(
|
||||
ax_cpu,
|
||||
x_nums,
|
||||
metrics["cpu"]["min"],
|
||||
metrics["cpu"]["avg"],
|
||||
metrics["cpu"]["max"],
|
||||
"CPU %",
|
||||
"tab:orange",
|
||||
)
|
||||
add_baseline_hline(ax_cpu, baseline.get("cpu") if baseline else None)
|
||||
ax_cpu.set_ylabel("CPU %")
|
||||
ax_cpu.legend(fontsize=8)
|
||||
ax_cpu.set_ylim(bottom=0)
|
||||
ax_cpu.set_title("Peak CPU usage (lower is better)")
|
||||
|
||||
# --- Row 2, col 1: Memory ---
|
||||
plot_band(
|
||||
ax_mem,
|
||||
x_nums,
|
||||
mem_min_mb,
|
||||
mem_avg_mb,
|
||||
mem_max_mb,
|
||||
"Memory",
|
||||
"tab:green",
|
||||
)
|
||||
if baseline and baseline.get("mem") and baseline["mem"][0] is not None:
|
||||
add_baseline_hline(
|
||||
ax_mem, tuple(None if v is None else v / 1024.0 for v in baseline["mem"])
|
||||
)
|
||||
ax_mem.set_ylabel("Memory (MB)")
|
||||
ax_mem.legend(fontsize=8)
|
||||
ax_mem.set_ylim(bottom=0)
|
||||
ax_mem.set_title("Peak memory usage (lower is better)")
|
||||
|
||||
# --- X-axis formatting: date-scaled, version labels on bottom row ---
|
||||
for ax in (ax_cpu, ax_mem):
|
||||
apply_date_xaxis(ax, x_nums, versions)
|
||||
|
||||
fig.suptitle("skim benchmark history", fontsize=13, fontweight="bold")
|
||||
plt.tight_layout()
|
||||
|
||||
os.makedirs(os.path.dirname(out_path) or ".", exist_ok=True)
|
||||
plt.savefig(out_path, dpi=150)
|
||||
print(f"wrote plots to {out_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Reference in a new issue