firefox/tridactyl.tridactyl
1
0
Fork 0
mirror of https://github.com/tridactyl/tridactyl.git synced 2026-09-10 07:16:33 -04:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Revert "Add HNSW library"

Browse source 
This reverts commit 5ba8438ef7.
This commit is contained in:
Oliver Blanthorn 2024-06-10 13:05:43 +02:00
parent 5ba8438ef7
commit 51fce783b5
No known key found for this signature in database
GPG key ID: 2BB8C36BB504BFF3
4 changed files with 0 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
package.json
View file

@ -21,7 +21,6 @@
"editor-adapter": "^0.0.5",
"esbuild": "^0.20.2",
"fuse.js": "^7.0.0",
"hnsw": "^1.0.2",
"nearley": "^2.20.1",
"ramda": "^0.29.1",
"semver-compare": "^1.0.0",

21
scratch.js
View file

@ -36,14 +36,6 @@ async function measureTime(fn, ...args) {
return { result, executionTime }
}
function reshapeArray(flatArray, n, l) {
const result = []
for (let i = 0; i < flatArray.length; i += n) {
result.push(flatArray.slice(i, i + n))
}
return result
}
// extractor = await tri.pipeline('feature-extraction', 'Supabase/gte-small')
extractor = await tri.pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2')
@ -51,24 +43,11 @@ funcs = tri.metadata.everything.getFile("src/excmds.ts").getFunctions().filter(f
docEmbeds = await extractor(funcs.map(f => f[0] + ": " + f[1].doc.slice(0, 512)), { pooling: 'mean', normalize: true }) // need to check max length. takes a few seconds
unflat = reshapeArray(docEmbeds.data, docEmbeds.dims[1], docEmbeds.dims[0])
vectors = funcs.map(f=>f[0]).map((f, i) => {return {id: f, vector: unflat[i]}})
// vectors = funcs.map(f=>f[0]).map((f, i) => {return {id: f, vector: unflat[i]}})
// max neighbours, max nodes to visit during build, vector dimension, metric
hnsw = new tri.HNSW(10, 200, docEmbeds.dims[1], 'cosine')
await hnsw.buildIndex(vectors)
probe = 'easter egg'
outputProbe = await extractor([probe], { pooling: 'mean', normalize: true }) // approx 10 ms
sims = cosineSimilarity(outputProbe, docEmbeds) // approx 60 ms
Array.from(funcs.map(f=>f[0]).entries()).sort((l, r) => sims[l[0]] < sims[r[0]]).map(x=>x[1])
hnsw.searchKNN(outputProbe.data, 5)
await measureTime( async _ => await extractor(["melons"], { pooling: 'mean', normalize: true })) // 10 ms
await measureTime( async _ => cosineSimilarity(outputProbe, docEmbeds)) // 60 ms
await measureTime( async _ => hnsw.searchKNN(outputProbe.data, 5)) // 1536 ms lol

2
src/background.ts
View file

@ -32,7 +32,6 @@ import * as Logging from "@src/lib/logging"
import * as Proxy from "@src/lib/proxy"
import { tabsProxy } from "@src/lib/tabs"
import { pipeline, env } from "@xenova/transformers"
import { HNSW } from "hnsw"
import * as metadata from "@src/.metadata.generated"
env.allowLocalModels = false
@ -62,7 +61,6 @@ env.allowLocalModels = false
metadata,
tabs: tabsProxy,
pipeline,
HNSW,
})
import { HintingCmds } from "@src/background/hinting"

12
yarn.lock
View file

@ -4189,13 +4189,6 @@ highlight.js@^10.2.0:
resolved "https://registry.yarnpkg.com/highlight.js/-/highlight.js-10.4.1.tgz#d48fbcf4a9971c4361b3f95f302747afe19dbad0"
integrity sha512-yR5lWvNz7c85OhVAEAeFhVCc/GV4C30Fjzc/rCP0aCWzc1UUOPUk55dK/qdwTZHBvMZo+eZ2jpk62ndX/xMFlg==
hnsw@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/hnsw/-/hnsw-1.0.2.tgz#7bfa0c768c3a286f4985bce5a3afad400e2e0967"
integrity sha512-CTAx49pCpi+aBAJWbL0DDJp50fFrHRc1To6NRP8u2VX68PkcZA7grzOrcC5XhTVkl8cGqw9oodvqSVYHL4vwAQ==
dependencies:
idb "^7.1.1"
hosted-git-info@^2.1.4:
version "2.8.9"
resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.9.tgz#dffc0bf9a21c02209090f2aa69429e1414daf3f9"
@ -4273,11 +4266,6 @@ iconv-lite@0.4.24:
dependencies:
safer-buffer ">= 2.1.2 < 3"
idb@^7.1.1:
version "7.1.1"
resolved "https://registry.yarnpkg.com/idb/-/idb-7.1.1.tgz#d910ded866d32c7ced9befc5bfdf36f572ced72b"
integrity sha512-gchesWBzyvGHRO9W8tzUWFDycow5gwjvFKfyV9FF32Y7F50yZMp7mP+T2mJIWFx49zicqyC4uefHM17o6xKIVQ==
ieee754@^1.1.13:
version "1.2.1"
resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352"