add quant tester

Signed-off-by: Matt Williams <m@technovangelist.com>

2024-08-20-quant-tester/.gitignore

@@ -0,0 +1,175 @@
+# Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore
+
+# Logs
+
+logs
+_.log
+npm-debug.log_
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Caches
+
+.cache
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+
+report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
+
+# Runtime data
+
+pids
+_.pid
+_.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+
+lib-cov
+
+# Coverage directory used by tools like istanbul
+
+coverage
+*.lcov
+
+# nyc test coverage
+
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+
+bower_components
+
+# node-waf configuration
+
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+
+build/Release
+
+# Dependency directories
+
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+
+web_modules/
+
+# TypeScript cache
+
+*.tsbuildinfo
+
+# Optional npm cache directory
+
+.npm
+
+# Optional eslint cache
+
+.eslintcache
+
+# Optional stylelint cache
+
+.stylelintcache
+
+# Microbundle cache
+
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+
+.node_repl_history
+
+# Output of 'npm pack'
+
+*.tgz
+
+# Yarn Integrity file
+
+.yarn-integrity
+
+# dotenv environment variable files
+
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+
+.parcel-cache
+
+# Next.js build output
+
+.next
+out
+
+# Nuxt.js build / generate output
+
+.nuxt
+dist
+
+# Gatsby files
+
+# Comment in the public line in if your project uses Gatsby and not Next.js
+
+# https://nextjs.org/blog/next-9-1#public-directory-support
+
+# public
+
+# vuepress build output
+
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+
+.temp
+
+# Docusaurus cache and generated files
+
+.docusaurus
+
+# Serverless directories
+
+.serverless/
+
+# FuseBox cache
+
+.fusebox/
+
+# DynamoDB Local files
+
+.dynamodb/
+
+# TernJS port file
+
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+
+.vscode-test
+
+# yarn v2
+
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.*
+
+# IntelliJ based IDEs
+.idea
+
+# Finder (MacOS) folder config
+.DS_Store

2024-08-20-quant-tester/README.md

@@ -0,0 +1,15 @@
+# quant-tester
+
+To install dependencies:
+
+```bash
+bun install
+```
+
+To run:
+
+```bash
+bun run index.ts
+```
+
+This project was created using `bun init` in bun v1.1.24. [Bun](https://bun.sh) is a fast all-in-one JavaScript runtime.

2024-08-20-quant-tester/index.ts

@@ -0,0 +1,77 @@
+import ollama from "ollama";
+import inquirer, { QuestionCollection} from "inquirer";
+
+const qrepeat = 5;
+
+const allmodels = await ollama.list();
+
+function processModelNames(
+	allmodels: { name: string; details: { parameter_size: string } }[],
+) {
+	// Extract names and process
+	const names = allmodels
+		.map((model) => `${model.name}:${model.details.parameter_size}`)
+		.map((name) => {
+      const splitname = name.split(":")
+      return `${splitname[0]} - ${splitname[2]}`
+    } )
+		.filter((value, index, self) => self.indexOf(value) === index)
+		.sort();
+	return names;
+}
+
+async function getSelectedModel(names: string[]) {
+	const questions: QuestionCollection = [
+		{
+			type: "list",
+			name: "selectedName",
+			message: "Select a model name:",
+			choices: names,
+		},
+	];
+	const answer = await inquirer.prompt(questions);
+	return answer.selectedName;
+}
+
+async function getSelectedQuants(name: string, size: string,  allmodels: { name: string, details: { parameter_size: string, quantization_level: string}}[]) {
+  const modelNames = allmodels.filter((model) => model.name.split(":")[0] === name && model.details.parameter_size === size).map((model) => model.name);
+
+  return modelNames;
+}
+async function getUserPrompt(): Promise<string> {
+	const questions: QuestionCollection = [
+		{
+			type: "input",
+			name: "userPrompt",
+			message: "Enter a prompt you want to test:",
+		},
+	];
+	const answer = await inquirer.prompt(questions);
+	return answer.userPrompt;
+}
+
+const names = processModelNames(allmodels.models);
+
+const selected = await getSelectedModel(names);
+const modelName = selected.split(" - ")[0];
+const selectedSize = selected.split(" - ")[1];
+
+
+const quants = await getSelectedQuants(modelName, selectedSize, allmodels.models);
+
+const userPrompt = await getUserPrompt();
+
+for (const quant of quants) {
+  for (let index = 0; index < qrepeat; index++) {
+    let format = ""
+    if (userPrompt.toLowerCase().includes("json")) {
+      format = "json"
+    }
+    const output = await ollama.generate({model: quant, prompt: userPrompt, format: format});
+    console.log(`\n${quant} round ${index + 1}:\n${output.response}\nGenerated in ${(output.eval_duration / 1000000000).toFixed(2)} seconds\n`);
+    if (index < qrepeat - 1) {
+      console.log('--------\n');
+    }
+  }
+  console.log('========\n');
+}

2024-08-20-quant-tester/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "quant-tester",
+  "module": "index.ts",
+  "type": "module",
+  "devDependencies": {
+    "@types/bun": "latest"
+  },
+  "peerDependencies": {
+    "typescript": "^5.0.0"
+  },
+  "dependencies": {
+    "@types/inquirer": "^9.0.7",
+    "cheerio": "^1.0.0",
+    "inquirer": "^10.1.8",
+    "ollama": "^0.5.8"
+  }
+}

2024-08-20-quant-tester/tsconfig.json

@@ -0,0 +1,27 @@
+{
+  "compilerOptions": {
+    // Enable latest features
+    "lib": ["ESNext", "DOM"],
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleDetection": "force",
+    "jsx": "react-jsx",
+    "allowJs": true,
+
+    // Bundler mode
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "verbatimModuleSyntax": true,
+    "noEmit": true,
+
+    // Best practices
+    "strict": true,
+    "skipLibCheck": true,
+    "noFallthroughCasesInSwitch": true,
+
+    // Some stricter flags (disabled by default)
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
+    "noPropertyAccessFromIndexSignature": false
+  }
+}