add feb 22

Signed-off-by: Matt Williams <m@technovangelist.com>

2024-02-22 embeddings/README.md

@@ -0,0 +1,15 @@
+# embeddings
+
+To install dependencies:
+
+```bash
+bun install
+```
+
+To run:
+
+```bash
+bun run embed.ts
+```
+
+This project was created using `bun init` in bun v1.0.25. [Bun](https://bun.sh) is a fast all-in-one JavaScript runtime.

2024-02-22 embeddings/embed.py

@@ -0,0 +1,32 @@
+import ollama
+
+def read_file(file_path):
+    with open(file_path, 'r') as file:
+        return file.read()
+
+
+def split_into_chunks(text, chunk_size):
+    words = text.split()
+    chunks = []
+    current_chunk = []
+
+    for word in words:
+        current_chunk.append(word)
+        if len(current_chunk) >= chunk_size:
+            chunks.append(' '.join(current_chunk))
+            current_chunk = []
+
+    if current_chunk:  
+        chunks.append(' '.join(current_chunk))
+
+    return chunks
+
+file_path = '/Users/matt/Downloads/warandpeace.txt'  #
+long_string = read_file(file_path)
+chunk_size = 500
+chunks = split_into_chunks(long_string, chunk_size)
+
+for i, chunk in enumerate(chunks, start=1):
+    response = ollama.embeddings(model='nomic-embed-text', prompt=chunk)
+    print(i)
+    

2024-02-22 embeddings/embed.ts

@@ -0,0 +1,41 @@
+
+import { Ollama } from "ollama";
+
+const ollama = new Ollama();
+
+function splitIntoChunks(text: string, chunkSize: number): string[] {
+  const words: string[] = text.split(/\s+/);
+  const chunks: string[] = [];
+  let currentChunk: string[] = [];
+  
+  for (const word of words) {
+    currentChunk.push(word);
+    if (currentChunk.length >= chunkSize) {
+      chunks.push(currentChunk.join(' '));
+      currentChunk = [];
+    }
+  }
+  if (currentChunk.length > 0) {
+    chunks.push(currentChunk.join(' '));
+  }
+  return chunks;
+}
+
+
+const path = "/Users/matt/Downloads/warandpeace.txt";
+const file = Bun.file(path);
+
+const chunks = splitIntoChunks(await file.text(), 500);
+console.log('done reading')
+
+
+let i = 0
+for await (const c of chunks) {
+  i++;
+  const b = await ollama.embeddings({
+    model: "llama2",
+    prompt: c
+  })
+  console.log(i);
+}
+

2024-02-22 embeddings/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "embeddings",
+  "module": "embed.ts",
+  "type": "module",
+  "devDependencies": {
+    "@types/bun": "latest"
+  },
+  "peerDependencies": {
+    "typescript": "^5.0.0"
+  },
+  "dependencies": {
+    "ollama": "^0.4.6"
+  }
+}

2024-02-22 embeddings/simple.py

@@ -0,0 +1,5 @@
+import ollama
+
+out = ollama.embeddings(model="nomic-embed-text", prompt="hi hi")
+
+print(out)

2024-02-22 embeddings/test.py

@@ -0,0 +1,10 @@
+import ollama
+
+stream = ollama.chat(
+    model='mistral',
+    messages=[{'role': 'user', 'content': 'Why is the sky blue?'}],
+    stream=True,
+)
+
+for chunk in stream:
+  print(chunk['message']['content'], end='', flush=True)

2024-02-22 embeddings/tsconfig.json

@@ -0,0 +1,22 @@
+{
+  "compilerOptions": {
+    "lib": ["ESNext"],
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleDetection": "force",
+    "jsx": "react-jsx",
+    "allowJs": true,
+
+    /* Bundler mode */
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "verbatimModuleSyntax": true,
+    "noEmit": true,
+
+    /* Linting */
+    "skipLibCheck": true,
+    "strict": true,
+    "noFallthroughCasesInSwitch": true,
+    "forceConsistentCasingInFileNames": true
+  }
+}