From daf22f63292fb931ce7c579f3ad98417e22ddd1b Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Tue, 1 Oct 2024 17:54:57 -0700 Subject: [PATCH] Add websearch example Signed-off-by: Matt Williams --- 2024-10-01-websearch/deno.json | 8 + 2024-10-01-websearch/deno.lock | 438 +++++++++++++++++++++++++++++++++ 2024-10-01-websearch/main.ts | 62 +++++ 3 files changed, 508 insertions(+) create mode 100644 2024-10-01-websearch/deno.json create mode 100644 2024-10-01-websearch/deno.lock create mode 100644 2024-10-01-websearch/main.ts diff --git a/2024-10-01-websearch/deno.json b/2024-10-01-websearch/deno.json new file mode 100644 index 0000000..5b320c2 --- /dev/null +++ b/2024-10-01-websearch/deno.json @@ -0,0 +1,8 @@ +{ + "tasks": { + "dev": "deno run --watch main.ts" + }, + "imports": { + "@std/assert": "jsr:@std/assert@1" + } +} diff --git a/2024-10-01-websearch/deno.lock b/2024-10-01-websearch/deno.lock new file mode 100644 index 0000000..610b7ee --- /dev/null +++ b/2024-10-01-websearch/deno.lock @@ -0,0 +1,438 @@ +{ + "version": "4", + "specifiers": { + "npm:@types/node@*": "22.5.4", + "npm:cheerio@*": "0.12.4", + "npm:chromadb@*": "1.8.1", + "npm:ht@*": "0.0.2", + "npm:htl@*": "0.3.1", + "npm:htm@*": "3.1.1", + "npm:html-to-text@*": "9.0.5", + "npm:html-to@*": "0.0.1", + "npm:html@*": "1.0.0", + "npm:htmlparser2@*": "8.0.2", + "npm:ollama@*": "0.5.9" + }, + "npm": { + "@selderee/plugin-htmlparser2@0.11.0": { + "integrity": "sha512-P33hHGdldxGabLFjPPpaTxVolMrzrcegejx+0GxjrIb9Zv48D8yAIA/QTDR2dFl7Uz7urX8aX6+5bCZslr+gWQ==", + "dependencies": [ + "domhandler@5.0.3", + "selderee" + ] + }, + "@types/node@22.5.4": { + "integrity": "sha512-FDuKUJQm/ju9fT/SeX/6+gBzoPzlVCzfzmGkwKvRHQVxi4BntVbyIwf6a4Xn62mrvndLiml6z/UBXIdEVjQLXg==", + "dependencies": [ + "undici-types" + ] + }, + "ansi-regex@5.0.1": { + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" + }, + "ansi-styles@4.3.0": { + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": [ + "color-convert" + ] + }, + "arguments-extended@0.0.3": { + "integrity": "sha512-MNYdPKgCiywbgHAmNsYr1tSNLtfbSdwE1akZV+33hU9A8RG0lO5HAK9oMnw7y7bjYUhc04dJpcIBMUaPPYYtXg==", + "dependencies": [ + "extended", + "is-extended" + ] + }, + "array-extended@0.0.11": { + "integrity": "sha512-Fe4Ti2YgM1onQgrcCD8dUhFuZgHQxzqylSl1C5IDJVVVqY5D07h8RghIXL9sZ6COZ0e+oTL5IusTv5eXABJ9Kw==", + "dependencies": [ + "arguments-extended", + "extended", + "is-extended" + ] + }, + "boolbase@1.0.0": { + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==" + }, + "buffer-from@1.1.2": { + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" + }, + "cheerio-select@2.1.0": { + "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", + "dependencies": [ + "boolbase", + "css-select", + "css-what", + "domelementtype@2.3.0", + "domhandler@5.0.3", + "domutils@3.1.0" + ] + }, + "cheerio@0.12.4": { + "integrity": "sha512-XZStQ+OVs2AcUeMdpBUI51tGN0F5Bmu0EQ+5PC8RI4yd+MD0DrgIgP5ejuK8j5TRtVinhZbQrwbCIU07EN7exQ==", + "dependencies": [ + "cheerio-select", + "entities@0.5.0", + "htmlparser2@3.1.4", + "underscore" + ] + }, + "chromadb@1.8.1": { + "integrity": "sha512-NpbYydbg4Uqt/9BXKgkZXn0fqpsh2Z1yjhkhKH+rcHMoq0pwI18BFSU2QU7Fk/ZypwGefW2AvqyE/3ZJIgy4QA==", + "dependencies": [ + "cliui", + "isomorphic-fetch" + ] + }, + "cliui@8.0.1": { + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dependencies": [ + "string-width", + "strip-ansi", + "wrap-ansi" + ] + }, + "color-convert@2.0.1": { + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": [ + "color-name" + ] + }, + "color-name@1.1.4": { + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "concat-stream@1.6.2": { + "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", + "dependencies": [ + "buffer-from", + "inherits", + "readable-stream@2.3.8", + "typedarray" + ] + }, + "core-util-is@1.0.3": { + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" + }, + "css-select@5.1.0": { + "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==", + "dependencies": [ + "boolbase", + "css-what", + "domhandler@5.0.3", + "domutils@3.1.0", + "nth-check" + ] + }, + "css-what@6.1.0": { + "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==" + }, + "declare.js@0.0.8": { + "integrity": "sha512-O659hy1gcHef7JnwtqdQlrj2c5DAEgtxm8pgFXofW7eUE1L4FjsSLlziovWcrOJAOFlEPaOJshY+0hBWCG/AnA==" + }, + "deepmerge@4.3.1": { + "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==" + }, + "dom-serializer@2.0.0": { + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "dependencies": [ + "domelementtype@2.3.0", + "domhandler@5.0.3", + "entities@4.5.0" + ] + }, + "domelementtype@1.3.1": { + "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==" + }, + "domelementtype@2.3.0": { + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==" + }, + "domhandler@2.0.3": { + "integrity": "sha512-D8+qeCUo6LpYvBZsmXWunDJ9zOD8mVg6EwZIdgxmnT+xGWRsReF/TwsZ5EzlIJDipxlE6qJh1dXt9oeplGN3Bg==", + "dependencies": [ + "domelementtype@1.3.1" + ] + }, + "domhandler@5.0.3": { + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "dependencies": [ + "domelementtype@2.3.0" + ] + }, + "domutils@1.1.6": { + "integrity": "sha512-ZeagMzMKyk9GSFMqV3x3uHgRN36hLpSOF6LIRXmftce0UUqFsAx/azJAJ4Jc+9DYKmwROH5HLOcOu1OPARWwNg==", + "dependencies": [ + "domelementtype@1.3.1" + ] + }, + "domutils@3.1.0": { + "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==", + "dependencies": [ + "dom-serializer", + "domelementtype@2.3.0", + "domhandler@5.0.3" + ] + }, + "emoji-regex@8.0.0": { + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "entities@0.5.0": { + "integrity": "sha512-T5XQtlzuW+PfeSsGp3uryfYQof820zYbnUnUDEkwUVIAfgYeixIN16c4jh8gs0SqJUTGLU0XD6QsvjEPbmdwzQ==" + }, + "entities@4.5.0": { + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==" + }, + "extended@0.0.6": { + "integrity": "sha512-rvAV3BDGsV1SYGzUOu7aO0k82quhfl0QAyZudYhAcTeIr1rPbBnyOhOlkCLwLpDfP7HyKAWAPNSjRb9p7lE3rg==", + "dependencies": [ + "extender" + ] + }, + "extender@0.0.10": { + "integrity": "sha512-iPLUHZJaNW6RuOShQX33ZpewEUIlijFBcsXnKWyiYERKWPsFxfKgx8J0xRz29hKQWPFFPACgBW6cHM7Ke1pfaA==", + "dependencies": [ + "declare.js" + ] + }, + "ht@0.0.2": { + "integrity": "sha512-eYULiMGWs9kEz7xPeC+NP97maMi/k8uPd9Xqbd7B0JDVQkIfAlqudsYb5NP5AopG5QqPbW/H51Xv/5l8xBpfnA==", + "dependencies": [ + "array-extended", + "declare.js", + "extended", + "is-extended" + ] + }, + "htl@0.3.1": { + "integrity": "sha512-1LBtd+XhSc+++jpOOt0lCcEycXs/zTQSupOISnVAUmvGBpV7DH+C2M6hwV7zWYfpTMMg9ch4NO0lHiOTAMHdVA==" + }, + "htm@3.1.1": { + "integrity": "sha512-983Vyg8NwUE7JkZ6NmOqpCZ+sh1bKv2iYTlUkzlWmA5JD2acKoxd4KVxbMmxX/85mtfdnDmTFoNKcg5DGAvxNQ==" + }, + "html-to-text@9.0.5": { + "integrity": "sha512-qY60FjREgVZL03vJU6IfMV4GDjGBIoOyvuFdpBDIX9yTlDw0TjxVBQp+P8NvpdIXNJvfWBTNul7fsAQJq2FNpg==", + "dependencies": [ + "@selderee/plugin-htmlparser2", + "deepmerge", + "dom-serializer", + "htmlparser2@8.0.2", + "selderee" + ] + }, + "html-to@0.0.1": { + "integrity": "sha512-7S3xuvqvdPv0boQP5WrdBHl/I6v5RT24KfvYstd8l2YU1Mz8yOAcOZ/IfwURDSCNP0Z2W53I1WOzyW/4UHLzwA==", + "dependencies": [ + "cheerio", + "through" + ] + }, + "html@1.0.0": { + "integrity": "sha512-lw/7YsdKiP3kk5PnR1INY17iJuzdAtJewxr14ozKJWbbR97znovZ0mh+WEMZ8rjc3lgTK+ID/htTjuyGKB52Kw==", + "dependencies": [ + "concat-stream" + ] + }, + "htmlparser2@3.1.4": { + "integrity": "sha512-45m656EiWxsqzeKl3UZIk9CW47Jodbl2JaawHMHuH8JKU2BF5q0UV+aq89kUjc+6SG5LSkJBW/UlrI1+AlKdHg==", + "dependencies": [ + "domelementtype@1.3.1", + "domhandler@2.0.3", + "domutils@1.1.6", + "readable-stream@1.0.34" + ] + }, + "htmlparser2@8.0.2": { + "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", + "dependencies": [ + "domelementtype@2.3.0", + "domhandler@5.0.3", + "domutils@3.1.0", + "entities@4.5.0" + ] + }, + "inherits@2.0.4": { + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "is-extended@0.0.10": { + "integrity": "sha512-qp+HR+L9QXbgFurvqiVgD+JiGyUboRgICNzCXmbiLtZBFVSNFbxRsI4q7Be9mCWTO5PoO1IxoWp5sl+j5b83FA==", + "dependencies": [ + "extended" + ] + }, + "is-fullwidth-code-point@3.0.0": { + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" + }, + "isarray@0.0.1": { + "integrity": "sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==" + }, + "isarray@1.0.0": { + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + }, + "isomorphic-fetch@3.0.0": { + "integrity": "sha512-qvUtwJ3j6qwsF3jLxkZ72qCgjMysPzDfeV240JHiGZsANBYd+EEuu35v7dfrJ9Up0Ak07D7GGSkGhCHTqg/5wA==", + "dependencies": [ + "node-fetch", + "whatwg-fetch" + ] + }, + "leac@0.6.0": { + "integrity": "sha512-y+SqErxb8h7nE/fiEX07jsbuhrpO9lL8eca7/Y1nuWV2moNlXhyd59iDGcRf6moVyDMbmTNzL40SUyrFU/yDpg==" + }, + "node-fetch@2.7.0": { + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dependencies": [ + "whatwg-url" + ] + }, + "nth-check@2.1.1": { + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "dependencies": [ + "boolbase" + ] + }, + "ollama@0.5.9": { + "integrity": "sha512-F/KZuDRC+ZsVCuMvcOYuQ6zj42/idzCkkuknGyyGVmNStMZ/sU3jQpvhnl4SyC0+zBzLiKNZJnJeuPFuieWZvQ==", + "dependencies": [ + "whatwg-fetch" + ] + }, + "parseley@0.12.1": { + "integrity": "sha512-e6qHKe3a9HWr0oMRVDTRhKce+bRO8VGQR3NyVwcjwrbhMmFCX9KszEV35+rn4AdilFAq9VPxP/Fe1wC9Qjd2lw==", + "dependencies": [ + "leac", + "peberminta" + ] + }, + "peberminta@0.9.0": { + "integrity": "sha512-XIxfHpEuSJbITd1H3EeQwpcZbTLHc+VVr8ANI9t5sit565tsI4/xK3KWTUFE2e6QiangUkh3B0jihzmGnNrRsQ==" + }, + "process-nextick-args@2.0.1": { + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, + "readable-stream@1.0.34": { + "integrity": "sha512-ok1qVCJuRkNmvebYikljxJA/UEsKwLl2nI1OmaqAu4/UE+h0wKCHok4XkL/gvi39OacXvw59RJUOFUkDib2rHg==", + "dependencies": [ + "core-util-is", + "inherits", + "isarray@0.0.1", + "string_decoder@0.10.31" + ] + }, + "readable-stream@2.3.8": { + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dependencies": [ + "core-util-is", + "inherits", + "isarray@1.0.0", + "process-nextick-args", + "safe-buffer", + "string_decoder@1.1.1", + "util-deprecate" + ] + }, + "safe-buffer@5.1.2": { + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "selderee@0.11.0": { + "integrity": "sha512-5TF+l7p4+OsnP8BCCvSyZiSPc4x4//p5uPwK8TCnVPJYRmU2aYKMpOXvw8zM5a5JvuuCGN1jmsMwuU2W02ukfA==", + "dependencies": [ + "parseley" + ] + }, + "string-width@4.2.3": { + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": [ + "emoji-regex", + "is-fullwidth-code-point", + "strip-ansi" + ] + }, + "string_decoder@0.10.31": { + "integrity": "sha512-ev2QzSzWPYmy9GuqfIVildA4OdcGLeFZQrq5ys6RtiuF+RQQiZWr8TZNyAcuVXyQRYfEO+MsoB/1BuQVhOJuoQ==" + }, + "string_decoder@1.1.1": { + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": [ + "safe-buffer" + ] + }, + "strip-ansi@6.0.1": { + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": [ + "ansi-regex" + ] + }, + "through@2.3.8": { + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==" + }, + "tr46@0.0.3": { + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, + "typedarray@0.0.6": { + "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==" + }, + "underscore@1.4.4": { + "integrity": "sha512-ZqGrAgaqqZM7LGRzNjLnw5elevWb5M8LEoDMadxIW3OWbcv72wMMgKdwOKpd5Fqxe8choLD8HN3iSj3TUh/giQ==" + }, + "undici-types@6.19.8": { + "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==" + }, + "util-deprecate@1.0.2": { + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + }, + "webidl-conversions@3.0.1": { + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "whatwg-fetch@3.6.20": { + "integrity": "sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==" + }, + "whatwg-url@5.0.0": { + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dependencies": [ + "tr46", + "webidl-conversions" + ] + }, + "wrap-ansi@7.0.0": { + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dependencies": [ + "ansi-styles", + "string-width", + "strip-ansi" + ] + } + }, + "redirects": { + "https://esm.sh/cheerio": "https://esm.sh/cheerio@1.0.0" + }, + "remote": { + "https://esm.sh/cheerio@1.0.0": "1f89cf6ff45292d5c2ed56a6a20380c1560790d1afc454e56bab2f8ae9423fb0", + "https://esm.sh/v135/boolbase@1.0.0/denonext/boolbase.mjs": "4e3bd67e9b1c5c55094eae98345d0107c6a44ef57bd3d4b9579698fa44722280", + "https://esm.sh/v135/cheerio-select@2.1.0/denonext/cheerio-select.mjs": "5b882e2ef61d79044badb386948403758e0b171f54bcd2f4975c502c8096a2df", + "https://esm.sh/v135/cheerio@1.0.0/denonext/cheerio.mjs": "e181e06351dc645f7c8de4ef0d7aaf4132548a200070d1357a8ddd57ea8faef0", + "https://esm.sh/v135/css-select@5.1.0/denonext/css-select.mjs": "7f4fb680eff2223a68fb5a1efa12e905077d7e6c44ae66c01fc437fe7af58f8b", + "https://esm.sh/v135/css-what@6.1.0/denonext/css-what.mjs": "283d02df6fef73d3223b55324b559b363dd0e4c008ea1efbcf9f14b8f2642202", + "https://esm.sh/v135/dom-serializer@2.0.0/denonext/dom-serializer.mjs": "700001f5fdb9f72fc7d1f169be700489ddd7f19489b58b6e8691ff1fca9000fe", + "https://esm.sh/v135/domelementtype@2.3.0/denonext/domelementtype.mjs": "371936c356d5ac797f2ce3a66b98dfc73e6fff0e095b2858b85668e6ad7d10e0", + "https://esm.sh/v135/domhandler@5.0.3/denonext/domhandler.mjs": "8c6e56c4596bbe7e9fd9d615f8fcdec7f11075f217bdc966a02ebe89c01b9f6f", + "https://esm.sh/v135/domutils@3.1.0/denonext/domutils.mjs": "2a86554cc5f543a068bbf6ea6ed0647f4ad4c3ccb555f0da9fae09180cd7b611", + "https://esm.sh/v135/encoding-sniffer@0.2.0/denonext/encoding-sniffer.mjs": "08c03ae2c199028b500c657e42390f51de870415a005c8fabde9cd9045511451", + "https://esm.sh/v135/entities@4.5.0/denonext/entities.mjs": "a9e8f9c22022c34755d1189030ac30c68545ccf8532fa32654c50fe9d90a13a1", + "https://esm.sh/v135/entities@4.5.0/denonext/lib/decode.js": "7fea6d8bd725edbbf7ea05031d2ea1bbbc1166dc11e3345d541198dd2dc16f1e", + "https://esm.sh/v135/entities@4.5.0/denonext/lib/escape.js": "7ebdc622bf3618bab25db40da4a49e2b9d03f044745f125f0bc3359f2d060def", + "https://esm.sh/v135/htmlparser2@9.1.0/denonext/htmlparser2.mjs": "e5186144ca1dd8829bda984fd778500356f241597c67e2c674213c684168e985", + "https://esm.sh/v135/iconv-lite@0.6.3/denonext/iconv-lite.mjs": "768e37377191ab3c7414bbb15fce202a328c411da0429764984c79b8bc65abd4", + "https://esm.sh/v135/nth-check@2.1.1/denonext/nth-check.mjs": "638b4f5a22236cd05c7d1d43e5c6ea719695c4a8bc7beccdf8d97a434bea96dc", + "https://esm.sh/v135/parse5-htmlparser2-tree-adapter@7.0.0/denonext/parse5-htmlparser2-tree-adapter.mjs": "2e8c24c6859e24e7c0f1e634574f8c014860868a603e85430b98212460e25091", + "https://esm.sh/v135/parse5-parser-stream@7.1.2/denonext/parse5-parser-stream.mjs": "ea8541ac707228dc40625fc0e4618a92bb51510fd5f4d4dfbaf833bb881162e3", + "https://esm.sh/v135/parse5@7.1.2/denonext/parse5.mjs": "35bb04ec36a1c25c8cd8137296d64d16fd523a8ad1c2b63c41ba867fcd455c36", + "https://esm.sh/v135/safer-buffer@2.1.2/denonext/safer-buffer.mjs": "ce0e787812c668ba082ad5b75958490c714b6e05836bd5b6013d9f75727c006f", + "https://esm.sh/v135/undici@6.19.6/denonext/undici.mjs": "ae6d6e0028a3f9543cd50e2911ae6f66b1d2e6bca262a39581867359cba99ad6", + "https://esm.sh/v135/whatwg-encoding@3.1.1/denonext/whatwg-encoding.mjs": "ecb7f2fe9fe4686e801ffaf4fa924e5860cbc3d07594a89e639d5c2ed20ca067", + "https://esm.sh/v135/whatwg-mimetype@4.0.0/denonext/whatwg-mimetype.mjs": "52577070194b4b1ebc78e6e9b457a078ca28d6f8477457bce914da555d92a5bb" + }, + "workspace": { + "dependencies": [ + "jsr:@std/assert@1" + ] + } +} diff --git a/2024-10-01-websearch/main.ts b/2024-10-01-websearch/main.ts new file mode 100644 index 0000000..ec37274 --- /dev/null +++ b/2024-10-01-websearch/main.ts @@ -0,0 +1,62 @@ +import ollama from "npm:ollama"; +import * as cheerio from "https://esm.sh/cheerio"; + +const searchUrl = Deno.env.get("SEARCH_URL"); +const query = Deno.args.join(" "); + +console.log(`Query: ${query}`); +const urls = await getNewsUrls(query); +const alltexts = await getCleanedText(urls); +await answerQuery(query, alltexts); + +async function getNewsUrls(query: string) { + const searchResults = await fetch(`${searchUrl}?q=${query}&format=json`); + const searchResultsJson: { results: Array<{ url: string }> } = + await searchResults.json(); + const urls = searchResultsJson.results + .map((result) => result.url) + .slice(0, 5); + return urls; +} + +async function getCleanedText(urls: string[]) { + const texts = []; + for await (const url of urls) { + const getUrl = await fetch(url); + console.log(`Fetching ${url}`); + const html = await getUrl.text(); + const text = htmlToText(html); + texts.push(`Source: ${url}\n${text}\n\n`); + } + return texts; +} + + +function htmlToText(html: string) { + const $ = cheerio.load(html); + $("script, source, style, head, img, svg, a, form, link, iframe").remove(); + $("*").removeClass(); + $("*").each((_, el) => { + if (el.type === "tag" || el.type === "script" || el.type === "style") { + for (const attr of Object.keys(el.attribs || {})) { + if (attr.startsWith("data-")) { + $(el).removeAttr(attr); + }}} + }); + const text = $("body").text().replace(/\s+/g, " "); + return text; +} + +async function answerQuery(query: string, texts: string[]) { + const result = await ollama.generate({ + model: "llama3.2:1b", + prompt: `${query}. Summarize the information and provide an answer. Use only the information in the following articles to answer the question: ${texts.join("\n\n")}`, + stream: true, + options: { + num_ctx: 16000, + }}); + for await (const chunk of result) { + if (chunk.done !== true) { + await Deno.stdout.write(new TextEncoder().encode(chunk.response)); + }} +}