about to gen embeddings

test.lua

@@ -4,58 +4,74 @@ package.path = (arg[0]:match("@?(.*/)") or arg[0]:match("@?(.*\\)")) .. "lib" ..
 local utility = require "utility"
 local json = utility.require("dkjson")
 
-local prompt = [[You are organizing text files.
+local file_name = arg[1]
 
-Return ONLY valid JSON with:
-- title
-- summary (short)
-- tags (array)
-- category
 
-Do not return JSON wrapped in Markdown.
 
-Text:
+local function strip_markdown(text)
-]]
+  local tab = text:split("\n")
+  table.remove(tab, 1)    -- codeblock opening
+  table.remove(tab, #tab) -- end of codeblock
+  print(table.concat(tab, "\n"))
+end
 
-local prompt = [[Classify and tag this document.
-Return JSON: category, tags (array), summary (short)
 
-]]
 
-local prompt = [[Return JSON: category, tags (array), summary (short)
+local tree
+tree = function(path, fn)
+  utility.list(path or ".", function(path_name)
+    if utility.is_file(path_name) then
+      fn(path_name)
+    else
+      tree(path .. utility.path_separator .. path_name, fn)
+    end
+  end)
+end
 
-]]
+tree(".", function(path_name)
+  print(path_name)
+end)
+os.exit()
 
-local prompt = [[Return YAML: category, tags (array), summary (short)
 
-]]
 
-local file_contents = utility.open(arg[1], "r", function(file)
+local prompt = [[Return JSON: category, tags (array), summary (short)]]
+-- local prompt = [[Return YAML: category, tags (array), summary (short)]]
+
+if prompt:sub(-1) ~= "\n" then
+  prompt = prompt .. "\n\n"
+end
+
+local file_contents = utility.open(file_name, "r", function(file)
   return file:read("*all")
 end)
 prompt = prompt .. file_contents
 
--- local model = "dolphin-phi:2.7b"
 local model = "gemma3:4b"
+-- local output = utility.capture_safe("ollama run " .. model .. " --nowordwrap " .. prompt:enquote())
 
-local output = utility.capture_safe("ollama run " .. model .. " --nowordwrap " .. prompt:enquote())
+local embedding_model = "nomic-embed-text"
+local output = utility.capture_safe("ollama run " .. embedding_model .. " " .. file_contents:enquote())
 
--- local output = utility.capture_safe("ollama run nomic-embed-text " .. file_contents:enquote())
+output = output:sub(1, -2) -- strip extra newline from utility.capture_safe
 
 print(output)
 
-print('--- div ---')
+-- print(strip_markdown(output))
 
---local a = output:find("\n")
+local embedding_metatable = {
---output = output:sub(a, -1)
+  __tojson = function(self, state)
---local b = output:find("\n"
+    return self.vector
+  end
+}
 
-local outtable = output:split("\n")
+local output = {
-table.remove(outtable, 1)
+  file_name = file_name,
-table.remove(outtable, #outtable)
+  vector = setmetatable({ vector = output }, embedding_metatable),
-table.remove(outtable, #outtable)
+}
-print(table.concat(outtable, "\n"))
 
-local file = utility.open("dump.txt", "w", function(file)
+print(json.encode(output, { indent = true }))
-  file:write(output)
+
-end)
+-- local file = utility.open("dump.txt", "w", function(file)
+--   file:write(output)
+-- end)