ReadMe.md

@@ -23,7 +23,7 @@ sources. Example:
 
 ```json
 {
-  "source name (unused by the script)":{
+  "source name":{
     "filters":{
       "blacklist":".git",
       "extension_whitelist":"md"
@@ -36,12 +36,11 @@ sources. Example:
 }
 ```
 
-The filters are based on `utility.tree`'s filter options. `initialize_command`
-is only run the first time (optional), while `refresh_command` is run each time
-(optional). Commands will be run in the specified `path`. `strip_frontmatter` is
-for removing YAML frontmatter (common in Markdown files) if truthy.
-
-Filters are optional.
+The filters are based on `utility.tree`'s filter options (optional).
+`initialize_command` is only run the first time (optional),
+while `refresh_command` is run each time (optional).
+Commands will be run in the specified `path`.
+`strip_frontmatter` will remove YAML frontmatter (common in Markdown files).
 
 ### `synopsis_generator.lua`
 Chooses a random file within `notebook`, and generates a novel synopsis from it.

lib/utility.lua

@@ -18,7 +18,7 @@ if package.config:sub(1, 1) == "\\" then
   }
 else
   utility = {
-    OS = "UNIX-like",
+    OS = "Linux",
     path_separator = "/",
     temp_directory = "/tmp/",
     commands = {
@@ -299,6 +299,18 @@ utility.file_size = function(file_path)
   return utility.open(file_path, "rb", function(file) return file:seek("end") end)
 end
 
+utility.sha512sum = function(file_path)
+  local sha512sum
+  if utility.OS == "Linux" then
+    sha512sum = os.capture_safe("shasum -p -t -a 512 " .. file_path:enquote()) -- TODO check this
+  elseif utility.OS == "macOS" then
+    sha512sum = os.capture_safe("shasum -U -a 512 " .. file_path:enquote())
+  elseif utility.OS == "Windows" then
+    error("utility.sha512sum() not implemented for Windows.")
+  end
+  return sha512sum:sub(1, 128)
+end
+
 
 
 utility.escape_quotes_and_escapes = function(input)
@@ -570,4 +582,6 @@ end
 
 
 
+if utility.capture_safe("uname"):find("Darwin") == 1 then utility.OS = "macOS" end
+
 return utility

refresh_sources.lua

@@ -21,8 +21,21 @@ if not config.models then
   utility.save_config()
 end
 
-local refresh_file_list = function(data_source)
-  timing.mark("Assembling file list.")
+local embeddings
+local embeddings_file_path = "PRIVATE_DATA/memory/+embeddings.json"
+if not utility.path_exists(embeddings_file_path) then
+  os.execute("mkdir -p PRIVATE_DATA/memory")
+  utility.save_data({
+    files = {},
+    vectors = {},
+  }, embeddings_file_path)
+end
+embeddings = utility.load_data(embeddings_file_path)
+
+
+
+local refresh_file_list = function(source_name, data_source)
+  timing.mark("Assembling file list for " .. source_name .. ".")
 
   local full_path = "PRIVATE_DATA" .. utility.path_separator .. data_source.path
   if data_source.initialize_command and (not config.models.initialized_sources[data_source.path]) then
@@ -47,7 +60,7 @@ local refresh_file_list = function(data_source)
     file_list[#file_list + 1] = file_name
   end)
 
-  timing.mark("Finished assembling file list.")
+  timing.mark("Finished assembling file list for " .. source_name .. ".")
   return file_list
 end
 
@@ -97,36 +110,70 @@ local process_file = function(data_source, file_name)
     end
   end
 
-  local embeddings = {}
+  local new_embeddings = {}
   for i = 1, #chunks do
-    embeddings[i] = generate_embeddings(chunks[i])
+    new_embeddings[i] = generate_embeddings(chunks[i]) or {}
   end
 
-  if embeddings[1] == nil then
-    -- TODO average all other embeddings to this one
-  end
-
-  return chunks, embeddings
-end
-
--- TODO somewhere near the top of this function chain needs to be the ability to recognize a file as unchanged and skip re-generating embeddings
---  this means running the same shasum command I will put below on that source file before trying to generate an embedding
---    something something merging new changes into the old data structure means we need to load that data structure first so we only replace modified chunks
-local refresh_sources = function()
-  local sources = utility.load_data("PRIVATE_DATA/sources.json")
-  for _, data_source in pairs(sources) do
-    local file_list = refresh_file_list(data_source)
-    timing.mark("Generating embeddings.")
-    for _, file_name in ipairs(file_list) do
-      local chunks, embeddings = process_file(file_name)
-      -- TODO decide how these will be stored
-      local sum_file_path = "PRIVATE_DATA/.tmp.2b65c19b-0883-49ca-8247-b1fe7760f922"
-      utility.write_file(sum_file_path, text)
-      -- -p ensures compatibility across OSes, -t ensures it is read as text (some OSes default differently), -a 512 ensures it is a 512-bit SHA2 sum
-      local sha512sum = utility.capture_safe("shasum -p -t -a 512 " .. sum_file_path)
-      -- for chunks, save to a specific local tmp file to shasum, then mv that file based on the sum
-      -- for embeddings, store sum = embedding ? (look at how generate_embeddings.lua worked)
+  if #new_embeddings[1] == 0 then
+    -- average all embeddings to make the core file embedding
+    local count = #new_embeddings[2]
+    for vector_index = 1, count do
+      local total = 0
+      for chunk = 2, #new_embeddings do
+        total = total + new_embeddings[chunk][vector_index]
+      end
+      new_embeddings[1][vector_index] = total / count
     end
-    timing.mark("Finished generating embeddings.")
   end
+
+  return chunks, new_embeddings
 end
+
+local refresh_sources = function()
+  local tmp_file_path = "PRIVATE_DATA/.tmp.2b65c19b-0883-49ca-8247-b1fe7760f922"
+  os.execute("mkdir -p PRIVATE_DATA/memory")
+
+  local sources = utility.load_data("PRIVATE_DATA/sources.json")
+  for source_name, data_source in pairs(sources) do
+    local file_list = refresh_file_list(source_name, data_source)
+
+    timing.mark("Generating embeddings for " .. source_name .. ".")
+    for _, file_name in ipairs(file_list) do
+      local function loop()
+        local sha512sum = utility.sha512sum(tmp_file_path)
+        if embeddings.vectors[sha512sum] then return end
+
+        local file_chunks, file_embeddings = process_file(data_source, file_name)
+
+        local file_sums = {}
+        for i = 1, #file_chunks do
+          local function loop()
+            local text = file_chunks[i]
+            local current_embedding = file_embeddings[i]
+
+            utility.write_file(tmp_file_path, text)
+
+            sha512sum = utility.sha512sum(tmp_file_path)
+            file_sums[#file_sums + 1] = sha512sum
+            if embeddings.vectors[sha512sum] then return end
+
+            os.execute(utility.commands.move .. tmp_file_path:enquote() .. " " .. "PRIVATE_DATA/memory/")
+            embeddings.vectors[sha512sum] = current_embedding
+          end
+          loop()
+        end
+
+        embeddings.files[file_name] = file_sums
+      end
+      loop()
+    end
+
+    timing.mark("Finished generating embeddings for " .. source_name .. ".")
+  end
+
+  utility.save_data(embeddings)
+  os.execute("rm " .. tmp_file_path:enquote())
+end
+
+refresh_sources()

test.lua

@@ -6,13 +6,7 @@ local json = utility.require("dkjson")
 
 local file_name = arg[1]
 
-print("---")
-print(utility.capture_safe("shasum -p"))
-print("---")
-
-print("---")
-print(utility.capture_safe("shasum -U -a 512 test.lua"))
-print("---")
+print(utility.OS)
 
 os.exit(0)
 
@@ -23,9 +17,6 @@ if prompt:sub(-1) ~= "\n" then
   prompt = prompt .. "\n\n"
 end
 
-local file_contents = utility.open(file_name, "r", function(file)
-  return file:read("*all")
-end)
+local file_contents = utility.read_file(file_name)
 prompt = prompt .. file_contents
-
--- output = output:sub(1, -2) -- strip extra newline from utility.capture_safe
+-- print(utility.llm_prompt(prompt))