very WIP, doesn't detect duplicates properly.. everything else seems fine?

ReadMe.md

@@ -42,6 +42,21 @@ while `refresh_command` is run each time (optional).
 Commands will be run in the specified `path`.
 `strip_frontmatter` will remove YAML frontmatter (common in Markdown files).
 
+The embeddings are stored like so:
+
+```json
+{
+  "files":{
+    "PRIVATE_DATA/source_path/path/to/file.ext":["sha512sum", "another sum"]
+  },
+  "vectors":{
+    "sha512sum":[0.5, 0, 1, -0.5, -1, ...]
+  }
+}
+```
+
+TODO
+
 ### `synopsis_generator.lua`
 Chooses a random file within `notebook`, and generates a novel synopsis from it.
 

lib/log.lua

@@ -0,0 +1,75 @@
+local stored_messages = {}
+local display_levels = {}
+local enable_logging = true
+local message_limit = math.huge
+
+-- USAGE:
+--   -- set arbitrary log levels to be printed immediately
+--   log{ info = true, warning = true, ducksauce = true, }
+--   -- send anything (except nil) to a log level
+--   log("bacom", true, "text", 5, function() end, {})
+--   -- print all messages saved at a particular log level
+--   log("error")
+--   -- toggle ALL logging on and off with a boolean
+--   log(false) -- this deletes all stored messages
+--   -- set a maximum number of stored messages per log level
+--   log(50) -- warning: reaching large limits makes logging expensive
+--   -- do anything to the stored messages (this example deletes them all)
+--  log(function(messages) return {} end)
+
+return function(options, ...)
+  local options_type = type(options)
+
+  if (options_type == "string") and enable_logging then
+    if not stored_messages[options] then stored_messages[options] = {} end
+    local message_table = stored_messages[options]
+
+    -- turn log message into text
+    local tab = {...}
+    for i = 1, #tab do
+      tab[i] = tostring(tab[i])
+    end
+    local current_message = table.concat(tab, "\t")
+
+    if #current_message == 0 then
+      -- print all stored_messages of specified level
+      for i = 1, #message_table do
+        print(message_table[i])
+      end
+
+    else
+      -- store message (and print if in display_levels)
+      message_table[#message_table + 1] = current_message
+      if #message_table > message_limit then table.remove(message_table, 1) end
+      if display_levels[options] then print(current_message) end
+    end
+
+  -- set display_levels
+  elseif options_type == "table" then
+    for k,v in pairs(options) do
+      display_levels[k] = v
+    end
+
+  elseif options_type == "number" then
+    message_limit = options
+    for k,v in pairs(stored_messages) do
+      while #v > message_limit do
+        table.remove(v, 1)
+      end
+    end
+
+  -- turn all logging off and delete stored messages; or turn it on
+  elseif options_type == "boolean" then
+    if options then
+      enable_logging = true
+    else
+      enable_logging = false
+      stored_messages = {}
+    end
+
+  -- arbitrary access :D
+  elseif options_type == "function" then
+    local result = options(stored_messages)
+    if type(result) == "table" then stored_messages = result end
+  end
+end

lib/timing.lua

@@ -40,6 +40,7 @@ timing.mark = function(label)
   if #timing > 1 then
     timing.display(#timing)
   end
+  print("", "", label)
 end
 
 timing.estimate = function(current_position, total_operations)

lib/utility.lua

@@ -221,7 +221,7 @@ utility.list = function(path, func)
 
   local run = function(fn)
     for line in output:gmatch("[^\r\n]+") do -- thanks to https://stackoverflow.com/a/32847589
-      if not (line == "." or line == "..") then
+      if not ((line == ".") or (line == "..")) then
         fn(line)
       end
     end
@@ -239,7 +239,8 @@ utility.ls = function(...)
   return utility.list(...)
 end
 
-utility.tree = function(path, options, fn)
+local tree
+tree = function(path, options, fn)
   if type(options) == "function" then
     fn = options
     options = {}
@@ -258,7 +259,16 @@ utility.tree = function(path, options, fn)
 
       fn(path_name)
     else
-      utility.tree(path .. utility.path_separator .. path_name, options, fn)
+      tree(path .. utility.path_separator .. path_name, options, fn)
+    end
+  end)
+end
+utility.tree = function(path, options, fn)
+  tree(path, options, function(path_name)
+    if path_name:find(path) == 1 then
+      fn(path_name)
+    else
+      fn(path .. utility.path_separator .. path_name)
     end
   end)
 end

refresh_sources.lua

@@ -4,6 +4,7 @@ package.path = (arg[0]:match("@?(.*/)") or arg[0]:match("@?(.*\\)")) .. "lib" ..
 local utility = require "utility"
 
 local json = utility.require("dkjson")
+local log = utility.require("log")
 local text_processing = utility.require("text_processing")
 local timing = utility.require("timing")
 
@@ -21,16 +22,42 @@ if not config.models then
   utility.save_config()
 end
 
+log{
+  -- info = true,
+  -- warning = true,
+  -- debug = true,
+  -- files = true, -- debugging why the wrong files are selected
+  sha = true, -- what the fuck is going on with sha sums?
+}
+
+local memory_path = "PRIVATE_DATA" .. utility.path_separator .. "memory"
+local embeddings_file_path = memory_path .. utility.path_separator .. "+embeddings.json"
+local tmp_file_path = "PRIVATE_DATA" .. utility.path_separator .. ".tmp.2b65c19b-0883-49ca-8247-b1fe7760f922"
+
 local embeddings
-local embeddings_file_path = "PRIVATE_DATA/memory/+embeddings.json"
 if not utility.path_exists(embeddings_file_path) then
-  os.execute("mkdir -p PRIVATE_DATA/memory")
+  os.execute("mkdir -p " .. memory_path:enquote())
   utility.save_data({
     files = {},
     vectors = {},
   }, embeddings_file_path)
 end
 embeddings = utility.load_data(embeddings_file_path)
+local function embeddings_debug()
+  log("debug", "Embeddings loaded.", embeddings, embeddings.files, embeddings.vectors)
+  local file_count = 0
+  for k,v in pairs(embeddings.files) do
+    file_count = file_count + 1
+  end
+  local vector_count = 0
+  for k,v in pairs(embeddings.vectors) do
+    vector_count = vector_count + 1
+  end
+  log("debug", file_count .. " files.")
+  log("debug", vector_count .. " vectors.")
+  -- os.exit(1)
+end
+embeddings_debug()
 
 
 
@@ -57,6 +84,7 @@ local refresh_file_list = function(source_name, data_source)
 
   local file_list = {}
   utility.tree("PRIVATE_DATA" .. utility.path_separator .. data_source.path, compiled_filters, function(file_name)
+    log("files", file_name)
     file_list[#file_list + 1] = file_name
   end)
 
@@ -74,7 +102,7 @@ local generate_embeddings = function(text)
   return json.decode(result)
 end
 
--- returns nothing for empty files
+-- returns nothing for empty files and errors
 local process_file = function(data_source, file_name)
   local text = utility.read_file(file_name)
 
@@ -83,7 +111,7 @@ local process_file = function(data_source, file_name)
   end
 
   if #text == 0 then
-    print(file_name .. "\n is empty and being skipped.")
+    log("empty", file_name .. "\n is empty and being skipped.")
     return
   end
 
@@ -107,21 +135,20 @@ local process_file = function(data_source, file_name)
 
   local new_embeddings = {}
   for i = 1, #chunks do
-    -- print("Embedding length:", #chunks[i])
+    log("debug", "Embedding length:", #chunks[i])
     new_embeddings[i] = generate_embeddings(chunks[i]) or {}
   end
 
   if #new_embeddings[1] == 0 then
-    -- print(file_name)
+    log("debug", "Vector lengths:")
-    -- print("Vector lengths:")
+    for e = 1, #new_embeddings do
-    -- for e = 1, #new_embeddings do
+      log("debug", "", e, #new_embeddings[e])
-    --   print("", e, #new_embeddings[e])
+    end
-    -- end
     if #new_embeddings == 1 then
       -- Ollama very rarely errors with:
       --   Error: do embedding request: Post "http://127.0.0.1:53441/v1/embeddings": EOF
-      -- but it is inconsistent and re-running will eventually fix it. (See issue #7 if present.)
+      -- but it is inconsistent and re-running will eventually fix it.
-      print(file_name .. "\n encountered an embedding error and will be skipped this run only.")
+      log("warning", file_name .. "\n encountered an embedding error and will be skipped this run only.")
       return
     end
 
@@ -139,57 +166,87 @@ local process_file = function(data_source, file_name)
   return chunks, new_embeddings
 end
 
+-- returns nothing for errors
+local memorize_file = function(data_source, file_name)
+  local file_chunks, file_embeddings = process_file(data_source, file_name)
+  if not file_chunks then return end
+
+  local file_sums = {}
+  for i = 1, #file_chunks do
+    local function loop()
+      local text = file_chunks[i]
+      local current_embedding = file_embeddings[i]
+
+      utility.write_file(tmp_file_path, text)
+
+      local sha512sum = utility.sha512sum(tmp_file_path)
+      log("sha", "New sum? " .. sha512sum, tostring(embeddings.vectors[sha512sum]))
+      file_sums[#file_sums + 1] = sha512sum
+      if embeddings.vectors[sha512sum] then
+        log("sha", "Detected previously extant sum, skipping.")
+        return
+      end
+
+      os.execute(utility.commands.move .. tmp_file_path:enquote()
+        .. " " .. (memory_path .. utility.path_separator .. sha512sum):enquote())
+      embeddings.vectors[sha512sum] = current_embedding
+      log("sha", "Saved new sum!")
+    end
+    loop()
+  end
+
+  return file_sums
+end
+
 local refresh_sources = function()
-  local tmp_file_path = "PRIVATE_DATA/.tmp.2b65c19b-0883-49ca-8247-b1fe7760f922"
+  os.execute("mkdir -p " .. memory_path:enquote())
-  os.execute("mkdir -p PRIVATE_DATA/memory")
+
+  timing.mark("Checking memory for missing embeddings.")
+  utility.list(memory_path, function(file_name)
+    log("files", file_name)
+    if (file_name == ".DS_Store") or (file_name == "+embeddings.json") then return end
+
+    local file_path = memory_path .. utility.path_separator .. file_name
+    local sha512sum = utility.sha512sum(file_path)
+    log("sha", "New sum? " .. sha512sum, tostring(embeddings.vectors[sha512sum]))
+    if not embeddings.vectors[sha512sum] then
+      log("sha", "Saved new sum!")
+      memorize_file({}, file_path)
+    end
+  end)
+  timing.mark("Finished checking memory for missing embeddings.")
 
   local sources = utility.load_data("PRIVATE_DATA/sources.json")
   for source_name, data_source in pairs(sources) do
     local file_list = refresh_file_list(source_name, data_source)
 
-    timing.mark("Generating embeddings for " .. source_name .. ".")
+    timing.mark("Generating embeddings from source \"" .. source_name .. "\"")
     for f = 1, #file_list do
       local file_name = file_list[f]
       local function loop()
         local sha512sum = utility.sha512sum(file_name)
+        log("sha", file_name, sha512sum, tostring(embeddings.vectors[sha512sum]))
         if embeddings.vectors[sha512sum] then
+          log("debug", file_name .. "\n has already been embedded, skipping.")
           -- add file reference if it was missing
           if not embeddings.files[file_name] then
-            embeddings.files[file_name] = sha512sum
+            embeddings.files[file_name] = { sha512sum }
           end
           return
         end
 
-        local file_chunks, file_embeddings = process_file(data_source, file_name)
+        log("debug", "Generating embeddings for " .. file_name .. ".")
-        if not file_chunks then return end
+        local file_sums = memorize_file(data_source, file_name)
-
-        local file_sums = {}
-        for i = 1, #file_chunks do
-          local function loop()
-            local text = file_chunks[i]
-            local current_embedding = file_embeddings[i]
-
-            utility.write_file(tmp_file_path, text)
-
-            sha512sum = utility.sha512sum(tmp_file_path)
-            file_sums[#file_sums + 1] = sha512sum
-            if embeddings.vectors[sha512sum] then return end
-
-            os.execute(utility.commands.move .. tmp_file_path:enquote() .. " " .. "PRIVATE_DATA/memory/" .. sha512sum)
-            embeddings.vectors[sha512sum] = current_embedding
-          end
-          loop()
-        end
-
         embeddings.files[file_name] = file_sums
       end
       loop()
-      print("Finished " .. utility.leftpad(f, #tostring(#file_list), "0") .. "/" .. #file_list
+      log("info", "Finished " .. utility.leftpad(f, #tostring(#file_list), "0")
+        .. "/" .. #file_list
         .. " (" .. utility.leftpad(math.floor(f / #file_list * 100), 3, "0")
         .. "%) ETA: " .. timing.estimate(f, #file_list))
     end
 
-    timing.mark("Finished generating embeddings for " .. source_name .. ".")
+    timing.mark("Finished generating embeddings from source \"" .. source_name .. "\"")
   end
 
   utility.save_data(embeddings)
@@ -202,3 +259,4 @@ refresh_sources()
 timing.mark("Finished.")
 print("")
 timing.display()
+log("warning")