synopsis_generator.lua updated, mostly

ReadMe.md

@@ -96,3 +96,6 @@ and `initialized_sources` to store which sources have been initialized.
 ## Tasks
 - [ ] The whitelisting/blacklisting of tree should be in list too.
 - [ ] synopsis_generator should be able to blacklist files it already tried?
+- [ ] `refresh_sources.lua` doesn't check for defined files that don't exist
+  anymore, does it?
+  - [ ] I think it checks for every other possibility, but this needs checking.

lib/utility.lua

@@ -375,10 +375,10 @@ end
 
 
 
+local config_path = utility.path .. "config.json"
 local config, config_lock
 utility.get_config = function(skip_lock)
   if not config then
-    local config_path = utility.path .. "config.json"
     if utility.is_file(config_path) then
       if not skip_lock then
         config_lock = utility.get_lock(config_path)
@@ -396,7 +396,6 @@ end
 
 utility.save_config = function()
   if config then
-    local config_path = utility.path .. "config.json"
     if not config_lock then
       print("Warning: A config lock file was not established.")
     end
@@ -412,6 +411,31 @@ utility.save_config = function()
   end
 end
 
+utility.get_config_with_defaults = function(defaults)
+  local config = utility.get_config()
+
+  local loop, changes_made
+  loop = function(config_level, default_level)
+    for k,v in pairs(default_level) do
+      if not config_level[k] then
+        config_level[k] = v
+        changes_made = true
+      elseif type(v) == "table" then
+        loop(config_level[k], v)
+      end
+    end
+  end
+  loop(config, defaults)
+
+  if changes_made then
+    utility.save_config()
+  else
+    utility.release_lock(config_path)
+  end
+
+  return config
+end
+
 
 
 local data_file_locations = {}

list_file_extensions.lua

@@ -0,0 +1,19 @@
+#!/usr/bin/env luajit
+
+package.path = (arg[0]:match("@?(.*/)") or arg[0]:match("@?(.*\\)")) .. "lib" .. package.config:sub(1, 1) .. "?.lua;" .. package.path
+local utility = require "utility"
+local json = utility.require("dkjson")
+
+local search_path = arg[1]
+
+local file_extensions = {}
+utility.tree(search_path, function(file_path)
+  local _, _, extension = utility.split_path_components(file_path)
+  if extension then
+    file_extensions[extension] = true
+  end
+end)
+
+for extension in pairs(file_extensions) do
+  print(extension)
+end

refresh_sources.lua

@@ -8,19 +8,16 @@ local log = utility.require("log")
 local text_processing = utility.require("text_processing")
 local timing = utility.require("timing")
 
--- TODO make utility have a function for getting/setting defaults where locking is only used to set defaults if they aren't present
 -- TODO there should be a check function for a lock so a warning/error can be dumped?
-local config = utility.get_config("no-lock")
+local config = utility.get_config_with_defaults{
-if not config.models then
+  models = {
-  config.models = {
     embedding = {
       model = "qwen3-embedding:0.6b",
       max_chunk_size = 32768,
     },
     initialized_sources = {},
-  }
+  },
-  utility.save_config()
+}
-end
 
 log{
   info = true,
@@ -34,7 +31,6 @@ local memory_path = "PRIVATE_DATA" .. utility.path_separator .. "memory"
 local embeddings_file_path = memory_path .. utility.path_separator .. "+embeddings.json"
 local tmp_file_path = "PRIVATE_DATA" .. utility.path_separator .. ".tmp.2b65c19b-0883-49ca-8247-b1fe7760f922"
 
-local embeddings
 if not utility.path_exists(embeddings_file_path) then
   os.execute("mkdir -p " .. memory_path:enquote())
   utility.save_data({
@@ -42,7 +38,7 @@ if not utility.path_exists(embeddings_file_path) then
     vectors = {},
   }, embeddings_file_path)
 end
-embeddings = utility.load_data(embeddings_file_path)
+local embeddings = utility.load_data(embeddings_file_path)
 if log.debug then
   log("debug", "Embeddings loaded.", embeddings, embeddings.files, embeddings.vectors)
   local file_count = 0
@@ -257,7 +253,7 @@ local refresh_sources = function()
 
         if embeddings.vectors[sha512sum] then
           log("debug", file_name .. "\n has already been embedded, skipping.")
-          -- add file reference if it was missing
+          -- add file reference if it was missing (also handles recognition of duplicate files)
           if not embeddings.files[file_name] then
             embeddings.files[file_name] = { sha512sum }
           end

synopsis_generator.lua

@@ -7,49 +7,49 @@ local json = utility.require("dkjson")
 local prompts = utility.require("prompts")
 local text_processing = utility.require("text_processing")
 
-local model = "gemma4:12b-mlx"
+local config = utility.get_config_with_defaults{
-local minimum_bytes = 1000
+  models = {
-local maximum_bytes = 40000
+    synopsis_generator = {
+      model = "gemma4:12b-mlx",
+      minimum_bytes = 1000,
+      maximum_bytes = 40000,
+    },
+  },
+}
 
-local NOTEBOOK_PATH = "PRIVATE_DATA/notebook"
+local memory_path = "PRIVATE_DATA" .. utility.path_separator .. "memory"
-local blacklist = utility.enumerate{ ".git", ".gitattributes", ".gitignore", ".gitkeep", ".DS_Store", }
+local embeddings_file_path = memory_path .. utility.path_separator .. "+embeddings.json"
-local extension_blacklist = utility.enumerate{ "gif", "jpg", "jpeg", "mp4", "pdf", "png", "webp", }
+local data_path = "PRIVATE_DATA" .. utility.path_separator .. "synopses"
-
+local data_location = data_path .. utility.path_separator .. "+file_list.json"
-local data_location = "PRIVATE_DATA/synopsis_generator_file_list.json"
-local file_list
-if utility.path_exists(data_location) then
-  file_list = utility.load_data(data_location)
-else
-  file_list = {}
-end
 
 local refresh_file_list = function()
-  os.execute("cd " .. NOTEBOOK_PATH:enquot() .. " && git pull origin")
+  assert(utility.path_exists(embeddings_file_path), "Run \"./refresh_sources.lua\" to set up memory.")
+  local embeddings = utility.load_data(embeddings_file_path)
 
-  local new_files_list = {}
+  local minimum_bytes = config.models.synopsis_generator.minimum_bytes
-  utility.tree(NOTEBOOK_PATH, {
+  local maximum_bytes = config.models.synopsis_generator.maximum_bytes
-    blacklist = blacklist,
+
-    extension_blacklist = extension_blacklist,
+  local file_list = {}
-  }, function(file_name)
+  for file_name, sha512sum_list in pairs(embeddings.files) do
     local file_size = utility.file_size(file_name)
     if file_size >= minimum_bytes and file_size <= maximum_bytes then
       local text = text_processing.strip_frontmatter(utility.read_file(file_name))
       if #text >= minimum_bytes and #text <= maximum_bytes then
-        new_files_list[#new_files_list + 1] = file_name
+        file_list[#file_list + 1] = { file_name = file_name, }
+      end
     end
   end
-  end)
 
-  file_list = new_files_list
+  utility.save_data(file_list, data_location)
-  utility.save_data(new_files_list, data_location)
+  return file_list
 end
 
 local generate_and_score = function(file_name, text)
   print("Writing synopsis...")
-  local synopsis = utility.llm_prompt(prompts.synopsis_prompt .. text, model)
+  local synopsis = utility.llm_prompt(prompts.synopsis_prompt .. text, config.models.synopsis_generator.model)
   print(synopsis)
   print("Scoring synopsis...")
-  local scoring, thinking = utility.llm_prompt(prompts.scoring_prompt .. synopsis, model)
+  local scoring, thinking = utility.llm_prompt(prompts.scoring_prompt .. synopsis, config.models.synopsis_generator.model)
   print(scoring)
   local scoring_decoded = json.decode(scoring) -- likely will not work because it consistently returns Markdown instead of JSON
   if not scoring_decoded then
@@ -62,24 +62,23 @@ local generate_and_score = function(file_name, text)
     thinking = thinking,
   }
 
-  utility.save_data(object, "PRIVATE_DATA/synopses/" .. utility.uuid() .. ".json")
+  utility.save_data(object, data_path .. utility.path_separator .. utility.uuid() .. ".json")
 end
 
 local export_ordered_list_of_prompts = function()
-  local path = "PRIVATE_DATA/synopses"
   local items = {}
   local item_order = {}
 
-  utility.list(path, function(path_name)
+  utility.list(data_path, function(path_name)
-    local full_path = path .. utility.path_separator .. path_name
+    local full_path = data_path .. utility.path_separator .. path_name
-    if path_name:find("%.json") then
+    if full_path == data_location then return end
+
     local object = utility.load_data(full_path)
     items[path_name] = object
     if type(object.scoring) == "table" then
       local mean_score, total_score = utility.mean(object.scoring)
       item_order[#item_order + 1] = { path_name = path_name, total_score = total_score, mean_score = mean_score, }
     end
-    end
   end)
 
   table.sort(item_order, function(A,B) return A.mean_score > B.mean_score end)
@@ -87,7 +86,7 @@ local export_ordered_list_of_prompts = function()
   local output = {
     "---",
     "title: Ordered Synopses (" .. #item_order .. " items)",
-    "author: [\"" .. model .. "\", \"Tangent\", \"Ollama\"]",
+    "author: [\"" .. config.models.synopsis_generator.model .. "\", \"Tangent\", \"Ollama\"]",
     "publisher: \"synopsis_generator.lua\"",
     "---",
     "",
@@ -99,6 +98,8 @@ local export_ordered_list_of_prompts = function()
     local tab = text:split("\n")
 
     for index, line in ipairs(tab) do
+      -- ensure no output heading is an H1
+      -- TODO make H6 get turned into a bold line
       if line:sub(1, 1) == "#" then
         tab[index] = "#" .. tab[index]
       end
@@ -114,25 +115,27 @@ end
 
 
 
-os.execute("mkdir -p PRIVATE_DATA/synopses")
+os.execute("mkdir -p " .. data_path:enquote())
+
+local file_list
+if utility.path_exists(data_location) then
+  file_list = utility.load_data(data_location)
+end
 
 if arg[1] == "refresh_file_list" then
   print("Refresing file list...")
-  refresh_file_list()
+  file_list = refresh_file_list()
   os.exit(0)
 elseif arg[1] == "export_ordered_list_of_prompts" then
   export_ordered_list_of_prompts()
   os.exit(0)
 end
 
-print(#file_list .. " files to select from.")
+assert(file_list and (#file_list > 0), "Run \"./synopsis_generator.lua refresh_file_list\" first.")
-if #file_list == 0 then
-  print("Run \"./synopsis_generator.lua refresh_file_list\" first.")
-  os.exit(1)
-end
 
+print(#file_list .. " files to select from.")
 while true do
-  local file_name = file_list[math.random(1, #file_list)]
+  local file_name = file_list[math.random(1, #file_list)].file_name
   local text = utility.read_file(file_name)
   print(file_name .. " chosen.")
   generate_and_score(file_name, text)