From 20015af69ceaec2b11f15ecd008c89d19a7383d8 Mon Sep 17 00:00:00 2001
From: tangent <rose@tangentfox.com>
Date: Tue, 15 Sep 2026 07:39:15 -0600
Subject: [PATCH 01/21] Add refresh_sources.lua

This file needs to be chmodded.
---
 refresh_sources.lua | 62 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 62 insertions(+)
 create mode 100644 refresh_sources.lua

diff --git a/refresh_sources.lua b/refresh_sources.lua
new file mode 100644
index 0000000..8b3c95e
--- /dev/null
+++ b/refresh_sources.lua
@@ -0,0 +1,62 @@
+#!/usr/bin/env luajit
+
+package.path = (arg[0]:match("@?(.*/)") or arg[0]:match("@?(.*\\)")) .. "lib" .. package.config:sub(1, 1) .. "?.lua;" .. package.path
+local utility = require "utility"
+
+local timing = utility.require("timing")
+
+-- TODO make utility have a function for getting/setting defaults where locking is only used to set defaults if they aren't present
+local config = utility.get_config("no-lock")
+if not config.models then
+  config.models = {
+    embedding = {
+      model = "qwen3-embedding:0.6b",
+      max_chunk_size = 32768,
+    }
+  }
+  utility.save_config()
+end
+
+local refresht_file_list = function(data_source)
+  timing.mark("Assembling file list.")
+
+  os.execute("cd " .. ("PRIVATE_DATA" .. utility.path_separator .. data_source.path):enquote() .. " && " .. data_source.refresh_command)
+
+  local compiled_filters = {}
+  for name, object in pairs(data_source.filters) do
+    compiled_filters[name] = utility.enumerate(object)
+  end
+
+  local file_list = {}
+  utility.tree(data_source.path, compiled_filters, function(file_name)
+    file_list[#file_list + 1] = file_name
+  end)
+
+  return file_list
+end
+
+local generate_embeddings = function(text)
+  if #text > config.models.embedding.max_chunk_size then
+    return nil, "generate_embeddings() must only be passed appropriately-sized chunks!"
+  end
+
+  local result = utility.llm_prompt(text, config.models.embedding.model)
+
+  -- skipping unnecessary decoding to an array and putting back into text for saving
+  -- this would cause problems if we intended to use the results immediately
+  return setmetatable({}, {
+    __tojson = function() return result end,
+  })
+end
+
+local process_file = function(file_name)
+  local text = utility.read_file(file_name)
+  if w
+end
+
+local to_be_named_generate_file_embeddings = function(text)
+  if #text > config.models.embedding.max_chunk_size then
+    -- TODO, split, process each chunk, return both a total and each chunk
+  end
+  -- return just the file and its embedding
+end

From 692d4f5690dc9eaa1056173702b46f67c06dc9d6 Mon Sep 17 00:00:00 2001
From: tangent <rose@tangentfox.com>
Date: Tue, 15 Sep 2026 07:39:28 -0600
Subject: [PATCH 02/21] Update .gitignore

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 38cb679..5b1d3f5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
 .DS_Store
 PRIVATE_DATA/**
 !PRIVATE_DATA/.gitkeep
+config.json

From 7b4608ec7e3ab73a3ea00b0ff9602ecbd56597a2 Mon Sep 17 00:00:00 2001
From: tangent <rose@tangentfox.com>
Date: Tue, 15 Sep 2026 08:14:35 -0600
Subject: [PATCH 03/21] Update refresh_sources.lua

---
 refresh_sources.lua | 73 ++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 63 insertions(+), 10 deletions(-)

diff --git a/refresh_sources.lua b/refresh_sources.lua
index 8b3c95e..f39b259 100644
--- a/refresh_sources.lua
+++ b/refresh_sources.lua
@@ -3,6 +3,8 @@
 package.path = (arg[0]:match("@?(.*/)") or arg[0]:match("@?(.*\\)")) .. "lib" .. package.config:sub(1, 1) .. "?.lua;" .. package.path
 local utility = require "utility"
 
+local json = utility.require("dkjson")
+local text_processing = utility.require("text_processing")
 local timing = utility.require("timing")
 
 -- TODO make utility have a function for getting/setting defaults where locking is only used to set defaults if they aren't present
@@ -17,7 +19,9 @@ if not config.models then
   utility.save_config()
 end
 
-local refresht_file_list = function(data_source)
+-- TODO recognize uninitialized source and initialize it
+--  this would need to be in the config
+local refresh_file_list = function(data_source)
   timing.mark("Assembling file list.")
 
   os.execute("cd " .. ("PRIVATE_DATA" .. utility.path_separator .. data_source.path):enquote() .. " && " .. data_source.refresh_command)
@@ -32,9 +36,11 @@ local refresht_file_list = function(data_source)
     file_list[#file_list + 1] = file_name
   end)
 
+  timing.mark("Finished assembling file list.")
   return file_list
 end
 
+-- returns nothing when too much text is sent
 local generate_embeddings = function(text)
   if #text > config.models.embedding.max_chunk_size then
     return nil, "generate_embeddings() must only be passed appropriately-sized chunks!"
@@ -42,21 +48,68 @@ local generate_embeddings = function(text)
 
   local result = utility.llm_prompt(text, config.models.embedding.model)
 
-  -- skipping unnecessary decoding to an array and putting back into text for saving
-  -- this would cause problems if we intended to use the results immediately
-  return setmetatable({}, {
+  result = setmetatable({}, {
     __tojson = function() return result end,
   })
+
+  return json.decode(result)
 end
 
-local process_file = function(file_name)
+-- returns nothing for empty files
+local process_file = function(data_source, file_name)
   local text = utility.read_file(file_name)
-  if w
+
+  if data_source.strip_frontmatter then
+    text = text_processing.strip_frontmatter(text)
+  end
+
+  if #text == 0 then
+    print(file_name .. "\n is empty and being skipped.")
+    return
+  end
+
+  local chunk_size = config.models.embedding.max_chunk_size
+  local half_chunk_size = math.floor(chunk_size / 2)
+  local chunks = { text }
+
+  while #text > chunk_size do
+    local first_chunk = text:sub(1, chunk_size)
+    local overlap_chunk = text:sub(half_chunk_size, chunk_size + half_chunk_size)
+
+    chunks[#chunks + 1] = first_chunk
+    chunks[#chunks + 1] = overlap_chunk
+
+    text = text:sub(chunk_size)
+    if #text > half_chunk_size and (not #text > chunk_size) then
+      -- last chunk would be skipped if we didn't handle this here
+      chunks[#chunks + 1] = text
+    end
+  end
+
+  local embeddings = {}
+  for i = 1, #chunks do
+    embeddings[i] = generate_embeddings(chunks[i])
+  end
+
+  if embeddings[1] == nil then
+    -- TODO average all other embeddings to this one
+  end
+
+  return chunks, embeddings
 end
 
-local to_be_named_generate_file_embeddings = function(text)
-  if #text > config.models.embedding.max_chunk_size then
-    -- TODO, split, process each chunk, return both a total and each chunk
+-- TODO somewhere near the top of this function chain needs to be the ability to recognize a file as unchanged and skip re-generating embeddings
+--  this means running the same shasum command I will put below on that source file before trying to generate an embedding
+--    something something merging new changes into the old data structure means we need to load that data structure first so we only replace modified chunks
+local refresh_sources = function()
+  local sources = utility.load_data("PRIVATE_DATA/sources.json")
+  for _, data_source in pairs(sources) do
+    local file_list = refresh_file_list(data_source)
+    for _, file_name in ipairs(file_list) do
+      local chunks, embeddings = process_file(file_name)
+      -- TODO decide how these will be stored
+      -- for chunks, save to a specific local tmp file to shasum, then mv that file based on the sum
+      -- for embeddings, store sum = embedding ? (look at how generate_embeddings.lua worked)
+    end
   end
-  -- return just the file and its embedding
 end

From 012d3abe5a899b610cb7b9db7bbb0c9259811fe4 Mon Sep 17 00:00:00 2001
From: tangent <rose@tangentfox.com>
Date: Tue, 15 Sep 2026 08:22:02 -0600
Subject: [PATCH 04/21] Update ReadMe.md

---
 ReadMe.md | 32 ++++++++++++++++++++++++++++----
 1 file changed, 28 insertions(+), 4 deletions(-)

diff --git a/ReadMe.md b/ReadMe.md
index 6c4beb6..b889130 100644
--- a/ReadMe.md
+++ b/ReadMe.md
@@ -4,20 +4,44 @@ Finding useful things to do with local LLMs.
 All scripts work with data within `PRIVATE_DATA/` so that private data can't be
 accidentally committed.
 
-###### `cosine_similarity.lua`
+### `cosine_similarity.lua`
 Opens `embeddings.json` and creates `similarities.json` with a sorted list of
 comparisons.
 
-###### `generate_embeddings.lua`
+### `generate_embeddings.lua`
 Opens every file on its whitelist within the `notebook` directory, and generates
 embeddings (placed in `embeddings.json`). When files are too long, it truncates
 them.
 
-###### `least_similar.lua`
+### `least_similar.lua`
 Opens `similarities.json`, reverses the sort order, and saves it as
 `differences.json`.
 
-###### `synopsis_generator.lua`
+### `refresh_sources.lua`
+Creates/Maintains a store of chunked data with embeddings based on configurable
+sources. Example:
+
+```json
+{
+  "source name (unused by the script)":{
+    "filters":{
+      "blacklist":".git",
+      "extension_whitelist":"md"
+    },
+    "initialize_command":"git clone REMOTE",
+    "path":"will be created before initialize_command is run",
+    "strip_frontmatter":true,
+    "refresh_command":"git reset --hard origin/main"
+  }
+}
+```
+
+The filters are based on `utility.tree`'s filter options. `initialize_command`
+is only run the first time (optional), while `refresh_command` is run each time
+(optional). Commands will be run in the specified `path`. `strip_frontmatter` is
+for removing YAML frontmatter (common in Markdown files) if truthy.
+
+### `synopsis_generator.lua`
 Chooses a random file within `notebook`, and generates a novel synopsis from it.
 
 Arguments:

From dde5283acafff1b1a896998506fbc5a5d6b50b72 Mon Sep 17 00:00:00 2001
From: tangent <rose@tangentfox.com>
Date: Tue, 15 Sep 2026 08:22:41 -0600
Subject: [PATCH 05/21] Update ReadMe.md

---
 ReadMe.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ReadMe.md b/ReadMe.md
index b889130..5eb5b28 100644
--- a/ReadMe.md
+++ b/ReadMe.md
@@ -28,7 +28,7 @@ sources. Example:
       "blacklist":".git",
       "extension_whitelist":"md"
     },
-    "initialize_command":"git clone REMOTE",
+    "initialize_command":"git clone REMOTE .",
     "path":"will be created before initialize_command is run",
     "strip_frontmatter":true,
     "refresh_command":"git reset --hard origin/main"

From bfe93e33f98ede13b31908e60abd8bbf49e64933 Mon Sep 17 00:00:00 2001
From: tangent <rose@tangentfox.com>
Date: Tue, 15 Sep 2026 08:36:19 -0600
Subject: [PATCH 06/21] Update refresh_sources.lua

---
 refresh_sources.lua | 29 +++++++++++++++++++++++------
 1 file changed, 23 insertions(+), 6 deletions(-)

diff --git a/refresh_sources.lua b/refresh_sources.lua
index f39b259..02a1fb0 100644
--- a/refresh_sources.lua
+++ b/refresh_sources.lua
@@ -8,27 +8,38 @@ local text_processing = utility.require("text_processing")
 local timing = utility.require("timing")
 
 -- TODO make utility have a function for getting/setting defaults where locking is only used to set defaults if they aren't present
+-- TODO there should be a check function for a lock so a warning/error can be dumped?
 local config = utility.get_config("no-lock")
 if not config.models then
   config.models = {
     embedding = {
       model = "qwen3-embedding:0.6b",
       max_chunk_size = 32768,
-    }
+    },
+    initialized_sources = {},
   }
   utility.save_config()
 end
 
--- TODO recognize uninitialized source and initialize it
---  this would need to be in the config
 local refresh_file_list = function(data_source)
   timing.mark("Assembling file list.")
 
-  os.execute("cd " .. ("PRIVATE_DATA" .. utility.path_separator .. data_source.path):enquote() .. " && " .. data_source.refresh_command)
+  local full_path = "PRIVATE_DATA" .. utility.path_separator .. data_source.path
+  if data_source.initialize_command and (not config.models.initialized_sources[data_source.path]) then
+    -- NOTE this is where we'd want to check/obtain a lock on the config so we can safely run this
+    os.execute("mkdir -p " .. full_path:enquote() .. " && cd " .. full_path:enquote() .. " && " .. data_source.initialize_command)
+    config.models.initialized_sources[data_source.path] = true
+    utility.save_config()
+  end
+  if data_source.refresh_command then
+    os.execute("cd " .. full_path:enquote() .. " && " .. data_source.refresh_command)
+  end
 
   local compiled_filters = {}
-  for name, object in pairs(data_source.filters) do
-    compiled_filters[name] = utility.enumerate(object)
+  if data_source.filters then
+    for name, object in pairs(data_source.filters) do
+      compiled_filters[name] = utility.enumerate(object)
+    end
   end
 
   local file_list = {}
@@ -105,11 +116,17 @@ local refresh_sources = function()
   local sources = utility.load_data("PRIVATE_DATA/sources.json")
   for _, data_source in pairs(sources) do
     local file_list = refresh_file_list(data_source)
+    timing.mark("Generating embeddings.")
     for _, file_name in ipairs(file_list) do
       local chunks, embeddings = process_file(file_name)
       -- TODO decide how these will be stored
+      local sum_file_path = "PRIVATE_DATA/.tmp.2b65c19b-0883-49ca-8247-b1fe7760f922"
+      utility.write_file(sum_file_path, text)
+      -- -p ensures compatibility across OSes, -t ensures it is read as text (some OSes default differently), -a 512 ensures it is a 512-bit SHA2 sum
+      local sha512sum = utility.capture_safe("shasum -p -t -a 512 " .. sum_file_path)
       -- for chunks, save to a specific local tmp file to shasum, then mv that file based on the sum
       -- for embeddings, store sum = embedding ? (look at how generate_embeddings.lua worked)
     end
+    timing.mark("Finished generating embeddings.")
   end
 end

From 5b0520a7ce484fbf1bfa3400d0dc64bb35047372 Mon Sep 17 00:00:00 2001
From: tangent <rose@tangentfox.com>
Date: Tue, 15 Sep 2026 08:37:03 -0600
Subject: [PATCH 07/21] Update ReadMe.md

---
 ReadMe.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ReadMe.md b/ReadMe.md
index 5eb5b28..7e86c20 100644
--- a/ReadMe.md
+++ b/ReadMe.md
@@ -41,6 +41,8 @@ is only run the first time (optional), while `refresh_command` is run each time
 (optional). Commands will be run in the specified `path`. `strip_frontmatter` is
 for removing YAML frontmatter (common in Markdown files) if truthy.
 
+Filters are optional.
+
 ### `synopsis_generator.lua`
 Chooses a random file within `notebook`, and generates a novel synopsis from it.
 

From 915319ee1c637740d77b4db7188ebaea719c279b Mon Sep 17 00:00:00 2001
From: tangent <rose@tangentfox.com>
Date: Tue, 15 Sep 2026 09:16:00 -0600
Subject: [PATCH 08/21] Delete PRIVATE_DATA/.gitkeep

---
 PRIVATE_DATA/.gitkeep | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 PRIVATE_DATA/.gitkeep

diff --git a/PRIVATE_DATA/.gitkeep b/PRIVATE_DATA/.gitkeep
deleted file mode 100644
index e69de29..0000000

From 425cae4c208516476bc03674202c839d2af7a0cd Mon Sep 17 00:00:00 2001
From: tangent <rose@tangentfox.com>
Date: Tue, 15 Sep 2026 09:16:12 -0600
Subject: [PATCH 09/21] Update .gitignore

---
 .gitignore | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 5b1d3f5..7ead4ff 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,3 @@
 .DS_Store
 PRIVATE_DATA/**
-!PRIVATE_DATA/.gitkeep
 config.json

From 1b7c211aa2ddeefa72fd67aa48fdecdaea8ad97a Mon Sep 17 00:00:00 2001
From: tangent <rose@tangentfox.com>
Date: Tue, 15 Sep 2026 09:25:04 -0600
Subject: [PATCH 10/21] Update test.lua

---
 test.lua | 45 ++++++---------------------------------------
 1 file changed, 6 insertions(+), 39 deletions(-)

diff --git a/test.lua b/test.lua
index a44e1ab..705358f 100755
--- a/test.lua
+++ b/test.lua
@@ -6,16 +6,13 @@ local json = utility.require("dkjson")
 
 local file_name = arg[1]
 
+print("---")
+print(utility.capture_safe("shasum -p"))
+print("---")
 
-
-local function strip_markdown(text)
-  local tab = text:split("\n")
-  table.remove(tab, 1)    -- codeblock opening
-  table.remove(tab, #tab) -- end of codeblock
-  print(table.concat(tab, "\n"))
-end
-
-
+print("---")
+print(utility.capture_safe("shasum -U -t -a 512 test.lua"))
+print("---")
 
 local prompt = [[Return JSON: category, tags (array), summary (short)]]
 -- local prompt = [[Return YAML: category, tags (array), summary (short)]]
@@ -29,34 +26,4 @@ local file_contents = utility.open(file_name, "r", function(file)
 end)
 prompt = prompt .. file_contents
 
-local model = "gemma3:4b"
--- local output = utility.capture_safe("ollama run " .. model .. " --nowordwrap " .. prompt:enquote())
-
--- local embedding_model = "nomic-embed-text"
--- local output = utility.capture_safe("ollama run " .. embedding_model .. " " .. file_contents:enquote())
-
 -- output = output:sub(1, -2) -- strip extra newline from utility.capture_safe
-
--- strip YAML frontmatter (if present)
---   can error, will return nil & error message
-local function strip_frontmatter(text)
-  local tab = text:split("\n")
-  if tab[1] == "---" then
-    table.remove(tab, 1)
-    while true do
-      local done = tab[1] == "---"
-      table.remove(tab, 1)
-      if done then
-        return table.concat(tab, "\n")
-      elseif #tab < 1 then
-        return nil, "Invalid YAML frontmatter."
-      end
-    end
-  end
-  return text
-end
-
--- print(output)
-
-print(strip_frontmatter(file_contents))
--- print(strip_markdown(output))

From 1e79c9ed991a16b086e07252fdbb967b9efd048d Mon Sep 17 00:00:00 2001
From: tangent <rose@tangentfox.com>
Date: Tue, 15 Sep 2026 09:27:46 -0600
Subject: [PATCH 11/21] macOS is weird

---
 test.lua | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/test.lua b/test.lua
index 705358f..f01eee3 100755
--- a/test.lua
+++ b/test.lua
@@ -11,9 +11,11 @@ print(utility.capture_safe("shasum -p"))
 print("---")
 
 print("---")
-print(utility.capture_safe("shasum -U -t -a 512 test.lua"))
+print(utility.capture_safe("shasum -U -a 512 test.lua"))
 print("---")
 
+os.exit(0)
+
 local prompt = [[Return JSON: category, tags (array), summary (short)]]
 -- local prompt = [[Return YAML: category, tags (array), summary (short)]]
 

From c5aed984f869c387b0eccbd7bda41e45ec82b1c1 Mon Sep 17 00:00:00 2001
From: Tangent <rose@tangentfox.com>
Date: Tue, 15 Sep 2026 11:02:54 -0600
Subject: [PATCH 12/21] should be ready to go

---
 ReadMe.md           |  13 +++---
 lib/utility.lua     |  16 ++++++-
 refresh_sources.lua | 107 +++++++++++++++++++++++++++++++-------------
 test.lua            |  15 ++-----
 4 files changed, 101 insertions(+), 50 deletions(-)

diff --git a/ReadMe.md b/ReadMe.md
index 7e86c20..9320f59 100644
--- a/ReadMe.md
+++ b/ReadMe.md
@@ -23,7 +23,7 @@ sources. Example:
 
 ```json
 {
-  "source name (unused by the script)":{
+  "source name":{
     "filters":{
       "blacklist":".git",
       "extension_whitelist":"md"
@@ -36,12 +36,11 @@ sources. Example:
 }
 ```
 
-The filters are based on `utility.tree`'s filter options. `initialize_command`
-is only run the first time (optional), while `refresh_command` is run each time
-(optional). Commands will be run in the specified `path`. `strip_frontmatter` is
-for removing YAML frontmatter (common in Markdown files) if truthy.
-
-Filters are optional.
+The filters are based on `utility.tree`'s filter options (optional).
+`initialize_command` is only run the first time (optional),
+while `refresh_command` is run each time (optional).
+Commands will be run in the specified `path`.
+`strip_frontmatter` will remove YAML frontmatter (common in Markdown files).
 
 ### `synopsis_generator.lua`
 Chooses a random file within `notebook`, and generates a novel synopsis from it.
diff --git a/lib/utility.lua b/lib/utility.lua
index fb9d28a..9e7bde8 100644
--- a/lib/utility.lua
+++ b/lib/utility.lua
@@ -18,7 +18,7 @@ if package.config:sub(1, 1) == "\\" then
   }
 else
   utility = {
-    OS = "UNIX-like",
+    OS = "Linux",
     path_separator = "/",
     temp_directory = "/tmp/",
     commands = {
@@ -299,6 +299,18 @@ utility.file_size = function(file_path)
   return utility.open(file_path, "rb", function(file) return file:seek("end") end)
 end
 
+utility.sha512sum = function(file_path)
+  local sha512sum
+  if utility.OS == "Linux" then
+    sha512sum = os.capture_safe("shasum -p -t -a 512 " .. file_path:enquote()) -- TODO check this
+  elseif utility.OS == "macOS" then
+    sha512sum = os.capture_safe("shasum -U -a 512 " .. file_path:enquote())
+  elseif utility.OS == "Windows" then
+    error("utility.sha512sum() not implemented for Windows.")
+  end
+  return sha512sum:sub(1, 128)
+end
+
 
 
 utility.escape_quotes_and_escapes = function(input)
@@ -570,4 +582,6 @@ end
 
 
 
+if utility.capture_safe("uname"):find("Darwin") == 1 then utility.OS = "macOS" end
+
 return utility
diff --git a/refresh_sources.lua b/refresh_sources.lua
index 02a1fb0..4cf7ceb 100644
--- a/refresh_sources.lua
+++ b/refresh_sources.lua
@@ -21,8 +21,21 @@ if not config.models then
   utility.save_config()
 end
 
-local refresh_file_list = function(data_source)
-  timing.mark("Assembling file list.")
+local embeddings
+local embeddings_file_path = "PRIVATE_DATA/memory/+embeddings.json"
+if not utility.path_exists(embeddings_file_path) then
+  os.execute("mkdir -p PRIVATE_DATA/memory")
+  utility.save_data({
+    files = {},
+    vectors = {},
+  }, embeddings_file_path)
+end
+embeddings = utility.load_data(embeddings_file_path)
+
+
+
+local refresh_file_list = function(source_name, data_source)
+  timing.mark("Assembling file list for " .. source_name .. ".")
 
   local full_path = "PRIVATE_DATA" .. utility.path_separator .. data_source.path
   if data_source.initialize_command and (not config.models.initialized_sources[data_source.path]) then
@@ -47,7 +60,7 @@ local refresh_file_list = function(data_source)
     file_list[#file_list + 1] = file_name
   end)
 
-  timing.mark("Finished assembling file list.")
+  timing.mark("Finished assembling file list for " .. source_name .. ".")
   return file_list
 end
 
@@ -97,36 +110,70 @@ local process_file = function(data_source, file_name)
     end
   end
 
-  local embeddings = {}
+  local new_embeddings = {}
   for i = 1, #chunks do
-    embeddings[i] = generate_embeddings(chunks[i])
+    new_embeddings[i] = generate_embeddings(chunks[i]) or {}
   end
 
-  if embeddings[1] == nil then
-    -- TODO average all other embeddings to this one
-  end
-
-  return chunks, embeddings
-end
-
--- TODO somewhere near the top of this function chain needs to be the ability to recognize a file as unchanged and skip re-generating embeddings
---  this means running the same shasum command I will put below on that source file before trying to generate an embedding
---    something something merging new changes into the old data structure means we need to load that data structure first so we only replace modified chunks
-local refresh_sources = function()
-  local sources = utility.load_data("PRIVATE_DATA/sources.json")
-  for _, data_source in pairs(sources) do
-    local file_list = refresh_file_list(data_source)
-    timing.mark("Generating embeddings.")
-    for _, file_name in ipairs(file_list) do
-      local chunks, embeddings = process_file(file_name)
-      -- TODO decide how these will be stored
-      local sum_file_path = "PRIVATE_DATA/.tmp.2b65c19b-0883-49ca-8247-b1fe7760f922"
-      utility.write_file(sum_file_path, text)
-      -- -p ensures compatibility across OSes, -t ensures it is read as text (some OSes default differently), -a 512 ensures it is a 512-bit SHA2 sum
-      local sha512sum = utility.capture_safe("shasum -p -t -a 512 " .. sum_file_path)
-      -- for chunks, save to a specific local tmp file to shasum, then mv that file based on the sum
-      -- for embeddings, store sum = embedding ? (look at how generate_embeddings.lua worked)
+  if #new_embeddings[1] == 0 then
+    -- average all embeddings to make the core file embedding
+    local count = #new_embeddings[2]
+    for vector_index = 1, count do
+      local total = 0
+      for chunk = 2, #new_embeddings do
+        total = total + new_embeddings[chunk][vector_index]
+      end
+      new_embeddings[1][vector_index] = total / count
     end
-    timing.mark("Finished generating embeddings.")
   end
+
+  return chunks, new_embeddings
 end
+
+local refresh_sources = function()
+  local tmp_file_path = "PRIVATE_DATA/.tmp.2b65c19b-0883-49ca-8247-b1fe7760f922"
+  os.execute("mkdir -p PRIVATE_DATA/memory")
+
+  local sources = utility.load_data("PRIVATE_DATA/sources.json")
+  for source_name, data_source in pairs(sources) do
+    local file_list = refresh_file_list(source_name, data_source)
+
+    timing.mark("Generating embeddings for " .. source_name .. ".")
+    for _, file_name in ipairs(file_list) do
+      local function loop()
+        local sha512sum = utility.sha512sum(tmp_file_path)
+        if embeddings.vectors[sha512sum] then return end
+
+        local file_chunks, file_embeddings = process_file(data_source, file_name)
+
+        local file_sums = {}
+        for i = 1, #file_chunks do
+          local function loop()
+            local text = file_chunks[i]
+            local current_embedding = file_embeddings[i]
+
+            utility.write_file(tmp_file_path, text)
+
+            sha512sum = utility.sha512sum(tmp_file_path)
+            file_sums[#file_sums + 1] = sha512sum
+            if embeddings.vectors[sha512sum] then return end
+
+            os.execute(utility.commands.move .. tmp_file_path:enquote() .. " " .. "PRIVATE_DATA/memory/")
+            embeddings.vectors[sha512sum] = current_embedding
+          end
+          loop()
+        end
+
+        embeddings.files[file_name] = file_sums
+      end
+      loop()
+    end
+
+    timing.mark("Finished generating embeddings for " .. source_name .. ".")
+  end
+
+  utility.save_data(embeddings)
+  os.execute("rm " .. tmp_file_path:enquote())
+end
+
+refresh_sources()
diff --git a/test.lua b/test.lua
index f01eee3..401a67f 100755
--- a/test.lua
+++ b/test.lua
@@ -6,13 +6,7 @@ local json = utility.require("dkjson")
 
 local file_name = arg[1]
 
-print("---")
-print(utility.capture_safe("shasum -p"))
-print("---")
-
-print("---")
-print(utility.capture_safe("shasum -U -a 512 test.lua"))
-print("---")
+print(utility.OS)
 
 os.exit(0)
 
@@ -23,9 +17,6 @@ if prompt:sub(-1) ~= "\n" then
   prompt = prompt .. "\n\n"
 end
 
-local file_contents = utility.open(file_name, "r", function(file)
-  return file:read("*all")
-end)
+local file_contents = utility.read_file(file_name)
 prompt = prompt .. file_contents
-
--- output = output:sub(1, -2) -- strip extra newline from utility.capture_safe
+-- print(utility.llm_prompt(prompt))

From 8ff72681e946fbfea5597abf61b82a8f84ab29e2 Mon Sep 17 00:00:00 2001
From: Tangent <rose@tangentfox.com>
Date: Tue, 15 Sep 2026 11:16:12 -0600
Subject: [PATCH 13/21] final bits

---
 lib/utility.lua     | 4 ++++
 refresh_sources.lua | 7 ++++++-
 2 files changed, 10 insertions(+), 1 deletion(-)
 mode change 100644 => 100755 refresh_sources.lua

diff --git a/lib/utility.lua b/lib/utility.lua
index 9e7bde8..8df3fe9 100644
--- a/lib/utility.lua
+++ b/lib/utility.lua
@@ -81,6 +81,10 @@ standard_library_addition(string, "split", function(s, delimiter)
   return result
 end)
 
+utility.leftpad = function(text, length, character)
+  return string.rep(character or " ", length - #(tostring(text))) .. text
+end
+
 
 
 utility.require = function(...)
diff --git a/refresh_sources.lua b/refresh_sources.lua
old mode 100644
new mode 100755
index 4cf7ceb..50a3493
--- a/refresh_sources.lua
+++ b/refresh_sources.lua
@@ -139,7 +139,8 @@ local refresh_sources = function()
     local file_list = refresh_file_list(source_name, data_source)
 
     timing.mark("Generating embeddings for " .. source_name .. ".")
-    for _, file_name in ipairs(file_list) do
+    for f = 1, #file_list do
+      local file_name = file_list[f]
       local function loop()
         local sha512sum = utility.sha512sum(tmp_file_path)
         if embeddings.vectors[sha512sum] then return end
@@ -167,6 +168,7 @@ local refresh_sources = function()
         embeddings.files[file_name] = file_sums
       end
       loop()
+      print("Finished " .. utility.leftpad(f, #tostring(#file_list), "0") .. "/" .. #file_list .. " (" .. leftpad(math.floor(i / #file_list * 100), 3, "0") .. "%)")
     end
 
     timing.mark("Finished generating embeddings for " .. source_name .. ".")
@@ -177,3 +179,6 @@ local refresh_sources = function()
 end
 
 refresh_sources()
+timing.mark("Finished.")
+print("")
+timing.display()

From d0bd2371749eefb31f4aef6e692d3ef7b3b712e8 Mon Sep 17 00:00:00 2001
From: Tangent <rose@tangentfox.com>
Date: Tue, 15 Sep 2026 13:16:57 -0600
Subject: [PATCH 14/21] fixed all the errors

---
 .gitignore          |  2 +-
 ReadMe.md           |  4 ++--
 lib/utility.lua     | 16 ++++++++--------
 refresh_sources.lua | 22 ++++++++++------------
 4 files changed, 21 insertions(+), 23 deletions(-)

diff --git a/.gitignore b/.gitignore
index 7ead4ff..3c2c238 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,3 @@
 .DS_Store
-PRIVATE_DATA/**
+PRIVATE_DATA/
 config.json
diff --git a/ReadMe.md b/ReadMe.md
index 9320f59..6804921 100644
--- a/ReadMe.md
+++ b/ReadMe.md
@@ -25,8 +25,8 @@ sources. Example:
 {
   "source name":{
     "filters":{
-      "blacklist":".git",
-      "extension_whitelist":"md"
+      "blacklist":[".git"],
+      "extension_whitelist":["md"]
     },
     "initialize_command":"git clone REMOTE .",
     "path":"will be created before initialize_command is run",
diff --git a/lib/utility.lua b/lib/utility.lua
index 8df3fe9..9d753db 100644
--- a/lib/utility.lua
+++ b/lib/utility.lua
@@ -249,13 +249,13 @@ utility.tree = function(path, options, fn)
     if options.blacklist and options.blacklist[path_name] then return end
     if options.whitelist and (not options.whitelist[path_name]) then return end
 
-    if options.extension_blacklist or options.extension_whitelist then
-      local _, _, extension = utility.split_path_components(path_name)
-      if options.extension_blacklist and options.extension_blacklist[extension] then return end
-      if options.extension_whitelist and (not options.extension_whitelist[extension]) then return end
-    end
-
     if utility.is_file(path_name) then
+      if options.extension_blacklist or options.extension_whitelist then
+        local _, _, extension = utility.split_path_components(path_name)
+        if options.extension_blacklist and options.extension_blacklist[extension] then return end
+        if options.extension_whitelist and (not options.extension_whitelist[extension]) then return end
+      end
+
       fn(path_name)
     else
       utility.tree(path .. utility.path_separator .. path_name, options, fn)
@@ -306,9 +306,9 @@ end
 utility.sha512sum = function(file_path)
   local sha512sum
   if utility.OS == "Linux" then
-    sha512sum = os.capture_safe("shasum -p -t -a 512 " .. file_path:enquote()) -- TODO check this
+    sha512sum = utility.capture_safe("shasum -p -t -a 512 " .. file_path:enquote()) -- TODO check this
   elseif utility.OS == "macOS" then
-    sha512sum = os.capture_safe("shasum -U -a 512 " .. file_path:enquote())
+    sha512sum = utility.capture_safe("shasum -U -a 512 " .. file_path:enquote())
   elseif utility.OS == "Windows" then
     error("utility.sha512sum() not implemented for Windows.")
   end
diff --git a/refresh_sources.lua b/refresh_sources.lua
index 50a3493..0896480 100755
--- a/refresh_sources.lua
+++ b/refresh_sources.lua
@@ -56,7 +56,7 @@ local refresh_file_list = function(source_name, data_source)
   end
 
   local file_list = {}
-  utility.tree(data_source.path, compiled_filters, function(file_name)
+  utility.tree("PRIVATE_DATA" .. utility.path_separator .. data_source.path, compiled_filters, function(file_name)
     file_list[#file_list + 1] = file_name
   end)
 
@@ -71,11 +71,6 @@ local generate_embeddings = function(text)
   end
 
   local result = utility.llm_prompt(text, config.models.embedding.model)
-
-  result = setmetatable({}, {
-    __tojson = function() return result end,
-  })
-
   return json.decode(result)
 end
 
@@ -98,13 +93,13 @@ local process_file = function(data_source, file_name)
 
   while #text > chunk_size do
     local first_chunk = text:sub(1, chunk_size)
-    local overlap_chunk = text:sub(half_chunk_size, chunk_size + half_chunk_size)
+    local overlap_chunk = text:sub(half_chunk_size, chunk_size + half_chunk_size - 1)
 
     chunks[#chunks + 1] = first_chunk
     chunks[#chunks + 1] = overlap_chunk
 
     text = text:sub(chunk_size)
-    if #text > half_chunk_size and (not #text > chunk_size) then
+    if (#text > half_chunk_size) and (not (#text > chunk_size)) then
       -- last chunk would be skipped if we didn't handle this here
       chunks[#chunks + 1] = text
     end
@@ -142,10 +137,11 @@ local refresh_sources = function()
     for f = 1, #file_list do
       local file_name = file_list[f]
       local function loop()
-        local sha512sum = utility.sha512sum(tmp_file_path)
+        local sha512sum = utility.sha512sum(file_name)
         if embeddings.vectors[sha512sum] then return end
 
         local file_chunks, file_embeddings = process_file(data_source, file_name)
+        if not file_chunks then return end
 
         local file_sums = {}
         for i = 1, #file_chunks do
@@ -159,7 +155,7 @@ local refresh_sources = function()
             file_sums[#file_sums + 1] = sha512sum
             if embeddings.vectors[sha512sum] then return end
 
-            os.execute(utility.commands.move .. tmp_file_path:enquote() .. " " .. "PRIVATE_DATA/memory/")
+            os.execute(utility.commands.move .. tmp_file_path:enquote() .. " " .. "PRIVATE_DATA/memory/" .. sha512sum)
             embeddings.vectors[sha512sum] = current_embedding
           end
           loop()
@@ -168,14 +164,16 @@ local refresh_sources = function()
         embeddings.files[file_name] = file_sums
       end
       loop()
-      print("Finished " .. utility.leftpad(f, #tostring(#file_list), "0") .. "/" .. #file_list .. " (" .. leftpad(math.floor(i / #file_list * 100), 3, "0") .. "%)")
+      print("Finished " .. utility.leftpad(f, #tostring(#file_list), "0") .. "/" .. #file_list .. " (" .. utility.leftpad(math.floor(f / #file_list * 100), 3, "0") .. "%)")
     end
 
     timing.mark("Finished generating embeddings for " .. source_name .. ".")
   end
 
   utility.save_data(embeddings)
-  os.execute("rm " .. tmp_file_path:enquote())
+  if utility.path_exists(tmp_file_path) then
+    os.execute("rm " .. tmp_file_path:enquote())
+  end
 end
 
 refresh_sources()

From 11d0e356a2b3ff899b375257d221c4cc331be7e7 Mon Sep 17 00:00:00 2001
From: Tangent <rose@tangentfox.com>
Date: Tue, 15 Sep 2026 19:11:38 -0600
Subject: [PATCH 15/21] debug lines, timing improved/corrected

---
 lib/timing.lua      | 30 ++++++++++++++++++++++--------
 refresh_sources.lua | 11 ++++++++++-
 2 files changed, 32 insertions(+), 9 deletions(-)

diff --git a/lib/timing.lua b/lib/timing.lua
index d272c1f..fb09794 100644
--- a/lib/timing.lua
+++ b/lib/timing.lua
@@ -1,18 +1,26 @@
 local timing = {}
 
+local function human_readable_time(delta)
+  if delta >= 2*7*24*60*60 then -- if more than 2 weeks
+    delta = tostring(math.floor(delta/(7*24*60*60))/10) .. " weeks"
+  elseif delta >= 2*24*60*60 then -- if more than 2 days
+    delta = tostring(math.floor(delta/(24*60*60))/10) .. " days"
+  elseif delta >= 2*60*60 then -- if more than 2 hours
+    delta = tostring(math.floor(delta/(60*60/10))/10) .. " hours"
+  elseif delta >= 2*60 then -- if more then 2 minutes
+    delta = tostring(math.floor(delta/(60/10))/10) .. " minutes"
+  else
+    delta = tostring(delta) .. " seconds"
+  end
+  return delta
+end
+
 timing.display = function(n)
   local function _display(n)
     local time = timing[n]
     local previous = timing[n - 1]
 
-    local delta = time.time - previous.time
-    if delta >= 2*60*60 then -- if more than 2 hours
-      delta = tostring(math.floor(delta/(60*60/10))/10) .. " hours"
-    elseif delta >= 120 then -- if more then 2 minutes
-      delta = tostring(math.floor(delta/(60/10))/10) .. " minutes"
-    else
-      delta = tostring(delta) .. " seconds"
-    end
+    local delta = human_readable_time(time.time - previous.time)
 
     print(delta, previous.label)
   end
@@ -34,4 +42,10 @@ timing.mark = function(label)
   end
 end
 
+timing.estimate = function(current_position, total_operations)
+  local delta = os.time() - timing[#timing].time
+  local estimate = delta * total_operations / current_position - delta
+  return human_readable_time(estimate)
+end
+
 return timing
diff --git a/refresh_sources.lua b/refresh_sources.lua
index 0896480..963b905 100755
--- a/refresh_sources.lua
+++ b/refresh_sources.lua
@@ -107,10 +107,17 @@ local process_file = function(data_source, file_name)
 
   local new_embeddings = {}
   for i = 1, #chunks do
+    print("Embedding length:", #chunks[i])
     new_embeddings[i] = generate_embeddings(chunks[i]) or {}
   end
 
   if #new_embeddings[1] == 0 then
+    print(file_name)
+    print("Vector lengths:")
+    for e = 1, #new_embeddings do
+      print("", e, #new_embeddings[e])
+    end
+
     -- average all embeddings to make the core file embedding
     local count = #new_embeddings[2]
     for vector_index = 1, count do
@@ -164,7 +171,9 @@ local refresh_sources = function()
         embeddings.files[file_name] = file_sums
       end
       loop()
-      print("Finished " .. utility.leftpad(f, #tostring(#file_list), "0") .. "/" .. #file_list .. " (" .. utility.leftpad(math.floor(f / #file_list * 100), 3, "0") .. "%)")
+      print("Finished " .. utility.leftpad(f, #tostring(#file_list), "0") .. "/" .. #file_list
+        .. " (" .. utility.leftpad(math.floor(f / #file_list * 100), 3, "0")
+        .. "%) ETA: " .. timing.estimate(f, #file_list))
     end
 
     timing.mark("Finished generating embeddings for " .. source_name .. ".")

From 214f82b0b1b9eb9ff5366551c4059191baf43678 Mon Sep 17 00:00:00 2001
From: Tangent <rose@tangentfox.com>
Date: Tue, 15 Sep 2026 22:28:53 -0600
Subject: [PATCH 16/21] remove debug printing, workaround for EOF Ollama error
 #7

---
 refresh_sources.lua | 25 +++++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/refresh_sources.lua b/refresh_sources.lua
index 963b905..3d4fdf0 100755
--- a/refresh_sources.lua
+++ b/refresh_sources.lua
@@ -107,15 +107,22 @@ local process_file = function(data_source, file_name)
 
   local new_embeddings = {}
   for i = 1, #chunks do
-    print("Embedding length:", #chunks[i])
+    -- print("Embedding length:", #chunks[i])
     new_embeddings[i] = generate_embeddings(chunks[i]) or {}
   end
 
   if #new_embeddings[1] == 0 then
-    print(file_name)
-    print("Vector lengths:")
-    for e = 1, #new_embeddings do
-      print("", e, #new_embeddings[e])
+    -- print(file_name)
+    -- print("Vector lengths:")
+    -- for e = 1, #new_embeddings do
+    --   print("", e, #new_embeddings[e])
+    -- end
+    if #new_embeddings == 1 then
+      -- Ollama very rarely errors with:
+      --   Error: do embedding request: Post "http://127.0.0.1:53441/v1/embeddings": EOF
+      -- but it is inconsistent and re-running will eventually fix it. (See issue #7 if present.)
+      print(file_name .. "\n encountered an embedding error and will be skipped this run only.")
+      return
     end
 
     -- average all embeddings to make the core file embedding
@@ -145,7 +152,13 @@ local refresh_sources = function()
       local file_name = file_list[f]
       local function loop()
         local sha512sum = utility.sha512sum(file_name)
-        if embeddings.vectors[sha512sum] then return end
+        if embeddings.vectors[sha512sum] then
+          -- add file reference if it was missing
+          if not embeddings.files[file_name] then
+            embeddings.files[file_name] = sha512sum
+          end
+          return
+        end
 
         local file_chunks, file_embeddings = process_file(data_source, file_name)
         if not file_chunks then return end

From 98112bde4a1c78c63837cacbcb285b4210cd3a28 Mon Sep 17 00:00:00 2001
From: Tangent <rose@tangentfox.com>
Date: Tue, 15 Sep 2026 22:30:46 -0600
Subject: [PATCH 17/21] close #6 Darwin check only runs on 'Linux' detection

This prevents Windows from erroring when trying to run uname.
---
 lib/utility.lua | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/utility.lua b/lib/utility.lua
index 9d753db..875b850 100644
--- a/lib/utility.lua
+++ b/lib/utility.lua
@@ -586,6 +586,8 @@ end
 
 
 
-if utility.capture_safe("uname"):find("Darwin") == 1 then utility.OS = "macOS" end
+if (utility.OS == "Linux") and (utility.capture_safe("uname"):find("Darwin") == 1) then
+  utility.OS = "macOS"
+end
 
 return utility

From 977920df611a4a053fbfef8f6fdcbfe3d5489140 Mon Sep 17 00:00:00 2001
From: Tangent <rose@tangentfox.com>
Date: Wed, 16 Sep 2026 12:48:43 -0600
Subject: [PATCH 18/21] fix #9 shasum error

---
 lib/utility.lua | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/lib/utility.lua b/lib/utility.lua
index 875b850..63e5e9f 100644
--- a/lib/utility.lua
+++ b/lib/utility.lua
@@ -305,9 +305,7 @@ end
 
 utility.sha512sum = function(file_path)
   local sha512sum
-  if utility.OS == "Linux" then
-    sha512sum = utility.capture_safe("shasum -p -t -a 512 " .. file_path:enquote()) -- TODO check this
-  elseif utility.OS == "macOS" then
+  if (utility.OS == "Linux") or (utility.OS == "macOS") then
     sha512sum = utility.capture_safe("shasum -U -a 512 " .. file_path:enquote())
   elseif utility.OS == "Windows" then
     error("utility.sha512sum() not implemented for Windows.")

From 470901d9b9fb1139092bfb6fa1573c70648112d0 Mon Sep 17 00:00:00 2001
From: Tangent <rose@tangentfox.com>
Date: Wed, 16 Sep 2026 16:03:14 -0600
Subject: [PATCH 19/21] very WIP, doesn't detect duplicates properly..
 everything else seems fine?

---
 ReadMe.md           |  15 +++++
 lib/log.lua         |  75 ++++++++++++++++++++++++
 lib/timing.lua      |   1 +
 lib/utility.lua     |  16 +++++-
 refresh_sources.lua | 136 +++++++++++++++++++++++++++++++-------------
 5 files changed, 201 insertions(+), 42 deletions(-)
 create mode 100644 lib/log.lua

diff --git a/ReadMe.md b/ReadMe.md
index 6804921..4ba1471 100644
--- a/ReadMe.md
+++ b/ReadMe.md
@@ -42,6 +42,21 @@ while `refresh_command` is run each time (optional).
 Commands will be run in the specified `path`.
 `strip_frontmatter` will remove YAML frontmatter (common in Markdown files).
 
+The embeddings are stored like so:
+
+```json
+{
+  "files":{
+    "PRIVATE_DATA/source_path/path/to/file.ext":["sha512sum", "another sum"]
+  },
+  "vectors":{
+    "sha512sum":[0.5, 0, 1, -0.5, -1, ...]
+  }
+}
+```
+
+TODO
+
 ### `synopsis_generator.lua`
 Chooses a random file within `notebook`, and generates a novel synopsis from it.
 
diff --git a/lib/log.lua b/lib/log.lua
new file mode 100644
index 0000000..fae2b37
--- /dev/null
+++ b/lib/log.lua
@@ -0,0 +1,75 @@
+local stored_messages = {}
+local display_levels = {}
+local enable_logging = true
+local message_limit = math.huge
+
+-- USAGE:
+--   -- set arbitrary log levels to be printed immediately
+--   log{ info = true, warning = true, ducksauce = true, }
+--   -- send anything (except nil) to a log level
+--   log("bacom", true, "text", 5, function() end, {})
+--   -- print all messages saved at a particular log level
+--   log("error")
+--   -- toggle ALL logging on and off with a boolean
+--   log(false) -- this deletes all stored messages
+--   -- set a maximum number of stored messages per log level
+--   log(50) -- warning: reaching large limits makes logging expensive
+--   -- do anything to the stored messages (this example deletes them all)
+--  log(function(messages) return {} end)
+
+return function(options, ...)
+  local options_type = type(options)
+
+  if (options_type == "string") and enable_logging then
+    if not stored_messages[options] then stored_messages[options] = {} end
+    local message_table = stored_messages[options]
+
+    -- turn log message into text
+    local tab = {...}
+    for i = 1, #tab do
+      tab[i] = tostring(tab[i])
+    end
+    local current_message = table.concat(tab, "\t")
+
+    if #current_message == 0 then
+      -- print all stored_messages of specified level
+      for i = 1, #message_table do
+        print(message_table[i])
+      end
+
+    else
+      -- store message (and print if in display_levels)
+      message_table[#message_table + 1] = current_message
+      if #message_table > message_limit then table.remove(message_table, 1) end
+      if display_levels[options] then print(current_message) end
+    end
+
+  -- set display_levels
+  elseif options_type == "table" then
+    for k,v in pairs(options) do
+      display_levels[k] = v
+    end
+
+  elseif options_type == "number" then
+    message_limit = options
+    for k,v in pairs(stored_messages) do
+      while #v > message_limit do
+        table.remove(v, 1)
+      end
+    end
+
+  -- turn all logging off and delete stored messages; or turn it on
+  elseif options_type == "boolean" then
+    if options then
+      enable_logging = true
+    else
+      enable_logging = false
+      stored_messages = {}
+    end
+
+  -- arbitrary access :D
+  elseif options_type == "function" then
+    local result = options(stored_messages)
+    if type(result) == "table" then stored_messages = result end
+  end
+end
diff --git a/lib/timing.lua b/lib/timing.lua
index fb09794..7c68b78 100644
--- a/lib/timing.lua
+++ b/lib/timing.lua
@@ -40,6 +40,7 @@ timing.mark = function(label)
   if #timing > 1 then
     timing.display(#timing)
   end
+  print("", "", label)
 end
 
 timing.estimate = function(current_position, total_operations)
diff --git a/lib/utility.lua b/lib/utility.lua
index 63e5e9f..73e4afe 100644
--- a/lib/utility.lua
+++ b/lib/utility.lua
@@ -221,7 +221,7 @@ utility.list = function(path, func)
 
   local run = function(fn)
     for line in output:gmatch("[^\r\n]+") do -- thanks to https://stackoverflow.com/a/32847589
-      if not (line == "." or line == "..") then
+      if not ((line == ".") or (line == "..")) then
         fn(line)
       end
     end
@@ -239,7 +239,8 @@ utility.ls = function(...)
   return utility.list(...)
 end
 
-utility.tree = function(path, options, fn)
+local tree
+tree = function(path, options, fn)
   if type(options) == "function" then
     fn = options
     options = {}
@@ -258,7 +259,16 @@ utility.tree = function(path, options, fn)
 
       fn(path_name)
     else
-      utility.tree(path .. utility.path_separator .. path_name, options, fn)
+      tree(path .. utility.path_separator .. path_name, options, fn)
+    end
+  end)
+end
+utility.tree = function(path, options, fn)
+  tree(path, options, function(path_name)
+    if path_name:find(path) == 1 then
+      fn(path_name)
+    else
+      fn(path .. utility.path_separator .. path_name)
     end
   end)
 end
diff --git a/refresh_sources.lua b/refresh_sources.lua
index 3d4fdf0..5b235a1 100755
--- a/refresh_sources.lua
+++ b/refresh_sources.lua
@@ -4,6 +4,7 @@ package.path = (arg[0]:match("@?(.*/)") or arg[0]:match("@?(.*\\)")) .. "lib" ..
 local utility = require "utility"
 
 local json = utility.require("dkjson")
+local log = utility.require("log")
 local text_processing = utility.require("text_processing")
 local timing = utility.require("timing")
 
@@ -21,16 +22,42 @@ if not config.models then
   utility.save_config()
 end
 
+log{
+  -- info = true,
+  -- warning = true,
+  -- debug = true,
+  -- files = true, -- debugging why the wrong files are selected
+  sha = true, -- what the fuck is going on with sha sums?
+}
+
+local memory_path = "PRIVATE_DATA" .. utility.path_separator .. "memory"
+local embeddings_file_path = memory_path .. utility.path_separator .. "+embeddings.json"
+local tmp_file_path = "PRIVATE_DATA" .. utility.path_separator .. ".tmp.2b65c19b-0883-49ca-8247-b1fe7760f922"
+
 local embeddings
-local embeddings_file_path = "PRIVATE_DATA/memory/+embeddings.json"
 if not utility.path_exists(embeddings_file_path) then
-  os.execute("mkdir -p PRIVATE_DATA/memory")
+  os.execute("mkdir -p " .. memory_path:enquote())
   utility.save_data({
     files = {},
     vectors = {},
   }, embeddings_file_path)
 end
 embeddings = utility.load_data(embeddings_file_path)
+local function embeddings_debug()
+  log("debug", "Embeddings loaded.", embeddings, embeddings.files, embeddings.vectors)
+  local file_count = 0
+  for k,v in pairs(embeddings.files) do
+    file_count = file_count + 1
+  end
+  local vector_count = 0
+  for k,v in pairs(embeddings.vectors) do
+    vector_count = vector_count + 1
+  end
+  log("debug", file_count .. " files.")
+  log("debug", vector_count .. " vectors.")
+  -- os.exit(1)
+end
+embeddings_debug()
 
 
 
@@ -57,6 +84,7 @@ local refresh_file_list = function(source_name, data_source)
 
   local file_list = {}
   utility.tree("PRIVATE_DATA" .. utility.path_separator .. data_source.path, compiled_filters, function(file_name)
+    log("files", file_name)
     file_list[#file_list + 1] = file_name
   end)
 
@@ -74,7 +102,7 @@ local generate_embeddings = function(text)
   return json.decode(result)
 end
 
--- returns nothing for empty files
+-- returns nothing for empty files and errors
 local process_file = function(data_source, file_name)
   local text = utility.read_file(file_name)
 
@@ -83,7 +111,7 @@ local process_file = function(data_source, file_name)
   end
 
   if #text == 0 then
-    print(file_name .. "\n is empty and being skipped.")
+    log("empty", file_name .. "\n is empty and being skipped.")
     return
   end
 
@@ -107,21 +135,20 @@ local process_file = function(data_source, file_name)
 
   local new_embeddings = {}
   for i = 1, #chunks do
-    -- print("Embedding length:", #chunks[i])
+    log("debug", "Embedding length:", #chunks[i])
     new_embeddings[i] = generate_embeddings(chunks[i]) or {}
   end
 
   if #new_embeddings[1] == 0 then
-    -- print(file_name)
-    -- print("Vector lengths:")
-    -- for e = 1, #new_embeddings do
-    --   print("", e, #new_embeddings[e])
-    -- end
+    log("debug", "Vector lengths:")
+    for e = 1, #new_embeddings do
+      log("debug", "", e, #new_embeddings[e])
+    end
     if #new_embeddings == 1 then
       -- Ollama very rarely errors with:
       --   Error: do embedding request: Post "http://127.0.0.1:53441/v1/embeddings": EOF
-      -- but it is inconsistent and re-running will eventually fix it. (See issue #7 if present.)
-      print(file_name .. "\n encountered an embedding error and will be skipped this run only.")
+      -- but it is inconsistent and re-running will eventually fix it.
+      log("warning", file_name .. "\n encountered an embedding error and will be skipped this run only.")
       return
     end
 
@@ -139,57 +166,87 @@ local process_file = function(data_source, file_name)
   return chunks, new_embeddings
 end
 
+-- returns nothing for errors
+local memorize_file = function(data_source, file_name)
+  local file_chunks, file_embeddings = process_file(data_source, file_name)
+  if not file_chunks then return end
+
+  local file_sums = {}
+  for i = 1, #file_chunks do
+    local function loop()
+      local text = file_chunks[i]
+      local current_embedding = file_embeddings[i]
+
+      utility.write_file(tmp_file_path, text)
+
+      local sha512sum = utility.sha512sum(tmp_file_path)
+      log("sha", "New sum? " .. sha512sum, tostring(embeddings.vectors[sha512sum]))
+      file_sums[#file_sums + 1] = sha512sum
+      if embeddings.vectors[sha512sum] then
+        log("sha", "Detected previously extant sum, skipping.")
+        return
+      end
+
+      os.execute(utility.commands.move .. tmp_file_path:enquote()
+        .. " " .. (memory_path .. utility.path_separator .. sha512sum):enquote())
+      embeddings.vectors[sha512sum] = current_embedding
+      log("sha", "Saved new sum!")
+    end
+    loop()
+  end
+
+  return file_sums
+end
+
 local refresh_sources = function()
-  local tmp_file_path = "PRIVATE_DATA/.tmp.2b65c19b-0883-49ca-8247-b1fe7760f922"
-  os.execute("mkdir -p PRIVATE_DATA/memory")
+  os.execute("mkdir -p " .. memory_path:enquote())
+
+  timing.mark("Checking memory for missing embeddings.")
+  utility.list(memory_path, function(file_name)
+    log("files", file_name)
+    if (file_name == ".DS_Store") or (file_name == "+embeddings.json") then return end
+
+    local file_path = memory_path .. utility.path_separator .. file_name
+    local sha512sum = utility.sha512sum(file_path)
+    log("sha", "New sum? " .. sha512sum, tostring(embeddings.vectors[sha512sum]))
+    if not embeddings.vectors[sha512sum] then
+      log("sha", "Saved new sum!")
+      memorize_file({}, file_path)
+    end
+  end)
+  timing.mark("Finished checking memory for missing embeddings.")
 
   local sources = utility.load_data("PRIVATE_DATA/sources.json")
   for source_name, data_source in pairs(sources) do
     local file_list = refresh_file_list(source_name, data_source)
 
-    timing.mark("Generating embeddings for " .. source_name .. ".")
+    timing.mark("Generating embeddings from source \"" .. source_name .. "\"")
     for f = 1, #file_list do
       local file_name = file_list[f]
       local function loop()
         local sha512sum = utility.sha512sum(file_name)
+        log("sha", file_name, sha512sum, tostring(embeddings.vectors[sha512sum]))
         if embeddings.vectors[sha512sum] then
+          log("debug", file_name .. "\n has already been embedded, skipping.")
           -- add file reference if it was missing
           if not embeddings.files[file_name] then
-            embeddings.files[file_name] = sha512sum
+            embeddings.files[file_name] = { sha512sum }
           end
           return
         end
 
-        local file_chunks, file_embeddings = process_file(data_source, file_name)
-        if not file_chunks then return end
-
-        local file_sums = {}
-        for i = 1, #file_chunks do
-          local function loop()
-            local text = file_chunks[i]
-            local current_embedding = file_embeddings[i]
-
-            utility.write_file(tmp_file_path, text)
-
-            sha512sum = utility.sha512sum(tmp_file_path)
-            file_sums[#file_sums + 1] = sha512sum
-            if embeddings.vectors[sha512sum] then return end
-
-            os.execute(utility.commands.move .. tmp_file_path:enquote() .. " " .. "PRIVATE_DATA/memory/" .. sha512sum)
-            embeddings.vectors[sha512sum] = current_embedding
-          end
-          loop()
-        end
-
+        log("debug", "Generating embeddings for " .. file_name .. ".")
+        local file_sums = memorize_file(data_source, file_name)
         embeddings.files[file_name] = file_sums
       end
       loop()
-      print("Finished " .. utility.leftpad(f, #tostring(#file_list), "0") .. "/" .. #file_list
+      log("info", "Finished " .. utility.leftpad(f, #tostring(#file_list), "0")
+        .. "/" .. #file_list
         .. " (" .. utility.leftpad(math.floor(f / #file_list * 100), 3, "0")
         .. "%) ETA: " .. timing.estimate(f, #file_list))
     end
 
-    timing.mark("Finished generating embeddings for " .. source_name .. ".")
+    timing.mark("Finished generating embeddings from source \"" .. source_name .. "\"")
   end
 
   utility.save_data(embeddings)
@@ -202,3 +259,4 @@ refresh_sources()
 timing.mark("Finished.")
 print("")
 timing.display()
+log("warning")

From f229e7331e6228b3fc1f325e8e5962a53341f7eb Mon Sep 17 00:00:00 2001
From: Tangent <rose@tangentfox.com>
Date: Wed, 16 Sep 2026 17:56:47 -0600
Subject: [PATCH 20/21] finally found my stupid trip-up, writing modified files

---
 lib/utility.lua        | 6 +++---
 refresh_sources.lua    | 2 +-
 synopsis_generator.lua | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/lib/utility.lua b/lib/utility.lua
index 73e4afe..a2d9d38 100644
--- a/lib/utility.lua
+++ b/lib/utility.lua
@@ -279,10 +279,10 @@ utility.read_file = function(file_name)
   end)
 end
 
-utility.write_file = function(file_name, text)
+utility.write_file = function(file_name, ...)
   return utility.open(file_name, "w", function(file)
-    file:write(text)
-    file:write("\n")
+    file:write(table.concat{...})
+    -- file:write("\n") -- I need to make sure /I/ handle this instead of trying to automate it
   end)
 end
 
diff --git a/refresh_sources.lua b/refresh_sources.lua
index 5b235a1..0bed99f 100755
--- a/refresh_sources.lua
+++ b/refresh_sources.lua
@@ -225,7 +225,7 @@ local refresh_sources = function()
       local file_name = file_list[f]
       local function loop()
         local sha512sum = utility.sha512sum(file_name)
-        log("sha", file_name, sha512sum, tostring(embeddings.vectors[sha512sum]))
+        log("sha", file_name, sha512sum, "\n Sum present? " .. tostring(embeddings.vectors[sha512sum]))
         if embeddings.vectors[sha512sum] then
           log("debug", file_name .. "\n has already been embedded, skipping.")
           -- add file reference if it was missing
diff --git a/synopsis_generator.lua b/synopsis_generator.lua
index af000a9..eedd156 100755
--- a/synopsis_generator.lua
+++ b/synopsis_generator.lua
@@ -108,7 +108,7 @@ local export_ordered_list_of_prompts = function()
     output[#output + 1] = "## Scoring\n\n```json\n" .. json.encode(item.scoring, { indent = true, }) .. "\n```\n"
   end
 
-  utility.write_file("PRIVATE_DATA/Ordered Synopses.md", table.concat(output, "\n"))
+  utility.write_file("PRIVATE_DATA/Ordered Synopses.md", table.concat(output, "\n"), "\n")
   os.execute("pandoc \"PRIVATE_DATA/Ordered Synopses.md\" -o \"PRIVATE_DATA/Ordered Synopses.epub\"")
 end
 

From f1b5fbc615ad2df0ba88e8cd8b026715b5bf93c4 Mon Sep 17 00:00:00 2001
From: Tangent <rose@tangentfox.com>
Date: Wed, 16 Sep 2026 17:59:24 -0600
Subject: [PATCH 21/21] Lua got confused by varargs

---
 lib/utility.lua     | 3 ++-
 refresh_sources.lua | 6 +++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/lib/utility.lua b/lib/utility.lua
index a2d9d38..cd6406c 100644
--- a/lib/utility.lua
+++ b/lib/utility.lua
@@ -280,8 +280,9 @@ utility.read_file = function(file_name)
 end
 
 utility.write_file = function(file_name, ...)
+  local text = table.concat{...}
   return utility.open(file_name, "w", function(file)
-    file:write(table.concat{...})
+    file:write(text)
     -- file:write("\n") -- I need to make sure /I/ handle this instead of trying to automate it
   end)
 end
diff --git a/refresh_sources.lua b/refresh_sources.lua
index 0bed99f..ab88a9d 100755
--- a/refresh_sources.lua
+++ b/refresh_sources.lua
@@ -23,11 +23,11 @@ if not config.models then
 end
 
 log{
-  -- info = true,
-  -- warning = true,
+  info = true,
+  warning = true,
   -- debug = true,
   -- files = true, -- debugging why the wrong files are selected
-  sha = true, -- what the fuck is going on with sha sums?
+  -- sha = true, -- what the fuck is going on with sha sums?
 }
 
 local memory_path = "PRIVATE_DATA" .. utility.path_separator .. "memory"