more refinement

ReadMe.md

@@ -16,3 +16,13 @@ them.
 ###### `least_similar.lua`
 Opens `similarities.json`, reverses the sort order, and saves it as
 `differences.json`.
+
+###### `synopsis_generator.lua`
+Chooses a random file within `notebook`, and generates a novel synopsis from it.
+
+Arguments:
+- `refresh_file_list`: Refreshes the cached file list to choose from.
+- `export_ordered_list_of_prompts`: Makes an epub to review generated synopses.
+
+## Tasks
+- [ ] synopsis_generator cache should include file sizes to make script running easier/faster

lib/text_processing.lua

@@ -0,0 +1,35 @@
+local text_processing = {}
+
+-- strip YAML frontmatter (if present)
+--   on error, will return nil & error message
+text_processing.strip_frontmatter = function(text)
+  local tab = text:split("\n")
+  if tab[1] == "---" then
+    table.remove(tab, 1)
+    while true do
+      local done = tab[1] == "---"
+      table.remove(tab, 1)
+      if done then
+        return table.concat(tab, "\n")
+      elseif #tab < 1 then
+        return nil, "Invalid YAML frontmatter."
+      end
+    end
+  end
+  return text
+end
+
+-- strip Markdown formatting of a JSON block
+--  just returns the string if it doesn't match
+text_processing.strip_markdown_codeblock = function(text)
+  local tab = text:split("\n")
+  if tab[1] == "```json" then
+    table.remove(tab, 1)
+    table.remove(tab, #tab)
+    return table.concat(tab, "\n")
+  else
+    return text
+  end
+end
+
+return text_processing

synopsis_generator.lua

@@ -5,8 +5,9 @@ local utility = require "utility"
 
 local json = utility.require("dkjson")
 local prompts = utility.require("prompts")
+local text_processing = utility.require("text_processing")
 
-local default_model = "gemma4:12b-mlx"
+local model = "gemma4:12b-mlx"
 local minimum_bytes = 1000
 local maximum_bytes = 40000
 
@@ -17,91 +18,13 @@ else
   files = {}
 end
 
--- strip YAML frontmatter (if present)
---   can error, will return nil & error message
-local function strip_frontmatter(text)
-  local tab = text:split("\n")
-  if tab[1] == "---" then
-    table.remove(tab, 1)
-    while true do
-      local done = tab[1] == "---"
-      table.remove(tab, 1)
-      if done then
-        return table.concat(tab, "\n")
-      elseif #tab < 1 then
-        return nil, "Invalid YAML frontmatter."
-      end
-    end
-  end
-  return text
-end
-
--- strip Markdown formatting of a JSON block
---  just returns the string if it doesn't match
-local function strip_markdown_codeblock(text)
-  local tab = text:split("\n")
-  if tab[1] == "```json" then
-    table.remove(tab, 1)
-    table.remove(tab, #tab)
-    return table.concat(tab, "\n")
-  else
-    return text
-  end
-end
-
-local strip_reasoning = function(text, reasoning_lines)
-  if not reasoning_lines then reasoning_lines = {} end
-  local tab = text:split("\n")
-  table.remove(tab, 1) -- remove "Thinking..."
-
-  while true do
-    local done = tab[1] == "...done thinking."
-    local line = table.remove(tab, 1)
-    if done then
-      table.remove(tab, 1) -- remove newline after end of thinking
-      return table.concat(tab, "\n")
-    elseif #tab < 1 then
-      return text -- no reasoning output
-    else
-      reasoning_lines[#reasoning_lines + 1] = line -- export thinking lines
-    end
-  end
-end
-
-local send_prompt = function(text, model)
-  if type(text) == "table" then text = table.concat(text, "\n") end
-
-  local tmp_file_name = utility.tmp_file_name()
-  utility.open(tmp_file_name, "w", function(file)
-    file:write(text)
-  end)
-
-  -- word wrap breaks the raw output badly, so I need to implement my own for terminal output somehow
-  local output = utility.capture_safe("cat " .. tmp_file_name:enquote() .. " | ollama run " .. (model or default_model) .. " --nowordwrap")
-  os.execute("ollama stop " .. (model or default_model)) -- NOTE this makes things slower, but more stable
-  os.execute("rm " .. tmp_file_name)
-  if not output then error("ollama failed to generate output") end
-  output = output:sub(1, -2) -- strip extra newline from utility.capture_safe
-
-  local thinking = {}
-  output = strip_reasoning(output, thinking)
-
-  return output, thinking
-end
-
 local refresh_file_list = function()
-  local blacklist = { -- I'm only blacklisting binary formats because funny results happen with really invalid texts
-    "jpg", "mp4", "pdf", "png", "webp", "jpeg", "gif",
-  } for _, name in ipairs(blacklist) do blacklist[name] = true end
-
   local new_files_list = {}
   utility.tree("PRIVATE_DATA/notebook", {
-    blacklist = utility.enumerate{".git", ".gitignore", ".gitkeep", ".DS_Store"}
+    blacklist = utility.enumerate{ ".git", ".gitattributes", ".gitignore", ".gitkeep", ".DS_Store", },
+    extension_blacklist = utility.enumerate{ "gif", "jpg", "jpeg", "mp4", "pdf", "png", "webp", },
   }, function(file_name)
-    local _, _, extension = utility.split_path_components(file_name)
-    if not blacklist[extension] then
     new_files_list[#new_files_list + 1] = file_name
-    end
   end)
   files = new_files_list
   utility.save_data(new_files_list, "PRIVATE_DATA/file_list.json")
@@ -109,14 +32,14 @@ end
 
 local generate_and_score = function(file_name, text)
   print("Writing synopsis...")
-  local synopsis = send_prompt(prompts.synopsis_prompt .. text)
+  local synopsis = utility.llm_prompt(prompts.synopsis_prompt .. text, model)
   print(synopsis)
   print("Scoring synopsis...")
-  local scoring, thinking = send_prompt(prompts.scoring_prompt .. synopsis)
+  local scoring, thinking = utility.llm_prompt(prompts.scoring_prompt .. synopsis, model)
   print(scoring)
   local scoring_decoded = json.decode(scoring) -- likely will not work because it consistently returns Markdown instead of JSON
   if not scoring_decoded then
-    scoring_decoded = json.decode(strip_markdown_codeblock(scoring))
+    scoring_decoded = json.decode(text_processing.strip_markdown_codeblock(scoring))
   end
 
   local object = {
@@ -137,23 +60,6 @@ if arg[1] == "refresh_file_list" then
   refresh_file_list()
 end
 
-if arg[1] == "repair_synopsis_exports" then
-  local path = "PRIVATE_DATA/synopses"
-  utility.list(path, function(path_name)
-    path_name = path .. utility.path_separator .. path_name
-    if path_name:find("%.json") then
-      local object = utility.load_data(path_name)
-      if type(object.scoring) == "table" then return end -- don't fuck with working pieces
-      local decoded = json.decode(strip_markdown_codeblock(object.scoring))
-      if decoded then
-        object.scoring = decoded
-        utility.save_data(object, path_name)
-      end
-    end
-  end)
-  os.exit(0)
-end
-
 if arg[1] == "export_ordered_list_of_prompts" then
   local path = "PRIVATE_DATA/synopses"
   local items = {}
@@ -210,7 +116,7 @@ while true do
   local file_size = utility.file_size(file_name)
   if file_size > minimum_bytes and file_size <= maximum_bytes then
     local text = utility.read_file(file_name)
-    text = strip_frontmatter(text)
+    text = text_processing.strip_frontmatter(text)
     if #text > minimum_bytes and #text <= maximum_bytes then
       print(file_name .. " chosen.")
       generate_and_score(file_name, text) -- kind of the main function, innit?