tools/llm-tricks
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

more refinement

Browse Source
This commit is contained in:
tangent
2026-09-14 19:08:49 -06:00
parent 707776c12d
commit 49a22eb2c4
3 changed files with 54 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files
synopsis_generator.lua
+9 -103
View File
@@ -5,8 +5,9 @@ local utility = require "utility"
local json = utility.require("dkjson")
local prompts = utility.require("prompts")
local text_processing = utility.require("text_processing")
local default_model = "gemma4:12b-mlx"
local model = "gemma4:12b-mlx"
local minimum_bytes = 1000
local maximum_bytes = 40000
@@ -17,91 +18,13 @@ else
files = {}
end
-- strip YAML frontmatter (if present)
-- can error, will return nil & error message
local function strip_frontmatter(text)
local tab = text:split("\n")
if tab[1] == "---" then
table.remove(tab, 1)
while true do
local done = tab[1] == "---"
table.remove(tab, 1)
if done then
return table.concat(tab, "\n")
elseif #tab < 1 then
return nil, "Invalid YAML frontmatter."
end
end
end
return text
end
-- strip Markdown formatting of a JSON block
-- just returns the string if it doesn't match
local function strip_markdown_codeblock(text)
local tab = text:split("\n")
if tab[1] == "```json" then
table.remove(tab, 1)
table.remove(tab, #tab)
return table.concat(tab, "\n")
else
return text
end
end
local strip_reasoning = function(text, reasoning_lines)
if not reasoning_lines then reasoning_lines = {} end
local tab = text:split("\n")
table.remove(tab, 1) -- remove "Thinking..."
while true do
local done = tab[1] == "...done thinking."
local line = table.remove(tab, 1)
if done then
table.remove(tab, 1) -- remove newline after end of thinking
return table.concat(tab, "\n")
elseif #tab < 1 then
return text -- no reasoning output
else
reasoning_lines[#reasoning_lines + 1] = line -- export thinking lines
end
end
end
local send_prompt = function(text, model)
if type(text) == "table" then text = table.concat(text, "\n") end
local tmp_file_name = utility.tmp_file_name()
utility.open(tmp_file_name, "w", function(file)
file:write(text)
end)
-- word wrap breaks the raw output badly, so I need to implement my own for terminal output somehow
local output = utility.capture_safe("cat " .. tmp_file_name:enquote() .. " | ollama run " .. (model or default_model) .. " --nowordwrap")
os.execute("ollama stop " .. (model or default_model)) -- NOTE this makes things slower, but more stable
os.execute("rm " .. tmp_file_name)
if not output then error("ollama failed to generate output") end
output = output:sub(1, -2) -- strip extra newline from utility.capture_safe
local thinking = {}
output = strip_reasoning(output, thinking)
return output, thinking
end
local refresh_file_list = function()
local blacklist = { -- I'm only blacklisting binary formats because funny results happen with really invalid texts
"jpg", "mp4", "pdf", "png", "webp", "jpeg", "gif",
} for _, name in ipairs(blacklist) do blacklist[name] = true end
local new_files_list = {}
utility.tree("PRIVATE_DATA/notebook", {
blacklist = utility.enumerate{".git", ".gitignore", ".gitkeep", ".DS_Store"}
blacklist = utility.enumerate{ ".git", ".gitattributes", ".gitignore", ".gitkeep", ".DS_Store", },
extension_blacklist = utility.enumerate{ "gif", "jpg", "jpeg", "mp4", "pdf", "png", "webp", },
}, function(file_name)
local _, _, extension = utility.split_path_components(file_name)
if not blacklist[extension] then
new_files_list[#new_files_list + 1] = file_name
end
new_files_list[#new_files_list + 1] = file_name
end)
files = new_files_list
utility.save_data(new_files_list, "PRIVATE_DATA/file_list.json")
@@ -109,14 +32,14 @@ end
local generate_and_score = function(file_name, text)
print("Writing synopsis...")
local synopsis = send_prompt(prompts.synopsis_prompt .. text)
local synopsis = utility.llm_prompt(prompts.synopsis_prompt .. text, model)
print(synopsis)
print("Scoring synopsis...")
local scoring, thinking = send_prompt(prompts.scoring_prompt .. synopsis)
local scoring, thinking = utility.llm_prompt(prompts.scoring_prompt .. synopsis, model)
print(scoring)
local scoring_decoded = json.decode(scoring) -- likely will not work because it consistently returns Markdown instead of JSON
if not scoring_decoded then
scoring_decoded = json.decode(strip_markdown_codeblock(scoring))
scoring_decoded = json.decode(text_processing.strip_markdown_codeblock(scoring))
end
local object = {
@@ -137,23 +60,6 @@ if arg[1] == "refresh_file_list" then
refresh_file_list()
end
if arg[1] == "repair_synopsis_exports" then
local path = "PRIVATE_DATA/synopses"
utility.list(path, function(path_name)
path_name = path .. utility.path_separator .. path_name
if path_name:find("%.json") then
local object = utility.load_data(path_name)
if type(object.scoring) == "table" then return end -- don't fuck with working pieces
local decoded = json.decode(strip_markdown_codeblock(object.scoring))
if decoded then
object.scoring = decoded
utility.save_data(object, path_name)
end
end
end)
os.exit(0)
end
if arg[1] == "export_ordered_list_of_prompts" then
local path = "PRIVATE_DATA/synopses"
local items = {}
@@ -210,7 +116,7 @@ while true do
local file_size = utility.file_size(file_name)
if file_size > minimum_bytes and file_size <= maximum_bytes then
local text = utility.read_file(file_name)
text = strip_frontmatter(text)
text = text_processing.strip_frontmatter(text)
if #text > minimum_bytes and #text <= maximum_bytes then
print(file_name .. " chosen.")
generate_and_score(file_name, text) -- kind of the main function, innit?