diff --git a/ReadMe.md b/ReadMe.md index 3017786..2e40604 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -16,3 +16,13 @@ them. ###### `least_similar.lua` Opens `similarities.json`, reverses the sort order, and saves it as `differences.json`. + +###### `synopsis_generator.lua` +Chooses a random file within `notebook`, and generates a novel synopsis from it. + +Arguments: +- `refresh_file_list`: Refreshes the cached file list to choose from. +- `export_ordered_list_of_prompts`: Makes an epub to review generated synopses. + +## Tasks +- [ ] synopsis_generator cache should include file sizes to make script running easier/faster diff --git a/lib/text_processing.lua b/lib/text_processing.lua new file mode 100644 index 0000000..e459cbc --- /dev/null +++ b/lib/text_processing.lua @@ -0,0 +1,35 @@ +local text_processing = {} + +-- strip YAML frontmatter (if present) +-- on error, will return nil & error message +text_processing.strip_frontmatter = function(text) + local tab = text:split("\n") + if tab[1] == "---" then + table.remove(tab, 1) + while true do + local done = tab[1] == "---" + table.remove(tab, 1) + if done then + return table.concat(tab, "\n") + elseif #tab < 1 then + return nil, "Invalid YAML frontmatter." + end + end + end + return text +end + +-- strip Markdown formatting of a JSON block +-- just returns the string if it doesn't match +text_processing.strip_markdown_codeblock = function(text) + local tab = text:split("\n") + if tab[1] == "```json" then + table.remove(tab, 1) + table.remove(tab, #tab) + return table.concat(tab, "\n") + else + return text + end +end + +return text_processing diff --git a/synopsis_generator.lua b/synopsis_generator.lua index cf8f88e..ddfb1f1 100755 --- a/synopsis_generator.lua +++ b/synopsis_generator.lua @@ -5,8 +5,9 @@ local utility = require "utility" local json = utility.require("dkjson") local prompts = utility.require("prompts") +local text_processing = utility.require("text_processing") -local default_model = "gemma4:12b-mlx" +local model = "gemma4:12b-mlx" local minimum_bytes = 1000 local maximum_bytes = 40000 @@ -17,91 +18,13 @@ else files = {} end --- strip YAML frontmatter (if present) --- can error, will return nil & error message -local function strip_frontmatter(text) - local tab = text:split("\n") - if tab[1] == "---" then - table.remove(tab, 1) - while true do - local done = tab[1] == "---" - table.remove(tab, 1) - if done then - return table.concat(tab, "\n") - elseif #tab < 1 then - return nil, "Invalid YAML frontmatter." - end - end - end - return text -end - --- strip Markdown formatting of a JSON block --- just returns the string if it doesn't match -local function strip_markdown_codeblock(text) - local tab = text:split("\n") - if tab[1] == "```json" then - table.remove(tab, 1) - table.remove(tab, #tab) - return table.concat(tab, "\n") - else - return text - end -end - -local strip_reasoning = function(text, reasoning_lines) - if not reasoning_lines then reasoning_lines = {} end - local tab = text:split("\n") - table.remove(tab, 1) -- remove "Thinking..." - - while true do - local done = tab[1] == "...done thinking." - local line = table.remove(tab, 1) - if done then - table.remove(tab, 1) -- remove newline after end of thinking - return table.concat(tab, "\n") - elseif #tab < 1 then - return text -- no reasoning output - else - reasoning_lines[#reasoning_lines + 1] = line -- export thinking lines - end - end -end - -local send_prompt = function(text, model) - if type(text) == "table" then text = table.concat(text, "\n") end - - local tmp_file_name = utility.tmp_file_name() - utility.open(tmp_file_name, "w", function(file) - file:write(text) - end) - - -- word wrap breaks the raw output badly, so I need to implement my own for terminal output somehow - local output = utility.capture_safe("cat " .. tmp_file_name:enquote() .. " | ollama run " .. (model or default_model) .. " --nowordwrap") - os.execute("ollama stop " .. (model or default_model)) -- NOTE this makes things slower, but more stable - os.execute("rm " .. tmp_file_name) - if not output then error("ollama failed to generate output") end - output = output:sub(1, -2) -- strip extra newline from utility.capture_safe - - local thinking = {} - output = strip_reasoning(output, thinking) - - return output, thinking -end - local refresh_file_list = function() - local blacklist = { -- I'm only blacklisting binary formats because funny results happen with really invalid texts - "jpg", "mp4", "pdf", "png", "webp", "jpeg", "gif", - } for _, name in ipairs(blacklist) do blacklist[name] = true end - local new_files_list = {} utility.tree("PRIVATE_DATA/notebook", { - blacklist = utility.enumerate{".git", ".gitignore", ".gitkeep", ".DS_Store"} + blacklist = utility.enumerate{ ".git", ".gitattributes", ".gitignore", ".gitkeep", ".DS_Store", }, + extension_blacklist = utility.enumerate{ "gif", "jpg", "jpeg", "mp4", "pdf", "png", "webp", }, }, function(file_name) - local _, _, extension = utility.split_path_components(file_name) - if not blacklist[extension] then - new_files_list[#new_files_list + 1] = file_name - end + new_files_list[#new_files_list + 1] = file_name end) files = new_files_list utility.save_data(new_files_list, "PRIVATE_DATA/file_list.json") @@ -109,14 +32,14 @@ end local generate_and_score = function(file_name, text) print("Writing synopsis...") - local synopsis = send_prompt(prompts.synopsis_prompt .. text) + local synopsis = utility.llm_prompt(prompts.synopsis_prompt .. text, model) print(synopsis) print("Scoring synopsis...") - local scoring, thinking = send_prompt(prompts.scoring_prompt .. synopsis) + local scoring, thinking = utility.llm_prompt(prompts.scoring_prompt .. synopsis, model) print(scoring) local scoring_decoded = json.decode(scoring) -- likely will not work because it consistently returns Markdown instead of JSON if not scoring_decoded then - scoring_decoded = json.decode(strip_markdown_codeblock(scoring)) + scoring_decoded = json.decode(text_processing.strip_markdown_codeblock(scoring)) end local object = { @@ -137,23 +60,6 @@ if arg[1] == "refresh_file_list" then refresh_file_list() end -if arg[1] == "repair_synopsis_exports" then - local path = "PRIVATE_DATA/synopses" - utility.list(path, function(path_name) - path_name = path .. utility.path_separator .. path_name - if path_name:find("%.json") then - local object = utility.load_data(path_name) - if type(object.scoring) == "table" then return end -- don't fuck with working pieces - local decoded = json.decode(strip_markdown_codeblock(object.scoring)) - if decoded then - object.scoring = decoded - utility.save_data(object, path_name) - end - end - end) - os.exit(0) -end - if arg[1] == "export_ordered_list_of_prompts" then local path = "PRIVATE_DATA/synopses" local items = {} @@ -210,7 +116,7 @@ while true do local file_size = utility.file_size(file_name) if file_size > minimum_bytes and file_size <= maximum_bytes then local text = utility.read_file(file_name) - text = strip_frontmatter(text) + text = text_processing.strip_frontmatter(text) if #text > minimum_bytes and #text <= maximum_bytes then print(file_name .. " chosen.") generate_and_score(file_name, text) -- kind of the main function, innit?