diff --git a/.gitignore b/.gitignore index 8f38cf2..15d9563 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ intermediates/** !intermediates/.gitkeep cover_briefs/** !cover_briefs/.gitkeep +for_gemini/** +!for_gemini/.gitkeep diff --git a/ReadMe.md b/ReadMe.md index 0820b1b..b8b692c 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -1,3 +1,3 @@ -- [ ] Rename cover to process -- [ ] Remove code that saves intermediates (comment it out) +- [x] Rename cover to process +- [x] Remove code that saves intermediates (comment it out) diff --git a/count_characters.lua b/count_characters.lua new file mode 100755 index 0000000..dd88908 --- /dev/null +++ b/count_characters.lua @@ -0,0 +1,32 @@ +#!/usr/bin/env luajit + +package.path = (arg[0]:match("@?(.*/)") or arg[0]:match("@?(.*\\)")) .. "lib" .. package.config:sub(1, 1) .. "?.lua;" .. package.path +local utility = require "utility" + +local maximum_bytes = 40000 + +local read_all = function(file_name) + return utility.open(file_name, "r", function(file) + return file:read("*all") + end) +end + +local list = function(path, fn) + utility.list(path or ".", function(path_name) + if not path_name:find("%.txt") then return end + if utility.is_file(path .. "/" .. path_name) then + fn(path .. "/" .. path_name) + end + end) +end + +for _, path in ipairs{ ".", "extracted_texts", } do + list(path, function(file_name) + local path, name, extension = utility.split_path_components(file_name) + local text = read_all(file_name) + + print(math.floor(#text/maximum_bytes + 1), math.floor(#text/1000) .. "k", name) + end) + print("Press enter.") + io.read("*line") +end diff --git a/for_gemini/.gitkeep b/for_gemini/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/intermediates/.gitkeep b/intermediates/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/lib/utility.lua b/lib/utility.lua new file mode 100644 index 0000000..88e7e20 --- /dev/null +++ b/lib/utility.lua @@ -0,0 +1,426 @@ +math.randomseed(os.time()) + +local utility + +if package.config:sub(1, 1) == "\\" then + utility = { + OS = "Windows", + path_separator = "\\", + temp_directory = "C:\\Windows\\Temp\\", + commands = { + recursive_remove = "rmdir /s /q ", + list = "dir /w /b ", + which = "where ", + move = "move ", + silence_output = " >nul 2>nul", + silence_errors = " 2>nul", + }, + } +else + utility = { + OS = "UNIX-like", + path_separator = "/", + temp_directory = "/tmp/", + commands = { + recursive_remove = "rm -r ", + list = "ls -1a ", + which = "which ", + move = "mv ", + silence_output = " >/dev/null 2>/dev/null", + silence_errors = " 2>/dev/null", + }, + } +end + +utility.version = "1.4.0" +-- WARNING: This will return "./" if the original script is called locally instead of with an absolute path! +if arg[0] ~= nil then + utility.path = (arg[0]:match("@?(.*/)") or arg[0]:match("@?(.*\\)")) -- inspired by discussion in https://stackoverflow.com/q/6380820 +else + utility.path = "./" +end + +utility.require = function(...) + -- if libraries adjacent to this one aren't already loadable, make sure they are! + if not package.path:find(utility.path, 1, true) then + package.path = utility.path .. "?.lua;" .. package.path + end + return require(...) +end + +-- errors if specified program isn't in the path +local _required_program_cache = {} +utility.required_program = function(name) + if _required_program_cache[name] then + return true + end + if os.execute(utility.commands.which .. tostring(name) .. utility.commands.silence_output) == 0 then + _required_program_cache[name] = true + else + error("\n\n" .. tostring(name) .. " must be installed and in the path\n") + end +end + + + +-- always uses outputting to a temporary file to guarantee safety +utility.capture_safe = function(command, get_status) + local file_name = utility.tmp_file_name() + command = command .. " > " .. file_name + if get_status then + command = command .. "\necho $? >> " .. file_name + end + os.execute(command) + + local file = io.open(file_name, "r") + local output = file:read("*all") + file:close() + os.execute("rm " .. file_name) + + if get_status then + local start, finish = output:find("\n.-\n$") + return tonumber(output:sub(start + 1, finish - 1)), output:sub(1, start) + end + + return output +end +-- WARNING DEPRECATED +utility.capture = function(...) + print("WARNING: Use utility.capture_safe or utility.capture_unsafe. This function will be removed.") + return utility.capture_safe(...) +end + +-- can hang indefinitely; not always available +utility.capture_unsafe = function(command) + if io.popen then + local file = assert(io.popen(command, 'r')) + local output = assert(file:read('*all')) + file:close() + return output + else + print("WARNING: io.popen not available, using a temporary file to receive output from:\n", command) + return utility.capture_safe(command) + end +end + + + +local function standard_library_addition(tab, name, func) + if tab[name] then + print("WARNING: " .. tab .. "." .. name .. " was defined by another library. lua-utility may encounter errors due to a differing implementation.") + else + tab[name] = func + end +end + + + +-- trim6 from Lua users wiki (best all-round pure Lua performance) +standard_library_addition(string, "trim", function(s) + return s:match'^()%s*$' and '' or s:match'^%s*(.*%S)' +end) + +standard_library_addition(string, "enquote", function(s) + return "\"" .. s:gsub("\"", "\\\"") .. "\"" +end) + +standard_library_addition(string, "split", function(s, delimiter) + local result = {} + for item in s:gsplit(delimiter) do + result[#result + 1] = item + end + return result +end) + +standard_library_addition(string, "gsplit", function(s, delimiter) + local function escape_special_characters(s) + local special_characters = "[()%%.[^$%]*+%-?]" + if s == nil then return end + return (s:gsub(special_characters, "%%%1")) + end + + delimiter = delimiter or "," + if s:sub(-#delimiter) ~= delimiter then s = s .. delimiter end + return s:gmatch("(.-)" .. escape_special_characters(delimiter)) +end) + + + +-- modified from my fork of lume +utility.uuid = function() + local fn = function(x) + local r = math.random(16) - 1 + r = (x == "x") and (r + 1) or (r % 4) + 9 + return ("0123456789abcdef"):sub(r, r) + end + return (("xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx"):gsub("[xy]", fn)) +end + +utility.tmp_file_name = function() + return utility.temp_directory .. utility.uuid() +end + + + +-- while I could replace this with a better implementation, I'm used to how it works and I might break existing scripts +utility.make_safe_file_name = function(file_name) + file_name = file_name:gsub("[%\"%:%\\%!%@%#%$%%%^%*%=%{%}%|%;%<%>%?%/]", "") -- everything except the & + file_name = file_name:gsub(" %&", ",") -- replacing & with a comma works for 99% of things + file_name = file_name:gsub("%&", ",") -- replacing & with a comma works for 99% of things + file_name = file_name:gsub("[%s+]", " ") -- more than one space in succession should be a single space + return file_name +end + +utility.split_path_components = function(file_path) + local path, name, extension = string.match(file_path, "(.-)([^\\/]-%.?([^%.\\/]*))$") + if name == extension then + extension = nil + end + return path, name, extension +end + +-- wrapper around io.open to prevent leaving a file handle open accidentally +-- throws errors instead of returning them +-- usage: utility.open(file_name, mode, function(file_handle) --[[ your code ]] end) +-- or utility.open(file_name, mode)(function(file_handle) --[[ your code ]] end) +utility.open = function(file_name, mode, func) + local file, err = io.open(file_name, mode) + if not file then error(err) end + if func then + local success, result = pcall(function() return func(file) end) + file:close() + if not success then error(result) end + return result + else + return function(fn) + local success, result = pcall(function() return fn(file) end) + file:close() + if not success then + error(result) + end + return result + end + end +end + +-- run a function on each file name in a directory +-- example list items: utility.list(".", print) OR utility.list(".")(print) +utility.list = function(path, func) + local command = utility.commands.list + if path then + command = command .. path:enquote() + end + + local output = utility.capture_safe(command) + + local run = function(fn) + for line in output:gmatch("[^\r\n]+") do -- thanks to https://stackoverflow.com/a/32847589 + if not (line == "." or line == "..") then + fn(line) + end + end + end + + if func then + run(func) + else + return run + end +end +-- WARNING DEPRECATED +utility.ls = function(...) + print("WARNING: Use utility.list. This function will be removed.") + return utility.list(...) +end + +utility.path_exists = function(file_name) + local file = io.open(file_name, "r") + if file then file:close() return true else return false end +end +-- WARNING DEPRECATED +utility.file_exists = function(...) + print("WARNING: Use utility.path_exists instead, or utility.is_file to check for a file existing.") + return utility.path_exists(...) +end + +utility.is_file = function(file_name) + local file = io.open(file_name, "r") + if file then + file:close() + file = io.open(file_name, "a") + if file then + file:close() + return true + end + end + return false +end + +utility.file_size = function(file_path) + return utility.open(file_path, "rb", function(file) return file:seek("end") end) +end + + + +utility.escape_quotes_and_escapes = function(input) + -- the order of these commands is important and must be preserved + input = input:gsub("\\", "\\\\") + input = input:gsub("\"", "\\\"") + return input +end + + + +-- only use for brief loads/saves, as this will block until a lock can be established +-- returns a UUID that can be checked on release to make sure unforeseen errors did not occur +utility.get_lock = function(file_path) + local lock_obtained, lock_uuid, lock_file_path = false, utility.uuid(), file_path .. ".lock" + repeat + if not utility.is_file(lock_file_path) then + pcall(function() + utility.open(lock_file_path, "w", function(file) + file:write(lock_uuid) + end) + utility.open(lock_file_path, "r", function(file) + if file:read("*all") == lock_uuid then + lock_obtained = true + end + end) + end) + end + if not lock_obtained then + print("Trying to obtain file lock: " .. lock_file_path) + os.execute("sleep 1") + end + until lock_obtained + return lock_uuid +end + +-- specifying lock_uuid is optional, to error if a conflict occurred despite the lock (should not be possible) +utility.release_lock = function(file_path, lock_uuid) + local lock_file_path = file_path .. ".lock" + if lock_uuid then + utility.open(lock_file_path, "r", function(file) + if not file:read("*all") == lock_uuid then + error("\n\n Lock UUID changed while lock was obtained. Data loss may have occurred. \n\n") + end + end) + end + os.execute("rm " .. lock_file_path:enquote()) +end + + + +local config, config_lock +utility.get_config = function(skip_lock) + if not config then + local config_path = utility.path .. "config.json" + if utility.is_file(config_path) then + if not skip_lock then + config_lock = utility.get_lock(config_path) + end + utility.open(config_path, "r", function(config_file) + local json = utility.require("dkjson") + config = json.decode(config_file:read("*all")) + end) + else + config = {} + end + end + return config +end + +utility.save_config = function() + if config then + local config_path = utility.path .. "config.json" + if not config_lock then + print("Warning: A config lock file was not established.") + end + utility.open(config_path, "w", function(config_file) + local json = utility.require("dkjson") + config_file:write(json.encode(config, { indent = true })) + end) + if config_lock then + utility.release_lock(config_path, config_lock) + end + else + error("utility config not loaded") + end +end + + + +utility.deepcopy = function(tab) + local _type = type(tab) + local copy + if _type == "table" then + copy = {} + for key, value in next, tab, nil do + copy[utility.deepcopy(key)] = utility.deepcopy(value) + end + setmetatable(copy, utility.deepcopy(getmetatable(tab))) + else + copy = tab + end + return copy +end + +utility.enumerate = function(list) + local result = {} + for _, value in ipairs(list) do + result[value] = { name = value } + end + return result +end + +local _ +_, utility.inspect = pcall(function() return utility.require("inspect") end) +if _ then + utility.print_table = function(tab) + print(utility.inspect(tab)) + end +else + utility.inspect = nil + -- much simpler (and worse) print_table as fallback + utility.print_table = function(tab, depth) + depth = depth or 0 + if type(tab) == "table" then + for k, v in pairs(tab) do + print(string.rep(" ", depth) .. tostring(k) .. ":") + if type(v) == "table" then + utility.print_table(v, depth + 1) + else + print(string.rep(" ", depth + 1) .. tostring(v)) + end + end + else + print(string.rep(" ", depth) .. tostring(tab)) + end + end +end + +utility.list_reverse = function(tab) + for i = 1, math.floor(#tab/2) do + tab[i], tab[#tab-i+1] = tab[#tab-i+1], tab[i] + end +end + + + +-- a super common need I'm encountering is wanting content from a URL without side effects +utility.curl_read = function(download_url, curl_options) + utility.required_program("curl") + local tmp_file_name = utility.tmp_file_name() + local command = "curl " + if curl_options then + command = command .. curl_options .. " " + end + os.execute(command .. download_url:enquote() .. " > " .. tmp_file_name) + local file_contents + utility.open(tmp_file_name, "r", function(file) + file_contents = file:read("*all") + end) + return file_contents +end + +return utility diff --git a/make_cover_briefs.lua b/make_cover_briefs.lua index a309eeb..c764f4b 100755 --- a/make_cover_briefs.lua +++ b/make_cover_briefs.lua @@ -4,7 +4,7 @@ package.path = (arg[0]:match("@?(.*/)") or arg[0]:match("@?(.*\\)")) .. "lib" .. local utility = require "utility" local default_model = "gemma4:12b-mlx" -local maximum_bytes = 50000 +local maximum_bytes = 40000 local partial_prompt = [[You are extracting information for a later cover design process. @@ -75,7 +75,8 @@ Avoid minor plot events. ]] local gemini_prompt = [[ -Generate an ebook cover inset within an empty white border 10% larger than the cover using the following cover brief: + +Generate a flate image of an ebook cover inset within an empty white border 10% larger than the cover using the following cover brief: ]] @@ -134,6 +135,7 @@ local send_prompt = function(text, model) local start_time = os.time() -- word wrap breaks the raw output badly, so I need to implement my own for terminal output somehow local output = utility.capture_safe("cat " .. tmp_file_name:enquote() .. " | ollama run " .. (model or default_model) .. " --nowordwrap") + os.execute("ollama stop " .. (model or default_model)) -- NOTE this makes things slower, but more stable local delta = math.floor( (os.time() - start_time) / 60 * 10 ) / 10 print("Took " .. delta .. " minutes.") timings[#timings + 1] = delta @@ -149,7 +151,12 @@ end local tree tree = function(path, fn) utility.list(path or ".", function(path_name) - local blacklist = { ".git", ".gitkeep", ".DS_Store", } + local blacklist = { + [".git"] = true, + [".gitkeep"] = true, + [".gitignore"] = true, + [".DS_Store"] = true, + } if blacklist[path_name] then return end if utility.is_file(path_name) then fn(path_name) @@ -161,10 +168,12 @@ end tree("extracted_texts", function(file_name) local path, name, extension = utility.split_path_components(file_name) + if utility.path_exists("cover_briefs/" .. name) then return end + local text = read_all(file_name) local result - print(name) + print(name) if #text > maximum_bytes then local outputs = {} -- TODO check if this would work just as well starting from 1 instead.. @@ -178,13 +187,13 @@ tree("extracted_texts", function(file_name) text = table.concat(outputs, "\n\n") -- TODO handle if there is too much context! - write_all("intermediates/" .. name .. ".txt", text) + write_all("intermediates/" .. name, text) end print("Generating cover brief.") result = send_prompt(cover_brief_prompt .. text) - write_all("cover_briefs/" .. name .. ".txt", result) - write_all("cover_briefs/for_gemini" .. name .. ".txt", name .. gemini_prompt .. result) + write_all("cover_briefs/" .. name, result) + write_all("for_gemini/" .. name, name:sub(1, -5) .. gemini_prompt .. result) end) timing_report()