almost final form?
This commit is contained in:
@@ -5,3 +5,5 @@ intermediates/**
|
||||
!intermediates/.gitkeep
|
||||
cover_briefs/**
|
||||
!cover_briefs/.gitkeep
|
||||
for_gemini/**
|
||||
!for_gemini/.gitkeep
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
|
||||
- [ ] Rename cover to process
|
||||
- [ ] Remove code that saves intermediates (comment it out)
|
||||
- [x] Rename cover to process
|
||||
- [x] Remove code that saves intermediates (comment it out)
|
||||
|
||||
Executable
+32
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env luajit
|
||||
|
||||
package.path = (arg[0]:match("@?(.*/)") or arg[0]:match("@?(.*\\)")) .. "lib" .. package.config:sub(1, 1) .. "?.lua;" .. package.path
|
||||
local utility = require "utility"
|
||||
|
||||
local maximum_bytes = 40000
|
||||
|
||||
local read_all = function(file_name)
|
||||
return utility.open(file_name, "r", function(file)
|
||||
return file:read("*all")
|
||||
end)
|
||||
end
|
||||
|
||||
local list = function(path, fn)
|
||||
utility.list(path or ".", function(path_name)
|
||||
if not path_name:find("%.txt") then return end
|
||||
if utility.is_file(path .. "/" .. path_name) then
|
||||
fn(path .. "/" .. path_name)
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
for _, path in ipairs{ ".", "extracted_texts", } do
|
||||
list(path, function(file_name)
|
||||
local path, name, extension = utility.split_path_components(file_name)
|
||||
local text = read_all(file_name)
|
||||
|
||||
print(math.floor(#text/maximum_bytes + 1), math.floor(#text/1000) .. "k", name)
|
||||
end)
|
||||
print("Press enter.")
|
||||
io.read("*line")
|
||||
end
|
||||
+426
@@ -0,0 +1,426 @@
|
||||
math.randomseed(os.time())
|
||||
|
||||
local utility
|
||||
|
||||
if package.config:sub(1, 1) == "\\" then
|
||||
utility = {
|
||||
OS = "Windows",
|
||||
path_separator = "\\",
|
||||
temp_directory = "C:\\Windows\\Temp\\",
|
||||
commands = {
|
||||
recursive_remove = "rmdir /s /q ",
|
||||
list = "dir /w /b ",
|
||||
which = "where ",
|
||||
move = "move ",
|
||||
silence_output = " >nul 2>nul",
|
||||
silence_errors = " 2>nul",
|
||||
},
|
||||
}
|
||||
else
|
||||
utility = {
|
||||
OS = "UNIX-like",
|
||||
path_separator = "/",
|
||||
temp_directory = "/tmp/",
|
||||
commands = {
|
||||
recursive_remove = "rm -r ",
|
||||
list = "ls -1a ",
|
||||
which = "which ",
|
||||
move = "mv ",
|
||||
silence_output = " >/dev/null 2>/dev/null",
|
||||
silence_errors = " 2>/dev/null",
|
||||
},
|
||||
}
|
||||
end
|
||||
|
||||
utility.version = "1.4.0"
|
||||
-- WARNING: This will return "./" if the original script is called locally instead of with an absolute path!
|
||||
if arg[0] ~= nil then
|
||||
utility.path = (arg[0]:match("@?(.*/)") or arg[0]:match("@?(.*\\)")) -- inspired by discussion in https://stackoverflow.com/q/6380820
|
||||
else
|
||||
utility.path = "./"
|
||||
end
|
||||
|
||||
utility.require = function(...)
|
||||
-- if libraries adjacent to this one aren't already loadable, make sure they are!
|
||||
if not package.path:find(utility.path, 1, true) then
|
||||
package.path = utility.path .. "?.lua;" .. package.path
|
||||
end
|
||||
return require(...)
|
||||
end
|
||||
|
||||
-- errors if specified program isn't in the path
|
||||
local _required_program_cache = {}
|
||||
utility.required_program = function(name)
|
||||
if _required_program_cache[name] then
|
||||
return true
|
||||
end
|
||||
if os.execute(utility.commands.which .. tostring(name) .. utility.commands.silence_output) == 0 then
|
||||
_required_program_cache[name] = true
|
||||
else
|
||||
error("\n\n" .. tostring(name) .. " must be installed and in the path\n")
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
|
||||
-- always uses outputting to a temporary file to guarantee safety
|
||||
utility.capture_safe = function(command, get_status)
|
||||
local file_name = utility.tmp_file_name()
|
||||
command = command .. " > " .. file_name
|
||||
if get_status then
|
||||
command = command .. "\necho $? >> " .. file_name
|
||||
end
|
||||
os.execute(command)
|
||||
|
||||
local file = io.open(file_name, "r")
|
||||
local output = file:read("*all")
|
||||
file:close()
|
||||
os.execute("rm " .. file_name)
|
||||
|
||||
if get_status then
|
||||
local start, finish = output:find("\n.-\n$")
|
||||
return tonumber(output:sub(start + 1, finish - 1)), output:sub(1, start)
|
||||
end
|
||||
|
||||
return output
|
||||
end
|
||||
-- WARNING DEPRECATED
|
||||
utility.capture = function(...)
|
||||
print("WARNING: Use utility.capture_safe or utility.capture_unsafe. This function will be removed.")
|
||||
return utility.capture_safe(...)
|
||||
end
|
||||
|
||||
-- can hang indefinitely; not always available
|
||||
utility.capture_unsafe = function(command)
|
||||
if io.popen then
|
||||
local file = assert(io.popen(command, 'r'))
|
||||
local output = assert(file:read('*all'))
|
||||
file:close()
|
||||
return output
|
||||
else
|
||||
print("WARNING: io.popen not available, using a temporary file to receive output from:\n", command)
|
||||
return utility.capture_safe(command)
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
|
||||
local function standard_library_addition(tab, name, func)
|
||||
if tab[name] then
|
||||
print("WARNING: " .. tab .. "." .. name .. " was defined by another library. lua-utility may encounter errors due to a differing implementation.")
|
||||
else
|
||||
tab[name] = func
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
|
||||
-- trim6 from Lua users wiki (best all-round pure Lua performance)
|
||||
standard_library_addition(string, "trim", function(s)
|
||||
return s:match'^()%s*$' and '' or s:match'^%s*(.*%S)'
|
||||
end)
|
||||
|
||||
standard_library_addition(string, "enquote", function(s)
|
||||
return "\"" .. s:gsub("\"", "\\\"") .. "\""
|
||||
end)
|
||||
|
||||
standard_library_addition(string, "split", function(s, delimiter)
|
||||
local result = {}
|
||||
for item in s:gsplit(delimiter) do
|
||||
result[#result + 1] = item
|
||||
end
|
||||
return result
|
||||
end)
|
||||
|
||||
standard_library_addition(string, "gsplit", function(s, delimiter)
|
||||
local function escape_special_characters(s)
|
||||
local special_characters = "[()%%.[^$%]*+%-?]"
|
||||
if s == nil then return end
|
||||
return (s:gsub(special_characters, "%%%1"))
|
||||
end
|
||||
|
||||
delimiter = delimiter or ","
|
||||
if s:sub(-#delimiter) ~= delimiter then s = s .. delimiter end
|
||||
return s:gmatch("(.-)" .. escape_special_characters(delimiter))
|
||||
end)
|
||||
|
||||
|
||||
|
||||
-- modified from my fork of lume
|
||||
utility.uuid = function()
|
||||
local fn = function(x)
|
||||
local r = math.random(16) - 1
|
||||
r = (x == "x") and (r + 1) or (r % 4) + 9
|
||||
return ("0123456789abcdef"):sub(r, r)
|
||||
end
|
||||
return (("xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx"):gsub("[xy]", fn))
|
||||
end
|
||||
|
||||
utility.tmp_file_name = function()
|
||||
return utility.temp_directory .. utility.uuid()
|
||||
end
|
||||
|
||||
|
||||
|
||||
-- while I could replace this with a better implementation, I'm used to how it works and I might break existing scripts
|
||||
utility.make_safe_file_name = function(file_name)
|
||||
file_name = file_name:gsub("[%\"%:%\\%!%@%#%$%%%^%*%=%{%}%|%;%<%>%?%/]", "") -- everything except the &
|
||||
file_name = file_name:gsub(" %&", ",") -- replacing & with a comma works for 99% of things
|
||||
file_name = file_name:gsub("%&", ",") -- replacing & with a comma works for 99% of things
|
||||
file_name = file_name:gsub("[%s+]", " ") -- more than one space in succession should be a single space
|
||||
return file_name
|
||||
end
|
||||
|
||||
utility.split_path_components = function(file_path)
|
||||
local path, name, extension = string.match(file_path, "(.-)([^\\/]-%.?([^%.\\/]*))$")
|
||||
if name == extension then
|
||||
extension = nil
|
||||
end
|
||||
return path, name, extension
|
||||
end
|
||||
|
||||
-- wrapper around io.open to prevent leaving a file handle open accidentally
|
||||
-- throws errors instead of returning them
|
||||
-- usage: utility.open(file_name, mode, function(file_handle) --[[ your code ]] end)
|
||||
-- or utility.open(file_name, mode)(function(file_handle) --[[ your code ]] end)
|
||||
utility.open = function(file_name, mode, func)
|
||||
local file, err = io.open(file_name, mode)
|
||||
if not file then error(err) end
|
||||
if func then
|
||||
local success, result = pcall(function() return func(file) end)
|
||||
file:close()
|
||||
if not success then error(result) end
|
||||
return result
|
||||
else
|
||||
return function(fn)
|
||||
local success, result = pcall(function() return fn(file) end)
|
||||
file:close()
|
||||
if not success then
|
||||
error(result)
|
||||
end
|
||||
return result
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
-- run a function on each file name in a directory
|
||||
-- example list items: utility.list(".", print) OR utility.list(".")(print)
|
||||
utility.list = function(path, func)
|
||||
local command = utility.commands.list
|
||||
if path then
|
||||
command = command .. path:enquote()
|
||||
end
|
||||
|
||||
local output = utility.capture_safe(command)
|
||||
|
||||
local run = function(fn)
|
||||
for line in output:gmatch("[^\r\n]+") do -- thanks to https://stackoverflow.com/a/32847589
|
||||
if not (line == "." or line == "..") then
|
||||
fn(line)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
if func then
|
||||
run(func)
|
||||
else
|
||||
return run
|
||||
end
|
||||
end
|
||||
-- WARNING DEPRECATED
|
||||
utility.ls = function(...)
|
||||
print("WARNING: Use utility.list. This function will be removed.")
|
||||
return utility.list(...)
|
||||
end
|
||||
|
||||
utility.path_exists = function(file_name)
|
||||
local file = io.open(file_name, "r")
|
||||
if file then file:close() return true else return false end
|
||||
end
|
||||
-- WARNING DEPRECATED
|
||||
utility.file_exists = function(...)
|
||||
print("WARNING: Use utility.path_exists instead, or utility.is_file to check for a file existing.")
|
||||
return utility.path_exists(...)
|
||||
end
|
||||
|
||||
utility.is_file = function(file_name)
|
||||
local file = io.open(file_name, "r")
|
||||
if file then
|
||||
file:close()
|
||||
file = io.open(file_name, "a")
|
||||
if file then
|
||||
file:close()
|
||||
return true
|
||||
end
|
||||
end
|
||||
return false
|
||||
end
|
||||
|
||||
utility.file_size = function(file_path)
|
||||
return utility.open(file_path, "rb", function(file) return file:seek("end") end)
|
||||
end
|
||||
|
||||
|
||||
|
||||
utility.escape_quotes_and_escapes = function(input)
|
||||
-- the order of these commands is important and must be preserved
|
||||
input = input:gsub("\\", "\\\\")
|
||||
input = input:gsub("\"", "\\\"")
|
||||
return input
|
||||
end
|
||||
|
||||
|
||||
|
||||
-- only use for brief loads/saves, as this will block until a lock can be established
|
||||
-- returns a UUID that can be checked on release to make sure unforeseen errors did not occur
|
||||
utility.get_lock = function(file_path)
|
||||
local lock_obtained, lock_uuid, lock_file_path = false, utility.uuid(), file_path .. ".lock"
|
||||
repeat
|
||||
if not utility.is_file(lock_file_path) then
|
||||
pcall(function()
|
||||
utility.open(lock_file_path, "w", function(file)
|
||||
file:write(lock_uuid)
|
||||
end)
|
||||
utility.open(lock_file_path, "r", function(file)
|
||||
if file:read("*all") == lock_uuid then
|
||||
lock_obtained = true
|
||||
end
|
||||
end)
|
||||
end)
|
||||
end
|
||||
if not lock_obtained then
|
||||
print("Trying to obtain file lock: " .. lock_file_path)
|
||||
os.execute("sleep 1")
|
||||
end
|
||||
until lock_obtained
|
||||
return lock_uuid
|
||||
end
|
||||
|
||||
-- specifying lock_uuid is optional, to error if a conflict occurred despite the lock (should not be possible)
|
||||
utility.release_lock = function(file_path, lock_uuid)
|
||||
local lock_file_path = file_path .. ".lock"
|
||||
if lock_uuid then
|
||||
utility.open(lock_file_path, "r", function(file)
|
||||
if not file:read("*all") == lock_uuid then
|
||||
error("\n\n Lock UUID changed while lock was obtained. Data loss may have occurred. \n\n")
|
||||
end
|
||||
end)
|
||||
end
|
||||
os.execute("rm " .. lock_file_path:enquote())
|
||||
end
|
||||
|
||||
|
||||
|
||||
local config, config_lock
|
||||
utility.get_config = function(skip_lock)
|
||||
if not config then
|
||||
local config_path = utility.path .. "config.json"
|
||||
if utility.is_file(config_path) then
|
||||
if not skip_lock then
|
||||
config_lock = utility.get_lock(config_path)
|
||||
end
|
||||
utility.open(config_path, "r", function(config_file)
|
||||
local json = utility.require("dkjson")
|
||||
config = json.decode(config_file:read("*all"))
|
||||
end)
|
||||
else
|
||||
config = {}
|
||||
end
|
||||
end
|
||||
return config
|
||||
end
|
||||
|
||||
utility.save_config = function()
|
||||
if config then
|
||||
local config_path = utility.path .. "config.json"
|
||||
if not config_lock then
|
||||
print("Warning: A config lock file was not established.")
|
||||
end
|
||||
utility.open(config_path, "w", function(config_file)
|
||||
local json = utility.require("dkjson")
|
||||
config_file:write(json.encode(config, { indent = true }))
|
||||
end)
|
||||
if config_lock then
|
||||
utility.release_lock(config_path, config_lock)
|
||||
end
|
||||
else
|
||||
error("utility config not loaded")
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
|
||||
utility.deepcopy = function(tab)
|
||||
local _type = type(tab)
|
||||
local copy
|
||||
if _type == "table" then
|
||||
copy = {}
|
||||
for key, value in next, tab, nil do
|
||||
copy[utility.deepcopy(key)] = utility.deepcopy(value)
|
||||
end
|
||||
setmetatable(copy, utility.deepcopy(getmetatable(tab)))
|
||||
else
|
||||
copy = tab
|
||||
end
|
||||
return copy
|
||||
end
|
||||
|
||||
utility.enumerate = function(list)
|
||||
local result = {}
|
||||
for _, value in ipairs(list) do
|
||||
result[value] = { name = value }
|
||||
end
|
||||
return result
|
||||
end
|
||||
|
||||
local _
|
||||
_, utility.inspect = pcall(function() return utility.require("inspect") end)
|
||||
if _ then
|
||||
utility.print_table = function(tab)
|
||||
print(utility.inspect(tab))
|
||||
end
|
||||
else
|
||||
utility.inspect = nil
|
||||
-- much simpler (and worse) print_table as fallback
|
||||
utility.print_table = function(tab, depth)
|
||||
depth = depth or 0
|
||||
if type(tab) == "table" then
|
||||
for k, v in pairs(tab) do
|
||||
print(string.rep(" ", depth) .. tostring(k) .. ":")
|
||||
if type(v) == "table" then
|
||||
utility.print_table(v, depth + 1)
|
||||
else
|
||||
print(string.rep(" ", depth + 1) .. tostring(v))
|
||||
end
|
||||
end
|
||||
else
|
||||
print(string.rep(" ", depth) .. tostring(tab))
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
utility.list_reverse = function(tab)
|
||||
for i = 1, math.floor(#tab/2) do
|
||||
tab[i], tab[#tab-i+1] = tab[#tab-i+1], tab[i]
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
|
||||
-- a super common need I'm encountering is wanting content from a URL without side effects
|
||||
utility.curl_read = function(download_url, curl_options)
|
||||
utility.required_program("curl")
|
||||
local tmp_file_name = utility.tmp_file_name()
|
||||
local command = "curl "
|
||||
if curl_options then
|
||||
command = command .. curl_options .. " "
|
||||
end
|
||||
os.execute(command .. download_url:enquote() .. " > " .. tmp_file_name)
|
||||
local file_contents
|
||||
utility.open(tmp_file_name, "r", function(file)
|
||||
file_contents = file:read("*all")
|
||||
end)
|
||||
return file_contents
|
||||
end
|
||||
|
||||
return utility
|
||||
+16
-7
@@ -4,7 +4,7 @@ package.path = (arg[0]:match("@?(.*/)") or arg[0]:match("@?(.*\\)")) .. "lib" ..
|
||||
local utility = require "utility"
|
||||
|
||||
local default_model = "gemma4:12b-mlx"
|
||||
local maximum_bytes = 50000
|
||||
local maximum_bytes = 40000
|
||||
|
||||
local partial_prompt = [[You are extracting information for a later cover design process.
|
||||
|
||||
@@ -75,7 +75,8 @@ Avoid minor plot events.
|
||||
]]
|
||||
|
||||
local gemini_prompt = [[
|
||||
Generate an ebook cover inset within an empty white border 10% larger than the cover using the following cover brief:
|
||||
|
||||
Generate a flate image of an ebook cover inset within an empty white border 10% larger than the cover using the following cover brief:
|
||||
|
||||
]]
|
||||
|
||||
@@ -134,6 +135,7 @@ local send_prompt = function(text, model)
|
||||
local start_time = os.time()
|
||||
-- word wrap breaks the raw output badly, so I need to implement my own for terminal output somehow
|
||||
local output = utility.capture_safe("cat " .. tmp_file_name:enquote() .. " | ollama run " .. (model or default_model) .. " --nowordwrap")
|
||||
os.execute("ollama stop " .. (model or default_model)) -- NOTE this makes things slower, but more stable
|
||||
local delta = math.floor( (os.time() - start_time) / 60 * 10 ) / 10
|
||||
print("Took " .. delta .. " minutes.")
|
||||
timings[#timings + 1] = delta
|
||||
@@ -149,7 +151,12 @@ end
|
||||
local tree
|
||||
tree = function(path, fn)
|
||||
utility.list(path or ".", function(path_name)
|
||||
local blacklist = { ".git", ".gitkeep", ".DS_Store", }
|
||||
local blacklist = {
|
||||
[".git"] = true,
|
||||
[".gitkeep"] = true,
|
||||
[".gitignore"] = true,
|
||||
[".DS_Store"] = true,
|
||||
}
|
||||
if blacklist[path_name] then return end
|
||||
if utility.is_file(path_name) then
|
||||
fn(path_name)
|
||||
@@ -161,10 +168,12 @@ end
|
||||
|
||||
tree("extracted_texts", function(file_name)
|
||||
local path, name, extension = utility.split_path_components(file_name)
|
||||
if utility.path_exists("cover_briefs/" .. name) then return end
|
||||
|
||||
local text = read_all(file_name)
|
||||
local result
|
||||
print(name)
|
||||
|
||||
print(name)
|
||||
if #text > maximum_bytes then
|
||||
local outputs = {}
|
||||
-- TODO check if this would work just as well starting from 1 instead..
|
||||
@@ -178,13 +187,13 @@ tree("extracted_texts", function(file_name)
|
||||
|
||||
text = table.concat(outputs, "\n\n")
|
||||
-- TODO handle if there is too much context!
|
||||
write_all("intermediates/" .. name .. ".txt", text)
|
||||
write_all("intermediates/" .. name, text)
|
||||
end
|
||||
|
||||
print("Generating cover brief.")
|
||||
result = send_prompt(cover_brief_prompt .. text)
|
||||
write_all("cover_briefs/" .. name .. ".txt", result)
|
||||
write_all("cover_briefs/for_gemini" .. name .. ".txt", name .. gemini_prompt .. result)
|
||||
write_all("cover_briefs/" .. name, result)
|
||||
write_all("for_gemini/" .. name, name:sub(1, -5) .. gemini_prompt .. result)
|
||||
end)
|
||||
|
||||
timing_report()
|
||||
|
||||
Reference in New Issue
Block a user