tools/llm-tricks
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

better caching

Browse Source
This commit is contained in:
tangent
2026-09-14 19:57:33 -06:00
parent b5250e302d
commit f8c911bcd4
2 changed files with 22 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ReadMe.md
+1 -1
View File
@@ -25,5 +25,5 @@ Arguments:
- `export_ordered_list_of_prompts`: Makes an epub to review generated synopses. - `export_ordered_list_of_prompts`: Makes an epub to review generated synopses.
## Tasks ## Tasks
- [ ] synopsis_generator cache should include file sizes to make script running easier/faster
- [ ] The whitelisting/blacklisting of tree should be in list too. - [ ] The whitelisting/blacklisting of tree should be in list too.
- [ ] synopsis_generator should be able to blacklist files it already tried?
synopsis_generator.lua
+21 -22
View File
@@ -11,26 +11,33 @@ local model = "gemma4:12b-mlx"
local minimum_bytes = 1000 local minimum_bytes = 1000
local maximum_bytes = 40000 local maximum_bytes = 40000
local files local data_location = "PRIVATE_DATA/synopsis_generator_file_list.json"
if utility.path_exists("PRIVATE_DATA/file_list.json") then local file_list
files = utility.load_data("PRIVATE_DATA/file_list.json") if utility.path_exists(data_location) then
file_list = utility.load_data(data_location)
else else
files = {} file_list = {}
end end
local refresh_file_list = function() local refresh_file_list = function()
os.execute("cd PRIVATE_DATA/notebook && git pull origin")
local new_files_list = {} local new_files_list = {}
utility.tree("PRIVATE_DATA/notebook", { utility.tree("PRIVATE_DATA/notebook", {
blacklist = utility.enumerate{ ".git", ".gitattributes", ".gitignore", ".gitkeep", ".DS_Store", }, blacklist = utility.enumerate{ ".git", ".gitattributes", ".gitignore", ".gitkeep", ".DS_Store", },
extension_blacklist = utility.enumerate{ "gif", "jpg", "jpeg", "mp4", "pdf", "png", "webp", }, extension_blacklist = utility.enumerate{ "gif", "jpg", "jpeg", "mp4", "pdf", "png", "webp", },
}, function(file_name) }, function(file_name)
local file_size = utility.file_size(file_name) local file_size = utility.file_size(file_name)
if file_size > minimum_bytes and file_size <= maximum_bytes then if file_size >= minimum_bytes and file_size <= maximum_bytes then
new_files_list[#new_files_list + 1] = file_name local text = text_processing.strip_frontmatter(utility.read_file(file_name))
if #text >= minimum_bytes and #text <= maximum_bytes then
new_files_list[#new_files_list + 1] = file_name
end
end end
end) end)
files = new_files_list
utility.save_data(new_files_list, "PRIVATE_DATA/file_list.json") file_list = new_files_list
utility.save_data(new_files_list, data_location)
end end
local generate_and_score = function(file_name, text) local generate_and_score = function(file_name, text)
@@ -114,23 +121,15 @@ elseif arg[1] == "export_ordered_list_of_prompts" then
os.exit(0) os.exit(0)
end end
print(#files .. " files to select from.") print(#file_list .. " files to select from.")
if #files == 0 then if #file_list == 0 then
print("Run \"./synopsis_generator.lua refresh_file_list\" first.") print("Run \"./synopsis_generator.lua refresh_file_list\" first.")
os.exit(1) os.exit(1)
end end
while true do while true do
print("Selecting a file...") local file_name = file_list[math.random(1, #file_list)]
local file_name = files[math.random(1, #files)] local text = utility.read_file(file_name)
local file_size = utility.file_size(file_name) print(file_name .. " chosen.")
if file_size > minimum_bytes and file_size <= maximum_bytes then generate_and_score(file_name, text)
local text = utility.read_file(file_name)
text = text_processing.strip_frontmatter(text)
if #text > minimum_bytes and #text <= maximum_bytes then
print(file_name .. " chosen.")
generate_and_score(file_name, text) -- kind of the main function, innit?
-- os.exit(0)
end
end
end end