mark additional compression done

This is a duplicate of the ReadMe portion of
addbb5e3d4
from 2026-08-31

ReadMe.md

@@ -58,13 +58,14 @@ generator to check accuracy.
 
 ## Tasks
 - [ ] Add timestamps to when a prompt is sent.
-- [ ] Specify ETA when sending a prompt (2.5-8 minutes depending on length). (Show as end time as well as relative time.)
+- [ ] Specify ETA when sending a prompt (use numbers from current estimator depending on length). (Show as end time as well as relative time.)
 - [ ] option to delete input files or intermediate files at end of each major operation segment (or after its finisher..)
-- [x] instead of placeholder directories, `mkdir -p` should be used to only make them when necessary
-- [x] make script resumable instead of having to restart per book
-- [x] dry run to estimate total time before running full script, print starting ETA
-- [x] Build a list of files to work on and current states before doing anything by looping over source directories
-- [x] Combine everything into one script to manage things
 - [ ] Allow specifying a filter on what to process
 - [ ] Allow changing the maximum bytes per prompt dynamically (run smaller while I'm doing other things, run full while I'm asleep)
-- [x] specify config format
+- [ ] `argparse` + ability to
+  - [ ] only convert ebooks or
+  - [ ] only generate cover briefs or
+  - [ ] only generate single pass cover briefs or
+  - [ ] list how many are left only
+- [x] I have an item with over 300 parts in it. Another level of compression needs to be added!
+- [ ] Change Gemini prompt because it is *obsessed* with putting colons EVERYWHERE.

lib/prompts.lua

@@ -61,10 +61,16 @@ prompts.cover_brief = [[Generate a cover brief from the following:
 
 ]]
 
-prompts.for_gemini = [[
+prompts.prior_gemini_prompt = [[
 
 Generate an ebook cover inset within an empty white border 10% larger than the cover using the following cover brief. It should be a flat graphic design file, with no artifacts of a physical object. The aspect ratio of an ebook is tall, not wide. The white border is very important, and should take up 10% of the area of the image. The title and author should only appear once on the cover. It must be sutiable for printing. Do not include the word "by" when adding the author's name to the cover.
 
 ]]
 
+prompts.for_gemini = [[
+
+Generate an ebook cover using the following cover brief. It should be a flat graphic design file, with no artifacts of a physical object. The aspect ratio of an ebook is tall, not wide. The title and author should only appear once on the cover. It must be sutiable for printing. Do not include the word "by" when adding the author's name to the cover. If an underscore is present in the title, that is an encoding mistake, it should be a colon. The author is separated from the title by a hyphen, the last hyphen on the title line.
+
+]]
+
 return prompts

make_cover_briefs.lua

@@ -173,6 +173,45 @@ local convert_ebooks = function(books)
   end
 end
 
+-- TODO deal with the fact that this function is a clone with only a couple line changes
+local compress_it_again = function(name, data, sections)
+  local condensed_sections = {}
+
+  while #sections > 2 do -- this intentionally can skip the end of a book?
+    local sections_slice = {}
+    for i = 1, 10 do
+      if #sections >= 1 then
+        sections_slice[#sections_slice + 1] = table.remove(sections, 1)
+      end
+    end
+
+    local intermediate_condensation_file = "intermediates/" .. name .. "-" .. (#condensed_sections + 1) .. "-super-condensed.txt"
+    if utility.path_exists(intermediate_condensation_file) then
+      condensed_sections[#condensed_sections + 1] = read_all(intermediate_condensation_file)
+      print("Loaded condensed summary segment. " .. #sections .. " sections remaining.")
+    else
+      print("Condensing summary. " .. #sections .. " sections remaining.")
+      local text = send_prompt(prompts.combine_summaries .. table.concat(sections_slice, "\n\n"))
+      condensed_sections[#condensed_sections + 1] = text
+      print(#text .. " characters added to final context.")
+      write_all(intermediate_condensation_file, text)
+    end
+  end
+
+  text = table.concat(condensed_sections, "\n\n")
+  local condensed_intermediate_file = "intermediates/" .. name .. " SUPER CONDENSED.txt"
+  write_all(condensed_intermediate_file, text)
+  data.condensed_intermediate_file = condensed_intermediate_file
+
+  -- if deleted any sooner, could break resuming
+  for i = 1, #condensed_sections do
+    local intermediate_condensation_file = "intermediates/" .. name .. "-" .. i .. "-super-condensed.txt"
+    os.execute("rm " .. intermediate_condensation_file:enquote())
+  end
+
+  return text
+end
+
 local condense_summary = function(name, data, sections)
   local condensed_sections = {}
 
@@ -202,6 +241,10 @@ local condense_summary = function(name, data, sections)
   write_all(condensed_intermediate_file, text)
   data.condensed_intermediate_file = condensed_intermediate_file
 
+  if #condensed_sections > 15 then
+    text = compress_it_again(name, data, condensed_sections)
+  end
+
   -- if deleted any sooner, could break resuming
   for i = 1, #condensed_sections do
     local intermediate_condensation_file = "intermediates/" .. name .. "-" .. i .. "-condensed.txt"
@@ -279,6 +322,7 @@ local estimate_time_required = function(books)
   for name, data in pairs(books) do
     local function loop()
       if data.cover_brief_file then return end
+      if not data.extracted_file then return end
       sum = sum + 60 * (1.5 + data.extracted_file_size/(config.llm_covers.maximum_bytes/6))
       count = count + 1
     end