make_cover_briefs.lua

@@ -170,17 +170,10 @@ tree = function(path, fn)
   end)
 end
 
-tree("extracted_texts", function(file_name)
-  local path, name, extension = utility.split_path_components(file_name)
-  if utility.path_exists("cover_briefs/" .. name) then return end
-
-  local text = read_all(file_name)
+local process_intermediates = function(text, name)
   local result
-
-  -- TODO there is no way to recognize a dangling intermediate and pickup from there
-  print(name)
   local outputs = {}
-  if #text > maximum_bytes then
+
   -- TODO check if this would work just as well starting from 1 instead..
   for i = 0, #text/maximum_bytes do
     print("Processing section " .. (i + 1) .. " of " .. math.floor(#text/maximum_bytes + 1))
@@ -188,7 +181,7 @@ tree("extracted_texts", function(file_name)
     local intermediate_name = "intermediates/" .. name:sub(1, -5) .. "-" .. tostring(#outputs + 1) .. ".txt"
     if utility.path_exists(intermediate_name) then
       result = read_all(intermediate_name)
-        print("Loaded from intermediates.")
+      print("Loaded from file.")
     else
       local piece = text:sub(i * maximum_bytes, (i + 1) * maximum_bytes - 1)
       result = send_prompt(partial_prompt .. piece)
@@ -201,15 +194,38 @@ tree("extracted_texts", function(file_name)
 
   text = table.concat(outputs, "\n\n")
   write_all("intermediates/" .. name, text)
-    print("Saved full intermediate prompt.")
+  print("Saved primary intermediate prompt.")
 
   for i = 0, #text/maximum_bytes do
     local intermediate_name = "intermediates/" .. name:sub(1, -5) .. "-" .. tostring(#outputs + 1) .. ".txt"
     if utility.path_exists(intermediate_name) then
       os.execute("rm " .. intermediate_name:enquote())
     end
-      print("Removed excess intermediate texts.")
   end
+  print("Removed redundant intermediate prompts.")
+
+  return result, outputs, text
+end
+
+tree("extracted_texts", function(file_name)
+  local path, name, extension = utility.split_path_components(file_name)
+  if utility.path_exists("cover_briefs/" .. name) then return end
+
+  local text = read_all(file_name)
+  local result
+
+  -- TODO there is no way to recognize a dangling intermediate and pickup from there
+  print(name)
+  local outputs
+
+  -- restore previous intermediate
+  -- (this is where I realized I should redo how I'm processing all of this)
+  if utility.path_exists("intermediates/" .. name) then
+  end
+
+  if #text > maximum_bytes then
+    -- previously overwrites result, text, outputs
+    result, outputs, text = process_intermediates(text, name)
   end
 
   -- handle too much context (works with up to 100 slices)