debug lines, timing improved/corrected

lib/timing.lua

@@ -1,18 +1,26 @@
 local timing = {}
 
+local function human_readable_time(delta)
+  if delta >= 2*7*24*60*60 then -- if more than 2 weeks
+    delta = tostring(math.floor(delta/(7*24*60*60))/10) .. " weeks"
+  elseif delta >= 2*24*60*60 then -- if more than 2 days
+    delta = tostring(math.floor(delta/(24*60*60))/10) .. " days"
+  elseif delta >= 2*60*60 then -- if more than 2 hours
+    delta = tostring(math.floor(delta/(60*60/10))/10) .. " hours"
+  elseif delta >= 2*60 then -- if more then 2 minutes
+    delta = tostring(math.floor(delta/(60/10))/10) .. " minutes"
+  else
+    delta = tostring(delta) .. " seconds"
+  end
+  return delta
+end
+
 timing.display = function(n)
   local function _display(n)
     local time = timing[n]
     local previous = timing[n - 1]
 
-    local delta = time.time - previous.time
-    if delta >= 2*60*60 then -- if more than 2 hours
-      delta = tostring(math.floor(delta/(60*60/10))/10) .. " hours"
-    elseif delta >= 120 then -- if more then 2 minutes
-      delta = tostring(math.floor(delta/(60/10))/10) .. " minutes"
-    else
-      delta = tostring(delta) .. " seconds"
-    end
+    local delta = human_readable_time(time.time - previous.time)
 
     print(delta, previous.label)
   end
@@ -34,4 +42,10 @@ timing.mark = function(label)
   end
 end
 
+timing.estimate = function(current_position, total_operations)
+  local delta = os.time() - timing[#timing].time
+  local estimate = delta * total_operations / current_position - delta
+  return human_readable_time(estimate)
+end
+
 return timing

refresh_sources.lua

@@ -107,10 +107,17 @@ local process_file = function(data_source, file_name)
 
   local new_embeddings = {}
   for i = 1, #chunks do
+    print("Embedding length:", #chunks[i])
     new_embeddings[i] = generate_embeddings(chunks[i]) or {}
   end
 
   if #new_embeddings[1] == 0 then
+    print(file_name)
+    print("Vector lengths:")
+    for e = 1, #new_embeddings do
+      print("", e, #new_embeddings[e])
+    end
+
     -- average all embeddings to make the core file embedding
     local count = #new_embeddings[2]
     for vector_index = 1, count do
@@ -164,7 +171,9 @@ local refresh_sources = function()
         embeddings.files[file_name] = file_sums
       end
       loop()
-      print("Finished " .. utility.leftpad(f, #tostring(#file_list), "0") .. "/" .. #file_list .. " (" .. utility.leftpad(math.floor(f / #file_list * 100), 3, "0") .. "%)")
+      print("Finished " .. utility.leftpad(f, #tostring(#file_list), "0") .. "/" .. #file_list
+        .. " (" .. utility.leftpad(math.floor(f / #file_list * 100), 3, "0")
+        .. "%) ETA: " .. timing.estimate(f, #file_list))
     end
 
     timing.mark("Finished generating embeddings for " .. source_name .. ".")