timings measured, larger model

generate_embeddings.lua

@@ -7,8 +7,47 @@ local json = utility.require("dkjson")
 local PATH = "PRIVATE_DATA/notebook"
 local whitelist = { md = true, }
 
-local embedding_model = "nomic-embed-text"
-local maximum_file_size = 2048
+-- local embedding_model = "nomic-embed-text"
+-- local maximum_file_size = 2048
+local embedding_model = "qwen3-embedding:0.6b"
+local maximum_file_size = 32768
+
+
+
+local timing = {}
+local function display_timing(n)
+  local function _display(n)
+    local time = timing[n]
+    local previous = timing[n - 1]
+
+    local delta = time.time - previous.time
+    if delta >= 2*60*60 then -- 2 hours
+      delta = tostring(math.floor(delta/(60*60/10))/10) .. " hours"
+    elseif delta >= 120 then -- 2 minutes
+      delta = tostring(math.floor(delta/(60/10))/10) .. " minutes"
+    else
+      delta = tostring(delta) .. " seconds"
+    end
+
+    print(delta, previous.label)
+  end
+
+  if n then
+    _display(n)
+  else
+    print("All measured timings:")
+    for i = 2, #timing do
+      _display(i)
+    end
+  end
+end
+
+local function mark_timing(label)
+  timing[#timing + 1] = { label = label, time = os.time(), }
+  if #timing > 1 then
+    display_timing(#timing)
+  end
+end
 
 -- strip YAML frontmatter (if present)
 --   can error, will return nil & error message
@@ -32,6 +71,7 @@ end
 local tree
 tree = function(path, fn)
   utility.list(path or ".", function(path_name)
+    if path_name == ".git" then return end
     if utility.is_file(path_name) then
       fn(path_name)
     else
@@ -49,6 +89,7 @@ end
 local file_list = {}
 local embeddings = {}
 
+mark_timing("Assembling file list.")
 tree(PATH, function(file_name)
   local path, name, extension = utility.split_path_components(file_name)
   if not extension then
@@ -62,6 +103,7 @@ tree(PATH, function(file_name)
   file_list[#file_list + 1] = file_name
 end)
 
+mark_timing("Generating embeddings.")
 for i = 1, #file_list do
   local function _run()
     local file_name = file_list[i]
@@ -115,8 +157,13 @@ for i = 1, #file_list do
   _run()
 end
 
+mark_timing("Outputting embeddings in JSON.")
 utility.open("PRIVATE_DATA/embeddings.json", "w", function(file)
   local output = json.encode(embeddings, { indent = true })
   file:write(output)
   file:write("\n")
 end)
+
+mark_timing("Finished.")
+print("")
+display_timing()