diff --git a/generate_embeddings.lua b/generate_embeddings.lua index 4087707..d242090 100755 --- a/generate_embeddings.lua +++ b/generate_embeddings.lua @@ -7,8 +7,47 @@ local json = utility.require("dkjson") local PATH = "PRIVATE_DATA/notebook" local whitelist = { md = true, } -local embedding_model = "nomic-embed-text" -local maximum_file_size = 2048 +-- local embedding_model = "nomic-embed-text" +-- local maximum_file_size = 2048 +local embedding_model = "qwen3-embedding:0.6b" +local maximum_file_size = 32768 + + + +local timing = {} +local function display_timing(n) + local function _display(n) + local time = timing[n] + local previous = timing[n - 1] + + local delta = time.time - previous.time + if delta >= 2*60*60 then -- 2 hours + delta = tostring(math.floor(delta/(60*60/10))/10) .. " hours" + elseif delta >= 120 then -- 2 minutes + delta = tostring(math.floor(delta/(60/10))/10) .. " minutes" + else + delta = tostring(delta) .. " seconds" + end + + print(delta, previous.label) + end + + if n then + _display(n) + else + print("All measured timings:") + for i = 2, #timing do + _display(i) + end + end +end + +local function mark_timing(label) + timing[#timing + 1] = { label = label, time = os.time(), } + if #timing > 1 then + display_timing(#timing) + end +end -- strip YAML frontmatter (if present) -- can error, will return nil & error message @@ -32,6 +71,7 @@ end local tree tree = function(path, fn) utility.list(path or ".", function(path_name) + if path_name == ".git" then return end if utility.is_file(path_name) then fn(path_name) else @@ -49,6 +89,7 @@ end local file_list = {} local embeddings = {} +mark_timing("Assembling file list.") tree(PATH, function(file_name) local path, name, extension = utility.split_path_components(file_name) if not extension then @@ -62,6 +103,7 @@ tree(PATH, function(file_name) file_list[#file_list + 1] = file_name end) +mark_timing("Generating embeddings.") for i = 1, #file_list do local function _run() local file_name = file_list[i] @@ -115,8 +157,13 @@ for i = 1, #file_list do _run() end +mark_timing("Outputting embeddings in JSON.") utility.open("PRIVATE_DATA/embeddings.json", "w", function(file) local output = json.encode(embeddings, { indent = true }) file:write(output) file:write("\n") end) + +mark_timing("Finished.") +print("") +display_timing()