tools/llm-tricks
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

very WIP, doesn't detect duplicates properly.. everything else seems fine?

Browse Source
This commit is contained in:
tangent
2026-09-16 16:03:14 -06:00
parent 977920df61
commit 470901d9b9
5 changed files with 201 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
refresh_sources.lua
+97 -39
View File
@@ -4,6 +4,7 @@ package.path = (arg[0]:match("@?(.*/)") or arg[0]:match("@?(.*\\)")) .. "lib" ..
local utility = require "utility"
local json = utility.require("dkjson")
local log = utility.require("log")
local text_processing = utility.require("text_processing")
local timing = utility.require("timing")
@@ -21,16 +22,42 @@ if not config.models then
utility.save_config()
end
log{
-- info = true,
-- warning = true,
-- debug = true,
-- files = true, -- debugging why the wrong files are selected
sha = true, -- what the fuck is going on with sha sums?
}
local memory_path = "PRIVATE_DATA" .. utility.path_separator .. "memory"
local embeddings_file_path = memory_path .. utility.path_separator .. "+embeddings.json"
local tmp_file_path = "PRIVATE_DATA" .. utility.path_separator .. ".tmp.2b65c19b-0883-49ca-8247-b1fe7760f922"
local embeddings
local embeddings_file_path = "PRIVATE_DATA/memory/+embeddings.json"
if not utility.path_exists(embeddings_file_path) then
os.execute("mkdir -p PRIVATE_DATA/memory")
os.execute("mkdir -p " .. memory_path:enquote())
utility.save_data({
files = {},
vectors = {},
}, embeddings_file_path)
end
embeddings = utility.load_data(embeddings_file_path)
local function embeddings_debug()
log("debug", "Embeddings loaded.", embeddings, embeddings.files, embeddings.vectors)
local file_count = 0
for k,v in pairs(embeddings.files) do
file_count = file_count + 1
end
local vector_count = 0
for k,v in pairs(embeddings.vectors) do
vector_count = vector_count + 1
end
log("debug", file_count .. " files.")
log("debug", vector_count .. " vectors.")
-- os.exit(1)
end
embeddings_debug()
@@ -57,6 +84,7 @@ local refresh_file_list = function(source_name, data_source)
local file_list = {}
utility.tree("PRIVATE_DATA" .. utility.path_separator .. data_source.path, compiled_filters, function(file_name)
log("files", file_name)
file_list[#file_list + 1] = file_name
end)
@@ -74,7 +102,7 @@ local generate_embeddings = function(text)
return json.decode(result)
end
-- returns nothing for empty files
-- returns nothing for empty files and errors
local process_file = function(data_source, file_name)
local text = utility.read_file(file_name)
@@ -83,7 +111,7 @@ local process_file = function(data_source, file_name)
end
if #text == 0 then
print(file_name .. "\n is empty and being skipped.")
log("empty", file_name .. "\n is empty and being skipped.")
return
end
@@ -107,21 +135,20 @@ local process_file = function(data_source, file_name)
local new_embeddings = {}
for i = 1, #chunks do
-- print("Embedding length:", #chunks[i])
log("debug", "Embedding length:", #chunks[i])
new_embeddings[i] = generate_embeddings(chunks[i]) or {}
end
if #new_embeddings[1] == 0 then
-- print(file_name)
-- print("Vector lengths:")
-- for e = 1, #new_embeddings do
-- print("", e, #new_embeddings[e])
-- end
log("debug", "Vector lengths:")
for e = 1, #new_embeddings do
log("debug", "", e, #new_embeddings[e])
end
if #new_embeddings == 1 then
-- Ollama very rarely errors with:
-- Error: do embedding request: Post "http://127.0.0.1:53441/v1/embeddings": EOF
-- but it is inconsistent and re-running will eventually fix it. (See issue #7 if present.)
print(file_name .. "\n encountered an embedding error and will be skipped this run only.")
-- but it is inconsistent and re-running will eventually fix it.
log("warning", file_name .. "\n encountered an embedding error and will be skipped this run only.")
return
end
@@ -139,57 +166,87 @@ local process_file = function(data_source, file_name)
return chunks, new_embeddings
end
-- returns nothing for errors
local memorize_file = function(data_source, file_name)
local file_chunks, file_embeddings = process_file(data_source, file_name)
if not file_chunks then return end
local file_sums = {}
for i = 1, #file_chunks do
local function loop()
local text = file_chunks[i]
local current_embedding = file_embeddings[i]
utility.write_file(tmp_file_path, text)
local sha512sum = utility.sha512sum(tmp_file_path)
log("sha", "New sum? " .. sha512sum, tostring(embeddings.vectors[sha512sum]))
file_sums[#file_sums + 1] = sha512sum
if embeddings.vectors[sha512sum] then
log("sha", "Detected previously extant sum, skipping.")
return
end
os.execute(utility.commands.move .. tmp_file_path:enquote()
.. " " .. (memory_path .. utility.path_separator .. sha512sum):enquote())
embeddings.vectors[sha512sum] = current_embedding
log("sha", "Saved new sum!")
end
loop()
end
return file_sums
end
local refresh_sources = function()
local tmp_file_path = "PRIVATE_DATA/.tmp.2b65c19b-0883-49ca-8247-b1fe7760f922"
os.execute("mkdir -p PRIVATE_DATA/memory")
os.execute("mkdir -p " .. memory_path:enquote())
timing.mark("Checking memory for missing embeddings.")
utility.list(memory_path, function(file_name)
log("files", file_name)
if (file_name == ".DS_Store") or (file_name == "+embeddings.json") then return end
local file_path = memory_path .. utility.path_separator .. file_name
local sha512sum = utility.sha512sum(file_path)
log("sha", "New sum? " .. sha512sum, tostring(embeddings.vectors[sha512sum]))
if not embeddings.vectors[sha512sum] then
log("sha", "Saved new sum!")
memorize_file({}, file_path)
end
end)
timing.mark("Finished checking memory for missing embeddings.")
local sources = utility.load_data("PRIVATE_DATA/sources.json")
for source_name, data_source in pairs(sources) do
local file_list = refresh_file_list(source_name, data_source)
timing.mark("Generating embeddings for " .. source_name .. ".")
timing.mark("Generating embeddings from source \"" .. source_name .. "\"")
for f = 1, #file_list do
local file_name = file_list[f]
local function loop()
local sha512sum = utility.sha512sum(file_name)
log("sha", file_name, sha512sum, tostring(embeddings.vectors[sha512sum]))
if embeddings.vectors[sha512sum] then
log("debug", file_name .. "\n has already been embedded, skipping.")
-- add file reference if it was missing
if not embeddings.files[file_name] then
embeddings.files[file_name] = sha512sum
embeddings.files[file_name] = { sha512sum }
end
return
end
local file_chunks, file_embeddings = process_file(data_source, file_name)
if not file_chunks then return end
local file_sums = {}
for i = 1, #file_chunks do
local function loop()
local text = file_chunks[i]
local current_embedding = file_embeddings[i]
utility.write_file(tmp_file_path, text)
sha512sum = utility.sha512sum(tmp_file_path)
file_sums[#file_sums + 1] = sha512sum
if embeddings.vectors[sha512sum] then return end
os.execute(utility.commands.move .. tmp_file_path:enquote() .. " " .. "PRIVATE_DATA/memory/" .. sha512sum)
embeddings.vectors[sha512sum] = current_embedding
end
loop()
end
log("debug", "Generating embeddings for " .. file_name .. ".")
local file_sums = memorize_file(data_source, file_name)
embeddings.files[file_name] = file_sums
end
loop()
print("Finished " .. utility.leftpad(f, #tostring(#file_list), "0") .. "/" .. #file_list
log("info", "Finished " .. utility.leftpad(f, #tostring(#file_list), "0")
.. "/" .. #file_list
.. " (" .. utility.leftpad(math.floor(f / #file_list * 100), 3, "0")
.. "%) ETA: " .. timing.estimate(f, #file_list))
end
timing.mark("Finished generating embeddings for " .. source_name .. ".")
timing.mark("Finished generating embeddings from source \"" .. source_name .. "\"")
end
utility.save_data(embeddings)
@@ -202,3 +259,4 @@ refresh_sources()
timing.mark("Finished.")
print("")
timing.display()
log("warning")