diff --git a/ReadMe.md b/ReadMe.md index 6804921..4ba1471 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -42,6 +42,21 @@ while `refresh_command` is run each time (optional). Commands will be run in the specified `path`. `strip_frontmatter` will remove YAML frontmatter (common in Markdown files). +The embeddings are stored like so: + +```json +{ + "files":{ + "PRIVATE_DATA/source_path/path/to/file.ext":["sha512sum", "another sum"] + }, + "vectors":{ + "sha512sum":[0.5, 0, 1, -0.5, -1, ...] + } +} +``` + +TODO + ### `synopsis_generator.lua` Chooses a random file within `notebook`, and generates a novel synopsis from it. diff --git a/lib/log.lua b/lib/log.lua new file mode 100644 index 0000000..fae2b37 --- /dev/null +++ b/lib/log.lua @@ -0,0 +1,75 @@ +local stored_messages = {} +local display_levels = {} +local enable_logging = true +local message_limit = math.huge + +-- USAGE: +-- -- set arbitrary log levels to be printed immediately +-- log{ info = true, warning = true, ducksauce = true, } +-- -- send anything (except nil) to a log level +-- log("bacom", true, "text", 5, function() end, {}) +-- -- print all messages saved at a particular log level +-- log("error") +-- -- toggle ALL logging on and off with a boolean +-- log(false) -- this deletes all stored messages +-- -- set a maximum number of stored messages per log level +-- log(50) -- warning: reaching large limits makes logging expensive +-- -- do anything to the stored messages (this example deletes them all) +-- log(function(messages) return {} end) + +return function(options, ...) + local options_type = type(options) + + if (options_type == "string") and enable_logging then + if not stored_messages[options] then stored_messages[options] = {} end + local message_table = stored_messages[options] + + -- turn log message into text + local tab = {...} + for i = 1, #tab do + tab[i] = tostring(tab[i]) + end + local current_message = table.concat(tab, "\t") + + if #current_message == 0 then + -- print all stored_messages of specified level + for i = 1, #message_table do + print(message_table[i]) + end + + else + -- store message (and print if in display_levels) + message_table[#message_table + 1] = current_message + if #message_table > message_limit then table.remove(message_table, 1) end + if display_levels[options] then print(current_message) end + end + + -- set display_levels + elseif options_type == "table" then + for k,v in pairs(options) do + display_levels[k] = v + end + + elseif options_type == "number" then + message_limit = options + for k,v in pairs(stored_messages) do + while #v > message_limit do + table.remove(v, 1) + end + end + + -- turn all logging off and delete stored messages; or turn it on + elseif options_type == "boolean" then + if options then + enable_logging = true + else + enable_logging = false + stored_messages = {} + end + + -- arbitrary access :D + elseif options_type == "function" then + local result = options(stored_messages) + if type(result) == "table" then stored_messages = result end + end +end diff --git a/lib/timing.lua b/lib/timing.lua index fb09794..7c68b78 100644 --- a/lib/timing.lua +++ b/lib/timing.lua @@ -40,6 +40,7 @@ timing.mark = function(label) if #timing > 1 then timing.display(#timing) end + print("", "", label) end timing.estimate = function(current_position, total_operations) diff --git a/lib/utility.lua b/lib/utility.lua index 63e5e9f..73e4afe 100644 --- a/lib/utility.lua +++ b/lib/utility.lua @@ -221,7 +221,7 @@ utility.list = function(path, func) local run = function(fn) for line in output:gmatch("[^\r\n]+") do -- thanks to https://stackoverflow.com/a/32847589 - if not (line == "." or line == "..") then + if not ((line == ".") or (line == "..")) then fn(line) end end @@ -239,7 +239,8 @@ utility.ls = function(...) return utility.list(...) end -utility.tree = function(path, options, fn) +local tree +tree = function(path, options, fn) if type(options) == "function" then fn = options options = {} @@ -258,7 +259,16 @@ utility.tree = function(path, options, fn) fn(path_name) else - utility.tree(path .. utility.path_separator .. path_name, options, fn) + tree(path .. utility.path_separator .. path_name, options, fn) + end + end) +end +utility.tree = function(path, options, fn) + tree(path, options, function(path_name) + if path_name:find(path) == 1 then + fn(path_name) + else + fn(path .. utility.path_separator .. path_name) end end) end diff --git a/refresh_sources.lua b/refresh_sources.lua index 3d4fdf0..5b235a1 100755 --- a/refresh_sources.lua +++ b/refresh_sources.lua @@ -4,6 +4,7 @@ package.path = (arg[0]:match("@?(.*/)") or arg[0]:match("@?(.*\\)")) .. "lib" .. local utility = require "utility" local json = utility.require("dkjson") +local log = utility.require("log") local text_processing = utility.require("text_processing") local timing = utility.require("timing") @@ -21,16 +22,42 @@ if not config.models then utility.save_config() end +log{ + -- info = true, + -- warning = true, + -- debug = true, + -- files = true, -- debugging why the wrong files are selected + sha = true, -- what the fuck is going on with sha sums? +} + +local memory_path = "PRIVATE_DATA" .. utility.path_separator .. "memory" +local embeddings_file_path = memory_path .. utility.path_separator .. "+embeddings.json" +local tmp_file_path = "PRIVATE_DATA" .. utility.path_separator .. ".tmp.2b65c19b-0883-49ca-8247-b1fe7760f922" + local embeddings -local embeddings_file_path = "PRIVATE_DATA/memory/+embeddings.json" if not utility.path_exists(embeddings_file_path) then - os.execute("mkdir -p PRIVATE_DATA/memory") + os.execute("mkdir -p " .. memory_path:enquote()) utility.save_data({ files = {}, vectors = {}, }, embeddings_file_path) end embeddings = utility.load_data(embeddings_file_path) +local function embeddings_debug() + log("debug", "Embeddings loaded.", embeddings, embeddings.files, embeddings.vectors) + local file_count = 0 + for k,v in pairs(embeddings.files) do + file_count = file_count + 1 + end + local vector_count = 0 + for k,v in pairs(embeddings.vectors) do + vector_count = vector_count + 1 + end + log("debug", file_count .. " files.") + log("debug", vector_count .. " vectors.") + -- os.exit(1) +end +embeddings_debug() @@ -57,6 +84,7 @@ local refresh_file_list = function(source_name, data_source) local file_list = {} utility.tree("PRIVATE_DATA" .. utility.path_separator .. data_source.path, compiled_filters, function(file_name) + log("files", file_name) file_list[#file_list + 1] = file_name end) @@ -74,7 +102,7 @@ local generate_embeddings = function(text) return json.decode(result) end --- returns nothing for empty files +-- returns nothing for empty files and errors local process_file = function(data_source, file_name) local text = utility.read_file(file_name) @@ -83,7 +111,7 @@ local process_file = function(data_source, file_name) end if #text == 0 then - print(file_name .. "\n is empty and being skipped.") + log("empty", file_name .. "\n is empty and being skipped.") return end @@ -107,21 +135,20 @@ local process_file = function(data_source, file_name) local new_embeddings = {} for i = 1, #chunks do - -- print("Embedding length:", #chunks[i]) + log("debug", "Embedding length:", #chunks[i]) new_embeddings[i] = generate_embeddings(chunks[i]) or {} end if #new_embeddings[1] == 0 then - -- print(file_name) - -- print("Vector lengths:") - -- for e = 1, #new_embeddings do - -- print("", e, #new_embeddings[e]) - -- end + log("debug", "Vector lengths:") + for e = 1, #new_embeddings do + log("debug", "", e, #new_embeddings[e]) + end if #new_embeddings == 1 then -- Ollama very rarely errors with: -- Error: do embedding request: Post "http://127.0.0.1:53441/v1/embeddings": EOF - -- but it is inconsistent and re-running will eventually fix it. (See issue #7 if present.) - print(file_name .. "\n encountered an embedding error and will be skipped this run only.") + -- but it is inconsistent and re-running will eventually fix it. + log("warning", file_name .. "\n encountered an embedding error and will be skipped this run only.") return end @@ -139,57 +166,87 @@ local process_file = function(data_source, file_name) return chunks, new_embeddings end +-- returns nothing for errors +local memorize_file = function(data_source, file_name) + local file_chunks, file_embeddings = process_file(data_source, file_name) + if not file_chunks then return end + + local file_sums = {} + for i = 1, #file_chunks do + local function loop() + local text = file_chunks[i] + local current_embedding = file_embeddings[i] + + utility.write_file(tmp_file_path, text) + + local sha512sum = utility.sha512sum(tmp_file_path) + log("sha", "New sum? " .. sha512sum, tostring(embeddings.vectors[sha512sum])) + file_sums[#file_sums + 1] = sha512sum + if embeddings.vectors[sha512sum] then + log("sha", "Detected previously extant sum, skipping.") + return + end + + os.execute(utility.commands.move .. tmp_file_path:enquote() + .. " " .. (memory_path .. utility.path_separator .. sha512sum):enquote()) + embeddings.vectors[sha512sum] = current_embedding + log("sha", "Saved new sum!") + end + loop() + end + + return file_sums +end + local refresh_sources = function() - local tmp_file_path = "PRIVATE_DATA/.tmp.2b65c19b-0883-49ca-8247-b1fe7760f922" - os.execute("mkdir -p PRIVATE_DATA/memory") + os.execute("mkdir -p " .. memory_path:enquote()) + + timing.mark("Checking memory for missing embeddings.") + utility.list(memory_path, function(file_name) + log("files", file_name) + if (file_name == ".DS_Store") or (file_name == "+embeddings.json") then return end + + local file_path = memory_path .. utility.path_separator .. file_name + local sha512sum = utility.sha512sum(file_path) + log("sha", "New sum? " .. sha512sum, tostring(embeddings.vectors[sha512sum])) + if not embeddings.vectors[sha512sum] then + log("sha", "Saved new sum!") + memorize_file({}, file_path) + end + end) + timing.mark("Finished checking memory for missing embeddings.") local sources = utility.load_data("PRIVATE_DATA/sources.json") for source_name, data_source in pairs(sources) do local file_list = refresh_file_list(source_name, data_source) - timing.mark("Generating embeddings for " .. source_name .. ".") + timing.mark("Generating embeddings from source \"" .. source_name .. "\"") for f = 1, #file_list do local file_name = file_list[f] local function loop() local sha512sum = utility.sha512sum(file_name) + log("sha", file_name, sha512sum, tostring(embeddings.vectors[sha512sum])) if embeddings.vectors[sha512sum] then + log("debug", file_name .. "\n has already been embedded, skipping.") -- add file reference if it was missing if not embeddings.files[file_name] then - embeddings.files[file_name] = sha512sum + embeddings.files[file_name] = { sha512sum } end return end - local file_chunks, file_embeddings = process_file(data_source, file_name) - if not file_chunks then return end - - local file_sums = {} - for i = 1, #file_chunks do - local function loop() - local text = file_chunks[i] - local current_embedding = file_embeddings[i] - - utility.write_file(tmp_file_path, text) - - sha512sum = utility.sha512sum(tmp_file_path) - file_sums[#file_sums + 1] = sha512sum - if embeddings.vectors[sha512sum] then return end - - os.execute(utility.commands.move .. tmp_file_path:enquote() .. " " .. "PRIVATE_DATA/memory/" .. sha512sum) - embeddings.vectors[sha512sum] = current_embedding - end - loop() - end - + log("debug", "Generating embeddings for " .. file_name .. ".") + local file_sums = memorize_file(data_source, file_name) embeddings.files[file_name] = file_sums end loop() - print("Finished " .. utility.leftpad(f, #tostring(#file_list), "0") .. "/" .. #file_list + log("info", "Finished " .. utility.leftpad(f, #tostring(#file_list), "0") + .. "/" .. #file_list .. " (" .. utility.leftpad(math.floor(f / #file_list * 100), 3, "0") .. "%) ETA: " .. timing.estimate(f, #file_list)) end - timing.mark("Finished generating embeddings for " .. source_name .. ".") + timing.mark("Finished generating embeddings from source \"" .. source_name .. "\"") end utility.save_data(embeddings) @@ -202,3 +259,4 @@ refresh_sources() timing.mark("Finished.") print("") timing.display() +log("warning")