diff --git a/lib/log.lua b/lib/log.lua index fae2b37..5bc89f8 100644 --- a/lib/log.lua +++ b/lib/log.lua @@ -1,11 +1,8 @@ -local stored_messages = {} -local display_levels = {} -local enable_logging = true -local message_limit = math.huge - -- USAGE: -- -- set arbitrary log levels to be printed immediately -- log{ info = true, warning = true, ducksauce = true, } +-- -- disable storage of messages depending on log level +-- log{ error = false, verbose = false, } -- -- send anything (except nil) to a log level -- log("bacom", true, "text", 5, function() end, {}) -- -- print all messages saved at a particular log level @@ -14,62 +11,72 @@ local message_limit = math.huge -- log(false) -- this deletes all stored messages -- -- set a maximum number of stored messages per log level -- log(50) -- warning: reaching large limits makes logging expensive +-- -- check if a log level is being displayed +-- if log.arbitrary_exit then log("arbitrary_exit") os.exit(1) end -- -- do anything to the stored messages (this example deletes them all) -- log(function(messages) return {} end) -return function(options, ...) - local options_type = type(options) +local microlog = {} -- display_levels are stored directly here +local stored_messages = {} +local enable_logging = true +local message_limit = math.huge - if (options_type == "string") and enable_logging then - if not stored_messages[options] then stored_messages[options] = {} end - local message_table = stored_messages[options] +return setmetatable(microlog, { + __call = function(options, ...) + local options_type = type(options) - -- turn log message into text - local tab = {...} - for i = 1, #tab do - tab[i] = tostring(tab[i]) - end - local current_message = table.concat(tab, "\t") + if (options_type == "string") and enable_logging then + if microlog[options] == false then return end + if not stored_messages[options] then stored_messages[options] = {} end + local message_table = stored_messages[options] - if #current_message == 0 then - -- print all stored_messages of specified level - for i = 1, #message_table do - print(message_table[i]) + -- turn log message into text + local tab = {...} + for i = 1, #tab do + tab[i] = tostring(tab[i]) + end + local current_message = table.concat(tab, "\t") + + if #current_message == 0 then + -- print all stored_messages of specified level + for i = 1, #message_table do + print(message_table[i]) + end + + else + -- store message (and print if in display_levels) + message_table[#message_table + 1] = current_message + if #message_table > message_limit then table.remove(message_table, 1) end + if microlog[options] then print(current_message) end end - else - -- store message (and print if in display_levels) - message_table[#message_table + 1] = current_message - if #message_table > message_limit then table.remove(message_table, 1) end - if display_levels[options] then print(current_message) end - end - - -- set display_levels - elseif options_type == "table" then - for k,v in pairs(options) do - display_levels[k] = v - end - - elseif options_type == "number" then - message_limit = options - for k,v in pairs(stored_messages) do - while #v > message_limit do - table.remove(v, 1) + -- set display_levels + elseif options_type == "table" then + for k,v in pairs(options) do + microlog[k] = v end - end - -- turn all logging off and delete stored messages; or turn it on - elseif options_type == "boolean" then - if options then - enable_logging = true - else - enable_logging = false - stored_messages = {} - end + elseif options_type == "number" then + message_limit = options + for k,v in pairs(stored_messages) do + while #v > message_limit do + table.remove(v, 1) + end + end - -- arbitrary access :D - elseif options_type == "function" then - local result = options(stored_messages) - if type(result) == "table" then stored_messages = result end + -- turn all logging off and delete stored messages; or turn it on + elseif options_type == "boolean" then + if options then + enable_logging = true + else + enable_logging = false + stored_messages = {} + end + + -- arbitrary access :D + elseif options_type == "function" then + local result = options(stored_messages) + if type(result) == "table" then stored_messages = result end + end end -end +}) diff --git a/refresh_sources.lua b/refresh_sources.lua index ab88a9d..4a8129e 100755 --- a/refresh_sources.lua +++ b/refresh_sources.lua @@ -25,7 +25,7 @@ end log{ info = true, warning = true, - -- debug = true, + debug = false, -- files = true, -- debugging why the wrong files are selected -- sha = true, -- what the fuck is going on with sha sums? } @@ -43,7 +43,7 @@ if not utility.path_exists(embeddings_file_path) then }, embeddings_file_path) end embeddings = utility.load_data(embeddings_file_path) -local function embeddings_debug() +if log.debug then log("debug", "Embeddings loaded.", embeddings, embeddings.files, embeddings.vectors) local file_count = 0 for k,v in pairs(embeddings.files) do @@ -57,7 +57,6 @@ local function embeddings_debug() log("debug", vector_count .. " vectors.") -- os.exit(1) end -embeddings_debug() @@ -93,12 +92,15 @@ local refresh_file_list = function(source_name, data_source) end -- returns nothing when too much text is sent -local generate_embeddings = function(text) - if #text > config.models.embedding.max_chunk_size then +local generate_embeddings = function(data_source, text) + local max_chunk_size = data_source.max_chunk_size or config.models.embedding.max_chunk_size + local model = data_source.embedding_model or config.models.embedding.model + + if #text > max_chunk_size then return nil, "generate_embeddings() must only be passed appropriately-sized chunks!" end - local result = utility.llm_prompt(text, config.models.embedding.model) + local result = utility.llm_prompt(text, model) return json.decode(result) end @@ -115,7 +117,7 @@ local process_file = function(data_source, file_name) return end - local chunk_size = config.models.embedding.max_chunk_size + local chunk_size = data_source.max_chunk_size or config.models.embedding.max_chunk_size local half_chunk_size = math.floor(chunk_size / 2) local chunks = { text } @@ -136,13 +138,15 @@ local process_file = function(data_source, file_name) local new_embeddings = {} for i = 1, #chunks do log("debug", "Embedding length:", #chunks[i]) - new_embeddings[i] = generate_embeddings(chunks[i]) or {} + new_embeddings[i] = generate_embeddings(data_source, chunks[i]) or {} end if #new_embeddings[1] == 0 then - log("debug", "Vector lengths:") - for e = 1, #new_embeddings do - log("debug", "", e, #new_embeddings[e]) + if log.debug then + log("debug", "Vector lengths:") + for e = 1, #new_embeddings do + log("debug", "", e, #new_embeddings[e]) + end end if #new_embeddings == 1 then -- Ollama very rarely errors with: