better logging

lib/log.lua

@@ -1,11 +1,8 @@
-local stored_messages = {}
-local display_levels = {}
-local enable_logging = true
-local message_limit = math.huge
-
 -- USAGE:
 --   -- set arbitrary log levels to be printed immediately
 --   log{ info = true, warning = true, ducksauce = true, }
+--   -- disable storage of messages depending on log level
+--   log{ error = false, verbose = false, }
 --   -- send anything (except nil) to a log level
 --   log("bacom", true, "text", 5, function() end, {})
 --   -- print all messages saved at a particular log level
@@ -14,13 +11,22 @@ local message_limit = math.huge
 --   log(false) -- this deletes all stored messages
 --   -- set a maximum number of stored messages per log level
 --   log(50) -- warning: reaching large limits makes logging expensive
+--  -- check if a log level is being displayed
+--  if log.arbitrary_exit then log("arbitrary_exit") os.exit(1) end
 --   -- do anything to the stored messages (this example deletes them all)
 --  log(function(messages) return {} end)
 
-return function(options, ...)
+local microlog = {} -- display_levels are stored directly here
+local stored_messages = {}
+local enable_logging = true
+local message_limit = math.huge
+
+return setmetatable(microlog, {
+  __call = function(options, ...)
     local options_type = type(options)
 
     if (options_type == "string") and enable_logging then
+      if microlog[options] == false then return end
       if not stored_messages[options] then stored_messages[options] = {} end
       local message_table = stored_messages[options]
 
@@ -41,13 +47,13 @@ return function(options, ...)
         -- store message (and print if in display_levels)
         message_table[#message_table + 1] = current_message
         if #message_table > message_limit then table.remove(message_table, 1) end
-      if display_levels[options] then print(current_message) end
+        if microlog[options] then print(current_message) end
       end
 
     -- set display_levels
     elseif options_type == "table" then
       for k,v in pairs(options) do
-      display_levels[k] = v
+        microlog[k] = v
       end
 
     elseif options_type == "number" then
@@ -73,3 +79,4 @@ return function(options, ...)
       if type(result) == "table" then stored_messages = result end
     end
   end
+})

refresh_sources.lua

@@ -25,7 +25,7 @@ end
 log{
   info = true,
   warning = true,
-  -- debug = true,
+  debug = false,
   -- files = true, -- debugging why the wrong files are selected
   -- sha = true, -- what the fuck is going on with sha sums?
 }
@@ -43,7 +43,7 @@ if not utility.path_exists(embeddings_file_path) then
   }, embeddings_file_path)
 end
 embeddings = utility.load_data(embeddings_file_path)
-local function embeddings_debug()
+if log.debug then
   log("debug", "Embeddings loaded.", embeddings, embeddings.files, embeddings.vectors)
   local file_count = 0
   for k,v in pairs(embeddings.files) do
@@ -57,7 +57,6 @@ local function embeddings_debug()
   log("debug", vector_count .. " vectors.")
   -- os.exit(1)
 end
-embeddings_debug()
 
 
 
@@ -93,12 +92,15 @@ local refresh_file_list = function(source_name, data_source)
 end
 
 -- returns nothing when too much text is sent
-local generate_embeddings = function(text)
+local generate_embeddings = function(data_source, text)
-  if #text > config.models.embedding.max_chunk_size then
+  local max_chunk_size = data_source.max_chunk_size or config.models.embedding.max_chunk_size
+  local model = data_source.embedding_model or config.models.embedding.model
+
+  if #text > max_chunk_size then
     return nil, "generate_embeddings() must only be passed appropriately-sized chunks!"
   end
 
-  local result = utility.llm_prompt(text, config.models.embedding.model)
+  local result = utility.llm_prompt(text, model)
   return json.decode(result)
 end
 
@@ -115,7 +117,7 @@ local process_file = function(data_source, file_name)
     return
   end
 
-  local chunk_size = config.models.embedding.max_chunk_size
+  local chunk_size = data_source.max_chunk_size or config.models.embedding.max_chunk_size
   local half_chunk_size = math.floor(chunk_size / 2)
   local chunks = { text }
 
@@ -136,14 +138,16 @@ local process_file = function(data_source, file_name)
   local new_embeddings = {}
   for i = 1, #chunks do
     log("debug", "Embedding length:", #chunks[i])
-    new_embeddings[i] = generate_embeddings(chunks[i]) or {}
+    new_embeddings[i] = generate_embeddings(data_source, chunks[i]) or {}
   end
 
   if #new_embeddings[1] == 0 then
+    if log.debug then
       log("debug", "Vector lengths:")
       for e = 1, #new_embeddings do
         log("debug", "", e, #new_embeddings[e])
       end
+    end
     if #new_embeddings == 1 then
       -- Ollama very rarely errors with:
       --   Error: do embedding request: Post "http://127.0.0.1:53441/v1/embeddings": EOF