better logging

lib/log.lua

@@ -1,11 +1,8 @@
-local stored_messages = {}
-local display_levels = {}
-local enable_logging = true
-local message_limit = math.huge
-
 -- USAGE:
 --   -- set arbitrary log levels to be printed immediately
 --   log{ info = true, warning = true, ducksauce = true, }
+--   -- disable storage of messages depending on log level
+--   log{ error = false, verbose = false, }
 --   -- send anything (except nil) to a log level
 --   log("bacom", true, "text", 5, function() end, {})
 --   -- print all messages saved at a particular log level
@@ -14,62 +11,72 @@ local message_limit = math.huge
 --   log(false) -- this deletes all stored messages
 --   -- set a maximum number of stored messages per log level
 --   log(50) -- warning: reaching large limits makes logging expensive
+--  -- check if a log level is being displayed
+--  if log.arbitrary_exit then log("arbitrary_exit") os.exit(1) end
 --   -- do anything to the stored messages (this example deletes them all)
 --  log(function(messages) return {} end)
 
-return function(options, ...)
-  local options_type = type(options)
+local microlog = {} -- display_levels are stored directly here
+local stored_messages = {}
+local enable_logging = true
+local message_limit = math.huge
 
-  if (options_type == "string") and enable_logging then
-    if not stored_messages[options] then stored_messages[options] = {} end
-    local message_table = stored_messages[options]
+return setmetatable(microlog, {
+  __call = function(options, ...)
+    local options_type = type(options)
 
-    -- turn log message into text
-    local tab = {...}
-    for i = 1, #tab do
-      tab[i] = tostring(tab[i])
-    end
-    local current_message = table.concat(tab, "\t")
+    if (options_type == "string") and enable_logging then
+      if microlog[options] == false then return end
+      if not stored_messages[options] then stored_messages[options] = {} end
+      local message_table = stored_messages[options]
 
-    if #current_message == 0 then
-      -- print all stored_messages of specified level
-      for i = 1, #message_table do
-        print(message_table[i])
+      -- turn log message into text
+      local tab = {...}
+      for i = 1, #tab do
+        tab[i] = tostring(tab[i])
+      end
+      local current_message = table.concat(tab, "\t")
+
+      if #current_message == 0 then
+        -- print all stored_messages of specified level
+        for i = 1, #message_table do
+          print(message_table[i])
+        end
+
+      else
+        -- store message (and print if in display_levels)
+        message_table[#message_table + 1] = current_message
+        if #message_table > message_limit then table.remove(message_table, 1) end
+        if microlog[options] then print(current_message) end
       end
 
-    else
-      -- store message (and print if in display_levels)
-      message_table[#message_table + 1] = current_message
-      if #message_table > message_limit then table.remove(message_table, 1) end
-      if display_levels[options] then print(current_message) end
-    end
-
-  -- set display_levels
-  elseif options_type == "table" then
-    for k,v in pairs(options) do
-      display_levels[k] = v
-    end
-
-  elseif options_type == "number" then
-    message_limit = options
-    for k,v in pairs(stored_messages) do
-      while #v > message_limit do
-        table.remove(v, 1)
+    -- set display_levels
+    elseif options_type == "table" then
+      for k,v in pairs(options) do
+        microlog[k] = v
       end
-    end
 
-  -- turn all logging off and delete stored messages; or turn it on
-  elseif options_type == "boolean" then
-    if options then
-      enable_logging = true
-    else
-      enable_logging = false
-      stored_messages = {}
-    end
+    elseif options_type == "number" then
+      message_limit = options
+      for k,v in pairs(stored_messages) do
+        while #v > message_limit do
+          table.remove(v, 1)
+        end
+      end
 
-  -- arbitrary access :D
-  elseif options_type == "function" then
-    local result = options(stored_messages)
-    if type(result) == "table" then stored_messages = result end
+    -- turn all logging off and delete stored messages; or turn it on
+    elseif options_type == "boolean" then
+      if options then
+        enable_logging = true
+      else
+        enable_logging = false
+        stored_messages = {}
+      end
+
+    -- arbitrary access :D
+    elseif options_type == "function" then
+      local result = options(stored_messages)
+      if type(result) == "table" then stored_messages = result end
+    end
   end
-end
+})

refresh_sources.lua

@@ -25,7 +25,7 @@ end
 log{
   info = true,
   warning = true,
-  -- debug = true,
+  debug = false,
   -- files = true, -- debugging why the wrong files are selected
   -- sha = true, -- what the fuck is going on with sha sums?
 }
@@ -43,7 +43,7 @@ if not utility.path_exists(embeddings_file_path) then
   }, embeddings_file_path)
 end
 embeddings = utility.load_data(embeddings_file_path)
-local function embeddings_debug()
+if log.debug then
   log("debug", "Embeddings loaded.", embeddings, embeddings.files, embeddings.vectors)
   local file_count = 0
   for k,v in pairs(embeddings.files) do
@@ -57,7 +57,6 @@ local function embeddings_debug()
   log("debug", vector_count .. " vectors.")
   -- os.exit(1)
 end
-embeddings_debug()
 
 
 
@@ -93,12 +92,15 @@ local refresh_file_list = function(source_name, data_source)
 end
 
 -- returns nothing when too much text is sent
-local generate_embeddings = function(text)
-  if #text > config.models.embedding.max_chunk_size then
+local generate_embeddings = function(data_source, text)
+  local max_chunk_size = data_source.max_chunk_size or config.models.embedding.max_chunk_size
+  local model = data_source.embedding_model or config.models.embedding.model
+
+  if #text > max_chunk_size then
     return nil, "generate_embeddings() must only be passed appropriately-sized chunks!"
   end
 
-  local result = utility.llm_prompt(text, config.models.embedding.model)
+  local result = utility.llm_prompt(text, model)
   return json.decode(result)
 end
 
@@ -115,7 +117,7 @@ local process_file = function(data_source, file_name)
     return
   end
 
-  local chunk_size = config.models.embedding.max_chunk_size
+  local chunk_size = data_source.max_chunk_size or config.models.embedding.max_chunk_size
   local half_chunk_size = math.floor(chunk_size / 2)
   local chunks = { text }
 
@@ -136,13 +138,15 @@ local process_file = function(data_source, file_name)
   local new_embeddings = {}
   for i = 1, #chunks do
     log("debug", "Embedding length:", #chunks[i])
-    new_embeddings[i] = generate_embeddings(chunks[i]) or {}
+    new_embeddings[i] = generate_embeddings(data_source, chunks[i]) or {}
   end
 
   if #new_embeddings[1] == 0 then
-    log("debug", "Vector lengths:")
-    for e = 1, #new_embeddings do
-      log("debug", "", e, #new_embeddings[e])
+    if log.debug then
+      log("debug", "Vector lengths:")
+      for e = 1, #new_embeddings do
+        log("debug", "", e, #new_embeddings[e])
+      end
     end
     if #new_embeddings == 1 then
       -- Ollama very rarely errors with: