refresh_sources.lua

@@ -0,0 +1,62 @@
+#!/usr/bin/env luajit
+
+package.path = (arg[0]:match("@?(.*/)") or arg[0]:match("@?(.*\\)")) .. "lib" .. package.config:sub(1, 1) .. "?.lua;" .. package.path
+local utility = require "utility"
+
+local timing = utility.require("timing")
+
+-- TODO make utility have a function for getting/setting defaults where locking is only used to set defaults if they aren't present
+local config = utility.get_config("no-lock")
+if not config.models then
+  config.models = {
+    embedding = {
+      model = "qwen3-embedding:0.6b",
+      max_chunk_size = 32768,
+    }
+  }
+  utility.save_config()
+end
+
+local refresht_file_list = function(data_source)
+  timing.mark("Assembling file list.")
+
+  os.execute("cd " .. ("PRIVATE_DATA" .. utility.path_separator .. data_source.path):enquote() .. " && " .. data_source.refresh_command)
+
+  local compiled_filters = {}
+  for name, object in pairs(data_source.filters) do
+    compiled_filters[name] = utility.enumerate(object)
+  end
+
+  local file_list = {}
+  utility.tree(data_source.path, compiled_filters, function(file_name)
+    file_list[#file_list + 1] = file_name
+  end)
+
+  return file_list
+end
+
+local generate_embeddings = function(text)
+  if #text > config.models.embedding.max_chunk_size then
+    return nil, "generate_embeddings() must only be passed appropriately-sized chunks!"
+  end
+
+  local result = utility.llm_prompt(text, config.models.embedding.model)
+
+  -- skipping unnecessary decoding to an array and putting back into text for saving
+  -- this would cause problems if we intended to use the results immediately
+  return setmetatable({}, {
+    __tojson = function() return result end,
+  })
+end
+
+local process_file = function(file_name)
+  local text = utility.read_file(file_name)
+  if w
+end
+
+local to_be_named_generate_file_embeddings = function(text)
+  if #text > config.models.embedding.max_chunk_size then
+    -- TODO, split, process each chunk, return both a total and each chunk
+  end
+  -- return just the file and its embedding
+end