fixed all the errors

.gitignore

@@ -1,3 +1,3 @@
 .DS_Store
-PRIVATE_DATA/**
+PRIVATE_DATA/
 config.json

ReadMe.md

@@ -25,8 +25,8 @@ sources. Example:
 {
   "source name":{
     "filters":{
-      "blacklist":".git",
-      "extension_whitelist":"md"
+      "blacklist":[".git"],
+      "extension_whitelist":["md"]
     },
     "initialize_command":"git clone REMOTE .",
     "path":"will be created before initialize_command is run",

lib/utility.lua

@@ -249,13 +249,13 @@ utility.tree = function(path, options, fn)
     if options.blacklist and options.blacklist[path_name] then return end
     if options.whitelist and (not options.whitelist[path_name]) then return end
 
-    if options.extension_blacklist or options.extension_whitelist then
-      local _, _, extension = utility.split_path_components(path_name)
-      if options.extension_blacklist and options.extension_blacklist[extension] then return end
-      if options.extension_whitelist and (not options.extension_whitelist[extension]) then return end
-    end
-
     if utility.is_file(path_name) then
+      if options.extension_blacklist or options.extension_whitelist then
+        local _, _, extension = utility.split_path_components(path_name)
+        if options.extension_blacklist and options.extension_blacklist[extension] then return end
+        if options.extension_whitelist and (not options.extension_whitelist[extension]) then return end
+      end
+
       fn(path_name)
     else
       utility.tree(path .. utility.path_separator .. path_name, options, fn)
@@ -306,9 +306,9 @@ end
 utility.sha512sum = function(file_path)
   local sha512sum
   if utility.OS == "Linux" then
-    sha512sum = os.capture_safe("shasum -p -t -a 512 " .. file_path:enquote()) -- TODO check this
+    sha512sum = utility.capture_safe("shasum -p -t -a 512 " .. file_path:enquote()) -- TODO check this
   elseif utility.OS == "macOS" then
-    sha512sum = os.capture_safe("shasum -U -a 512 " .. file_path:enquote())
+    sha512sum = utility.capture_safe("shasum -U -a 512 " .. file_path:enquote())
   elseif utility.OS == "Windows" then
     error("utility.sha512sum() not implemented for Windows.")
   end

refresh_sources.lua

@@ -56,7 +56,7 @@ local refresh_file_list = function(source_name, data_source)
   end
 
   local file_list = {}
-  utility.tree(data_source.path, compiled_filters, function(file_name)
+  utility.tree("PRIVATE_DATA" .. utility.path_separator .. data_source.path, compiled_filters, function(file_name)
     file_list[#file_list + 1] = file_name
   end)
 
@@ -71,11 +71,6 @@ local generate_embeddings = function(text)
   end
 
   local result = utility.llm_prompt(text, config.models.embedding.model)
-
-  result = setmetatable({}, {
-    __tojson = function() return result end,
-  })
-
   return json.decode(result)
 end
 
@@ -98,13 +93,13 @@ local process_file = function(data_source, file_name)
 
   while #text > chunk_size do
     local first_chunk = text:sub(1, chunk_size)
-    local overlap_chunk = text:sub(half_chunk_size, chunk_size + half_chunk_size)
+    local overlap_chunk = text:sub(half_chunk_size, chunk_size + half_chunk_size - 1)
 
     chunks[#chunks + 1] = first_chunk
     chunks[#chunks + 1] = overlap_chunk
 
     text = text:sub(chunk_size)
-    if #text > half_chunk_size and (not #text > chunk_size) then
+    if (#text > half_chunk_size) and (not (#text > chunk_size)) then
       -- last chunk would be skipped if we didn't handle this here
       chunks[#chunks + 1] = text
     end
@@ -142,10 +137,11 @@ local refresh_sources = function()
     for f = 1, #file_list do
       local file_name = file_list[f]
       local function loop()
-        local sha512sum = utility.sha512sum(tmp_file_path)
+        local sha512sum = utility.sha512sum(file_name)
         if embeddings.vectors[sha512sum] then return end
 
         local file_chunks, file_embeddings = process_file(data_source, file_name)
+        if not file_chunks then return end
 
         local file_sums = {}
         for i = 1, #file_chunks do
@@ -159,7 +155,7 @@ local refresh_sources = function()
             file_sums[#file_sums + 1] = sha512sum
             if embeddings.vectors[sha512sum] then return end
 
-            os.execute(utility.commands.move .. tmp_file_path:enquote() .. " " .. "PRIVATE_DATA/memory/")
+            os.execute(utility.commands.move .. tmp_file_path:enquote() .. " " .. "PRIVATE_DATA/memory/" .. sha512sum)
             embeddings.vectors[sha512sum] = current_embedding
           end
           loop()
@@ -168,14 +164,16 @@ local refresh_sources = function()
         embeddings.files[file_name] = file_sums
       end
       loop()
-      print("Finished " .. utility.leftpad(f, #tostring(#file_list), "0") .. "/" .. #file_list .. " (" .. leftpad(math.floor(i / #file_list * 100), 3, "0") .. "%)")
+      print("Finished " .. utility.leftpad(f, #tostring(#file_list), "0") .. "/" .. #file_list .. " (" .. utility.leftpad(math.floor(f / #file_list * 100), 3, "0") .. "%)")
     end
 
     timing.mark("Finished generating embeddings for " .. source_name .. ".")
   end
 
   utility.save_data(embeddings)
-  os.execute("rm " .. tmp_file_path:enquote())
+  if utility.path_exists(tmp_file_path) then
+    os.execute("rm " .. tmp_file_path:enquote())
+  end
 end
 
 refresh_sources()