tools/llm-tricks
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

target chunk size is possible now

Browse Source 
closes #4
This commit is contained in:
tangent
2026-09-16 23:16:29 -06:00
parent 8d0c79b34d
commit 77298dad0c
2 changed files with 9 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/log.lua
+1 -1
View File
@@ -22,7 +22,7 @@ local enable_logging = true
local message_limit = math.huge local message_limit = math.huge
return setmetatable(microlog, { return setmetatable(microlog, {
__call = function(options, ...) __call = function(self, options, ...)
local options_type = type(options) local options_type = type(options)
if (options_type == "string") and enable_logging then if (options_type == "string") and enable_logging then
refresh_sources.lua
+8 -1
View File
@@ -143,9 +143,16 @@ local process_file = function(data_source, file_name)
local chunk_size = data_source.max_chunk_size or config.models.embedding.max_chunk_size local chunk_size = data_source.max_chunk_size or config.models.embedding.max_chunk_size
local chunks = make_chunks(text, chunk_size) local chunks = make_chunks(text, chunk_size)
if data_source.target_chunk_size then
local extra_chunks = make_chunks(text, data_source.target_chunk_size)
for i = 2, #extra_chunks do
chunks[#chunks + 1] = extra_chunks[i]
end
end
local new_embeddings = {} local new_embeddings = {}
for i = 1, #chunks do for i = 1, #chunks do
log("debug", "Embedding length:", #chunks[i]) log("debug", "Embedding source length:", #chunks[i])
new_embeddings[i] = generate_embeddings(data_source, chunks[i]) or {} new_embeddings[i] = generate_embeddings(data_source, chunks[i]) or {}
end end