From 719951a3486587f04a9d028aeea7f74475c34a69 Mon Sep 17 00:00:00 2001 From: Tangent Date: Fri, 5 Jun 2026 16:50:12 -0600 Subject: [PATCH] most of this should've been committed a long time ago... --- .gitignore | 3 +- PRIVATE_DATA/.gitkeep | 0 ReadMe.md | 18 ++++++ cosine_similarity.lua | 67 ++++++++++++++++++++++ generate-embeddings.lua | 122 ---------------------------------------- least_similar.lua | 15 +++++ 6 files changed, 102 insertions(+), 123 deletions(-) create mode 100644 PRIVATE_DATA/.gitkeep create mode 100644 ReadMe.md create mode 100755 cosine_similarity.lua delete mode 100755 generate-embeddings.lua create mode 100755 least_similar.lua diff --git a/.gitignore b/.gitignore index 1d9adb5..38cb679 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .DS_Store -notebook/** +PRIVATE_DATA/** +!PRIVATE_DATA/.gitkeep diff --git a/PRIVATE_DATA/.gitkeep b/PRIVATE_DATA/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/ReadMe.md b/ReadMe.md new file mode 100644 index 0000000..3017786 --- /dev/null +++ b/ReadMe.md @@ -0,0 +1,18 @@ +# LLM Tricks +Trying to find *anything* useful to do with local LLMs. + +All scripts work with data within `PRIVATE_DATA/` so that private data can't be +accidentally committed. + +###### `cosine_similarity.lua` +Opens `embeddings.json` and creates `similarities.json` with a sorted list of +comparisons. + +###### `generate_embeddings.lua` +Opens every file on its whitelist within the `notebook` directory, and generates +embeddings (placed in `embeddings.json`). When files are too long, it truncates +them. + +###### `least_similar.lua` +Opens `similarities.json`, reverses the sort order, and saves it as +`differences.json`. diff --git a/cosine_similarity.lua b/cosine_similarity.lua new file mode 100755 index 0000000..7e3f9dc --- /dev/null +++ b/cosine_similarity.lua @@ -0,0 +1,67 @@ +#!/usr/bin/env luajit + +package.path = (arg[0]:match("@?(.*/)") or arg[0]:match("@?(.*\\)")) .. "lib" .. package.config:sub(1, 1) .. "?.lua;" .. package.path +local utility = require "utility" +local json = utility.require("dkjson") + +local function normalizing_cosine_similarity(a, b) + local dot, normalized_a, normalized_b = 0, 0, 0 + + for i = 1, #a do + dot = dot + a[i] * b[i] + normalized_a = normalized_a + a[i] * a[i] + normalized_b = normalized_b + b[i] * b[i] + end + + normalized_a = math.sqrt(normalized_a) + normalized_b = math.sqrt(normalized_b) + + if normalized_a == 0 or normalized_b == 0 then + return 0 + end + + return dot / (normalized_a * normalized_b) +end + +local function cosine_similarity(a, b) + local dot = 0 + + for i = 1, #a do + dot = dot + a[i] * b[i] + end + + return dot +end + + + +local embeddings = utility.open("PRIVATE_DATA/embeddings.json", "r", function(file) + return json.decode(file:read("*all")) +end) + +local embeddings_list = {} +for file_name, tab in pairs(embeddings) do + table.insert(embeddings_list, { file_name = file_name, vector = tab.vector, }) +end + +local total_comparisons = #embeddings_list * (#embeddings_list - 1) / 2 +local comparisons_completed = 0 + +local relations_list = {} +for i = 1, #embeddings_list - 1 do + for j = i + 1, #embeddings_list do + local a, b = embeddings_list[i], embeddings_list[j] + local similarity = cosine_similarity(a.vector, b.vector) + + table.insert(relations_list, { similarity = similarity, a = a.file_name, b = b.file_name, }) + + comparisons_completed = comparisons_completed + 1 + print(comparisons_completed .. "/" .. total_comparisons) + end +end + +table.sort(relations_list, function(a, b) return a.similarity > b.similarity end) + +utility.open("PRIVATE_DATA/similarities.json", "w", function(file) + file:write(json.encode(relations_list, { indent = true })) +end) diff --git a/generate-embeddings.lua b/generate-embeddings.lua deleted file mode 100755 index 7e52722..0000000 --- a/generate-embeddings.lua +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/env luajit - -package.path = (arg[0]:match("@?(.*/)") or arg[0]:match("@?(.*\\)")) .. "lib" .. package.config:sub(1, 1) .. "?.lua;" .. package.path -local utility = require "utility" -local json = utility.require("dkjson") - -local PATH = "notebook" -local whitelist = { md = true, } - -local embedding_model = "nomic-embed-text" -local maximum_file_size = 2048 - --- strip YAML frontmatter (if present) --- can error, will return nil & error message -local function strip_frontmatter(text) - local tab = text:split("\n") - if tab[1] == "---" then - table.remove(tab, 1) - while true do - local done = tab[1] == "---" - table.remove(tab, 1) - if done then - return table.concat(tab, "\n") - elseif #tab < 1 then - return nil, "Invalid YAML frontmatter." - end - end - end - return text -end - -local tree -tree = function(path, fn) - utility.list(path or ".", function(path_name) - if utility.is_file(path_name) then - fn(path_name) - else - tree(path .. utility.path_separator .. path_name, fn) - end - end) -end - -local function leftpad(text, length) - return string.rep("0", length - #(tostring(text))) .. text -end - - - -local file_list = {} -local embeddings = {} - -tree(PATH, function(file_name) - local path, name, extension = utility.split_path_components(file_name) - if not extension then - return - end - - if not whitelist[extension] then - return - end - - file_list[#file_list + 1] = file_name -end) - -for i = 1, #file_list do - local function _run() - local file_name = file_list[i] - - -- stripping YAML frontmatter before generating embeddings - local file_contents = utility.open(file_name, "r", function(file) - return file:read("*all") - end) - - local tmp_file_contents, error_message = strip_frontmatter(file_contents) - if tmp_file_contents == nil then - print("ERROR: " .. file_name .. " " .. error_message) - tmp_file_contents = file_contents - end - - if #tmp_file_contents == 0 then - print(file_name .. "\n is empty and will be skipped.") - return - end - - if #tmp_file_contents > maximum_file_size then - print(file_name .. "\n is too long. Only the first " .. maximum_file_size .. " bytes will be scanned.") - tmp_file_contents = tmp_file_contents:sub(1, maximum_file_size) - end - - local tmp_file_name = utility.tmp_file_name() - utility.open(tmp_file_name, "w", function(file) - file:write(tmp_file_contents) - end) - - local output = utility.capture_safe("cat " .. tmp_file_name:enquote() .. " | ollama run " .. embedding_model) - os.execute("rm " .. tmp_file_name) - output = output:sub(1, -2) -- strip extra newline from utility.capture_safe - - if #output == 0 then - print("Warning: " .. file_name .. " did not have embeddings generated.") - return - end - - output = setmetatable({ vector = output }, { - __tojson = function(self, state) - return self.vector - end - }) - - embeddings[file_name] = { vector = output } - - print("Finished " .. leftpad(i, #tostring(#file_list)) .. "/" .. #file_list .. " (" .. leftpad(math.floor(i / #file_list * 100), 3) .. "%)") - end - - _run() -end - -utility.open("embeddings.json", "w", function(file) - local output = json.encode(embeddings, { indent = true }) - file:write(output) - file:write("\n") -end) diff --git a/least_similar.lua b/least_similar.lua new file mode 100755 index 0000000..fbd3e9c --- /dev/null +++ b/least_similar.lua @@ -0,0 +1,15 @@ +#!/usr/bin/env luajit + +package.path = (arg[0]:match("@?(.*/)") or arg[0]:match("@?(.*\\)")) .. "lib" .. package.config:sub(1, 1) .. "?.lua;" .. package.path +local utility = require "utility" +local json = utility.require("dkjson") + +local similarities = utility.open("PRIVATE_DATA/similarities.json", "r", function(file) + return json.decode(file:read("*all")) +end) + +table.sort(similarities, function(a, b) return a.similarity < b.similarity end) + +utility.open("PRIVATE_DATA/differences.json", "w", function(file) + file:write(json.encode(similarities, { indent = true })) +end)