most of this should've been committed a long time ago...

.gitignore

@@ -1,2 +1,3 @@
 .DS_Store
-notebook/**
+PRIVATE_DATA/**
+!PRIVATE_DATA/.gitkeep

ReadMe.md

@@ -0,0 +1,18 @@
+# LLM Tricks
+Trying to find *anything* useful to do with local LLMs.
+
+All scripts work with data within `PRIVATE_DATA/` so that private data can't be
+accidentally committed.
+
+###### `cosine_similarity.lua`
+Opens `embeddings.json` and creates `similarities.json` with a sorted list of
+comparisons.
+
+###### `generate_embeddings.lua`
+Opens every file on its whitelist within the `notebook` directory, and generates
+embeddings (placed in `embeddings.json`). When files are too long, it truncates
+them.
+
+###### `least_similar.lua`
+Opens `similarities.json`, reverses the sort order, and saves it as
+`differences.json`.

cosine_similarity.lua

@@ -0,0 +1,67 @@
+#!/usr/bin/env luajit
+
+package.path = (arg[0]:match("@?(.*/)") or arg[0]:match("@?(.*\\)")) .. "lib" .. package.config:sub(1, 1) .. "?.lua;" .. package.path
+local utility = require "utility"
+local json = utility.require("dkjson")
+
+local function normalizing_cosine_similarity(a, b)
+  local dot, normalized_a, normalized_b = 0, 0, 0
+
+  for i = 1, #a do
+    dot = dot + a[i] * b[i]
+    normalized_a = normalized_a + a[i] * a[i]
+    normalized_b = normalized_b + b[i] * b[i]
+  end
+
+  normalized_a = math.sqrt(normalized_a)
+  normalized_b = math.sqrt(normalized_b)
+
+  if normalized_a == 0 or normalized_b == 0 then
+    return 0
+  end
+
+  return dot / (normalized_a * normalized_b)
+end
+
+local function cosine_similarity(a, b)
+  local dot = 0
+
+  for i = 1, #a do
+    dot = dot + a[i] * b[i]
+  end
+
+  return dot
+end
+
+
+
+local embeddings = utility.open("PRIVATE_DATA/embeddings.json", "r", function(file)
+  return json.decode(file:read("*all"))
+end)
+
+local embeddings_list = {}
+for file_name, tab in pairs(embeddings) do
+  table.insert(embeddings_list, { file_name = file_name, vector = tab.vector, })
+end
+
+local total_comparisons = #embeddings_list * (#embeddings_list - 1) / 2
+local comparisons_completed = 0
+
+local relations_list = {}
+for i = 1, #embeddings_list - 1 do
+  for j = i + 1, #embeddings_list do
+    local a, b = embeddings_list[i], embeddings_list[j]
+    local similarity = cosine_similarity(a.vector, b.vector)
+
+    table.insert(relations_list, { similarity = similarity, a = a.file_name, b = b.file_name, })
+
+    comparisons_completed = comparisons_completed + 1
+    print(comparisons_completed .. "/" .. total_comparisons)
+  end
+end
+
+table.sort(relations_list, function(a, b) return a.similarity > b.similarity end)
+
+utility.open("PRIVATE_DATA/similarities.json", "w", function(file)
+  file:write(json.encode(relations_list, { indent = true }))
+end)

generate_embeddings.lua

@@ -0,0 +1,122 @@
+#!/usr/bin/env luajit
+
+package.path = (arg[0]:match("@?(.*/)") or arg[0]:match("@?(.*\\)")) .. "lib" .. package.config:sub(1, 1) .. "?.lua;" .. package.path
+local utility = require "utility"
+local json = utility.require("dkjson")
+
+local PATH = "PRIVATE_DATA/notebook"
+local whitelist = { md = true, }
+
+local embedding_model = "nomic-embed-text"
+local maximum_file_size = 2048
+
+-- strip YAML frontmatter (if present)
+--   can error, will return nil & error message
+local function strip_frontmatter(text)
+  local tab = text:split("\n")
+  if tab[1] == "---" then
+    table.remove(tab, 1)
+    while true do
+      local done = tab[1] == "---"
+      table.remove(tab, 1)
+      if done then
+        return table.concat(tab, "\n")
+      elseif #tab < 1 then
+        return nil, "Invalid YAML frontmatter."
+      end
+    end
+  end
+  return text
+end
+
+local tree
+tree = function(path, fn)
+  utility.list(path or ".", function(path_name)
+    if utility.is_file(path_name) then
+      fn(path_name)
+    else
+      tree(path .. utility.path_separator .. path_name, fn)
+    end
+  end)
+end
+
+local function leftpad(text, length)
+  return string.rep("0", length - #(tostring(text))) .. text
+end
+
+
+
+local file_list = {}
+local embeddings = {}
+
+tree(PATH, function(file_name)
+  local path, name, extension = utility.split_path_components(file_name)
+  if not extension then
+    return
+  end
+
+  if not whitelist[extension] then
+    return
+  end
+
+  file_list[#file_list + 1] = file_name
+end)
+
+for i = 1, #file_list do
+  local function _run()
+    local file_name = file_list[i]
+
+    -- stripping YAML frontmatter before generating embeddings
+    local file_contents = utility.open(file_name, "r", function(file)
+      return file:read("*all")
+    end)
+
+    local tmp_file_contents, error_message = strip_frontmatter(file_contents)
+    if tmp_file_contents == nil then
+      print("ERROR: " .. file_name .. " " .. error_message)
+      tmp_file_contents = file_contents
+    end
+
+    if #tmp_file_contents == 0 then
+      print(file_name .. "\n is empty and will be skipped.")
+      return
+    end
+
+    if #tmp_file_contents > maximum_file_size then
+      print(file_name .. "\n is too long. Only the first " .. maximum_file_size .. " bytes will be scanned.")
+      tmp_file_contents = tmp_file_contents:sub(1, maximum_file_size)
+    end
+
+    local tmp_file_name = utility.tmp_file_name()
+    utility.open(tmp_file_name, "w", function(file)
+      file:write(tmp_file_contents)
+    end)
+
+    local output = utility.capture_safe("cat " .. tmp_file_name:enquote() .. " | ollama run " .. embedding_model)
+    os.execute("rm " .. tmp_file_name)
+    output = output:sub(1, -2) -- strip extra newline from utility.capture_safe
+
+    if #output == 0 then
+      print("Warning: " .. file_name .. " did not have embeddings generated.")
+      return
+    end
+
+    output = setmetatable({ vector = output }, {
+      __tojson = function(self, state)
+        return self.vector
+      end
+    })
+
+    embeddings[file_name] = { vector = output }
+
+    print("Finished " .. leftpad(i, #tostring(#file_list)) .. "/" .. #file_list .. " (" .. leftpad(math.floor(i / #file_list * 100), 3) .. "%)")
+  end
+
+  _run()
+end
+
+utility.open("PRIVATE_DATA/embeddings.json", "w", function(file)
+  local output = json.encode(embeddings, { indent = true })
+  file:write(output)
+  file:write("\n")
+end)

least_similar.lua

@@ -0,0 +1,15 @@
+#!/usr/bin/env luajit
+
+package.path = (arg[0]:match("@?(.*/)") or arg[0]:match("@?(.*\\)")) .. "lib" .. package.config:sub(1, 1) .. "?.lua;" .. package.path
+local utility = require "utility"
+local json = utility.require("dkjson")
+
+local similarities = utility.open("PRIVATE_DATA/similarities.json", "r", function(file)
+  return json.decode(file:read("*all"))
+end)
+
+table.sort(similarities, function(a, b) return a.similarity < b.similarity end)
+
+utility.open("PRIVATE_DATA/differences.json", "w", function(file)
+  file:write(json.encode(similarities, { indent = true }))
+end)