feat: add logic to store data by native code (#130)

* refactor: make logic for Database be abstract
* feat: add logic for DB by string.dump
* fix: run with async.void to run synchronously
* test: add tests for native feature
* feat!: sort candidates by path when score is same
  This is needed because candidates from SQLite is sorted by id, but ones from native is sorted by path.
* chore: clean up types
* feat: add lock/unlock feature to access DB
* test: use async version of busted
  And disable benchmark tests (fix later)
* test: add tests for file_lock
* chore: use more explicit names
* chore: use plenary.log instead
* fix: wait async functions definitely
* feat: add migrator
* chore: fix logging
* fix: detect emptiness of the table
* fix: deal with buffer with no names
* test: loosen the condition temporarily
* test: add tests for migrator
* fix: return true when the table is not empty
* feat: load sqlite lazily not to require in start
* chore: add logging to calculate time for fetching
* feat: add converter from native code to SQLite
* feat: warn when sqlite.lua is not available
* feat: add FrecencyMigrateDB to migrate DB
* docs: add note for native code logic
* test: ignore type bug

lua/frecency/database/native.lua

@@ -0,0 +1,173 @@
+local FileLock = require "frecency.file_lock"
+local wait = require "frecency.wait"
+local log = require "plenary.log"
+local async = require "plenary.async" --[[@as PlenaryAsync]]
+
+---@class FrecencyDatabaseNative: FrecencyDatabase
+---@field version "v1"
+---@field filename string
+---@field file_lock FrecencyFileLock
+---@field table FrecencyDatabaseNativeTable
+local Native = {}
+
+---@class FrecencyDatabaseNativeTable
+---@field version string
+---@field records table<string,FrecencyDatabaseNativeRecord>
+
+---@class FrecencyDatabaseNativeRecord
+---@field count integer
+---@field timestamps integer[]
+
+---@param fs FrecencyFS
+---@param config FrecencyDatabaseConfig
+---@return FrecencyDatabaseNative
+Native.new = function(fs, config)
+  local version = "v1"
+  local self = setmetatable({
+    config = config,
+    fs = fs,
+    table = { version = version, records = {} },
+    version = version,
+  }, { __index = Native })
+  self.filename = self.config.root .. "/file_frecency.bin"
+  self.file_lock = FileLock.new(self.filename)
+  wait(function()
+    self:load()
+  end)
+  return self
+end
+
+---@return boolean
+function Native:has_entry()
+  return not vim.tbl_isempty(self.table.records)
+end
+
+---@param paths string[]
+---@return nil
+function Native:insert_files(paths)
+  if #paths == 0 then
+    return
+  end
+  for _, path in ipairs(paths) do
+    self.table.records[path] = { count = 1, timestamps = { 0 } }
+  end
+  wait(function()
+    self:save()
+  end)
+end
+
+---@return string[]
+function Native:unlinked_entries()
+  local paths = {}
+  for file in pairs(self.table.records) do
+    if not self.fs:is_valid_path(file) then
+      table.insert(paths, file)
+    end
+  end
+  return paths
+end
+
+---@param paths string[]
+function Native:remove_files(paths)
+  for _, file in ipairs(paths) do
+    self.table.records[file] = nil
+  end
+  wait(function()
+    self:save()
+  end)
+end
+
+---@param path string
+---@param max_count integer
+---@param datetime string?
+function Native:update(path, max_count, datetime)
+  local record = self.table.records[path] or { count = 0, timestamps = {} }
+  record.count = record.count + 1
+  local now = self:now(datetime)
+  table.insert(record.timestamps, now)
+  if #record.timestamps > max_count then
+    local new_table = {}
+    for i = #record.timestamps - max_count + 1, #record.timestamps do
+      table.insert(new_table, record.timestamps[i])
+    end
+    record.timestamps = new_table
+  end
+  self.table.records[path] = record
+  wait(function()
+    self:save()
+  end)
+end
+
+---@param workspace string?
+---@param datetime string?
+---@return FrecencyDatabaseEntry[]
+function Native:get_entries(workspace, datetime)
+  -- TODO: check mtime of DB and reload it
+  -- self:load()
+  local now = self:now(datetime)
+  local items = {}
+  for path, record in pairs(self.table.records) do
+    if not workspace or path:find(workspace .. "/", 1, true) then
+      table.insert(items, {
+        path = path,
+        count = record.count,
+        ages = vim.tbl_map(function(v)
+          return (now - v) / 60
+        end, record.timestamps),
+      })
+    end
+  end
+  return items
+end
+
+---@private
+---@param datetime string?
+---@return integer
+function Native:now(datetime)
+  return datetime and vim.fn.strptime("%FT%T%z", datetime) or os.time()
+end
+
+---@async
+---@return nil
+function Native:load()
+  local start = os.clock()
+  local err, data = self.file_lock:with(function()
+    local err, st = async.uv.fs_stat(self.filename)
+    if err then
+      return nil
+    end
+    local fd
+    err, fd = async.uv.fs_open(self.filename, "r", tonumber("644", 8))
+    assert(not err)
+    local data
+    err, data = async.uv.fs_read(fd, st.size)
+    assert(not err)
+    assert(not async.uv.fs_close(fd))
+    return data
+  end)
+  assert(not err, err)
+  local tbl = loadstring(data or "")() --[[@as FrecencyDatabaseNativeTable?]]
+  if tbl and tbl.version == self.version then
+    self.table = tbl
+  end
+  log.debug(("load() takes %f seconds"):format(os.clock() - start))
+end
+
+---@async
+---@return nil
+function Native:save()
+  local start = os.clock()
+  local err = self.file_lock:with(function()
+    local f = assert(load("return " .. vim.inspect(self.table)))
+    local data = string.dump(f)
+    local err, fd = async.uv.fs_open(self.filename, "w", tonumber("644", 8))
+    assert(not err)
+    assert(not async.uv.fs_write(fd, data))
+    assert(not async.uv.fs_close(fd))
+    return nil
+  end)
+  assert(not err, err)
+  log.debug(("save() takes %f seconds"):format(os.clock() - start))
+end
+
+return Native

lua/frecency/database/sqlite.lua

@@ -0,0 +1,135 @@
+local sqlite = require "frecency.sqlite"
+local log = require "plenary.log"
+
+---@class FrecencySqliteDB: sqlite_db
+---@field files sqlite_tbl
+---@field timestamps sqlite_tbl
+
+---@class FrecencyFile
+---@field count integer
+---@field id integer
+---@field path string
+---@field score integer calculated from count and age
+
+---@class FrecencyTimestamp
+---@field age integer calculated from timestamp
+---@field file_id integer
+---@field id integer
+---@field timestamp number
+
+---@class FrecencyDatabaseSqlite: FrecencyDatabase
+---@field sqlite FrecencySqliteDB
+local Sqlite = {}
+
+---@param fs FrecencyFS
+---@param config FrecencyDatabaseConfig
+---@return FrecencyDatabaseSqlite
+Sqlite.new = function(fs, config)
+  local lib = sqlite.lib
+  local self = setmetatable(
+    { config = config, buf_registered_flag_name = "telescope_frecency_registered", fs = fs },
+    { __index = Sqlite }
+  )
+  self.sqlite = sqlite {
+    uri = self.config.root .. "/file_frecency.sqlite3",
+    files = { id = true, count = { "integer", default = 1, required = true }, path = "string" },
+    timestamps = {
+      id = true,
+      file_id = { "integer", reference = "files.id", on_delete = "cascade" },
+      timestamp = { "real", default = lib.julianday "now" },
+    },
+  }
+  return self
+end
+
+---@return boolean
+function Sqlite:has_entry()
+  return self.sqlite.files:count() > 0
+end
+
+---@param paths string[]
+---@return integer
+function Sqlite:insert_files(paths)
+  if #paths == 0 then
+    return 0
+  end
+  ---@param path string
+  return self.sqlite.files:insert(vim.tbl_map(function(path)
+    return { path = path, count = 0 } -- TODO: remove when sql.nvim#97 is closed
+  end, paths))
+end
+
+---@param workspace string?
+---@param datetime string?
+---@return FrecencyDatabaseEntry[]
+function Sqlite:get_entries(workspace, datetime)
+  local query = workspace and { contains = { path = { workspace .. "/*" } } } or {}
+  log.debug { query = query }
+  local files = self.sqlite.files:get(query) --[[@as FrecencyFile[] ]]
+  local lib = sqlite.lib
+  local age = lib.cast((lib.julianday(datetime) - lib.julianday "timestamp") * 24 * 60, "integer")
+  local timestamps = self.sqlite.timestamps:get { keys = { age = age, "id", "file_id" } } --[[@as FrecencyTimestamp[] ]]
+  ---@type table<integer,number[]>
+  local age_map = {}
+  for _, timestamp in ipairs(timestamps) do
+    if not age_map[timestamp.file_id] then
+      age_map[timestamp.file_id] = {}
+    end
+    table.insert(age_map[timestamp.file_id], timestamp.age)
+  end
+  local items = {}
+  for _, file in ipairs(files) do
+    table.insert(items, { path = file.path, count = file.count, ages = age_map[file.id] })
+  end
+  return items
+end
+
+---@param datetime string? ISO8601 format string
+---@return FrecencyTimestamp[]
+function Sqlite:get_timestamps(datetime)
+  local lib = sqlite.lib
+  local age = lib.cast((lib.julianday(datetime) - lib.julianday "timestamp") * 24 * 60, "integer")
+  return self.sqlite.timestamps:get { keys = { age = age, "id", "file_id" } }
+end
+
+---@param path string
+---@param count integer
+---@param datetime string?
+---@return nil
+function Sqlite:update(path, count, datetime)
+  local file = self.sqlite.files:get({ where = { path = path } })[1] --[[@as FrecencyFile?]]
+  local file_id
+  if file then
+    self.sqlite.files:update { where = { id = file.id }, set = { count = file.count + 1 } }
+    file_id = file.id
+  else
+    file_id = self.sqlite.files:insert { path = path }
+  end
+  self.sqlite.timestamps:insert {
+    file_id = file_id,
+    timestamp = datetime and sqlite.lib.julianday(datetime) or nil,
+  }
+  local timestamps = self.sqlite.timestamps:get { where = { file_id = file_id } } --[[@as FrecencyTimestamp[] ]]
+  local trim_at = timestamps[#timestamps - count + 1]
+  if trim_at then
+    self.sqlite.timestamps:remove { file_id = tostring(file_id), id = "<" .. tostring(trim_at.id) }
+  end
+end
+
+---@return integer[]
+function Sqlite:unlinked_entries()
+  ---@param file FrecencyFile
+  return self.sqlite.files:map(function(file)
+    if not self.fs:is_valid_path(file.path) then
+      return file.id
+    end
+  end)
+end
+
+---@param ids integer[]
+---@return nil
+function Sqlite:remove_files(ids)
+  self.sqlite.files:remove { id = ids }
+end
+
+return Sqlite