feat: access to DB as lazily as possible (#180)

* fix: fix prop attributes and names

* feat: load DB as lazily as possible

* fix: move util function to test module

* feat: use one coroutine to access DB

* test: fix to wait the table to be ready

* fix: avoid race conditions

Before this, it can run require("frecency").new() duplicatedly to wait
until frecency:setup() finishes.

lua/frecency/database.lua

@@ -1,5 +1,5 @@
+local Table = require "frecency.database.table"
 local FileLock = require "frecency.file_lock"
-local wait = require "frecency.wait"
 local watcher = require "frecency.watcher"
 local log = require "plenary.log"
 local async = require "plenary.async" --[[@as PlenaryAsync]]
@@ -19,23 +19,15 @@ local Path = require "plenary.path" --[[@as PlenaryPath]]
 ---@field score number
 
 ---@class FrecencyDatabase
----@field config FrecencyDatabaseConfig
+---@field tx PlenaryAsyncControlChannelTx
----@field file_lock FrecencyFileLock
+---@field private config FrecencyDatabaseConfig
----@field filename string
+---@field private file_lock FrecencyFileLock
----@field fs FrecencyFS
+---@field private filename string
----@field new fun(fs: FrecencyFS, config: FrecencyDatabaseConfig): FrecencyDatabase
+---@field private fs FrecencyFS
----@field table FrecencyDatabaseTable
+---@field private tbl FrecencyDatabaseTable
----@field version "v1"
+---@field private version "v1"
 local Database = {}
 
----@class FrecencyDatabaseTable
----@field version string
----@field records table<string,FrecencyDatabaseRecord>
-
----@class FrecencyDatabaseRecord
----@field count integer
----@field timestamps integer[]
-
 ---@param fs FrecencyFS
 ---@param config FrecencyDatabaseConfig
 ---@return FrecencyDatabase
@@ -44,21 +36,29 @@ Database.new = function(fs, config)
   local self = setmetatable({
     config = config,
     fs = fs,
-    table = { version = version, records = {} },
+    tbl = Table.new(version),
     version = version,
   }, { __index = Database })
   self.filename = Path.new(self.config.root, "file_frecency.bin").filename
   self.file_lock = FileLock.new(self.filename)
-  local tx, rx = async.control.channel.counter()
+  local rx
-  watcher.watch(self.filename, tx)
+  self.tx, rx = async.control.channel.mpsc()
-  wait(function()
+  self.tx.send "load"
-    self:load()
+  watcher.watch(self.filename, function()
+    self.tx.send "load"
   end)
   async.void(function()
     while true do
-      rx.last()
+      local mode = rx.recv()
-      log.debug "file changed. loading..."
+      log.debug("DB coroutine start:", mode)
+      if mode == "load" then
         self:load()
+      elseif mode == "save" then
+        self:save()
+      else
+        log.error("unknown mode: " .. mode)
+      end
+      log.debug("DB coroutine end:", mode)
     end
   end)()
   return self
@@ -66,7 +66,7 @@ end
 
 ---@return boolean
 function Database:has_entry()
-  return not vim.tbl_isempty(self.table.records)
+  return not vim.tbl_isempty(self.tbl.records)
 end
 
 ---@param paths string[]
@@ -76,17 +76,15 @@ function Database:insert_files(paths)
     return
   end
   for _, path in ipairs(paths) do
-    self.table.records[path] = { count = 1, timestamps = { 0 } }
+    self.tbl.records[path] = { count = 1, timestamps = { 0 } }
   end
-  wait(function()
+  self.tx.send "save"
-    self:save()
-  end)
 end
 
 ---@return string[]
 function Database:unlinked_entries()
   local paths = {}
-  for file in pairs(self.table.records) do
+  for file in pairs(self.tbl.records) do
     if not self.fs:is_valid_path(file) then
       table.insert(paths, file)
     end
@@ -97,18 +95,16 @@ end
 ---@param paths string[]
 function Database:remove_files(paths)
   for _, file in ipairs(paths) do
-    self.table.records[file] = nil
+    self.tbl.records[file] = nil
   end
-  wait(function()
+  self.tx.send "save"
-    self:save()
-  end)
 end
 
 ---@param path string
 ---@param max_count integer
 ---@param datetime string?
 function Database:update(path, max_count, datetime)
-  local record = self.table.records[path] or { count = 0, timestamps = {} }
+  local record = self.tbl.records[path] or { count = 0, timestamps = {} }
   record.count = record.count + 1
   local now = self:now(datetime)
   table.insert(record.timestamps, now)
@@ -119,10 +115,8 @@ function Database:update(path, max_count, datetime)
     end
     record.timestamps = new_table
   end
-  self.table.records[path] = record
+  self.tbl.records[path] = record
-  wait(function()
+  self.tx.send "save"
-    self:save()
-  end)
 end
 
 ---@param workspace string?
@@ -131,7 +125,7 @@ end
 function Database:get_entries(workspace, datetime)
   local now = self:now(datetime)
   local items = {}
-  for path, record in pairs(self.table.records) do
+  for path, record in pairs(self.tbl.records) do
     if self.fs:starts_with(path, workspace) then
       table.insert(items, {
         path = path,
@@ -154,12 +148,8 @@ function Database:now(datetime)
   if not datetime then
     return os.time()
   end
-  local epoch
-  wait(function()
   local tz_fix = datetime:gsub("+(%d%d):(%d%d)$", "+%1%2")
-    epoch = require("frecency.tests.util").time_piece(tz_fix)
+  return require("frecency.tests.util").time_piece(tz_fix)
-  end)
-  return epoch
 end
 
 ---@async
@@ -182,10 +172,8 @@ function Database:load()
     return data
   end)
   assert(not err, err)
-  local tbl = loadstring(data or "")() --[[@as FrecencyDatabaseTable?]]
+  local tbl = vim.F.npcall(loadstring(data or ""))
-  if tbl and tbl.version == self.version then
+  self.tbl:set(tbl)
-    self.table = tbl
-  end
   log.debug(("load() takes %f seconds"):format(os.clock() - start))
 end
 
@@ -194,7 +182,7 @@ end
 function Database:save()
   local start = os.clock()
   local err = self.file_lock:with(function()
-    self:raw_save(self.table)
+    self:raw_save(self.tbl:raw())
     local err, stat = async.uv.fs_stat(self.filename)
     assert(not err, err)
     watcher.update(stat)
@@ -205,7 +193,7 @@ function Database:save()
 end
 
 ---@async
----@param tbl FrecencyDatabaseTable
+---@param tbl FrecencyDatabaseRawTable
 function Database:raw_save(tbl)
   local f = assert(load("return " .. vim.inspect(tbl)))
   local data = string.dump(f)
@@ -218,13 +206,11 @@ end
 ---@param path string
 ---@return boolean
 function Database:remove_entry(path)
-  if not self.table.records[path] then
+  if not self.tbl.records[path] then
     return false
   end
-  self.table.records[path] = nil
+  self.tbl.records[path] = nil
-  wait(function()
+  self.tx.send "save"
-    self:save()
-  end)
   return true
 end
 

lua/frecency/database/table.lua

@@ -0,0 +1,55 @@
+local log = require "plenary.log"
+
+---@class FrecencyDatabaseRecord
+---@field count integer
+---@field timestamps integer[]
+
+---@class FrecencyDatabaseRawTable
+---@field version string
+---@field records table<string,FrecencyDatabaseRecord>
+
+---@class FrecencyDatabaseTable: FrecencyDatabaseRawTable
+---@field private is_ready boolean
+local Table = {}
+
+---@param version string
+---@return FrecencyDatabaseTable
+Table.new = function(version)
+  return setmetatable({ is_ready = false, version = version }, { __index = Table.__index })
+end
+
+function Table:__index(key)
+  if key == "records" and not rawget(self, "is_ready") then
+    local start = os.clock()
+    log.debug "waiting start"
+    Table.wait_ready(self)
+    log.debug(("waiting until DB become clean takes %f seconds"):format(os.clock() - start))
+  end
+  log.debug(("is_ready: %s, key: %s, value: %s"):format(rawget(self, "is_ready"), key, rawget(self, key)))
+  return vim.F.if_nil(rawget(self, key), Table[key])
+end
+
+function Table:raw()
+  return { version = self.version, records = self.records }
+end
+
+---@param raw_table? FrecencyDatabaseRawTable
+---@return nil
+function Table:set(raw_table)
+  local tbl = raw_table or { version = self.version, records = {} }
+  if self.version ~= tbl.version then
+    error "Invalid version"
+  end
+  self.is_ready = true
+  self.records = tbl.records
+end
+
+---This is for internal or testing use only.
+---@return nil
+function Table:wait_ready()
+  vim.wait(2000, function()
+    return rawget(self, "is_ready")
+  end)
+end
+
+return Table

lua/frecency/tests/frecency_spec.lua

@@ -27,6 +27,7 @@ local function with_files(files, cb_or_config, callback)
   log.debug(cfg)
   config.setup(cfg)
   local frecency = Frecency.new()
+  frecency.database.tbl:wait_ready()
   frecency.picker = Picker.new(
     frecency.database,
     frecency.entry_maker,

lua/frecency/tests/util.lua

@@ -2,6 +2,7 @@ local uv = vim.uv or vim.loop
 local async = require "plenary.async" --[[@as PlenaryAsync]]
 local Path = require "plenary.path"
 local Job = require "plenary.job"
+local wait = require "frecency.tests.wait"
 
 ---@return PlenaryPath
 ---@return fun(): nil close swwp all entries
@@ -37,9 +38,13 @@ end, 2)
 
 -- NOTE: vim.fn.strptime cannot be used in Lua loop
 local function time_piece(iso8601)
+  local epoch
+  wait(function()
     local stdout, code =
       AsyncJob { "perl", "-MTime::Piece", "-e", "print Time::Piece->strptime('" .. iso8601 .. "', '%FT%T%z')->epoch" }
-  return code == 0 and tonumber(stdout) or nil
+    epoch = code == 0 and tonumber(stdout) or nil
+  end)
+  return epoch
 end
 
 ---@param source table<string,{ count: integer, timestamps: string[] }>

lua/frecency/watcher.lua

@@ -36,8 +36,8 @@ Watcher.new = function()
 end
 
 ---@param path string
----@param tx PlenaryAsyncControlChannelTx
+---@param cb fun(): nil
-function Watcher:watch(path, tx)
+function Watcher:watch(path, cb)
   if self.handler then
     self.handler:stop()
   end
@@ -60,7 +60,7 @@ function Watcher:watch(path, tx)
       if self.mtime ~= mtime then
         log.debug(("mtime changed: %s -> %s"):format(self.mtime, mtime))
         self.mtime = mtime
-        tx.send()
+        cb()
       end
     end)()
   end)
@@ -70,11 +70,11 @@ local watcher = Watcher.new()
 
 return {
   ---@param path string
-  ---@param tx PlenaryAsyncControlChannelTx
+  ---@param cb fun(): nil
   ---@return nil
-  watch = function(path, tx)
+  watch = function(path, cb)
     log.debug("watch path: " .. path)
-    watcher:watch(path, tx)
+    watcher:watch(path, cb)
   end,
 
   ---@param stat FsStat

lua/telescope/_extensions/frecency.lua

@@ -8,15 +8,21 @@
 ---@field start fun(opts: FrecencyPickerOptions?): nil
 ---@field validate_database fun(force: boolean?): nil
 local frecency = setmetatable({}, {
+  ---@param self FrecencyInstance
+  ---@param key "complete"|"delete"|"register"|"start"|"validate_database"
+  ---@return function
   __index = function(self, key)
-    return function(...)
+    ---@return Frecency
-      local instance = rawget(self, "instance") --[[@as Frecency?]]
+    local function instance()
-      if not instance then
+      return rawget(self, "instance")
-        instance = require("frecency").new()
-        instance:setup()
-        rawset(self, "instance", instance)
     end
-      return instance[key](instance, ...)
+
+    return function(...)
+      if not instance() then
+        rawset(self, "instance", require("frecency").new())
+        instance():setup()
+      end
+      return instance()[key](instance(), ...)
     end
   end,
 })