From dde0b71e40a25d364f3de1ae59d532f42ec82e2d Mon Sep 17 00:00:00 2001 From: JINNOUCHI Yasushi Date: Thu, 21 Mar 2024 16:23:45 +0900 Subject: [PATCH] feat: access to DB as lazily as possible (#180) * fix: fix prop attributes and names * feat: load DB as lazily as possible * fix: move util function to test module * feat: use one coroutine to access DB * test: fix to wait the table to be ready * fix: avoid race conditions Before this, it can run require("frecency").new() duplicatedly to wait until frecency:setup() finishes. --- lua/frecency/database.lua | 100 +++++++++++-------------- lua/frecency/database/table.lua | 55 ++++++++++++++ lua/frecency/tests/frecency_spec.lua | 1 + lua/frecency/tests/util.lua | 11 ++- lua/frecency/{ => tests}/wait.lua | 0 lua/frecency/watcher.lua | 12 +-- lua/telescope/_extensions/frecency.lua | 18 +++-- 7 files changed, 125 insertions(+), 72 deletions(-) create mode 100644 lua/frecency/database/table.lua rename lua/frecency/{ => tests}/wait.lua (100%) diff --git a/lua/frecency/database.lua b/lua/frecency/database.lua index 9185b76..d2afdf9 100644 --- a/lua/frecency/database.lua +++ b/lua/frecency/database.lua @@ -1,5 +1,5 @@ +local Table = require "frecency.database.table" local FileLock = require "frecency.file_lock" -local wait = require "frecency.wait" local watcher = require "frecency.watcher" local log = require "plenary.log" local async = require "plenary.async" --[[@as PlenaryAsync]] @@ -19,23 +19,15 @@ local Path = require "plenary.path" --[[@as PlenaryPath]] ---@field score number ---@class FrecencyDatabase ----@field config FrecencyDatabaseConfig ----@field file_lock FrecencyFileLock ----@field filename string ----@field fs FrecencyFS ----@field new fun(fs: FrecencyFS, config: FrecencyDatabaseConfig): FrecencyDatabase ----@field table FrecencyDatabaseTable ----@field version "v1" +---@field tx PlenaryAsyncControlChannelTx +---@field private config FrecencyDatabaseConfig +---@field private file_lock FrecencyFileLock +---@field private filename string +---@field private fs FrecencyFS +---@field private tbl FrecencyDatabaseTable +---@field private version "v1" local Database = {} ----@class FrecencyDatabaseTable ----@field version string ----@field records table - ----@class FrecencyDatabaseRecord ----@field count integer ----@field timestamps integer[] - ---@param fs FrecencyFS ---@param config FrecencyDatabaseConfig ---@return FrecencyDatabase @@ -44,21 +36,29 @@ Database.new = function(fs, config) local self = setmetatable({ config = config, fs = fs, - table = { version = version, records = {} }, + tbl = Table.new(version), version = version, }, { __index = Database }) self.filename = Path.new(self.config.root, "file_frecency.bin").filename self.file_lock = FileLock.new(self.filename) - local tx, rx = async.control.channel.counter() - watcher.watch(self.filename, tx) - wait(function() - self:load() + local rx + self.tx, rx = async.control.channel.mpsc() + self.tx.send "load" + watcher.watch(self.filename, function() + self.tx.send "load" end) async.void(function() while true do - rx.last() - log.debug "file changed. loading..." - self:load() + local mode = rx.recv() + log.debug("DB coroutine start:", mode) + if mode == "load" then + self:load() + elseif mode == "save" then + self:save() + else + log.error("unknown mode: " .. mode) + end + log.debug("DB coroutine end:", mode) end end)() return self @@ -66,7 +66,7 @@ end ---@return boolean function Database:has_entry() - return not vim.tbl_isempty(self.table.records) + return not vim.tbl_isempty(self.tbl.records) end ---@param paths string[] @@ -76,17 +76,15 @@ function Database:insert_files(paths) return end for _, path in ipairs(paths) do - self.table.records[path] = { count = 1, timestamps = { 0 } } + self.tbl.records[path] = { count = 1, timestamps = { 0 } } end - wait(function() - self:save() - end) + self.tx.send "save" end ---@return string[] function Database:unlinked_entries() local paths = {} - for file in pairs(self.table.records) do + for file in pairs(self.tbl.records) do if not self.fs:is_valid_path(file) then table.insert(paths, file) end @@ -97,18 +95,16 @@ end ---@param paths string[] function Database:remove_files(paths) for _, file in ipairs(paths) do - self.table.records[file] = nil + self.tbl.records[file] = nil end - wait(function() - self:save() - end) + self.tx.send "save" end ---@param path string ---@param max_count integer ---@param datetime string? function Database:update(path, max_count, datetime) - local record = self.table.records[path] or { count = 0, timestamps = {} } + local record = self.tbl.records[path] or { count = 0, timestamps = {} } record.count = record.count + 1 local now = self:now(datetime) table.insert(record.timestamps, now) @@ -119,10 +115,8 @@ function Database:update(path, max_count, datetime) end record.timestamps = new_table end - self.table.records[path] = record - wait(function() - self:save() - end) + self.tbl.records[path] = record + self.tx.send "save" end ---@param workspace string? @@ -131,7 +125,7 @@ end function Database:get_entries(workspace, datetime) local now = self:now(datetime) local items = {} - for path, record in pairs(self.table.records) do + for path, record in pairs(self.tbl.records) do if self.fs:starts_with(path, workspace) then table.insert(items, { path = path, @@ -154,12 +148,8 @@ function Database:now(datetime) if not datetime then return os.time() end - local epoch - wait(function() - local tz_fix = datetime:gsub("+(%d%d):(%d%d)$", "+%1%2") - epoch = require("frecency.tests.util").time_piece(tz_fix) - end) - return epoch + local tz_fix = datetime:gsub("+(%d%d):(%d%d)$", "+%1%2") + return require("frecency.tests.util").time_piece(tz_fix) end ---@async @@ -182,10 +172,8 @@ function Database:load() return data end) assert(not err, err) - local tbl = loadstring(data or "")() --[[@as FrecencyDatabaseTable?]] - if tbl and tbl.version == self.version then - self.table = tbl - end + local tbl = vim.F.npcall(loadstring(data or "")) + self.tbl:set(tbl) log.debug(("load() takes %f seconds"):format(os.clock() - start)) end @@ -194,7 +182,7 @@ end function Database:save() local start = os.clock() local err = self.file_lock:with(function() - self:raw_save(self.table) + self:raw_save(self.tbl:raw()) local err, stat = async.uv.fs_stat(self.filename) assert(not err, err) watcher.update(stat) @@ -205,7 +193,7 @@ function Database:save() end ---@async ----@param tbl FrecencyDatabaseTable +---@param tbl FrecencyDatabaseRawTable function Database:raw_save(tbl) local f = assert(load("return " .. vim.inspect(tbl))) local data = string.dump(f) @@ -218,13 +206,11 @@ end ---@param path string ---@return boolean function Database:remove_entry(path) - if not self.table.records[path] then + if not self.tbl.records[path] then return false end - self.table.records[path] = nil - wait(function() - self:save() - end) + self.tbl.records[path] = nil + self.tx.send "save" return true end diff --git a/lua/frecency/database/table.lua b/lua/frecency/database/table.lua new file mode 100644 index 0000000..79a4945 --- /dev/null +++ b/lua/frecency/database/table.lua @@ -0,0 +1,55 @@ +local log = require "plenary.log" + +---@class FrecencyDatabaseRecord +---@field count integer +---@field timestamps integer[] + +---@class FrecencyDatabaseRawTable +---@field version string +---@field records table + +---@class FrecencyDatabaseTable: FrecencyDatabaseRawTable +---@field private is_ready boolean +local Table = {} + +---@param version string +---@return FrecencyDatabaseTable +Table.new = function(version) + return setmetatable({ is_ready = false, version = version }, { __index = Table.__index }) +end + +function Table:__index(key) + if key == "records" and not rawget(self, "is_ready") then + local start = os.clock() + log.debug "waiting start" + Table.wait_ready(self) + log.debug(("waiting until DB become clean takes %f seconds"):format(os.clock() - start)) + end + log.debug(("is_ready: %s, key: %s, value: %s"):format(rawget(self, "is_ready"), key, rawget(self, key))) + return vim.F.if_nil(rawget(self, key), Table[key]) +end + +function Table:raw() + return { version = self.version, records = self.records } +end + +---@param raw_table? FrecencyDatabaseRawTable +---@return nil +function Table:set(raw_table) + local tbl = raw_table or { version = self.version, records = {} } + if self.version ~= tbl.version then + error "Invalid version" + end + self.is_ready = true + self.records = tbl.records +end + +---This is for internal or testing use only. +---@return nil +function Table:wait_ready() + vim.wait(2000, function() + return rawget(self, "is_ready") + end) +end + +return Table diff --git a/lua/frecency/tests/frecency_spec.lua b/lua/frecency/tests/frecency_spec.lua index a571540..b1e52ea 100644 --- a/lua/frecency/tests/frecency_spec.lua +++ b/lua/frecency/tests/frecency_spec.lua @@ -27,6 +27,7 @@ local function with_files(files, cb_or_config, callback) log.debug(cfg) config.setup(cfg) local frecency = Frecency.new() + frecency.database.tbl:wait_ready() frecency.picker = Picker.new( frecency.database, frecency.entry_maker, diff --git a/lua/frecency/tests/util.lua b/lua/frecency/tests/util.lua index 2ab4436..354de0f 100644 --- a/lua/frecency/tests/util.lua +++ b/lua/frecency/tests/util.lua @@ -2,6 +2,7 @@ local uv = vim.uv or vim.loop local async = require "plenary.async" --[[@as PlenaryAsync]] local Path = require "plenary.path" local Job = require "plenary.job" +local wait = require "frecency.tests.wait" ---@return PlenaryPath ---@return fun(): nil close swwp all entries @@ -37,9 +38,13 @@ end, 2) -- NOTE: vim.fn.strptime cannot be used in Lua loop local function time_piece(iso8601) - local stdout, code = - AsyncJob { "perl", "-MTime::Piece", "-e", "print Time::Piece->strptime('" .. iso8601 .. "', '%FT%T%z')->epoch" } - return code == 0 and tonumber(stdout) or nil + local epoch + wait(function() + local stdout, code = + AsyncJob { "perl", "-MTime::Piece", "-e", "print Time::Piece->strptime('" .. iso8601 .. "', '%FT%T%z')->epoch" } + epoch = code == 0 and tonumber(stdout) or nil + end) + return epoch end ---@param source table diff --git a/lua/frecency/wait.lua b/lua/frecency/tests/wait.lua similarity index 100% rename from lua/frecency/wait.lua rename to lua/frecency/tests/wait.lua diff --git a/lua/frecency/watcher.lua b/lua/frecency/watcher.lua index ac5ba16..86dc712 100644 --- a/lua/frecency/watcher.lua +++ b/lua/frecency/watcher.lua @@ -36,8 +36,8 @@ Watcher.new = function() end ---@param path string ----@param tx PlenaryAsyncControlChannelTx -function Watcher:watch(path, tx) +---@param cb fun(): nil +function Watcher:watch(path, cb) if self.handler then self.handler:stop() end @@ -60,7 +60,7 @@ function Watcher:watch(path, tx) if self.mtime ~= mtime then log.debug(("mtime changed: %s -> %s"):format(self.mtime, mtime)) self.mtime = mtime - tx.send() + cb() end end)() end) @@ -70,11 +70,11 @@ local watcher = Watcher.new() return { ---@param path string - ---@param tx PlenaryAsyncControlChannelTx + ---@param cb fun(): nil ---@return nil - watch = function(path, tx) + watch = function(path, cb) log.debug("watch path: " .. path) - watcher:watch(path, tx) + watcher:watch(path, cb) end, ---@param stat FsStat diff --git a/lua/telescope/_extensions/frecency.lua b/lua/telescope/_extensions/frecency.lua index 71b5a2b..fecb56c 100644 --- a/lua/telescope/_extensions/frecency.lua +++ b/lua/telescope/_extensions/frecency.lua @@ -8,15 +8,21 @@ ---@field start fun(opts: FrecencyPickerOptions?): nil ---@field validate_database fun(force: boolean?): nil local frecency = setmetatable({}, { + ---@param self FrecencyInstance + ---@param key "complete"|"delete"|"register"|"start"|"validate_database" + ---@return function __index = function(self, key) + ---@return Frecency + local function instance() + return rawget(self, "instance") + end + return function(...) - local instance = rawget(self, "instance") --[[@as Frecency?]] - if not instance then - instance = require("frecency").new() - instance:setup() - rawset(self, "instance", instance) + if not instance() then + rawset(self, "instance", require("frecency").new()) + instance():setup() end - return instance[key](instance, ...) + return instance()[key](instance(), ...) end end, })