feat: check DB file has been changed (#143)

* refactor: unite logic for finder & async_finder

* chore: fix types

* chore: add sleep to show results at first

* refactor: fix to find results separatedly

* test: remove unnecessary ones and fix others

* test: add matrix for 0.9.x & Windows

* test: use forked plenary.log for Windows

* test: fix to use strptime in Windows

* test: run again if segmentation fault in Windows

* test: loosen timeout for Perl

* test: use the latest plenary.nvim again

* chore: fix types

* chore: change variable name

* feat: watch changes of DB to reload

* chore: add comments to steps

* test: copy whole modules for testing in Windows

* fix: make valid paths for Windows

* test: add tests for Native

* test: use robust way to calculate time

vim.fn.strptime cannot be used in Lua loop

* chore: fix comments

* refactor: simplify the code

* test: loosen condition to detect failures

* test: disable some logging

Many loggings make the test fail.

* test: run tests sequentially in Windows

* test: loosen timeout not to fail on Windows

.github/workflows/ci.yml

@@ -9,11 +9,11 @@ jobs:
       matrix:
         os:
           - ubuntu-latest
-          # TODO: nix seems not to work with SIP
+          - macos-latest
-          # - macos-latest
+          - windows-latest
-          # TODO: PlenaryBustedDirectory seems not to run on Windows
-          # - windows-latest
         version:
+          - v0.9.2
+          - v0.9.1
           - v0.9.0
           - nightly
     runs-on: ${{ matrix.os }}
@@ -41,7 +41,7 @@ jobs:
         with:
           neovim: true
           version: ${{ matrix.version }}
-      - name: Run tests
+      - name: Run tests (not for Windows)
         env:
           PLENARY_PATH: plenary.nvim
           TELESCOPE_PATH: telescope.nvim
@@ -53,8 +53,29 @@ jobs:
           MINIMAL_LUA=${TEST_DIR}minimal.lua
           NVIM=$(perl -e '$_ = $ENV{EXE}; s,\\,/,g; print')
           $NVIM --headless --clean -u $MINIMAL_LUA -c "PlenaryBustedDirectory $TEST_DIR {minimal_init = '$MINIMAL_LUA'}"
+        if: matrix.os != 'windows-latest'
+      - name: Run tests (for Windows)
+        shell: bash
+        env:
+          PLENARY_PATH: plenary.nvim
+          TELESCOPE_PATH: telescope.nvim
+          SQLITE_PATH: sqlite.lua
+          DEBUG_PLENARY: 1
+          EXE: ${{ steps.nvim.outputs.executable }}
+        run: |-
+          # HACK: This is needed because it fails to add runtimepath's.
+          cp -af $PLENARY_PATH/lua/plenary/ lua/
+          cp -af $TELESCOPE_PATH/lua/telescope/ lua/
+          cp -af $SQLITE_PATH/lua/sqlite/ lua/
+          TEST_DIR=lua/frecency/tests/
+          MINIMAL_LUA=${TEST_DIR}minimal.lua
+          NVIM=$(perl -e '$_ = $ENV{EXE}; s,\\,/,g; print')
+          $NVIM --headless --clean -u $MINIMAL_LUA -c "PlenaryBustedDirectory $TEST_DIR {minimal_init = '$MINIMAL_LUA', timeout = 180000, sequential = true}"
+        if: matrix.os == 'windows-latest'
       - name: Type Check Code Base
         uses: mrcjkb/lua-typecheck-action@v0.2.0
         with:
           checkLevel: Hint
           configpath: .luarc.json
+        # NOTE: This step needs nix that seems not to work with SIP (macOS)
+        if: matrix.os == 'ubuntu-latest'

lua/frecency/async_finder.lua

@@ -1,123 +0,0 @@
-local async = require "plenary.async" --[[@as PlenaryAsync]]
-
----@class FrecencyAsyncFinder
----@field closed boolean
----@field entries FrecencyEntry[]
----@field reflowed boolean
----@field rx PlenaryAsyncControlChannelRx
----@field state FrecencyState
----@overload fun(_: string, process_result: (fun(entry: FrecencyEntry): nil), process_complete: fun(): nil): nil
-local AsyncFinder = {}
-
----@param fs FrecencyFS
----@param state FrecencyState
----@param path string
----@param entry_maker fun(file: FrecencyFile): FrecencyEntry
----@param initial_results FrecencyFile[]
----@return FrecencyAsyncFinder
-AsyncFinder.new = function(state, fs, path, entry_maker, initial_results)
-  local self = setmetatable({ closed = false, entries = {}, reflowed = false, state = state }, {
-    __index = AsyncFinder,
-    ---@param self FrecencyAsyncFinder
-    __call = function(self, ...)
-      return self:find(...)
-    end,
-  })
-  local seen = {}
-  for i, file in ipairs(initial_results) do
-    local entry = entry_maker(file)
-    seen[entry.filename] = true
-    entry.index = i
-    table.insert(self.entries, entry)
-  end
-  local tx, rx = async.control.channel.mpsc()
-  self.rx = rx
-  async.run(function()
-    local index = #initial_results
-    local count = 0
-    for name in fs:scan_dir(path) do
-      if self.closed then
-        break
-      end
-      local fullpath = fs.joinpath(path, name)
-      if not seen[fullpath] then
-        seen[fullpath] = true
-        index = index + 1
-        count = count + 1
-        local entry = entry_maker { id = 0, count = 0, path = fs.joinpath(path, name), score = 0 }
-        if entry then
-          entry.index = index
-          table.insert(self.entries, entry)
-          tx.send(entry)
-          if count % 1000 == 0 then
-            self:reflow_results()
-            -- NOTE: This is needed not to lock text input.
-            async.util.sleep(50)
-          end
-        end
-      end
-    end
-    self:close()
-    tx.send(nil)
-  end)
-  return self
-end
-
----@param _ string
----@param process_result fun(entry: FrecencyEntry): nil
----@param process_complete fun(): nil
----@return nil
-function AsyncFinder:find(_, process_result, process_complete)
-  for _, entry in ipairs(self.entries) do
-    if process_result(entry) then
-      return
-    end
-  end
-  local last_index = self.entries[#self.entries].index
-  while true do
-    if self.closed then
-      break
-    end
-    local entry = self.rx.recv()
-    if not entry then
-      break
-    elseif entry.index > last_index and process_result(entry) then
-      return
-    end
-  end
-  process_complete()
-end
-
-function AsyncFinder:close()
-  self.closed = true
-end
-
----@return nil
-function AsyncFinder:reflow_results()
-  local picker = self.state:get()
-  if not picker then
-    return
-  end
-  local bufnr = picker.results_bufnr
-  local win = picker.results_win
-  if not bufnr or not win then
-    return
-  end
-  picker:clear_extra_rows(bufnr)
-  if picker.sorting_strategy == "descending" then
-    local manager = picker.manager
-    if not manager then
-      return
-    end
-    local worst_line = picker:get_row(manager:num_results())
-    ---@type WinInfo
-    local wininfo = vim.fn.getwininfo(win)[1]
-    local bottom = vim.api.nvim_buf_line_count(bufnr)
-    if not self.reflowed or worst_line > wininfo.botline then
-      self.reflowed = true
-      vim.api.nvim_win_set_cursor(win, { bottom, 0 })
-    end
-  end
-end
-
-return AsyncFinder

lua/frecency/database/native.lua

@@ -1,7 +1,9 @@
 local FileLock = require "frecency.file_lock"
 local wait = require "frecency.wait"
+local watcher = require "frecency.database.native.watcher"
 local log = require "plenary.log"
 local async = require "plenary.async" --[[@as PlenaryAsync]]
+local Path = require "plenary.path" --[[@as PlenaryPath]]
 
 ---@class FrecencyDatabaseNative: FrecencyDatabase
 ---@field version "v1"
@@ -29,11 +31,20 @@ Native.new = function(fs, config)
     table = { version = version, records = {} },
     version = version,
   }, { __index = Native })
-  self.filename = self.config.root .. "/file_frecency.bin"
+  self.filename = Path.new(self.config.root, "file_frecency.bin").filename
   self.file_lock = FileLock.new(self.filename)
+  local tx, rx = async.control.channel.counter()
+  watcher.watch(self.filename, tx)
   wait(function()
     self:load()
   end)
+  async.void(function()
+    while true do
+      rx.last()
+      log.debug "file changed. loading..."
+      self:load()
+    end
+  end)()
   return self
 end
 
@@ -102,8 +113,6 @@ end
 ---@param datetime string?
 ---@return FrecencyDatabaseEntry[]
 function Native:get_entries(workspace, datetime)
-  -- TODO: check mtime of DB and reload it
-  -- self:load()
   local now = self:now(datetime)
   local items = {}
   for path, record in pairs(self.table.records) do
@@ -120,11 +129,21 @@ function Native:get_entries(workspace, datetime)
   return items
 end
 
+-- TODO: remove this func
+-- This is a func for testing
 ---@private
 ---@param datetime string?
 ---@return integer
 function Native:now(datetime)
-  return datetime and vim.fn.strptime("%FT%T%z", datetime) or os.time()
+  if not datetime then
+    return os.time()
+  end
+  local epoch
+  wait(function()
+    local tz_fix = datetime:gsub("+(%d%d):(%d%d)$", "+%1%2")
+    epoch = require("frecency.tests.util").time_piece(tz_fix)
+  end)
+  return epoch
 end
 
 ---@async
@@ -132,17 +151,18 @@ end
 function Native:load()
   local start = os.clock()
   local err, data = self.file_lock:with(function()
-    local err, st = async.uv.fs_stat(self.filename)
+    local err, stat = async.uv.fs_stat(self.filename)
     if err then
       return nil
     end
     local fd
     err, fd = async.uv.fs_open(self.filename, "r", tonumber("644", 8))
-    assert(not err)
+    assert(not err, err)
     local data
-    err, data = async.uv.fs_read(fd, st.size)
+    err, data = async.uv.fs_read(fd, stat.size)
-    assert(not err)
+    assert(not err, err)
     assert(not async.uv.fs_close(fd))
+    watcher.update(stat)
     return data
   end)
   assert(not err, err)
@@ -158,16 +178,23 @@ end
 function Native:save()
   local start = os.clock()
   local err = self.file_lock:with(function()
-    local f = assert(load("return " .. vim.inspect(self.table)))
+    self:raw_save(self.table)
-    local data = string.dump(f)
+    local err, stat = async.uv.fs_stat(self.filename)
-    local err, fd = async.uv.fs_open(self.filename, "w", tonumber("644", 8))
+    assert(not err, err)
-    assert(not err)
+    watcher.update(stat)
-    assert(not async.uv.fs_write(fd, data))
-    assert(not async.uv.fs_close(fd))
     return nil
   end)
   assert(not err, err)
   log.debug(("save() takes %f seconds"):format(os.clock() - start))
 end
 
+function Native:raw_save(tbl)
+  local f = assert(load("return " .. vim.inspect(tbl)))
+  local data = string.dump(f)
+  local err, fd = async.uv.fs_open(self.filename, "w", tonumber("644", 8))
+  assert(not err, err)
+  assert(not async.uv.fs_write(fd, data))
+  assert(not async.uv.fs_close(fd))
+end
+
 return Native

lua/frecency/database/native/watcher.lua

@@ -0,0 +1,87 @@
+local async = require "plenary.async" --[[@as PlenaryAsync]]
+local log = require "plenary.log"
+local uv = vim.loop or vim.uv
+
+---@class FrecencyNativeWatcherMtime
+---@field sec integer
+---@field nsec integer
+local Mtime = {}
+
+---@param mtime FsStatMtime
+---@return FrecencyNativeWatcherMtime
+Mtime.new = function(mtime)
+  return setmetatable({ sec = mtime.sec, nsec = mtime.nsec }, Mtime)
+end
+
+---@param other FrecencyNativeWatcherMtime
+---@return boolean
+function Mtime:__eq(other)
+  return self.sec == other.sec and self.nsec == other.nsec
+end
+
+---@return string
+function Mtime:__tostring()
+  return string.format("%d.%d", self.sec, self.nsec)
+end
+
+---@class FrecencyNativeWatcher
+---@field handler UvFsEventHandle
+---@field path string
+---@field mtime FrecencyNativeWatcherMtime
+local Watcher = {}
+
+---@return FrecencyNativeWatcher
+Watcher.new = function()
+  return setmetatable({ path = "", mtime = Mtime.new { sec = 0, nsec = 0 } }, { __index = Watcher })
+end
+
+---@param path string
+---@param tx PlenaryAsyncControlChannelTx
+function Watcher:watch(path, tx)
+  if self.handler then
+    self.handler:stop()
+  end
+  self.handler = assert(uv.new_fs_event()) --[[@as UvFsEventHandle]]
+  self.handler:start(path, { recursive = true }, function(err, _, _)
+    if err then
+      log.debug("failed to watch path: " .. err)
+      return
+    end
+    async.void(function()
+      -- NOTE: wait for updating mtime
+      async.util.sleep(50)
+      local stat
+      err, stat = async.uv.fs_stat(path)
+      if err then
+        log.debug("failed to stat path: " .. err)
+        return
+      end
+      local mtime = Mtime.new(stat.mtime)
+      if self.mtime ~= mtime then
+        log.debug(("mtime changed: %s -> %s"):format(self.mtime, mtime))
+        self.mtime = mtime
+        tx.send()
+      end
+    end)()
+  end)
+end
+
+local watcher = Watcher.new()
+
+return {
+  ---@param path string
+  ---@param tx PlenaryAsyncControlChannelTx
+  ---@return nil
+  watch = function(path, tx)
+    log.debug("watch path: " .. path)
+    watcher:watch(path, tx)
+  end,
+
+  ---@param stat FsStat
+  ---@return nil
+  update = function(stat)
+    local mtime = Mtime.new(stat.mtime)
+    log.debug(("update mtime: %s -> %s"):format(watcher.mtime, mtime))
+    watcher.mtime = mtime
+  end,
+}

lua/frecency/entry_maker.lua

@@ -40,10 +40,12 @@ end
 ---@field score number
 ---@field display fun(entry: FrecencyEntry): string, table
 
+---@alias FrecencyEntryMakerInstance fun(file: FrecencyFile): FrecencyEntry
+
 ---@param filepath_formatter FrecencyFilepathFormatter
 ---@param workspace string?
 ---@param workspace_tag string?
----@return fun(file: FrecencyFile): FrecencyEntry
+---@return FrecencyEntryMakerInstance
 function EntryMaker:create(filepath_formatter, workspace, workspace_tag)
   local displayer = entry_display.create {
     separator = "",

lua/frecency/finder.lua

@@ -1,47 +1,191 @@
-local AsyncFinder = require "frecency.async_finder"
+local async = require "plenary.async" --[[@as PlenaryAsync]]
-local finders = require "telescope.finders"
 local log = require "plenary.log"
 
 ---@class FrecencyFinder
----@field private config FrecencyFinderConfig
+---@field config FrecencyFinderConfig
----@field private entry_maker FrecencyEntryMaker
+---@field closed boolean
----@field private fs FrecencyFS
+---@field entries FrecencyEntry[]
+---@field entry_maker FrecencyEntryMakerInstance
+---@field fs FrecencyFS
+---@field need_scandir boolean
+---@field path string?
+---@field private database FrecencyDatabase
+---@field private recency FrecencyRecency
+---@field private rx PlenaryAsyncControlChannelRx
+---@field private state FrecencyState
+---@field private tx PlenaryAsyncControlChannelTx
 local Finder = {}
 
 ---@class FrecencyFinderConfig
----@field chunk_size integer
+---@field chunk_size integer default: 1000
+---@field sleep_interval integer default: 50
 
----@param entry_maker FrecencyEntryMaker
+---@param database FrecencyDatabase
+---@param entry_maker FrecencyEntryMakerInstance
 ---@param fs FrecencyFS
+---@param need_scandir boolean
+---@param path string?
+---@param recency FrecencyRecency
+---@param state FrecencyState
 ---@param config FrecencyFinderConfig?
 ---@return FrecencyFinder
-Finder.new = function(entry_maker, fs, config)
+Finder.new = function(database, entry_maker, fs, need_scandir, path, recency, state, config)
-  return setmetatable(
+  local tx, rx = async.control.channel.mpsc()
-    { config = vim.tbl_extend("force", { chunk_size = 1000 }, config or {}), entry_maker = entry_maker, fs = fs },
+  return setmetatable({
-    { __index = Finder }
+    config = vim.tbl_extend("force", { chunk_size = 1000, sleep_interval = 50 }, config or {}),
-  )
+    closed = false,
+    database = database,
+    entries = {},
+    entry_maker = entry_maker,
+    fs = fs,
+    need_scandir = need_scandir,
+    path = path,
+    recency = recency,
+    rx = rx,
+    state = state,
+    tx = tx,
+  }, {
+    __index = Finder,
+    ---@param self FrecencyFinder
+    __call = function(self, ...)
+      return self:find(...)
+    end,
+  })
 end
 
----@class FrecencyFinderOptions
+---@param datetime string?
----@field need_scandir boolean
+---@return nil
----@field workspace string?
+function Finder:start(datetime)
----@field workspace_tag string?
+  async.void(function()
-
+    -- NOTE: return to the main loop to show the main window
----@param state FrecencyState
+    async.util.sleep(0)
----@param filepath_formatter FrecencyFilepathFormatter
+    local seen = {}
----@param initial_results table
+    for i, file in ipairs(self:get_results(self.path, datetime)) do
----@param opts FrecencyFinderOptions
+      local entry = self.entry_maker(file)
----@return table
+      seen[entry.filename] = true
-function Finder:start(state, filepath_formatter, initial_results, opts)
+      entry.index = i
-  local entry_maker = self.entry_maker:create(filepath_formatter, opts.workspace, opts.workspace_tag)
+      table.insert(self.entries, entry)
-  if not opts.need_scandir then
+      self.tx.send(entry)
-    return finders.new_table {
+    end
-      results = initial_results,
+    if self.need_scandir and self.path then
-      entry_maker = entry_maker,
+      -- NOTE: return to the main loop to show results from DB
-    }
+      async.util.sleep(self.config.sleep_interval)
+      self:scan_dir(seen)
+    end
+    self:close()
+    self.tx.send(nil)
+  end)()
+end
+
+---@param seen table<string, boolean>
+---@return nil
+function Finder:scan_dir(seen)
+  local count = 0
+  local index = #self.entries
+  for name in self.fs:scan_dir(self.path) do
+    if self.closed then
+      break
+    end
+    local fullpath = self.fs.joinpath(self.path, name)
+    if not seen[fullpath] then
+      seen[fullpath] = true
+      count = count + 1
+      local entry = self.entry_maker { id = 0, count = 0, path = fullpath, score = 0 }
+      if entry then
+        index = index + 1
+        entry.index = index
+        table.insert(self.entries, entry)
+        self.tx.send(entry)
+        if count % self.config.chunk_size == 0 then
+          self:reflow_results()
+          async.util.sleep(self.config.sleep_interval)
+        end
+      end
+    end
+  end
+end
+
+---@param _ string
+---@param process_result fun(entry: FrecencyEntry): nil
+---@param process_complete fun(): nil
+---@return nil
+function Finder:find(_, process_result, process_complete)
+  local index = 0
+  for _, entry in ipairs(self.entries) do
+    index = index + 1
+    if process_result(entry) then
+      return
+    end
+  end
+  local count = 0
+  while not self.closed do
+    count = count + 1
+    local entry = self.rx.recv()
+    if not entry then
+      break
+    elseif entry.index > index and process_result(entry) then
+      return
+    end
+  end
+  process_complete()
+end
+
+---@param workspace string?
+---@param datetime string?
+---@return FrecencyFile[]
+function Finder:get_results(workspace, datetime)
+  log.debug { workspace = workspace or "NONE" }
+  local start_fetch = os.clock()
+  local files = self.database:get_entries(workspace, datetime)
+  log.debug(("it takes %f seconds in fetching entries"):format(os.clock() - start_fetch))
+  local start_results = os.clock()
+  local elapsed_recency = 0
+  for _, file in ipairs(files) do
+    local start_recency = os.clock()
+    file.score = file.ages and self.recency:calculate(file.count, file.ages) or 0
+    file.ages = nil
+    elapsed_recency = elapsed_recency + (os.clock() - start_recency)
+  end
+  log.debug(("it takes %f seconds in calculating recency"):format(elapsed_recency))
+  log.debug(("it takes %f seconds in making results"):format(os.clock() - start_results))
+
+  local start_sort = os.clock()
+  table.sort(files, function(a, b)
+    return a.score > b.score or (a.score == b.score and a.path > b.path)
+  end)
+  log.debug(("it takes %f seconds in sorting"):format(os.clock() - start_sort))
+  return files
+end
+
+function Finder:close()
+  self.closed = true
+end
+
+function Finder:reflow_results()
+  local picker = self.state:get()
+  if not picker then
+    return
+  end
+  local bufnr = picker.results_bufnr
+  local win = picker.results_win
+  if not bufnr or not win then
+    return
+  end
+  picker:clear_extra_rows(bufnr)
+  if picker.sorting_strategy == "descending" then
+    local manager = picker.manager
+    if not manager then
+      return
+    end
+    local worst_line = picker:get_row(manager:num_results())
+    ---@type WinInfo
+    local wininfo = vim.fn.getwininfo(win)[1]
+    local bottom = vim.api.nvim_buf_line_count(bufnr)
+    if not self.reflowed or worst_line > wininfo.botline then
+      self.reflowed = true
+      vim.api.nvim_win_set_cursor(win, { bottom, 0 })
+    end
   end
-  log.debug { finder = opts }
-  return AsyncFinder.new(state, self.fs, opts.workspace, entry_maker, initial_results)
 end
 
 return Finder

lua/frecency/frecency.lua

@@ -2,7 +2,6 @@ local Sqlite = require "frecency.database.sqlite"
 local Native = require "frecency.database.native"
 local EntryMaker = require "frecency.entry_maker"
 local FS = require "frecency.fs"
-local Finder = require "frecency.finder"
 local Migrator = require "frecency.migrator"
 local Picker = require "frecency.picker"
 local Recency = require "frecency.recency"
@@ -14,7 +13,7 @@ local log = require "plenary.log"
 ---@field config FrecencyConfig
 ---@field private buf_registered table<integer, boolean> flag to indicate the buffer is registered to the database.
 ---@field private database FrecencyDatabase
----@field private finder FrecencyFinder
+---@field private entry_maker FrecencyEntryMaker
 ---@field private fs FrecencyFS
 ---@field private migrator FrecencyMigrator
 ---@field private picker FrecencyPicker
@@ -69,11 +68,10 @@ Frecency.new = function(opts)
   end
   self.database = Database.new(self.fs, { root = config.db_root })
   local web_devicons = WebDevicons.new(not config.disable_devicons)
-  local entry_maker = EntryMaker.new(self.fs, web_devicons, {
+  self.entry_maker = EntryMaker.new(self.fs, web_devicons, {
     show_filter_column = config.show_filter_column,
     show_scores = config.show_scores,
   })
-  self.finder = Finder.new(entry_maker, self.fs)
   self.recency = Recency.new()
   self.migrator = Migrator.new(self.fs, self.recency, self.config.db_root)
   return self
@@ -122,7 +120,7 @@ function Frecency:start(opts)
   local start = os.clock()
   log.debug "Frecency:start"
   opts = opts or {}
-  self.picker = Picker.new(self.database, self.finder, self.fs, self.recency, {
+  self.picker = Picker.new(self.database, self.entry_maker, self.fs, self.recency, {
     default_workspace_tag = self.config.default_workspace,
     editing_bufnr = vim.api.nvim_get_current_buf(),
     filter_delimiter = self.config.filter_delimiter,

lua/frecency/picker.lua

@@ -1,4 +1,5 @@
 local State = require "frecency.state"
+local Finder = require "frecency.finder"
 local log = require "plenary.log"
 local Path = require "plenary.path" --[[@as PlenaryPath]]
 local actions = require "telescope.actions"
@@ -11,12 +12,12 @@ local uv = vim.loop or vim.uv
 ---@class FrecencyPicker
 ---@field private config FrecencyPickerConfig
 ---@field private database FrecencyDatabase
----@field private finder FrecencyFinder
+---@field private entry_maker FrecencyEntryMaker
 ---@field private fs FrecencyFS
 ---@field private lsp_workspaces string[]
 ---@field private namespace integer
 ---@field private recency FrecencyRecency
----@field private results table[]
+---@field private state FrecencyState
 ---@field private workspace string?
 ---@field private workspace_tag_regex string
 local Picker = {}
@@ -37,21 +38,20 @@ local Picker = {}
 ---@field score number
 
 ---@param database FrecencyDatabase
----@param finder FrecencyFinder
+---@param entry_maker FrecencyEntryMaker
 ---@param fs FrecencyFS
 ---@param recency FrecencyRecency
 ---@param config FrecencyPickerConfig
 ---@return FrecencyPicker
-Picker.new = function(database, finder, fs, recency, config)
+Picker.new = function(database, entry_maker, fs, recency, config)
   local self = setmetatable({
     config = config,
     database = database,
-    finder = finder,
+    entry_maker = entry_maker,
     fs = fs,
     lsp_workspaces = {},
     namespace = vim.api.nvim_create_namespace "frecency",
     recency = recency,
-    results = {},
   }, { __index = Picker })
   local d = self.config.filter_delimiter
   self.workspace_tag_regex = "^%s*" .. d .. "(%S+)" .. d
@@ -70,6 +70,16 @@ end
 ---| fun(opts: FrecencyPickerOptions, path: string): string
 ---@field workspace string?
 
+---@param opts table
+---@param workspace string?
+---@param workspace_tag string?
+function Picker:finder(opts, workspace, workspace_tag)
+  local filepath_formatter = self:filepath_formatter(opts)
+  local entry_maker = self.entry_maker:create(filepath_formatter, workspace, workspace_tag)
+  local need_scandir = not not (workspace and self.config.show_unindexed)
+  return Finder.new(self.database, entry_maker, self.fs, need_scandir, workspace, self.recency, self.state)
+end
+
 ---@param opts FrecencyPickerOptions?
 function Picker:start(opts)
   opts = vim.tbl_extend("force", {
@@ -80,36 +90,25 @@ function Picker:start(opts)
   }, opts or {}) --[[@as FrecencyPickerOptions]]
   self.workspace = self:get_workspace(opts.cwd, self.config.initial_workspace_tag)
   log.debug { workspace = self.workspace }
-  self.results = self:fetch_results(self.workspace)
-
-  local state = State.new()
-
-  local filepath_formatter = self:filepath_formatter(opts)
-  local finder = self.finder:start(state, filepath_formatter, self.results, {
-    need_scandir = self.workspace and self.config.show_unindexed and true or false,
-    workspace = self.workspace,
-    workspace_tag = self.config.initial_workspace_tag,
-  })
 
+  self.state = State.new()
+  local finder = self:finder(opts, self.workspace, self.config.initial_workspace_tag)
   local picker = pickers.new(opts, {
     prompt_title = "Frecency",
     finder = finder,
     previewer = config_values.file_previewer(opts),
     sorter = sorters.get_substr_matcher(),
-    on_input_filter_cb = self:on_input_filter_cb(state, opts),
+    on_input_filter_cb = self:on_input_filter_cb(opts),
     attach_mappings = function(prompt_bufnr)
       return self:attach_mappings(prompt_bufnr)
     end,
   })
-  state:set(picker)
+  self.state:set(picker)
   picker:find()
+  finder:start()
   self:set_prompt_options(picker.prompt_bufnr)
 end
 
-function Picker:discard_results()
-  -- TODO: implement here when it needs to cache.
-end
-
 --- See :h 'complete-functions'
 ---@param findstart 1|0
 ---@param base string
@@ -178,34 +177,6 @@ function Picker:get_workspace(cwd, tag)
   end
 end
 
----@private
----@param workspace string?
----@param datetime string? ISO8601 format string
----@return FrecencyFile[]
-function Picker:fetch_results(workspace, datetime)
-  log.debug { workspace = workspace or "NONE" }
-  local start_fetch = os.clock()
-  local files = self.database:get_entries(workspace, datetime)
-  log.debug(("it takes %f seconds in fetching entries"):format(os.clock() - start_fetch))
-  local start_results = os.clock()
-  local elapsed_recency = 0
-  for _, file in ipairs(files) do
-    local start_recency = os.clock()
-    file.score = file.ages and self.recency:calculate(file.count, file.ages) or 0
-    file.ages = nil
-    elapsed_recency = elapsed_recency + (os.clock() - start_recency)
-  end
-  log.debug(("it takes %f seconds in calculating recency"):format(elapsed_recency))
-  log.debug(("it takes %f seconds in making results"):format(os.clock() - start_results))
-
-  local start_sort = os.clock()
-  table.sort(files, function(a, b)
-    return a.score > b.score or (a.score == b.score and a.path > b.path)
-  end)
-  log.debug(("it takes %f seconds in sorting"):format(os.clock() - start_sort))
-  return files
-end
-
 ---@private
 ---@return string?
 function Picker:get_lsp_workspace()
@@ -216,11 +187,9 @@ function Picker:get_lsp_workspace()
 end
 
 ---@private
----@param state FrecencyState
 ---@param picker_opts table
 ---@return fun(prompt: string): table
-function Picker:on_input_filter_cb(state, picker_opts)
+function Picker:on_input_filter_cb(picker_opts)
-  local filepath_formatter = self:filepath_formatter(picker_opts)
   return function(prompt)
     local workspace
     local start, finish, tag = prompt:find(self.workspace_tag_regex)
@@ -230,7 +199,7 @@ function Picker:on_input_filter_cb(state, picker_opts)
     else
       workspace = self:get_workspace(picker_opts.cwd, tag) or self.workspace
     end
-    local picker = state:get()
+    local picker = self.state:get()
     if picker then
       local buf = picker.prompt_bufnr
       vim.api.nvim_buf_clear_namespace(buf, self.namespace, 0, -1)
@@ -249,13 +218,7 @@ function Picker:on_input_filter_cb(state, picker_opts)
     end
     if self.workspace ~= workspace then
       self.workspace = workspace
-      self.results = self:fetch_results(workspace)
+      opts.updated_finder = self:finder(picker_opts, self.workspace, tag or self.config.initial_workspace_tag):start()
-      opts.updated_finder = self.finder:start(state, filepath_formatter, self.results, {
-        initial_results = self.results,
-        need_scandir = self.workspace and self.config.show_unindexed and true or false,
-        workspace = self.workspace,
-        workspace_tag = tag,
-      })
     end
     return opts
   end

lua/frecency/tests/async_finder_spec.lua

@@ -1,98 +0,0 @@
----@diagnostic disable: invisible
-local AsyncFinder = require "frecency.async_finder"
-local State = require "frecency.state"
-local FS = require "frecency.fs"
-local EntryMaker = require "frecency.entry_maker"
-local WebDevicons = require "frecency.web_devicons"
-local util = require "frecency.tests.util"
-
----@param files string[]
----@param initial_results string[]
----@param callback fun(async_finder: FrecencyAsyncFinder, dir: PlenaryPath): nil
-local function with_files(files, initial_results, callback)
-  local dir, close = util.make_tree(files)
-  local fs = FS.new { ignore_patterns = {} }
-  local web_devicons = WebDevicons.new(true)
-  local function filepath_formatter()
-    return function(name)
-      return name
-    end
-  end
-  local entry_maker = EntryMaker.new(fs, web_devicons, { show_filter_column = false, show_scores = false })
-    :create(filepath_formatter, dir:absolute())
-  local initials = vim.tbl_map(function(v)
-    return { path = (dir / v):absolute() }
-  end, initial_results)
-  local async_finder = AsyncFinder.new(State.new(), fs, dir:absolute(), entry_maker, initials)
-  callback(async_finder, dir)
-  close()
-end
-
-describe("async_finder", function()
-  local function run(async_finder)
-    local count = { process_result = 0, process_complete = 0 }
-    local results = {}
-    async_finder("", function(result)
-      count.process_result = count.process_result + 1
-      table.insert(results, result.filename)
-    end, function()
-      count.process_complete = count.process_complete + 1
-    end)
-    return count, results
-  end
-
-  describe("with no initial_results", function()
-    with_files({ "hoge1.txt", "hoge2.txt" }, {}, function(async_finder, dir)
-      describe("when run at the first time", function()
-        local count, results = run(async_finder)
-        it("called process_result() at 2 times", function()
-          assert.are.same(2, count.process_result)
-        end)
-        it("called process_complete() at 1 time", function()
-          assert.are.same(1, count.process_complete)
-        end)
-        it("returns the whole results", function()
-          assert.are.same({
-            dir:joinpath("hoge1.txt").filename,
-            dir:joinpath("hoge2.txt").filename,
-          }, results)
-        end)
-      end)
-
-      describe("when run again", function()
-        local count, results = run(async_finder)
-        it("called process_result() at 2 times", function()
-          assert.are.same(2, count.process_result)
-        end)
-        it("called process_complete() at 1 time", function()
-          assert.are.same(1, count.process_complete)
-        end)
-        it("returns the same results", function()
-          assert.are.same({
-            dir:joinpath("hoge1.txt").filename,
-            dir:joinpath("hoge2.txt").filename,
-          }, results)
-        end)
-      end)
-    end)
-  end)
-
-  describe("with initial_results", function()
-    with_files({ "fuga1.txt", "hoge1.txt", "hoge2.txt" }, { "fuga1.txt" }, function(async_finder, dir)
-      local count, results = run(async_finder)
-      it("called process_result() at 3 times", function()
-        assert.are.same(3, count.process_result)
-      end)
-      it("called process_complete() at 1 time", function()
-        assert.are.same(1, count.process_complete)
-      end)
-      it("returns the same results without duplications", function()
-        assert.are.same({
-          dir:joinpath("fuga1.txt").filename,
-          dir:joinpath("hoge1.txt").filename,
-          dir:joinpath("hoge2.txt").filename,
-        }, results)
-      end)
-    end)
-  end)
-end)

lua/frecency/tests/frecency_spec.lua

@@ -8,7 +8,7 @@ local Path = require "plenary.path"
 local use_sqlite
 
 ---@param files string[]
----@param callback fun(frecency: Frecency, dir: PlenaryPath): nil
+---@param callback fun(frecency: Frecency, finder: FrecencyFinder, dir: PlenaryPath): nil
 ---@return nil
 local function with_files(files, callback)
   local dir, close = util.make_tree(files)
@@ -16,12 +16,13 @@ local function with_files(files, callback)
   local frecency = Frecency.new { db_root = dir.filename, use_sqlite = use_sqlite }
   frecency.picker = Picker.new(
     frecency.database,
-    frecency.finder,
+    frecency.entry_maker,
     frecency.fs,
     frecency.recency,
     { editing_bufnr = 0, filter_delimiter = ":", show_unindexed = false, workspaces = {} }
   )
-  callback(frecency, dir)
+  local finder = frecency.picker:finder {}
+  callback(frecency, finder, dir)
   close()
 end
 
@@ -92,13 +93,13 @@ describe("frecency", function()
     describe(db, function()
       describe("register", function()
         describe("when opening files", function()
-          with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir)
+          with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, finder, dir)
             local register = make_register(frecency, dir)
             register("hoge1.txt", "2023-07-29T00:00:00+09:00")
             register("hoge2.txt", "2023-07-29T01:00:00+09:00")
 
             it("has valid records in DB", function()
-              local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00")
+              local results = finder:get_results(nil, "2023-07-29T02:00:00+09:00")
               assert.are.same({
                 { count = 1, path = filepath(dir, "hoge2.txt"), score = 10 },
                 { count = 1, path = filepath(dir, "hoge1.txt"), score = 10 },
@@ -108,14 +109,14 @@ describe("frecency", function()
         end)
 
         describe("when opening again", function()
-          with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir)
+          with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, finder, dir)
             local register = make_register(frecency, dir)
             register("hoge1.txt", "2023-07-29T00:00:00+09:00")
             register("hoge2.txt", "2023-07-29T01:00:00+09:00")
             register("hoge1.txt", "2023-07-29T02:00:00+09:00", true)
 
             it("increases the score", function()
-              local results = frecency.picker:fetch_results(nil, "2023-07-29T03:00:00+09:00")
+              local results = finder:get_results(nil, "2023-07-29T03:00:00+09:00")
               assert.are.same({
                 { count = 2, path = filepath(dir, "hoge1.txt"), score = 40 },
                 { count = 1, path = filepath(dir, "hoge2.txt"), score = 10 },
@@ -125,14 +126,14 @@ describe("frecency", function()
         end)
 
         describe("when opening again but the same instance", function()
-          with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir)
+          with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, finder, dir)
             local register = make_register(frecency, dir)
             register("hoge1.txt", "2023-07-29T00:00:00+09:00")
             register("hoge2.txt", "2023-07-29T01:00:00+09:00")
             register("hoge1.txt", "2023-07-29T02:00:00+09:00")
 
             it("does not increase the score", function()
-              local results = frecency.picker:fetch_results(nil, "2023-07-29T03:00:00+09:00")
+              local results = finder:get_results(nil, "2023-07-29T03:00:00+09:00")
               assert.are.same({
                 { count = 1, path = filepath(dir, "hoge2.txt"), score = 10 },
                 { count = 1, path = filepath(dir, "hoge1.txt"), score = 10 },
@@ -142,7 +143,7 @@ describe("frecency", function()
         end)
 
         describe("when opening more than 10 times", function()
-          with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir)
+          with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, finder, dir)
             local register = make_register(frecency, dir)
             register("hoge1.txt", "2023-07-29T00:00:00+09:00")
             register("hoge1.txt", "2023-07-29T00:01:00+09:00", true)
@@ -161,7 +162,7 @@ describe("frecency", function()
             register("hoge2.txt", "2023-07-29T00:11:00+09:00", true)
 
             it("calculates score from the recent 10 times", function()
-              local results = frecency.picker:fetch_results(nil, "2023-07-29T00:12:00+09:00")
+              local results = finder:get_results(nil, "2023-07-29T00:12:00+09:00")
               assert.are.same({
                 { count = 12, path = filepath(dir, "hoge2.txt"), score = 12 * (10 * 100) / 10 },
                 { count = 2, path = filepath(dir, "hoge1.txt"), score = 2 * (2 * 100) / 10 },
@@ -173,7 +174,7 @@ describe("frecency", function()
 
       describe("benchmark", function()
         describe("after registered over >5000 files", function()
-          with_files({}, function(frecency, dir)
+          with_files({}, function(frecency, finder, dir)
             with_fake_register(frecency, dir, function(register)
               -- TODO: 6000 records is too many to use with native?
               -- local file_count = 6000
@@ -187,10 +188,13 @@ describe("frecency", function()
               for i = 1, file_count do
                 local file = ("hoge%08d.txt"):format(i)
                 table.insert(expected, { count = 1, path = filepath(dir, file), score = 10 })
+                -- HACK: disable log because it fails with too many logging
+                log.new({ level = "info" }, true)
                 register(file, "2023-07-29T00:00:00+09:00")
+                log.new({}, true)
               end
               local start = os.clock()
-              local results = frecency.picker:fetch_results(nil, "2023-07-29T00:01:00+09:00")
+              local results = finder:get_results(nil, "2023-07-29T00:01:00+09:00")
               table.sort(results, function(a, b)
                 return a.path < b.path
               end)
@@ -211,13 +215,13 @@ describe("frecency", function()
 
       describe("validate_database", function()
         describe("when no files are unlinked", function()
-          with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir)
+          with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, finder, dir)
             local register = make_register(frecency, dir)
             register("hoge1.txt", "2023-07-29T00:00:00+09:00")
             register("hoge2.txt", "2023-07-29T00:01:00+09:00")
 
             it("removes no entries", function()
-              local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00")
+              local results = finder:get_results(nil, "2023-07-29T02:00:00+09:00")
               assert.are.same({
                 { count = 1, path = filepath(dir, "hoge2.txt"), score = 10 },
                 { count = 1, path = filepath(dir, "hoge1.txt"), score = 10 },
@@ -228,7 +232,9 @@ describe("frecency", function()
 
         describe("when with not force", function()
           describe("when files are unlinked but it is less than threshold", function()
-            with_files({ "hoge1.txt", "hoge2.txt", "hoge3.txt", "hoge4.txt", "hoge5.txt" }, function(frecency, dir)
+            with_files(
+              { "hoge1.txt", "hoge2.txt", "hoge3.txt", "hoge4.txt", "hoge5.txt" },
+              function(frecency, finder, dir)
                 local register = make_register(frecency, dir)
                 register("hoge1.txt", "2023-07-29T00:00:00+09:00")
                 register("hoge2.txt", "2023-07-29T00:01:00+09:00")
@@ -241,7 +247,7 @@ describe("frecency", function()
                 frecency:validate_database()
 
                 it("removes no entries", function()
-                local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00")
+                  local results = finder:get_results(nil, "2023-07-29T02:00:00+09:00")
                   table.sort(results, function(a, b)
                     return a.path < b.path
                   end)
@@ -253,12 +259,15 @@ describe("frecency", function()
                     { count = 1, path = filepath(dir, "hoge5.txt"), score = 10 },
                   }, results)
                 end)
-            end)
+              end
+            )
           end)
 
           describe("when files are unlinked and it is more than threshold", function()
             describe('when the user response "yes"', function()
-              with_files({ "hoge1.txt", "hoge2.txt", "hoge3.txt", "hoge4.txt", "hoge5.txt" }, function(frecency, dir)
+              with_files(
+                { "hoge1.txt", "hoge2.txt", "hoge3.txt", "hoge4.txt", "hoge5.txt" },
+                function(frecency, finder, dir)
                   local register = make_register(frecency, dir)
                   register("hoge1.txt", "2023-07-29T00:00:00+09:00")
                   register("hoge2.txt", "2023-07-29T00:01:00+09:00")
@@ -279,7 +288,7 @@ describe("frecency", function()
                   end)
 
                   it("removes entries", function()
-                  local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00")
+                    local results = finder:get_results(nil, "2023-07-29T02:00:00+09:00")
                     table.sort(results, function(a, b)
                       return a.path < b.path
                     end)
@@ -288,11 +297,14 @@ describe("frecency", function()
                       { count = 1, path = filepath(dir, "hoge5.txt"), score = 10 },
                     }, results)
                   end)
-              end)
+                end
+              )
             end)
 
             describe('when the user response "no"', function()
-              with_files({ "hoge1.txt", "hoge2.txt", "hoge3.txt", "hoge4.txt", "hoge5.txt" }, function(frecency, dir)
+              with_files(
+                { "hoge1.txt", "hoge2.txt", "hoge3.txt", "hoge4.txt", "hoge5.txt" },
+                function(frecency, finder, dir)
                   local register = make_register(frecency, dir)
                   register("hoge1.txt", "2023-07-29T00:00:00+09:00")
                   register("hoge2.txt", "2023-07-29T00:01:00+09:00")
@@ -313,7 +325,7 @@ describe("frecency", function()
                   end)
 
                   it("removes no entries", function()
-                  local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00")
+                    local results = finder:get_results(nil, "2023-07-29T02:00:00+09:00")
                     table.sort(results, function(a, b)
                       return a.path < b.path
                     end)
@@ -325,14 +337,15 @@ describe("frecency", function()
                       { count = 1, path = filepath(dir, "hoge5.txt"), score = 10 },
                     }, results)
                   end)
-              end)
+                end
+              )
             end)
           end)
         end)
 
         describe("when with force", function()
           describe("when db_safe_mode is true", function()
-            with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir)
+            with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, finder, dir)
               local register = make_register(frecency, dir)
               register("hoge1.txt", "2023-07-29T00:00:00+09:00")
               register("hoge2.txt", "2023-07-29T00:01:00+09:00")
@@ -347,7 +360,7 @@ describe("frecency", function()
               end)
 
               it("needs confirmation for removing entries", function()
-                local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00")
+                local results = finder:get_results(nil, "2023-07-29T02:00:00+09:00")
                 assert.are.same({
                   { count = 1, path = filepath(dir, "hoge2.txt"), score = 10 },
                 }, results)
@@ -356,7 +369,7 @@ describe("frecency", function()
           end)
 
           describe("when db_safe_mode is false", function()
-            with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir)
+            with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, finder, dir)
               local register = make_register(frecency, dir)
               register("hoge1.txt", "2023-07-29T00:00:00+09:00")
               register("hoge2.txt", "2023-07-29T00:01:00+09:00")
@@ -372,7 +385,7 @@ describe("frecency", function()
               end)
 
               it("needs no confirmation for removing entries", function()
-                local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00")
+                local results = finder:get_results(nil, "2023-07-29T02:00:00+09:00")
                 assert.are.same({
                   { count = 1, path = filepath(dir, "hoge2.txt"), score = 10 },
                 }, results)

lua/frecency/tests/migrator_spec.lua

@@ -6,6 +6,8 @@ local Sqlite = require "frecency.database.sqlite"
 local Native = require "frecency.database.native"
 local util = require "frecency.tests.util"
 local wait = require "frecency.wait"
+-- TODO: replace this with vim.system
+local Job = require "plenary.job"
 
 ---@param callback fun(migrator: FrecencyMigrator, sqlite: FrecencyDatabase): nil
 ---@return nil
@@ -19,13 +21,27 @@ local function with(callback)
   close()
 end
 
+local function strptime(iso8601)
+  local result = vim.fn.strptime("%FT%T%z", iso8601)
+  return result ~= 0 and result or nil
+end
+
+-- NOTE: Windows has no strptime
+local function time_piece(iso8601)
+  local stdout, code =
+    Job:new({ "perl", "-MTime::Piece", "-e", "print Time::Piece->strptime('" .. iso8601 .. "', '%FT%T%z')->epoch" })
+      :sync(30000)
+  return code == 0 and tonumber(stdout[1]) or nil
+end
+
 ---@param source table<string,{ count: integer, timestamps: string[] }>
 local function v1_table(source)
   local records = {}
   for path, record in pairs(source) do
     local timestamps = {}
     for _, timestamp in ipairs(record.timestamps) do
-      table.insert(timestamps, vim.fn.strptime("%FT%T%z", timestamp))
+      local iso8601 = timestamp .. "+0000"
+      table.insert(timestamps, strptime(iso8601) or time_piece(iso8601))
     end
     records[path] = { count = record.count, timestamps = timestamps }
   end
@@ -44,11 +60,11 @@ describe("migrator", function()
 
         it("has converted into a valid table", function()
           assert.are.same(
+            native.table,
             v1_table {
-              ["hoge1.txt"] = { count = 1, timestamps = { "2023-08-21T00:00:00+0000" } },
+              ["hoge1.txt"] = { count = 1, timestamps = { "2023-08-21T00:00:00" } },
-              ["hoge2.txt"] = { count = 1, timestamps = { "2023-08-21T00:00:00+0000" } },
+              ["hoge2.txt"] = { count = 1, timestamps = { "2023-08-21T00:00:00" } },
-            },
+            }
-            native.table
           )
         end)
       end)
@@ -75,13 +91,13 @@ describe("migrator", function()
 
         it("has converted into a valid table", function()
           assert.are.same(
+            native.table,
             v1_table {
-              ["hoge1.txt"] = { count = 4, timestamps = { "2023-08-21T00:03:00+0000", "2023-08-21T00:04:00+0000" } },
+              ["hoge1.txt"] = { count = 4, timestamps = { "2023-08-21T00:03:00", "2023-08-21T00:04:00" } },
-              ["hoge2.txt"] = { count = 3, timestamps = { "2023-08-21T00:06:00+0000", "2023-08-21T00:07:00+0000" } },
+              ["hoge2.txt"] = { count = 3, timestamps = { "2023-08-21T00:06:00", "2023-08-21T00:07:00" } },
-              ["hoge3.txt"] = { count = 2, timestamps = { "2023-08-21T00:08:00+0000", "2023-08-21T00:09:00+0000" } },
+              ["hoge3.txt"] = { count = 2, timestamps = { "2023-08-21T00:08:00", "2023-08-21T00:09:00" } },
-              ["hoge4.txt"] = { count = 1, timestamps = { "2023-08-21T00:10:00+0000" } },
+              ["hoge4.txt"] = { count = 1, timestamps = { "2023-08-21T00:10:00" } },
-            },
+            }
-            native.table
           )
         end)
       end)
@@ -92,10 +108,10 @@ describe("migrator", function()
     with(function(migrator, sqlite)
       local native = Native.new(migrator.fs, { root = migrator.root })
       native.table = v1_table {
-        ["hoge1.txt"] = { count = 4, timestamps = { "2023-08-21T00:03:00+0000", "2023-08-21T00:04:00+0000" } },
+        ["hoge1.txt"] = { count = 4, timestamps = { "2023-08-21T00:03:00", "2023-08-21T00:04:00" } },
-        ["hoge2.txt"] = { count = 3, timestamps = { "2023-08-21T00:06:00+0000", "2023-08-21T00:07:00+0000" } },
+        ["hoge2.txt"] = { count = 3, timestamps = { "2023-08-21T00:06:00", "2023-08-21T00:07:00" } },
-        ["hoge3.txt"] = { count = 2, timestamps = { "2023-08-21T00:08:00+0000", "2023-08-21T00:09:00+0000" } },
+        ["hoge3.txt"] = { count = 2, timestamps = { "2023-08-21T00:08:00", "2023-08-21T00:09:00" } },
-        ["hoge4.txt"] = { count = 1, timestamps = { "2023-08-21T00:10:00+0000" } },
+        ["hoge4.txt"] = { count = 1, timestamps = { "2023-08-21T00:10:00" } },
       }
       wait(function()
         native:save()
@@ -114,7 +130,7 @@ describe("migrator", function()
       ]]
 
       it("has converted into a valid DB", function()
-        assert.are.same({
+        assert.are.same(records, {
           { path = "hoge1.txt", count = 4, datetime = "2023-08-21 00:03:00" },
           { path = "hoge1.txt", count = 4, datetime = "2023-08-21 00:04:00" },
           { path = "hoge2.txt", count = 3, datetime = "2023-08-21 00:06:00" },
@@ -122,7 +138,7 @@ describe("migrator", function()
           { path = "hoge3.txt", count = 2, datetime = "2023-08-21 00:08:00" },
           { path = "hoge3.txt", count = 2, datetime = "2023-08-21 00:09:00" },
           { path = "hoge4.txt", count = 1, datetime = "2023-08-21 00:10:00" },
-        }, records)
+        })
       end)
     end)
   end)

lua/frecency/tests/minimal.lua

@@ -7,7 +7,6 @@ end
 if not vim.env.SQLITE_PATH then
   error "set $SQLITE_PATH to find telescope.nvim"
 end
-vim.opt.runtimepath:append "."
 vim.opt.runtimepath:append(vim.env.PLENARY_PATH)
 vim.opt.runtimepath:append(vim.env.TELESCOPE_PATH)
 vim.opt.runtimepath:append(vim.env.SQLITE_PATH)

lua/frecency/tests/native_spec.lua

@@ -0,0 +1,46 @@
+local FS = require "frecency.fs"
+local Native = require "frecency.database.native"
+local async = require "plenary.async" --[[@as PlenaryAsync]]
+local util = require "frecency.tests.util"
+async.tests.add_to_env()
+
+local function with_native(f)
+  local fs = FS.new { ignore_patterns = {} }
+  local dir, close = util.tmpdir()
+  dir:joinpath("file_frecency.bin"):touch()
+  return function()
+    local native = Native.new(fs, { root = dir.filename })
+    f(native)
+    close()
+  end
+end
+
+local function save_and_load(native, tbl, datetime)
+  native:raw_save(util.v1_table(tbl))
+  async.util.sleep(100)
+  local entries = native:get_entries(nil, datetime)
+  table.sort(entries, function(a, b)
+    return a.path < b.path
+  end)
+  return entries
+end
+
+a.describe("frecency.database.native", function()
+  a.describe("updated by another process", function()
+    a.it(
+      "returns valid entries",
+      with_native(function(native)
+        assert.are.same(
+          {
+            { path = "hoge1.txt", count = 1, ages = { 60 } },
+            { path = "hoge2.txt", count = 1, ages = { 60 } },
+          },
+          save_and_load(native, {
+            ["hoge1.txt"] = { count = 1, timestamps = { "2023-08-21T00:00:00+0000" } },
+            ["hoge2.txt"] = { count = 1, timestamps = { "2023-08-21T00:00:00+0000" } },
+          }, "2023-08-21T01:00:00+0000")
+        )
+      end)
+    )
+  end)
+end)

lua/frecency/tests/util.lua

@@ -1,5 +1,7 @@
 local uv = vim.uv or vim.loop
+local async = require "plenary.async" --[[@as PlenaryAsync]]
 local Path = require "plenary.path"
+local Job = require "plenary.job"
 
 ---@return PlenaryPath
 ---@return fun(): nil close swwp all entries
@@ -22,4 +24,35 @@ local function make_tree(entries)
   return dir, close
 end
 
-return { make_tree = make_tree, tmpdir = tmpdir }
+local AsyncJob = async.wrap(function(cmd, callback)
+  return Job:new({
+    command = cmd[1],
+    args = { select(2, unpack(cmd)) },
+    on_exit = function(self, code, _)
+      local stdout = code == 0 and table.concat(self:result(), "\n") or nil
+      callback(stdout, code)
+    end,
+  }):start()
+end, 2)
+
+-- NOTE: vim.fn.strptime cannot be used in Lua loop
+local function time_piece(iso8601)
+  local stdout, code =
+    AsyncJob { "perl", "-MTime::Piece", "-e", "print Time::Piece->strptime('" .. iso8601 .. "', '%FT%T%z')->epoch" }
+  return code == 0 and tonumber(stdout) or nil
+end
+
+---@param source table<string,{ count: integer, timestamps: string[] }>
+local function v1_table(source)
+  local records = {}
+  for path, record in pairs(source) do
+    local timestamps = {}
+    for _, iso8601 in ipairs(record.timestamps) do
+      table.insert(timestamps, time_piece(iso8601))
+    end
+    records[path] = { count = record.count, timestamps = timestamps }
+  end
+  return { version = "v1", records = records }
+end
+
+return { make_tree = make_tree, tmpdir = tmpdir, v1_table = v1_table, time_piece = time_piece }

lua/frecency/types.lua

@@ -55,6 +55,7 @@
 ---@field parent PlenaryPath
 ---@field path { sep: string }
 ---@field rm fun(self: PlenaryPath, opts: { recursive: boolean }?): nil
+---@field touch fun(self: PlenaryPath, opts: { parents: boolean }?): nil
 
 ---@class PlenaryScanDirOptions
 ---@field hidden boolean if true hidden files will be added
@@ -70,9 +71,11 @@
 
 ---@class PlenaryAsync
 ---@field control PlenaryAsyncControl
+---@field tests { add_to_env: fun(): nil }
 ---@field util PlenaryAsyncUtil
 ---@field uv PlenaryAsyncUv
 ---@field void fun(f: fun(): nil): fun(): nil
+---@field wrap fun(f: (fun(...): any), args: integer): (fun(...): any)
 local PlenaryAsync = {}
 
 ---@async
@@ -85,28 +88,42 @@ function PlenaryAsync.run(f) end
 
 ---@class PlenaryAsyncControlChannel
 ---@field mpsc fun(): PlenaryAsyncControlChannelTx, PlenaryAsyncControlChannelRx
+---@field counter fun(): PlenaryAsyncControlChannelTx, PlenaryAsyncControlChannelRx
 
 ---@class PlenaryAsyncControlChannelTx
----@field send fun(entry: FrecencyEntry?): nil
+---@field send fun(entry: any?): nil
 local PlenaryAsyncControlChannelTx = {}
 
 ---@class PlenaryAsyncControlChannelRx
 local PlenaryAsyncControlChannelRx = {}
 
 ---@async
----@return FrecencyEntry?
+---@return any?
 function PlenaryAsyncControlChannelRx.recv() end
 
+---@async
+---@return any?
+function PlenaryAsyncControlChannelRx.last() end
+
 ---@class PlenaryAsyncUtil
 local PlenaryAsyncUtil = {}
 
 ---@class PlenaryAsyncUv
 local PlenaryAsyncUv = {}
 
+---@class FsStatMtime
+---@field sec integer
+---@field nsec integer
+
+---@class FsStat
+---@field mtime FsStatMtime
+---@field size integer
+---@field type "file"|"directory"
+
 ---@async
 ---@param path string
 ---@return string? err
----@return { mtime: integer, size: integer, type: "file"|"directory" }
+---@return { mtime: FsStatMtime, size: integer, type: "file"|"directory" }
 function PlenaryAsyncUv.fs_stat(path) end
 
 ---@async
@@ -184,3 +201,8 @@ function PlenaryAsyncUtil.sleep(ms) end
 ---@class WinInfo
 ---@field topline integer
 ---@field botline integer
+
+---@class UvFsEventHandle
+---@field stop fun(self: UvFsEventHandle): nil
+---@field start fun(self: UvFsEventHandle, path: string, opts: { recursive: boolean }, cb: fun(err: string?, filename: string?, events: string[])): nil
+---@field close fun(self: UvFsEventHandle): nil