List up entries for workspace files by rg or fd (#156)

* feat: read workspace files by external commands

* fix: avoid errors when it manages invalid buffers

* feat: add workspace_scan_cmd to select a way

* docs: describe `workspace_scan_cmd` option

README.md

@@ -79,12 +79,18 @@ directories provided by the language server.
 
 - [telescope.nvim](https://github.com/nvim-telescope/telescope.nvim) (required)
 - [nvim-web-devicons](https://github.com/kyazdani42/nvim-web-devicons) (optional)
+- [ripgrep](https://github.com/BurntSushi/ripgrep) or [fd](https://github.com/sharkdp/fd) (optional)
 
 **NOTE:** The former version of this plugin has used [SQLite3][] database to
 store timestamps and file records. But the current build uses Lua native code
 to store them, so you can now remove [sqlite.lua][] from dependencies. See
 [*Remove dependency for sqlite.lua*][remove-sqlite] for the detail.
 
+**NOTE:** `ripgrep` or `fd` will be used to list up workspace files. They are
+extremely faster than the native Lua logic. If you don't have them, it
+fallbacks to Lua code automatically. See the detail for `workspace_scan_cmd`
+option.
+
 [SQLite3]: https://www.sqlite.org/index.html
 [sqlite.lua]: https://github.com/kkharji/sqlite.lua
 [remove-sqlite]: #user-content-remove-dependency-for-sqlitelua
@@ -203,6 +209,23 @@ See [default configuration](https://github.com/nvim-telescope/telescope.nvim#tel
   Use [sqlite.lua][] with `true` or native code with `false`. See [*Remove
   dependency for sqlite.lua*][remove-sqlite] for the detail.
 
+- `workspace_scan_cmd` (default: `nil`)
+
+  This option can be set values as `"LUA"|string[]|nil`. With the default
+  value: `nil`, it uses these way below to make entries for workspace files.
+  It tries in order until it works successfully.
+
+  1. `rg -0.g '!.git' --files`
+  2. `fdfind -0Htf`
+  3. `fd -0Htf`
+  4. Native Lua code (old way)
+
+  If you like another commands, set them to this option, like
+  `workspace_scan_cmd = { "find", ".", "-type", "f", "-print0" }`. This command
+  must use NUL characters for delimiters.
+
+  If you prefer Native Lua code, set `workspace_scan_cmd = "LUA"`.
+
 - `workspaces` (default: `{}`)
 
   This table contains mappings of `workspace_tag` -> `workspace_directory`. The

lua/frecency/finder.lua

@@ -5,20 +5,27 @@ local log = require "plenary.log"
 ---@field config FrecencyFinderConfig
 ---@field closed boolean
 ---@field entries FrecencyEntry[]
+---@field scanned_entries FrecencyEntry[]
 ---@field entry_maker FrecencyEntryMakerInstance
 ---@field fs FrecencyFS
----@field need_scandir boolean
 ---@field path string?
 ---@field private database FrecencyDatabase
----@field private recency FrecencyRecency
 ---@field private rx PlenaryAsyncControlChannelRx
----@field private state FrecencyState
 ---@field private tx PlenaryAsyncControlChannelTx
+---@field private scan_rx PlenaryAsyncControlChannelRx
+---@field private scan_tx PlenaryAsyncControlChannelTx
+---@field private need_scan_db boolean
+---@field private need_scan_dir boolean
+---@field private seen table<string, boolean>
+---@field private process VimSystemObj?
+---@field private recency FrecencyRecency
+---@field private state FrecencyState
 local Finder = {}
 
 ---@class FrecencyFinderConfig
----@field chunk_size integer default: 1000
+---@field chunk_size integer? default: 1000
----@field sleep_interval integer default: 50
+---@field sleep_interval integer? default: 50
+---@field workspace_scan_cmd "LUA"|string[]|nil default: nil
 
 ---@param database FrecencyDatabase
 ---@param entry_maker FrecencyEntryMakerInstance
@@ -31,19 +38,26 @@ local Finder = {}
 ---@return FrecencyFinder
 Finder.new = function(database, entry_maker, fs, need_scandir, path, recency, state, config)
   local tx, rx = async.control.channel.mpsc()
+  local scan_tx, scan_rx = async.control.channel.mpsc()
   return setmetatable({
     config = vim.tbl_extend("force", { chunk_size = 1000, sleep_interval = 50 }, config or {}),
     closed = false,
     database = database,
-    entries = {},
     entry_maker = entry_maker,
     fs = fs,
-    need_scandir = need_scandir,
     path = path,
     recency = recency,
-    rx = rx,
     state = state,
+
+    seen = {},
+    entries = {},
+    scanned_entries = {},
+    need_scan_db = true,
+    need_scan_dir = need_scandir and path,
+    rx = rx,
     tx = tx,
+    scan_rx = scan_rx,
+    scan_tx = scan_tx,
   }, {
     __index = Finder,
     ---@param self FrecencyFinder
@@ -56,80 +70,149 @@ end
 ---@param datetime string?
 ---@return nil
 function Finder:start(datetime)
+  local cmd = self.config.workspace_scan_cmd
+  local ok
+  if cmd ~= "LUA" and self.need_scan_dir then
+    ---@type string[][]
+    local cmds = cmd and { cmd } or { { "rg", "-0.g", "!.git", "--files" }, { "fdfind", "-0Htf" }, { "fd", "-0Htf" } }
+    for _, c in ipairs(cmds) do
+      ok = self:scan_dir_cmd(c)
+      if ok then
+        log.debug("scan_dir_cmd: " .. vim.inspect(c))
+        break
+      end
+    end
+  end
   async.void(function()
     -- NOTE: return to the main loop to show the main window
-    async.util.sleep(0)
+    async.util.scheduler()
-    local seen = {}
+    for _, file in ipairs(self:get_results(self.path, datetime)) do
-    for i, file in ipairs(self:get_results(self.path, datetime)) do
       local entry = self.entry_maker(file)
-      seen[entry.filename] = true
-      entry.index = i
-      table.insert(self.entries, entry)
       self.tx.send(entry)
     end
-    if self.need_scandir and self.path then
-      -- NOTE: return to the main loop to show results from DB
-      async.util.sleep(self.config.sleep_interval)
-      self:scan_dir(seen)
-    end
-    self:close()
     self.tx.send(nil)
+    if self.need_scan_dir and not ok then
+      log.debug "scan_dir_lua"
+      async.util.scheduler()
+      self:scan_dir_lua()
+    end
   end)()
 end
 
----@param seen table<string, boolean>
+---@param cmd string[]
+---@return boolean
+function Finder:scan_dir_cmd(cmd)
+  local ok
+  ---@diagnostic disable-next-line: assign-type-mismatch
+  ok, self.process = pcall(vim.system, cmd, {
+    cwd = self.path,
+    stdout = function(err, chunk)
+      if not self.closed and not err and chunk then
+        for name in chunk:gmatch "[^%z]+" do
+          local cleaned = name:gsub("^%./", "")
+          local fullpath = self.fs.joinpath(self.path, cleaned)
+          local entry = self.entry_maker { id = 0, count = 0, path = fullpath, score = 0 }
+          self.scan_tx.send(entry)
+        end
+      end
+    end,
+  }, function()
+    self.process = nil
+    self:close()
+    self.scan_tx.send(nil)
+  end)
+  if not ok then
+    self.process = nil
+  end
+  return ok
+end
+
+---@async
 ---@return nil
-function Finder:scan_dir(seen)
+function Finder:scan_dir_lua()
   local count = 0
-  local index = #self.entries
   for name in self.fs:scan_dir(self.path) do
     if self.closed then
       break
     end
     local fullpath = self.fs.joinpath(self.path, name)
-    if not seen[fullpath] then
+    local entry = self.entry_maker { id = 0, count = 0, path = fullpath, score = 0 }
-      seen[fullpath] = true
+    self.scan_tx.send(entry)
-      count = count + 1
+    count = count + 1
-      local entry = self.entry_maker { id = 0, count = 0, path = fullpath, score = 0 }
+    if count % self.config.chunk_size == 0 then
-      if entry then
+      async.util.sleep(self.config.sleep_interval)
-        index = index + 1
-        entry.index = index
-        table.insert(self.entries, entry)
-        self.tx.send(entry)
-        if count % self.config.chunk_size == 0 then
-          self:reflow_results()
-          async.util.sleep(self.config.sleep_interval)
-        end
-      end
     end
   end
+  self.scan_tx.send(nil)
 end
 
+---@async
 ---@param _ string
 ---@param process_result fun(entry: FrecencyEntry): nil
 ---@param process_complete fun(): nil
 ---@return nil
 function Finder:find(_, process_result, process_complete)
-  local index = 0
+  if self:process_table(process_result, self.entries) then
-  for _, entry in ipairs(self.entries) do
+    return
-    index = index + 1
-    if process_result(entry) then
-      return
-    end
   end
-  local count = 0
+  if self.need_scan_db then
-  while not self.closed do
+    if self:process_channel(process_result, self.entries, self.rx) then
-    count = count + 1
-    local entry = self.rx.recv()
-    if not entry then
-      break
-    elseif entry.index > index and process_result(entry) then
       return
     end
+    self.need_scan_db = false
+  end
+  if self:process_table(process_result, self.scanned_entries) then
+    return
+  end
+  if self.need_scan_dir then
+    if self:process_channel(process_result, self.scanned_entries, self.scan_rx, #self.entries) then
+      return
+    end
+    self.need_scan_dir = false
   end
   process_complete()
 end
 
+---@param process_result fun(entry: FrecencyEntry): nil
+---@param entries FrecencyEntry[]
+---@return boolean?
+function Finder:process_table(process_result, entries)
+  for _, entry in ipairs(entries) do
+    if process_result(entry) then
+      return true
+    end
+  end
+end
+
+---@async
+---@param process_result fun(entry: FrecencyEntry): nil
+---@param entries FrecencyEntry[]
+---@param rx PlenaryAsyncControlChannelRx
+---@param start_index integer?
+---@return boolean?
+function Finder:process_channel(process_result, entries, rx, start_index)
+  local index = #entries > 0 and entries[#entries].index or start_index or 0
+  local count = 0
+  while true do
+    local entry = rx.recv()
+    if not entry then
+      break
+    elseif not self.seen[entry.filename] then
+      self.seen[entry.filename] = true
+      index = index + 1
+      entry.index = index
+      table.insert(entries, entry)
+      if process_result(entry) then
+        return true
+      end
+    end
+    count = count + 1
+    if count % self.config.chunk_size == 0 then
+      self:reflow_results()
+    end
+  end
+end
+
 ---@param workspace string?
 ---@param datetime string?
 ---@return FrecencyFile[]
@@ -159,16 +242,22 @@ end
 
 function Finder:close()
   self.closed = true
+  if self.process then
+    self.process:kill(9)
+  end
 end
 
+---@async
+---@return nil
 function Finder:reflow_results()
   local picker = self.state:get()
   if not picker then
     return
   end
+  async.util.scheduler()
   local bufnr = picker.results_bufnr
   local win = picker.results_win
-  if not bufnr or not win then
+  if not bufnr or not win or not vim.api.nvim_buf_is_valid(bufnr) or not vim.api.nvim_win_is_valid(win) then
     return
   end
   picker:clear_extra_rows(bufnr)
@@ -178,7 +267,6 @@ function Finder:reflow_results()
       return
     end
     local worst_line = picker:get_row(manager:num_results())
-    ---@type WinInfo
     local wininfo = vim.fn.getwininfo(win)[1]
     local bottom = vim.api.nvim_buf_line_count(bufnr)
     if not self.reflowed or worst_line > wininfo.botline then

lua/frecency/frecency.lua

@@ -34,6 +34,7 @@ local Frecency = {}
 ---@field show_scores boolean? default: false
 ---@field show_unindexed boolean? default: true
 ---@field use_sqlite boolean? default: false
+---@field workspace_scan_cmd "LUA"|string[]|nil default: nil
 ---@field workspaces table<string, string>? default: {}
 
 ---@param opts FrecencyConfig?
@@ -54,6 +55,7 @@ Frecency.new = function(opts)
     show_scores = false,
     show_unindexed = true,
     use_sqlite = false,
+    workspace_scan_cmd = nil,
     workspaces = {},
   }, opts or {})
   local self = setmetatable({ buf_registered = {}, config = config }, { __index = Frecency })--[[@as Frecency]]
@@ -133,6 +135,7 @@ function Frecency:start(opts)
     filter_delimiter = self.config.filter_delimiter,
     initial_workspace_tag = opts.workspace,
     show_unindexed = self.config.show_unindexed,
+    workspace_scan_cmd = self.config.workspace_scan_cmd,
     workspaces = self.config.workspaces,
   })
   self.picker:start(vim.tbl_extend("force", self.config, opts))

lua/frecency/picker.lua

@@ -28,6 +28,7 @@ local Picker = {}
 ---@field filter_delimiter string
 ---@field initial_workspace_tag string?
 ---@field show_unindexed boolean
+---@field workspace_scan_cmd "LUA"|string[]|nil
 ---@field workspaces table<string, string>
 
 ---@class FrecencyPickerEntry
@@ -77,7 +78,16 @@ function Picker:finder(opts, workspace, workspace_tag)
   local filepath_formatter = self:filepath_formatter(opts)
   local entry_maker = self.entry_maker:create(filepath_formatter, workspace, workspace_tag)
   local need_scandir = not not (workspace and self.config.show_unindexed)
-  return Finder.new(self.database, entry_maker, self.fs, need_scandir, workspace, self.recency, self.state)
+  return Finder.new(
+    self.database,
+    entry_maker,
+    self.fs,
+    need_scandir,
+    workspace,
+    self.recency,
+    self.state,
+    { workspace_scan_cmd = self.config.workspace_scan_cmd }
+  )
 end
 
 ---@param opts FrecencyPickerOptions?

lua/frecency/types.lua

@@ -166,6 +166,10 @@ function PlenaryAsyncUv.fs_close(fd) end
 ---@return nil
 function PlenaryAsyncUtil.sleep(ms) end
 
+---@async
+---@return nil
+function PlenaryAsyncUtil.scheduler() end
+
 -- NOTE: types are for telescope.nvim
 
 ---@alias TelescopeEntryDisplayer fun(items: string[]): table
@@ -198,11 +202,20 @@ function PlenaryAsyncUtil.sleep(ms) end
 
 -- NOTE: types for default functions
 
----@class WinInfo
----@field topline integer
----@field botline integer
-
 ---@class UvFsEventHandle
 ---@field stop fun(self: UvFsEventHandle): nil
 ---@field start fun(self: UvFsEventHandle, path: string, opts: { recursive: boolean }, cb: fun(err: string?, filename: string?, events: string[])): nil
 ---@field close fun(self: UvFsEventHandle): nil
+
+--- @class VimSystemObj
+--- @field pid integer
+--- @field wait fun(self: VimSystemObj, timeout?: integer): VimSystemCompleted
+--- @field kill fun(self: VimSystemObj, signal: integer|string)
+--- @field write fun(self: VimSystemObj, data?: string|string[])
+--- @field is_closing fun(self: VimSystemObj): boolean?
+
+--- @class VimSystemCompleted
+--- @field code integer
+--- @field signal integer
+--- @field stdout? string
+--- @field stderr? string