kristoferssolo/telescope-frecency.nvim
1
0
Fork 0
mirror of https://github.com/kristoferssolo/telescope-frecency.nvim.git synced 2025-10-21 20:10:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: add logic to store data by native code (#130)

Browse Source 
* refactor: make logic for Database be abstract
* feat: add logic for DB by string.dump
* fix: run with async.void to run synchronously
* test: add tests for native feature
* feat!: sort candidates by path when score is same
  This is needed because candidates from SQLite is sorted by id, but ones from native is sorted by path.
* chore: clean up types
* feat: add lock/unlock feature to access DB
* test: use async version of busted
  And disable benchmark tests (fix later)
* test: add tests for file_lock
* chore: use more explicit names
* chore: use plenary.log instead
* fix: wait async functions definitely
* feat: add migrator
* chore: fix logging
* fix: detect emptiness of the table
* fix: deal with buffer with no names
* test: loosen the condition temporarily
* test: add tests for migrator
* fix: return true when the table is not empty
* feat: load sqlite lazily not to require in start
* chore: add logging to calculate time for fetching
* feat: add converter from native code to SQLite
* feat: warn when sqlite.lua is not available
* feat: add FrecencyMigrateDB to migrate DB
* docs: add note for native code logic
* test: ignore type bug
This commit is contained in:
JINNOUCHI Yasushi 2023-08-27 18:51:16 +09:00 committed by GitHub
parent 5d1a01be63
commit 9037d696e6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 1283 additions and 426 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.luarc.json
View File

@ -1,6 +1,7 @@
{
"diagnostics": {
"globals": [
"a",
"describe",
"it",
"vim"

49
README.md
View File

@ -31,7 +31,7 @@ The score is calculated using the age of the 10 most recent timestamps and the t
### Score calculation
```
```lua
score = frequency * recency_score / max_number_of_timestamps
```
## What about files that are neither 'frequent' _or_ 'recent' ?
@ -59,9 +59,11 @@ If the active buffer (prior to the finder being launched) is attached to an LSP
## Requirements
- [telescope.nvim](https://github.com/nvim-telescope/telescope.nvim) (required)
- [sqlite.lua](https://github.com/kkharji/sqlite.lua) (required)
- [sqlite.lua][] (required)
- [nvim-web-devicons](https://github.com/kyazdani42/nvim-web-devicons) (optional)
[sqlite.lua]: https://github.com/kkharji/sqlite.lua
Timestamps and file records are stored in an [SQLite3](https://www.sqlite.org/index.html) database for persistence and speed.
This plugin uses `sqlite.lua` to perform the database transactions.
@ -73,9 +75,9 @@ This plugin uses `sqlite.lua` to perform the database transactions.
use {
"nvim-telescope/telescope-frecency.nvim",
config = function()
require"telescope".load_extension("frecency")
require("telescope").load_extension "frecency"
end,
requires = {"kkharji/sqlite.lua"}
requires = { "kkharji/sqlite.lua" },
}
```
@ -85,9 +87,9 @@ use {
{
"nvim-telescope/telescope-frecency.nvim",
config = function()
require"telescope".load_extension("frecency")
require("telescope").load_extension "frecency"
end,
dependencies = {"kkharji/sqlite.lua"}
dependencies = { "kkharji/sqlite.lua" },
}
```
@ -102,7 +104,7 @@ If no database is found when running Neovim with the plugin installed, a new one
or to map to a key:
```lua
vim.api.nvim_set_keymap("n", "<leader><leader>", "<Cmd>lua require('telescope').extensions.frecency.frecency()<CR>", {noremap = true, silent = true})
vim.api.nvim_set_keymap("n", "<leader><leader>", "<Cmd>Telescope frecency<CR>")
```
Use a specific workspace tag:
@ -114,7 +116,7 @@ Use a specific workspace tag:
or
```lua
vim.api.nvim_set_keymap("n", "<leader><leader>", "<Cmd>lua require('telescope').extensions.frecency.frecency({ workspace = 'CWD' })<CR>", {noremap = true, silent = true})
vim.api.nvim_set_keymap("n", "<leader><leader>", "<Cmd>Telescope frecency workspace=CWD<CR>")
```
Filter tags are applied by typing the `:tag:` name (adding surrounding colons) in the finder query.
@ -124,7 +126,7 @@ Entering `:<Tab>` will trigger omnicompletion for available tags.
See [default configuration](https://github.com/nvim-telescope/telescope.nvim#telescope-defaults) for full details on configuring Telescope.
- `db_root` (default: `nil`)
- `db_root` (default: `vim.fn.stdpath "data"`)
Path to parent directory of custom database location.
Defaults to `$XDG_DATA_HOME/nvim` if unset.
@ -133,7 +135,7 @@ See [default configuration](https://github.com/nvim-telescope/telescope.nvim#tel
Default workspace tag to filter by e.g. `'CWD'` to filter by default to the current directory. Can be overridden at query time by specifying another filter like `':*:'`.
- `ignore_patterns` (default: `{"*.git/*", "*/tmp/*"}`)
- `ignore_patterns` (default: `{ "*.git/*", "*/tmp/*", "term://*" }`)
Patterns in this table control which files are indexed (and subsequently which you'll see in the finder results).
@ -141,7 +143,7 @@ See [default configuration](https://github.com/nvim-telescope/telescope.nvim#tel
To see the scores generated by the algorithm in the results, set this to `true`.
- `workspaces` (default: {})
- `workspaces` (default: `{}`)
This table contains mappings of `workspace_tag` -> `workspace_directory`
The key corresponds to the `:tag_name` used to select the filter in queries.
@ -164,10 +166,13 @@ See [default configuration](https://github.com/nvim-telescope/telescope.nvim#tel
show_filter_column = { "LSP", "CWD", "FOO" }
```
- `use_sqlite` (default: `true`) ***experimental feature***
Use [sqlite.lua] `true` or native code `false`. See [*Remove dependency for sqlite.lua*](#remove-dependency-for-sqlite.lua) for the detail.
### Example Configuration:
```
```lua
telescope.setup {
extensions = {
frecency = {
@ -187,12 +192,14 @@ telescope.setup {
}
```
### SQL database location
## Note for Database
The default location for the sqlite3 database is `$XDG_DATA_HOME/nvim` (eg `~/.local/share/nvim/` on linux).
### Location
The default location for the database is `$XDG_DATA_HOME/nvim` (eg `~/.local/share/nvim/` on linux).
This can be configured with the `db_root` config option.
### SQL database maintainance
### Maintainance
By default, frecency will prune files that no longer exist from the database.
In certain workflows, switching branches in a repository, that behaviour might not be desired.
@ -210,7 +217,17 @@ The command `FrecencyValidate` can be used to clean the database when `auto_vali
:FrecencyValidate!
```
### Highlight Groups
### Remove dependency for [sqlite.lua][]
***This is an experimental feature.***
In default, it uses SQLite3 library to access the DB. When `use_sqlite` option is set to `false`, it stores the whole data and saves them with encoding by `string.dump()` Lua function.
With this, we can remove the dependency for [sqlite.lua][] and obtain faster speed to open `:Telescope frecency`.
You can migrate from SQLite DB into native code by `:FrecencyMigrateDB` command. It converts data into native code, but does not delete the existent SQLite DB. You can use old SQLite logic by `use_sqlite = true` again.
## Highlight Groups
```vim
TelescopeBufferLoaded

141
lua/frecency/database.lua
View File

@ -1,136 +1,43 @@
local sqlite = require "sqlite"
local log = require "plenary.log"
---@diagnostic disable: missing-return, unused-local
---@class FrecencyDatabaseConfig
---@field root string
---@class FrecencySqlite: sqlite_db
---@field files sqlite_tbl
---@field timestamps sqlite_tbl
---@class FrecencyFile
---@field count integer
---@field id integer
---@field path string
---@field score integer calculated from count and age
---@class FrecencyTimestamp
---@field age integer calculated from timestamp
---@field file_id integer
---@field id integer
---@field timestamp number
---@class FrecencyDatabaseGetFilesOptions
---@field path string?
---@field workspace string?
---@class FrecencyDatabase
---@field config FrecencyDatabaseConfig
---@field private buf_registered_flag_name string
---@field private fs FrecencyFS
---@field private sqlite FrecencySqlite
---@field has_entry fun(): boolean
---@field new fun(fs: FrecencyFS, config: FrecencyDatabaseConfig): FrecencyDatabase
---@field protected fs FrecencyFS
local Database = {}
---@param fs FrecencyFS
---@param config FrecencyDatabaseConfig
---@return FrecencyDatabase
Database.new = function(fs, config)
local lib = sqlite.lib --[[@as sqlite_lib]]
local self = setmetatable(
{ config = config, buf_registered_flag_name = "telescope_frecency_registered", fs = fs },
{ __index = Database }
)
self.sqlite = sqlite {
uri = self.config.root .. "/file_frecency.sqlite3",
files = { id = true, count = { "integer", default = 1, required = true }, path = "string" },
timestamps = {
id = true,
file_id = { "integer", reference = "files.id", on_delete = "cascade" },
timestamp = { "real", default = lib.julianday "now" },
},
}
return self
end
---@return boolean
function Database:has_entry()
return self.sqlite.files:count() > 0
end
---@param paths string[]
---@return integer
function Database:insert_files(paths)
if #paths == 0 then
return 0
end
---@param path string
return self.sqlite.files:insert(vim.tbl_map(function(path)
return { path = path, count = 0 } -- TODO: remove when sql.nvim#97 is closed
end, paths))
end
---@return nil
function Database:insert_files(paths) end
---@param workspace string?
---@return FrecencyFile[]
function Database:get_files(workspace)
local query = workspace and { contains = { path = { workspace .. "/*" } } } or {}
log.debug { query = query }
return self.sqlite.files:get(query)
end
---@return integer[]|string[]
function Database:unlinked_entries() end
---@param datetime string? ISO8601 format string
---@return FrecencyTimestamp[]
function Database:get_timestamps(datetime)
local lib = sqlite.lib
local age = lib.cast((lib.julianday(datetime) - lib.julianday "timestamp") * 24 * 60, "integer")
return self.sqlite.timestamps:get { keys = { age = age, "id", "file_id" } }
end
---@param files integer[]|string[]
---@return nil
function Database:remove_files(files) end
---@param path string
---@return integer: id of the file entry
---@return boolean: whether the entry is inserted (true) or updated (false)
function Database:upsert_files(path)
local file = self.sqlite.files:get({ where = { path = path } })[1] --[[@as FrecencyFile?]]
if file then
self.sqlite.files:update { where = { id = file.id }, set = { count = file.count + 1 } }
return file.id, false
end
return self.sqlite.files:insert { path = path }, true
end
---@param file_id integer
---@param datetime string? ISO8601 format string
---@return integer
function Database:insert_timestamps(file_id, datetime)
return self.sqlite.timestamps:insert {
file_id = file_id,
timestamp = datetime and sqlite.lib.julianday(datetime) or nil,
}
end
---@param file_id integer
---@param max_count integer
function Database:trim_timestamps(file_id, max_count)
local timestamps = self.sqlite.timestamps:get { where = { file_id = file_id } } --[[@as FrecencyTimestamp[] ]]
local trim_at = timestamps[#timestamps - max_count + 1]
if trim_at then
self.sqlite.timestamps:remove { file_id = tostring(file_id), id = "<" .. tostring(trim_at.id) }
end
end
---@return integer[]
function Database:unlinked_entries()
---@param file FrecencyFile
return self.sqlite.files:map(function(file)
if not self.fs:is_valid_path(file.path) then
return file.id
end
end)
end
---@param ids integer[]
---@param datetime string?
---@return nil
function Database:remove_files(ids)
self.sqlite.files:remove { id = ids }
end
function Database:update(path, max_count, datetime) end
return Database
---@async
---@class FrecencyDatabaseEntry
---@field ages number[]
---@field count integer
---@field path string
---@field score number
---@param workspace string?
---@param datetime string?
---@return FrecencyDatabaseEntry[]
function Database:get_entries(workspace, datetime) end

173
lua/frecency/database/native.lua Normal file
View File

@ -0,0 +1,173 @@
local FileLock = require "frecency.file_lock"
local wait = require "frecency.wait"
local log = require "plenary.log"
local async = require "plenary.async" --[[@as PlenaryAsync]]
---@class FrecencyDatabaseNative: FrecencyDatabase
---@field version "v1"
---@field filename string
---@field file_lock FrecencyFileLock
---@field table FrecencyDatabaseNativeTable
local Native = {}
---@class FrecencyDatabaseNativeTable
---@field version string
---@field records table<string,FrecencyDatabaseNativeRecord>
---@class FrecencyDatabaseNativeRecord
---@field count integer
---@field timestamps integer[]
---@param fs FrecencyFS
---@param config FrecencyDatabaseConfig
---@return FrecencyDatabaseNative
Native.new = function(fs, config)
local version = "v1"
local self = setmetatable({
config = config,
fs = fs,
table = { version = version, records = {} },
version = version,
}, { __index = Native })
self.filename = self.config.root .. "/file_frecency.bin"
self.file_lock = FileLock.new(self.filename)
wait(function()
self:load()
end)
return self
end
---@return boolean
function Native:has_entry()
return not vim.tbl_isempty(self.table.records)
end
---@param paths string[]
---@return nil
function Native:insert_files(paths)
if #paths == 0 then
return
end
for _, path in ipairs(paths) do
self.table.records[path] = { count = 1, timestamps = { 0 } }
end
wait(function()
self:save()
end)
end
---@return string[]
function Native:unlinked_entries()
local paths = {}
for file in pairs(self.table.records) do
if not self.fs:is_valid_path(file) then
table.insert(paths, file)
end
end
return paths
end
---@param paths string[]
function Native:remove_files(paths)
for _, file in ipairs(paths) do
self.table.records[file] = nil
end
wait(function()
self:save()
end)
end
---@param path string
---@param max_count integer
---@param datetime string?
function Native:update(path, max_count, datetime)
local record = self.table.records[path] or { count = 0, timestamps = {} }
record.count = record.count + 1
local now = self:now(datetime)
table.insert(record.timestamps, now)
if #record.timestamps > max_count then
local new_table = {}
for i = #record.timestamps - max_count + 1, #record.timestamps do
table.insert(new_table, record.timestamps[i])
end
record.timestamps = new_table
end
self.table.records[path] = record
wait(function()
self:save()
end)
end
---@param workspace string?
---@param datetime string?
---@return FrecencyDatabaseEntry[]
function Native:get_entries(workspace, datetime)
-- TODO: check mtime of DB and reload it
-- self:load()
local now = self:now(datetime)
local items = {}
for path, record in pairs(self.table.records) do
if not workspace or path:find(workspace .. "/", 1, true) then
table.insert(items, {
path = path,
count = record.count,
ages = vim.tbl_map(function(v)
return (now - v) / 60
end, record.timestamps),
})
end
end
return items
end
---@private
---@param datetime string?
---@return integer
function Native:now(datetime)
return datetime and vim.fn.strptime("%FT%T%z", datetime) or os.time()
end
---@async
---@return nil
function Native:load()
local start = os.clock()
local err, data = self.file_lock:with(function()
local err, st = async.uv.fs_stat(self.filename)
if err then
return nil
end
local fd
err, fd = async.uv.fs_open(self.filename, "r", tonumber("644", 8))
assert(not err)
local data
err, data = async.uv.fs_read(fd, st.size)
assert(not err)
assert(not async.uv.fs_close(fd))
return data
end)
assert(not err, err)
local tbl = loadstring(data or "")() --[[@as FrecencyDatabaseNativeTable?]]
if tbl and tbl.version == self.version then
self.table = tbl
end
log.debug(("load() takes %f seconds"):format(os.clock() - start))
end
---@async
---@return nil
function Native:save()
local start = os.clock()
local err = self.file_lock:with(function()
local f = assert(load("return " .. vim.inspect(self.table)))
local data = string.dump(f)
local err, fd = async.uv.fs_open(self.filename, "w", tonumber("644", 8))
assert(not err)
assert(not async.uv.fs_write(fd, data))
assert(not async.uv.fs_close(fd))
return nil
end)
assert(not err, err)
log.debug(("save() takes %f seconds"):format(os.clock() - start))
end
return Native

135
lua/frecency/database/sqlite.lua Normal file
View File

@ -0,0 +1,135 @@
local sqlite = require "frecency.sqlite"
local log = require "plenary.log"
---@class FrecencySqliteDB: sqlite_db
---@field files sqlite_tbl
---@field timestamps sqlite_tbl
---@class FrecencyFile
---@field count integer
---@field id integer
---@field path string
---@field score integer calculated from count and age
---@class FrecencyTimestamp
---@field age integer calculated from timestamp
---@field file_id integer
---@field id integer
---@field timestamp number
---@class FrecencyDatabaseSqlite: FrecencyDatabase
---@field sqlite FrecencySqliteDB
local Sqlite = {}
---@param fs FrecencyFS
---@param config FrecencyDatabaseConfig
---@return FrecencyDatabaseSqlite
Sqlite.new = function(fs, config)
local lib = sqlite.lib
local self = setmetatable(
{ config = config, buf_registered_flag_name = "telescope_frecency_registered", fs = fs },
{ __index = Sqlite }
)
self.sqlite = sqlite {
uri = self.config.root .. "/file_frecency.sqlite3",
files = { id = true, count = { "integer", default = 1, required = true }, path = "string" },
timestamps = {
id = true,
file_id = { "integer", reference = "files.id", on_delete = "cascade" },
timestamp = { "real", default = lib.julianday "now" },
},
}
return self
end
---@return boolean
function Sqlite:has_entry()
return self.sqlite.files:count() > 0
end
---@param paths string[]
---@return integer
function Sqlite:insert_files(paths)
if #paths == 0 then
return 0
end
---@param path string
return self.sqlite.files:insert(vim.tbl_map(function(path)
return { path = path, count = 0 } -- TODO: remove when sql.nvim#97 is closed
end, paths))
end
---@param workspace string?
---@param datetime string?
---@return FrecencyDatabaseEntry[]
function Sqlite:get_entries(workspace, datetime)
local query = workspace and { contains = { path = { workspace .. "/*" } } } or {}
log.debug { query = query }
local files = self.sqlite.files:get(query) --[[@as FrecencyFile[] ]]
local lib = sqlite.lib
local age = lib.cast((lib.julianday(datetime) - lib.julianday "timestamp") * 24 * 60, "integer")
local timestamps = self.sqlite.timestamps:get { keys = { age = age, "id", "file_id" } } --[[@as FrecencyTimestamp[] ]]
---@type table<integer,number[]>
local age_map = {}
for _, timestamp in ipairs(timestamps) do
if not age_map[timestamp.file_id] then
age_map[timestamp.file_id] = {}
end
table.insert(age_map[timestamp.file_id], timestamp.age)
end
local items = {}
for _, file in ipairs(files) do
table.insert(items, { path = file.path, count = file.count, ages = age_map[file.id] })
end
return items
end
---@param datetime string? ISO8601 format string
---@return FrecencyTimestamp[]
function Sqlite:get_timestamps(datetime)
local lib = sqlite.lib
local age = lib.cast((lib.julianday(datetime) - lib.julianday "timestamp") * 24 * 60, "integer")
return self.sqlite.timestamps:get { keys = { age = age, "id", "file_id" } }
end
---@param path string
---@param count integer
---@param datetime string?
---@return nil
function Sqlite:update(path, count, datetime)
local file = self.sqlite.files:get({ where = { path = path } })[1] --[[@as FrecencyFile?]]
local file_id
if file then
self.sqlite.files:update { where = { id = file.id }, set = { count = file.count + 1 } }
file_id = file.id
else
file_id = self.sqlite.files:insert { path = path }
end
self.sqlite.timestamps:insert {
file_id = file_id,
timestamp = datetime and sqlite.lib.julianday(datetime) or nil,
}
local timestamps = self.sqlite.timestamps:get { where = { file_id = file_id } } --[[@as FrecencyTimestamp[] ]]
local trim_at = timestamps[#timestamps - count + 1]
if trim_at then
self.sqlite.timestamps:remove { file_id = tostring(file_id), id = "<" .. tostring(trim_at.id) }
end
end
---@return integer[]
function Sqlite:unlinked_entries()
---@param file FrecencyFile
return self.sqlite.files:map(function(file)
if not self.fs:is_valid_path(file.path) then
return file.id
end
end)
end
---@param ids integer[]
---@return nil
function Sqlite:remove_files(ids)
self.sqlite.files:remove { id = ids }
end
return Sqlite

5
lua/frecency/entry_maker.lua
View File

@ -24,7 +24,10 @@ EntryMaker.new = function(fs, web_devicons, config)
end, vim.api.nvim_list_bufs())
self.loaded = {}
for _, bufnr in ipairs(loaded_bufnrs) do
self.loaded[vim.api.nvim_buf_get_name(bufnr)] = true
local bufname = vim.api.nvim_buf_get_name(bufnr)
if bufname then
self.loaded[bufname] = true
end
end
return self
end

84
lua/frecency/file_lock.lua Normal file
View File

@ -0,0 +1,84 @@
local async = require "plenary.async" --[[@as PlenaryAsync]]
local log = require "plenary.log"
---@class FrecencyFileLock
---@field base string
---@field config FrecencyFileLockConfig
---@field filename string
local FileLock = {}
---@class FrecencyFileLockConfig
---@field retry integer default: 5
---@field interval integer default: 500
---@param path string
---@param opts FrecencyFileLockConfig?
---@return FrecencyFileLock
FileLock.new = function(path, opts)
local config = vim.tbl_extend("force", { retry = 5, interval = 500 }, opts or {})
local self = setmetatable({ config = config }, { __index = FileLock })
self.filename = path .. ".lock"
return self
end
---@async
---@return string? err
function FileLock:get()
local count = 0
local err, fd
while true do
count = count + 1
err, fd = async.uv.fs_open(self.filename, "wx", tonumber("600", 8))
if not err then
break
end
async.util.sleep(self.config.interval)
if count == self.config.retry then
log.debug(("file_lock get() failed: retry count reached: %d"):format(count))
return "failed to get lock"
end
log.debug(("file_lock get() retry: %d"):format(count))
end
err = async.uv.fs_close(fd)
if err then
log.debug("file_lock get() failed: " .. err)
return err
end
end
---@async
---@return string? err
function FileLock:release()
local err = async.uv.fs_stat(self.filename)
if err then
log.debug("file_lock release() not found: " .. err)
return "lock not found"
end
err = async.uv.fs_unlink(self.filename)
if err then
log.debug("file_lock release() unlink failed: " .. err)
return err
end
end
---@async
---@generic T
---@param f fun(): T
---@return string? err
---@return T
function FileLock:with(f)
local err = self:get()
if err then
return err, nil
end
local ok, result_or_err = pcall(f)
err = self:release()
if err then
return err, nil
elseif ok then
return nil, result_or_err
end
return result_or_err, nil
end
return FileLock

77
lua/frecency/frecency.lua
View File

@ -1,10 +1,13 @@
local Database = require "frecency.database"
local Sqlite = require "frecency.database.sqlite"
local Native = require "frecency.database.native"
local EntryMaker = require "frecency.entry_maker"
local FS = require "frecency.fs"
local Finder = require "frecency.finder"
local Migrator = require "frecency.migrator"
local Picker = require "frecency.picker"
local Recency = require "frecency.recency"
local WebDevicons = require "frecency.web_devicons"
local sqlite_module = require "frecency.sqlite"
local log = require "plenary.log"
---@class Frecency
@ -13,6 +16,7 @@ local log = require "plenary.log"
---@field private database FrecencyDatabase
---@field private finder FrecencyFinder
---@field private fs FrecencyFS
---@field private migrator FrecencyMigrator
---@field private picker FrecencyPicker
---@field private recency FrecencyRecency
local Frecency = {}
@ -29,6 +33,7 @@ local Frecency = {}
---@field show_filter_column boolean|string[]|nil default: true
---@field show_scores boolean? default: false
---@field show_unindexed boolean? default: true
---@field use_sqlite boolean? default: true
---@field workspaces table<string, string>? default: {}
---@param opts FrecencyConfig?
@ -47,10 +52,21 @@ Frecency.new = function(opts)
show_filter_column = true,
show_scores = false,
show_unindexed = true,
use_sqlite = true,
workspaces = {},
}, opts or {})
local self = setmetatable({ buf_registered = {}, config = config }, { __index = Frecency })--[[@as Frecency]]
self.fs = FS.new { ignore_patterns = config.ignore_patterns }
local Database
if not self.config.use_sqlite then
Database = Native
elseif not sqlite_module.can_use then
self:warn "use_sqlite = true, but sqlite module can not be found. It fallbacks to native code."
Database = Native
else
Database = Sqlite
end
self.database = Database.new(self.fs, { root = config.db_root })
local web_devicons = WebDevicons.new(not config.disable_devicons)
local entry_maker = EntryMaker.new(self.fs, web_devicons, {
@ -59,6 +75,7 @@ Frecency.new = function(opts)
})
self.finder = Finder.new(entry_maker, self.fs)
self.recency = Recency.new()
self.migrator = Migrator.new(self.fs, self.recency, self.config.db_root)
return self
end
@ -84,6 +101,10 @@ function Frecency:setup()
self:validate_database()
end
vim.api.nvim_create_user_command("FrecencyMigrateDB", function()
self:migrate_database()
end, { desc = "Migrate DB telescope-frecency to native code" })
local group = vim.api.nvim_create_augroup("TelescopeFrecency", {})
vim.api.nvim_create_autocmd({ "BufWinEnter", "BufWritePost" }, {
desc = "Update database for telescope-frecency",
@ -152,7 +173,6 @@ function Frecency:validate_database(force)
end)
end
---@private
---@param bufnr integer
---@param datetime string? ISO8601 format string
function Frecency:register(bufnr, datetime)
@ -160,15 +180,40 @@ function Frecency:register(bufnr, datetime)
if self.buf_registered[bufnr] or not self.fs:is_valid_path(path) then
return
end
local id, inserted = self.database:upsert_files(path)
self.database:insert_timestamps(id, datetime)
self.database:trim_timestamps(id, self.recency.config.max_count)
if inserted and self.picker then
self.picker:discard_results()
end
self.database:update(path, self.recency.config.max_count, datetime)
self.buf_registered[bufnr] = true
end
---@param to_sqlite boolean?
---@return nil
function Frecency:migrate_database(to_sqlite)
local prompt = to_sqlite and "migrate the DB into SQLite from native code?"
or "migrate the DB into native code from SQLite?"
vim.ui.select({ "y", "n" }, {
prompt = prompt,
---@param item "y"|"n"
---@return string
format_item = function(item)
return item == "y" and "Yes, Migrate it." or "No. Do nothing."
end,
}, function(item)
if item == "n" then
self:notify "migration aborted"
return
elseif to_sqlite then
if sqlite_module.can_use then
self.migrator:to_sqlite()
else
self:error "sqlite.lua is unavailable"
return
end
else
self.migrator:to_v1()
end
self:notify "migration finished successfully"
end)
end
---@private
---@param fmt string
---@param ... any?
@ -185,4 +230,20 @@ function Frecency:notify(fmt, ...)
vim.notify(self:message(fmt, ...))
end
---@private
---@param fmt string
---@param ... any?
---@return nil
function Frecency:warn(fmt, ...)
vim.notify(self:message(fmt, ...), vim.log.levels.WARN)
end
---@private
---@param fmt string
---@param ... any?
---@return nil
function Frecency:error(fmt, ...)
vim.notify(self:message(fmt, ...), vim.log.levels.ERROR)
end
return Frecency

83
lua/frecency/migrator.lua Normal file
View File

@ -0,0 +1,83 @@
local Sqlite = require "frecency.database.sqlite"
local Native = require "frecency.database.native"
local wait = require "frecency.wait"
---@class FrecencyMigrator
---@field fs FrecencyFS
---@field recency FrecencyRecency
---@field root string
local Migrator = {}
---@param fs FrecencyFS
---@param recency FrecencyRecency
---@param root string
---@return FrecencyMigrator
Migrator.new = function(fs, recency, root)
return setmetatable({ fs = fs, recency = recency, root = root }, { __index = Migrator })
end
---@return nil
function Migrator:to_v1()
local native = Native.new(self.fs, { root = self.root })
native.table = self:v1_table_from_sqlite()
wait(function()
native:save()
end)
end
---@return nil
function Migrator:to_sqlite()
local sqlite = Sqlite.new(self.fs, { root = self.root })
local native = Native.new(self.fs, { root = self.root })
for path, record in pairs(native.table.records) do
local file_id = sqlite.sqlite.files:insert { path = path, count = record.count }
sqlite.sqlite.timestamps:insert(vim.tbl_map(function(timestamp)
return { file_id = file_id, timestamp = ('julianday(datetime(%d, "unixepoch"))'):format(timestamp) }
end, record.timestamps))
end
end
---@private
---@return FrecencyDatabaseNativeTable
function Migrator:v1_table_from_sqlite()
local sqlite = Sqlite.new(self.fs, { root = self.root })
---@type FrecencyDatabaseNativeTable
local tbl = { version = "v1", records = {} }
local files = sqlite.sqlite.files:get {} --[[@as FrecencyFile[] ]]
---@type table<integer,string>
local path_map = {}
for _, file in ipairs(files) do
tbl.records[file.path] = { count = file.count, timestamps = { 0 } }
path_map[file.id] = file.path
end
-- local timestamps = sqlite.sqlite.timestamps:get { keys = { "id", "file_id", epoch = "unixepoch(timestamp)" } } --[[@as FrecencyTimestamp[] ]]
local timestamps = sqlite.sqlite.timestamps:get {
keys = { "id", "file_id", epoch = "cast(strftime('%s', timestamp) as integer)" },
} --[[@as FrecencyTimestamp[] ]]
table.sort(timestamps, function(a, b)
return a.id < b.id
end)
for _, timestamp in ipairs(timestamps) do
local path = path_map[timestamp.file_id]
if path then
local record = tbl.records[path]
if record then
if #record.timestamps == 1 and record.timestamps[1] == 0 then
record.timestamps = {}
end
---@diagnostic disable-next-line: undefined-field
table.insert(record.timestamps, timestamp.epoch)
if #record.timestamps > self.recency.config.max_count then
local new_table = {}
for i = #record.timestamps - self.recency.config.max_count + 1, #record.timestamps do
table.insert(new_table, record.timestamps[i])
end
record.timestamps = new_table
end
end
end
end
return tbl
end
return Migrator

25
lua/frecency/picker.lua
View File

@ -184,28 +184,15 @@ end
---@return FrecencyFile[]
function Picker:fetch_results(workspace, datetime)
log.debug { workspace = workspace or "NONE" }
local start_files = os.clock()
local files = self.database:get_files(workspace)
log.debug { files = #files }
log.debug(("it takes %f seconds in fetching files with workspace: %s"):format(os.clock() - start_files, workspace))
local start_timesatmps = os.clock()
local timestamps = self.database:get_timestamps(datetime)
log.debug { timestamps = #timestamps }
log.debug(("it takes %f seconds in fetching all timestamps"):format(os.clock() - start_timesatmps))
local start_fetch = os.clock()
local files = self.database:get_entries(workspace, datetime)
log.debug(("it takes %f seconds in fetching entries"):format(os.clock() - start_fetch))
local start_results = os.clock()
local elapsed_recency = 0
---@type table<integer,number[]>
local age_map = {}
for _, timestamp in ipairs(timestamps) do
if not age_map[timestamp.file_id] then
age_map[timestamp.file_id] = {}
end
table.insert(age_map[timestamp.file_id], timestamp.age)
end
for _, file in ipairs(files) do
local start_recency = os.clock()
local ages = age_map[file.id] --[[@as number[]?]]
file.score = ages and self.recency:calculate(file.count, ages) or 0
file.score = file.ages and self.recency:calculate(file.count, file.ages) or 0
file.ages = nil
elapsed_recency = elapsed_recency + (os.clock() - start_recency)
end
log.debug(("it takes %f seconds in calculating recency"):format(elapsed_recency))
@ -213,7 +200,7 @@ function Picker:fetch_results(workspace, datetime)
local start_sort = os.clock()
table.sort(files, function(a, b)
return a.score > b.score
return a.score > b.score or (a.score == b.score and a.path > b.path)
end)
log.debug(("it takes %f seconds in sorting"):format(os.clock() - start_sort))
return files

17
lua/frecency/sqlite.lua Normal file
View File

@ -0,0 +1,17 @@
---@class FrecencySqlite
---@field can_use boolean
---@field lib sqlite_lib
---@overload fun(opts: table): FrecencySqliteDB
return setmetatable({}, {
__index = function(_, k)
if k == "lib" then
return require("sqlite").lib
elseif k == "can_use" then
return not not vim.F.npcall(require, "sqlite")
end
end,
__call = function(_, opts)
return require "sqlite"(opts)
end,
}) --[[@as FrecencySqlite]]

135
lua/frecency/tests/file_lock_spec.lua Normal file
View File

@ -0,0 +1,135 @@
local FileLock = require "frecency.file_lock"
local util = require "frecency.tests.util"
local async = require "plenary.async" --[[@as PlenaryAsync]]
require("plenary.async").tests.add_to_env()
local function with_dir(f)
local dir, close = util.make_tree {}
local filename = (dir / "file_lock_test").filename
f(filename)
close()
end
a.describe("file_lock", function()
a.describe("get()", function()
a.describe("when no lock file", function()
with_dir(function(filename)
local fl = FileLock.new(filename, { retry = 1, interval = 10 })
a.it("gets successfully", function()
assert.is.Nil(fl:get())
end)
end)
end)
a.describe("when with a lock file", function()
with_dir(function(filename)
local fl = FileLock.new(filename, { retry = 1, interval = 10 })
a.it("fails to get", function()
assert.is.Nil(async.uv.fs_open(fl.filename, "wx", tonumber("600", 8)))
assert.are.same("failed to get lock", fl:get())
end)
end)
end)
a.describe("when getting twice", function()
with_dir(function(filename)
local fl = FileLock.new(filename, { retry = 1, interval = 10 })
a.it("fails to get", function()
assert.is.Nil(fl:get())
assert.are.same("failed to get lock", fl:get())
end)
end)
end)
end)
a.describe("release()", function()
a.describe("when no lock file", function()
with_dir(function(filename)
local fl = FileLock.new(filename, { retry = 1, interval = 10 })
a.it("fails to release", function()
assert.are.same("lock not found", fl:release())
end)
end)
end)
a.describe("when with a lock file", function()
with_dir(function(filename)
local fl = FileLock.new(filename, { retry = 1, interval = 10 })
a.it("releases successfully", function()
assert.is.Nil(fl:get())
assert.is.Nil(fl:release())
end)
end)
end)
a.describe("when releasing twice", function()
with_dir(function(filename)
local fl = FileLock.new(filename, { retry = 1, interval = 10 })
a.it("fails to release", function()
assert.is.Nil(fl:get())
assert.is.Nil(fl:release())
assert.are.same("lock not found", fl:release())
end)
end)
end)
end)
a.describe("with()", function()
a.describe("when get() fails", function()
with_dir(function(filename)
local fl = FileLock.new(filename, { retry = 1, interval = 10 })
a.it("fails with a valid err", function()
assert.is.Nil(fl:get())
assert.are.same(
"failed to get lock",
fl:with(function()
return nil
end)
)
end)
end)
end)
a.describe("when release() fails", function()
with_dir(function(filename)
local fl = FileLock.new(filename, { retry = 1, interval = 10 })
a.it("fails with a valid err", function()
assert.are.same(
"lock not found",
fl:with(function()
assert.is.Nil(async.uv.fs_unlink(fl.filename))
return nil
end)
)
end)
end)
end)
a.describe("when f() fails", function()
with_dir(function(filename)
local fl = FileLock.new(filename, { retry = 1, interval = 10 })
a.it("fails with a valid err", function()
assert.has.match(
": error in hoge function$",
fl:with(function()
error "error in hoge function"
end)
)
end)
end)
end)
a.describe("when no errors", function()
with_dir(function(filename)
local fl = FileLock.new(filename, { retry = 1, interval = 10 })
a.it("run successfully and returns valid results", function()
local err, result = fl:with(function()
return "hogehogeo"
end)
assert.is.Nil(err)
assert.are.same("hogehogeo", result)
end)
end)
end)
end)
end)

520
lua/frecency/tests/frecency_spec.lua
View File

@ -2,15 +2,18 @@
local Frecency = require "frecency.frecency"
local Picker = require "frecency.picker"
local util = require "frecency.tests.util"
local Path = require "plenary.path"
local log = require "plenary.log"
local Path = require "plenary.path"
local use_sqlite
---@param files string[]
---@param callback fun(frecency: Frecency, dir: PlenaryPath): nil
---@return nil
local function with_files(files, callback)
local dir, close = util.make_tree(files)
local frecency = Frecency.new { db_root = dir.filename }
log.debug { db_root = dir.filename }
local frecency = Frecency.new { db_root = dir.filename, use_sqlite = use_sqlite }
frecency.picker = Picker.new(
frecency.database,
frecency.finder,
@ -41,7 +44,6 @@ local function make_register(frecency, dir)
end
end
---comment
---@param frecency Frecency
---@param dir PlenaryPath
---@param callback fun(register: fun(file: string, datetime: string?): nil): nil
@ -86,292 +88,304 @@ local function with_fake_vim_ui_select(choice, callback)
end
describe("frecency", function()
describe("register", function()
describe("when opening files", function()
with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir)
local register = make_register(frecency, dir)
register("hoge1.txt", "2023-07-29T00:00:00+09:00")
register("hoge2.txt", "2023-07-29T01:00:00+09:00")
it("has valid records in DB", function()
local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00")
assert.are.same({
{ count = 1, id = 1, path = filepath(dir, "hoge1.txt"), score = 10 },
{ count = 1, id = 2, path = filepath(dir, "hoge2.txt"), score = 10 },
}, results)
end)
end)
end)
describe("when opening again", function()
with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir)
local register = make_register(frecency, dir)
register("hoge1.txt", "2023-07-29T00:00:00+09:00")
register("hoge2.txt", "2023-07-29T01:00:00+09:00")
register("hoge1.txt", "2023-07-29T02:00:00+09:00", true)
it("increases the score", function()
local results = frecency.picker:fetch_results(nil, "2023-07-29T03:00:00+09:00")
assert.are.same({
{ count = 2, id = 1, path = filepath(dir, "hoge1.txt"), score = 40 },
{ count = 1, id = 2, path = filepath(dir, "hoge2.txt"), score = 10 },
}, results)
end)
end)
end)
describe("when opening again but the same instance", function()
with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir)
local register = make_register(frecency, dir)
register("hoge1.txt", "2023-07-29T00:00:00+09:00")
register("hoge2.txt", "2023-07-29T01:00:00+09:00")
register("hoge1.txt", "2023-07-29T02:00:00+09:00")
it("does not increase the score", function()
local results = frecency.picker:fetch_results(nil, "2023-07-29T03:00:00+09:00")
assert.are.same({
{ count = 1, id = 1, path = filepath(dir, "hoge1.txt"), score = 10 },
{ count = 1, id = 2, path = filepath(dir, "hoge2.txt"), score = 10 },
}, results)
end)
end)
end)
describe("when opening more than 10 times", function()
with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir)
local register = make_register(frecency, dir)
register("hoge1.txt", "2023-07-29T00:00:00+09:00")
register("hoge1.txt", "2023-07-29T00:01:00+09:00", true)
register("hoge2.txt", "2023-07-29T00:00:00+09:00")
register("hoge2.txt", "2023-07-29T00:01:00+09:00", true)
register("hoge2.txt", "2023-07-29T00:02:00+09:00", true)
register("hoge2.txt", "2023-07-29T00:03:00+09:00", true)
register("hoge2.txt", "2023-07-29T00:04:00+09:00", true)
register("hoge2.txt", "2023-07-29T00:05:00+09:00", true)
register("hoge2.txt", "2023-07-29T00:06:00+09:00", true)
register("hoge2.txt", "2023-07-29T00:07:00+09:00", true)
register("hoge2.txt", "2023-07-29T00:08:00+09:00", true)
register("hoge2.txt", "2023-07-29T00:09:00+09:00", true)
register("hoge2.txt", "2023-07-29T00:10:00+09:00", true)
register("hoge2.txt", "2023-07-29T00:11:00+09:00", true)
it("calculates score from the recent 10 times", function()
local results = frecency.picker:fetch_results(nil, "2023-07-29T00:12:00+09:00")
assert.are.same({
{ count = 12, id = 2, path = filepath(dir, "hoge2.txt"), score = 12 * (10 * 100) / 10 },
{ count = 2, id = 1, path = filepath(dir, "hoge1.txt"), score = 2 * (2 * 100) / 10 },
}, results)
end)
end)
end)
end)
describe("benchmark", function()
describe("after registered over >5000 files", function()
with_files({}, function(frecency, dir)
with_fake_register(frecency, dir, function(register)
local file_count = 6000
if not os.getenv "CI" then
log.info "It works not on CI. Files is decreaed into 10 count."
file_count = 10
end
local expected = {}
for i = 1, file_count do
local file = ("hoge%08d.txt"):format(i)
table.insert(expected, { count = 1, id = i, path = filepath(dir, file), score = 10 })
register(file, "2023-07-29T00:00:00+09:00")
end
local start = os.clock()
local results = frecency.picker:fetch_results(nil, "2023-07-29T00:01:00+09:00")
table.sort(results, function(a, b)
return a.path < b.path
end)
local elapsed = os.clock() - start
log.info(("it takes %f seconds in fetching all results"):format(elapsed))
it("returns appropriate latency (<1.0 second)", function()
assert.are.is_true(elapsed < 1.0)
end)
it("returns valid response", function()
assert.are.same(expected, results)
end)
end)
end)
end)
end)
describe("validate_database", function()
describe("when no files are unlinked", function()
with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir)
local register = make_register(frecency, dir)
register("hoge1.txt", "2023-07-29T00:00:00+09:00")
register("hoge2.txt", "2023-07-29T00:01:00+09:00")
it("removes no entries", function()
local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00")
assert.are.same({
{ count = 1, id = 1, path = filepath(dir, "hoge1.txt"), score = 10 },
{ count = 1, id = 2, path = filepath(dir, "hoge2.txt"), score = 10 },
}, results)
end)
end)
end)
describe("when with not force", function()
describe("when files are unlinked but it is less than threshold", function()
with_files({ "hoge1.txt", "hoge2.txt", "hoge3.txt", "hoge4.txt", "hoge5.txt" }, function(frecency, dir)
local register = make_register(frecency, dir)
register("hoge1.txt", "2023-07-29T00:00:00+09:00")
register("hoge2.txt", "2023-07-29T00:01:00+09:00")
register("hoge3.txt", "2023-07-29T00:02:00+09:00")
register("hoge4.txt", "2023-07-29T00:03:00+09:00")
register("hoge5.txt", "2023-07-29T00:04:00+09:00")
frecency.config.db_validate_threshold = 3
dir:joinpath("hoge1.txt"):rm()
dir:joinpath("hoge2.txt"):rm()
frecency:validate_database()
it("removes no entries", function()
local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00")
table.sort(results, function(a, b)
return a.path < b.path
end)
assert.are.same({
{ count = 1, id = 1, path = filepath(dir, "hoge1.txt"), score = 10 },
{ count = 1, id = 2, path = filepath(dir, "hoge2.txt"), score = 10 },
{ count = 1, id = 3, path = filepath(dir, "hoge3.txt"), score = 10 },
{ count = 1, id = 4, path = filepath(dir, "hoge4.txt"), score = 10 },
{ count = 1, id = 5, path = filepath(dir, "hoge5.txt"), score = 10 },
}, results)
end)
end)
end)
describe("when files are unlinked and it is more than threshold", function()
describe('when the user response "yes"', function()
with_files({ "hoge1.txt", "hoge2.txt", "hoge3.txt", "hoge4.txt", "hoge5.txt" }, function(frecency, dir)
local function test(db)
describe(db, function()
describe("register", function()
describe("when opening files", function()
with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir)
local register = make_register(frecency, dir)
register("hoge1.txt", "2023-07-29T00:00:00+09:00")
register("hoge2.txt", "2023-07-29T00:01:00+09:00")
register("hoge3.txt", "2023-07-29T00:02:00+09:00")
register("hoge4.txt", "2023-07-29T00:03:00+09:00")
register("hoge5.txt", "2023-07-29T00:04:00+09:00")
frecency.config.db_validate_threshold = 3
dir:joinpath("hoge1.txt"):rm()
dir:joinpath("hoge2.txt"):rm()
dir:joinpath("hoge3.txt"):rm()
register("hoge2.txt", "2023-07-29T01:00:00+09:00")
with_fake_vim_ui_select("y", function(called)
frecency:validate_database()
it("called vim.ui.select()", function()
assert.are.same(1, called())
end)
end)
it("removes entries", function()
it("has valid records in DB", function()
local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00")
table.sort(results, function(a, b)
return a.path < b.path
end)
assert.are.same({
{ count = 1, id = 4, path = filepath(dir, "hoge4.txt"), score = 10 },
{ count = 1, id = 5, path = filepath(dir, "hoge5.txt"), score = 10 },
{ count = 1, path = filepath(dir, "hoge2.txt"), score = 10 },
{ count = 1, path = filepath(dir, "hoge1.txt"), score = 10 },
}, results)
end)
end)
end)
describe('when the user response "no"', function()
with_files({ "hoge1.txt", "hoge2.txt", "hoge3.txt", "hoge4.txt", "hoge5.txt" }, function(frecency, dir)
describe("when opening again", function()
with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir)
local register = make_register(frecency, dir)
register("hoge1.txt", "2023-07-29T00:00:00+09:00")
register("hoge2.txt", "2023-07-29T01:00:00+09:00")
register("hoge1.txt", "2023-07-29T02:00:00+09:00", true)
it("increases the score", function()
local results = frecency.picker:fetch_results(nil, "2023-07-29T03:00:00+09:00")
assert.are.same({
{ count = 2, path = filepath(dir, "hoge1.txt"), score = 40 },
{ count = 1, path = filepath(dir, "hoge2.txt"), score = 10 },
}, results)
end)
end)
end)
describe("when opening again but the same instance", function()
with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir)
local register = make_register(frecency, dir)
register("hoge1.txt", "2023-07-29T00:00:00+09:00")
register("hoge2.txt", "2023-07-29T01:00:00+09:00")
register("hoge1.txt", "2023-07-29T02:00:00+09:00")
it("does not increase the score", function()
local results = frecency.picker:fetch_results(nil, "2023-07-29T03:00:00+09:00")
assert.are.same({
{ count = 1, path = filepath(dir, "hoge2.txt"), score = 10 },
{ count = 1, path = filepath(dir, "hoge1.txt"), score = 10 },
}, results)
end)
end)
end)
describe("when opening more than 10 times", function()
with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir)
local register = make_register(frecency, dir)
register("hoge1.txt", "2023-07-29T00:00:00+09:00")
register("hoge1.txt", "2023-07-29T00:01:00+09:00", true)
register("hoge2.txt", "2023-07-29T00:00:00+09:00")
register("hoge2.txt", "2023-07-29T00:01:00+09:00", true)
register("hoge2.txt", "2023-07-29T00:02:00+09:00", true)
register("hoge2.txt", "2023-07-29T00:03:00+09:00", true)
register("hoge2.txt", "2023-07-29T00:04:00+09:00", true)
register("hoge2.txt", "2023-07-29T00:05:00+09:00", true)
register("hoge2.txt", "2023-07-29T00:06:00+09:00", true)
register("hoge2.txt", "2023-07-29T00:07:00+09:00", true)
register("hoge2.txt", "2023-07-29T00:08:00+09:00", true)
register("hoge2.txt", "2023-07-29T00:09:00+09:00", true)
register("hoge2.txt", "2023-07-29T00:10:00+09:00", true)
register("hoge2.txt", "2023-07-29T00:11:00+09:00", true)
it("calculates score from the recent 10 times", function()
local results = frecency.picker:fetch_results(nil, "2023-07-29T00:12:00+09:00")
assert.are.same({
{ count = 12, path = filepath(dir, "hoge2.txt"), score = 12 * (10 * 100) / 10 },
{ count = 2, path = filepath(dir, "hoge1.txt"), score = 2 * (2 * 100) / 10 },
}, results)
end)
end)
end)
end)
describe("benchmark", function()
describe("after registered over >5000 files", function()
with_files({}, function(frecency, dir)
with_fake_register(frecency, dir, function(register)
-- TODO: 6000 records is too many to use with native?
-- local file_count = 6000
local file_count = 600
if not os.getenv "CI" then
log.info "It works not on CI. Files is decreased into 10 count."
file_count = 10
end
local expected = {}
log.info(("making %d files and register them"):format(file_count))
for i = 1, file_count do
local file = ("hoge%08d.txt"):format(i)
table.insert(expected, { count = 1, path = filepath(dir, file), score = 10 })
register(file, "2023-07-29T00:00:00+09:00")
end
local start = os.clock()
local results = frecency.picker:fetch_results(nil, "2023-07-29T00:01:00+09:00")
table.sort(results, function(a, b)
return a.path < b.path
end)
local elapsed = os.clock() - start
log.info(("it takes %f seconds in fetching all results"):format(elapsed))
it("returns appropriate latency (<1.0 second)", function()
assert.are.is_true(elapsed < 1.0)
end)
it("returns valid response", function()
assert.are.same(expected, results)
end)
end)
end)
end)
end)
describe("validate_database", function()
describe("when no files are unlinked", function()
with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir)
local register = make_register(frecency, dir)
register("hoge1.txt", "2023-07-29T00:00:00+09:00")
register("hoge2.txt", "2023-07-29T00:01:00+09:00")
register("hoge3.txt", "2023-07-29T00:02:00+09:00")
register("hoge4.txt", "2023-07-29T00:03:00+09:00")
register("hoge5.txt", "2023-07-29T00:04:00+09:00")
frecency.config.db_validate_threshold = 3
dir:joinpath("hoge1.txt"):rm()
dir:joinpath("hoge2.txt"):rm()
dir:joinpath("hoge3.txt"):rm()
with_fake_vim_ui_select("n", function(called)
frecency:validate_database()
it("called vim.ui.select()", function()
assert.are.same(1, called())
end)
end)
it("removes no entries", function()
local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00")
table.sort(results, function(a, b)
return a.path < b.path
end)
assert.are.same({
{ count = 1, id = 1, path = filepath(dir, "hoge1.txt"), score = 10 },
{ count = 1, id = 2, path = filepath(dir, "hoge2.txt"), score = 10 },
{ count = 1, id = 3, path = filepath(dir, "hoge3.txt"), score = 10 },
{ count = 1, id = 4, path = filepath(dir, "hoge4.txt"), score = 10 },
{ count = 1, id = 5, path = filepath(dir, "hoge5.txt"), score = 10 },
{ count = 1, path = filepath(dir, "hoge2.txt"), score = 10 },
{ count = 1, path = filepath(dir, "hoge1.txt"), score = 10 },
}, results)
end)
end)
end)
end)
end)
describe("when with force", function()
describe("when db_safe_mode is true", function()
with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir)
local register = make_register(frecency, dir)
register("hoge1.txt", "2023-07-29T00:00:00+09:00")
register("hoge2.txt", "2023-07-29T00:01:00+09:00")
dir:joinpath("hoge1.txt"):rm()
describe("when with not force", function()
describe("when files are unlinked but it is less than threshold", function()
with_files({ "hoge1.txt", "hoge2.txt", "hoge3.txt", "hoge4.txt", "hoge5.txt" }, function(frecency, dir)
local register = make_register(frecency, dir)
register("hoge1.txt", "2023-07-29T00:00:00+09:00")
register("hoge2.txt", "2023-07-29T00:01:00+09:00")
register("hoge3.txt", "2023-07-29T00:02:00+09:00")
register("hoge4.txt", "2023-07-29T00:03:00+09:00")
register("hoge5.txt", "2023-07-29T00:04:00+09:00")
frecency.config.db_validate_threshold = 3
dir:joinpath("hoge1.txt"):rm()
dir:joinpath("hoge2.txt"):rm()
frecency:validate_database()
with_fake_vim_ui_select("y", function(called)
frecency:validate_database(true)
it("called vim.ui.select()", function()
assert.are.same(1, called())
it("removes no entries", function()
local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00")
table.sort(results, function(a, b)
return a.path < b.path
end)
assert.are.same({
{ count = 1, path = filepath(dir, "hoge1.txt"), score = 10 },
{ count = 1, path = filepath(dir, "hoge2.txt"), score = 10 },
{ count = 1, path = filepath(dir, "hoge3.txt"), score = 10 },
{ count = 1, path = filepath(dir, "hoge4.txt"), score = 10 },
{ count = 1, path = filepath(dir, "hoge5.txt"), score = 10 },
}, results)
end)
end)
end)
it("needs confirmation for removing entries", function()
local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00")
assert.are.same({
{ count = 1, id = 2, path = filepath(dir, "hoge2.txt"), score = 10 },
}, results)
describe("when files are unlinked and it is more than threshold", function()
describe('when the user response "yes"', function()
with_files({ "hoge1.txt", "hoge2.txt", "hoge3.txt", "hoge4.txt", "hoge5.txt" }, function(frecency, dir)
local register = make_register(frecency, dir)
register("hoge1.txt", "2023-07-29T00:00:00+09:00")
register("hoge2.txt", "2023-07-29T00:01:00+09:00")
register("hoge3.txt", "2023-07-29T00:02:00+09:00")
register("hoge4.txt", "2023-07-29T00:03:00+09:00")
register("hoge5.txt", "2023-07-29T00:04:00+09:00")
frecency.config.db_validate_threshold = 3
dir:joinpath("hoge1.txt"):rm()
dir:joinpath("hoge2.txt"):rm()
dir:joinpath("hoge3.txt"):rm()
with_fake_vim_ui_select("y", function(called)
frecency:validate_database()
it("called vim.ui.select()", function()
assert.are.same(1, called())
end)
end)
it("removes entries", function()
local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00")
table.sort(results, function(a, b)
return a.path < b.path
end)
assert.are.same({
{ count = 1, path = filepath(dir, "hoge4.txt"), score = 10 },
{ count = 1, path = filepath(dir, "hoge5.txt"), score = 10 },
}, results)
end)
end)
end)
describe('when the user response "no"', function()
with_files({ "hoge1.txt", "hoge2.txt", "hoge3.txt", "hoge4.txt", "hoge5.txt" }, function(frecency, dir)
local register = make_register(frecency, dir)
register("hoge1.txt", "2023-07-29T00:00:00+09:00")
register("hoge2.txt", "2023-07-29T00:01:00+09:00")
register("hoge3.txt", "2023-07-29T00:02:00+09:00")
register("hoge4.txt", "2023-07-29T00:03:00+09:00")
register("hoge5.txt", "2023-07-29T00:04:00+09:00")
frecency.config.db_validate_threshold = 3
dir:joinpath("hoge1.txt"):rm()
dir:joinpath("hoge2.txt"):rm()
dir:joinpath("hoge3.txt"):rm()
with_fake_vim_ui_select("n", function(called)
frecency:validate_database()
it("called vim.ui.select()", function()
assert.are.same(1, called())
end)
end)
it("removes no entries", function()
local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00")
table.sort(results, function(a, b)
return a.path < b.path
end)
assert.are.same({
{ count = 1, path = filepath(dir, "hoge1.txt"), score = 10 },
{ count = 1, path = filepath(dir, "hoge2.txt"), score = 10 },
{ count = 1, path = filepath(dir, "hoge3.txt"), score = 10 },
{ count = 1, path = filepath(dir, "hoge4.txt"), score = 10 },
{ count = 1, path = filepath(dir, "hoge5.txt"), score = 10 },
}, results)
end)
end)
end)
end)
end)
end)
describe("when db_safe_mode is false", function()
with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir)
local register = make_register(frecency, dir)
register("hoge1.txt", "2023-07-29T00:00:00+09:00")
register("hoge2.txt", "2023-07-29T00:01:00+09:00")
dir:joinpath("hoge1.txt"):rm()
describe("when with force", function()
describe("when db_safe_mode is true", function()
with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir)
local register = make_register(frecency, dir)
register("hoge1.txt", "2023-07-29T00:00:00+09:00")
register("hoge2.txt", "2023-07-29T00:01:00+09:00")
dir:joinpath("hoge1.txt"):rm()
with_fake_vim_ui_select("y", function(called)
frecency.config.db_safe_mode = false
frecency:validate_database(true)
with_fake_vim_ui_select("y", function(called)
frecency:validate_database(true)
it("did not call vim.ui.select()", function()
assert.are.same(0, called())
it("called vim.ui.select()", function()
assert.are.same(1, called())
end)
end)
it("needs confirmation for removing entries", function()
local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00")
assert.are.same({
{ count = 1, path = filepath(dir, "hoge2.txt"), score = 10 },
}, results)
end)
end)
end)
it("needs no confirmation for removing entries", function()
local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00")
assert.are.same({
{ count = 1, id = 2, path = filepath(dir, "hoge2.txt"), score = 10 },
}, results)
describe("when db_safe_mode is false", function()
with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir)
local register = make_register(frecency, dir)
register("hoge1.txt", "2023-07-29T00:00:00+09:00")
register("hoge2.txt", "2023-07-29T00:01:00+09:00")
dir:joinpath("hoge1.txt"):rm()
with_fake_vim_ui_select("y", function(called)
frecency.config.db_safe_mode = false
frecency:validate_database(true)
it("did not call vim.ui.select()", function()
assert.are.same(0, called())
end)
end)
it("needs no confirmation for removing entries", function()
local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00")
assert.are.same({
{ count = 1, path = filepath(dir, "hoge2.txt"), score = 10 },
}, results)
end)
end)
end)
end)
end)
end)
end)
end
use_sqlite = true
test "sqlite"
use_sqlite = false
test "native"
end)

129
lua/frecency/tests/migrator_spec.lua Normal file
View File

@ -0,0 +1,129 @@
---@diagnostic disable: undefined-field
local Migrator = require "frecency.migrator"
local FS = require "frecency.fs"
local Recency = require "frecency.recency"
local Sqlite = require "frecency.database.sqlite"
local Native = require "frecency.database.native"
local util = require "frecency.tests.util"
local wait = require "frecency.wait"
---@param callback fun(migrator: FrecencyMigrator, sqlite: FrecencyDatabase): nil
---@return nil
local function with(callback)
local dir, close = util.tmpdir()
local recency = Recency.new { max_count = 2 }
local fs = FS.new { ignore_patterns = {} }
local migrator = Migrator.new(fs, recency, dir.filename)
local sqlite = Sqlite.new(fs, { root = dir.filename })
callback(migrator, sqlite)
close()
end
---@param source table<string,{ count: integer, timestamps: string[] }>
local function v1_table(source)
local records = {}
for path, record in pairs(source) do
local timestamps = {}
for _, timestamp in ipairs(record.timestamps) do
table.insert(timestamps, vim.fn.strptime("%FT%T%z", timestamp))
end
records[path] = { count = record.count, timestamps = timestamps }
end
return { version = "v1", records = records }
end
describe("migrator", function()
describe("to_v1", function()
describe("when with simple database", function()
with(function(migrator, sqlite)
for _, path in ipairs { "hoge1.txt", "hoge2.txt" } do
sqlite:update(path, migrator.recency.config.max_count, "2023-08-21T00:00:00")
end
migrator:to_v1()
local native = Native.new(migrator.fs, { root = migrator.root })
it("has converted into a valid table", function()
assert.are.same(
v1_table {
["hoge1.txt"] = { count = 1, timestamps = { "2023-08-21T00:00:00+0000" } },
["hoge2.txt"] = { count = 1, timestamps = { "2023-08-21T00:00:00+0000" } },
},
native.table
)
end)
end)
end)
describe("when with more large database", function()
with(function(migrator, sqlite)
for i, path in ipairs {
"hoge1.txt",
"hoge1.txt",
"hoge1.txt",
"hoge1.txt",
"hoge2.txt",
"hoge2.txt",
"hoge2.txt",
"hoge3.txt",
"hoge3.txt",
"hoge4.txt",
} do
sqlite:update(path, migrator.recency.config.max_count, ("2023-08-21T00:%02d:00"):format(i))
end
migrator:to_v1()
local native = Native.new(migrator.fs, { root = migrator.root })
it("has converted into a valid table", function()
assert.are.same(
v1_table {
["hoge1.txt"] = { count = 4, timestamps = { "2023-08-21T00:03:00+0000", "2023-08-21T00:04:00+0000" } },
["hoge2.txt"] = { count = 3, timestamps = { "2023-08-21T00:06:00+0000", "2023-08-21T00:07:00+0000" } },
["hoge3.txt"] = { count = 2, timestamps = { "2023-08-21T00:08:00+0000", "2023-08-21T00:09:00+0000" } },
["hoge4.txt"] = { count = 1, timestamps = { "2023-08-21T00:10:00+0000" } },
},
native.table
)
end)
end)
end)
end)
describe("to_sqlite", function()
with(function(migrator, sqlite)
local native = Native.new(migrator.fs, { root = migrator.root })
native.table = v1_table {
["hoge1.txt"] = { count = 4, timestamps = { "2023-08-21T00:03:00+0000", "2023-08-21T00:04:00+0000" } },
["hoge2.txt"] = { count = 3, timestamps = { "2023-08-21T00:06:00+0000", "2023-08-21T00:07:00+0000" } },
["hoge3.txt"] = { count = 2, timestamps = { "2023-08-21T00:08:00+0000", "2023-08-21T00:09:00+0000" } },
["hoge4.txt"] = { count = 1, timestamps = { "2023-08-21T00:10:00+0000" } },
}
wait(function()
native:save()
end)
migrator:to_sqlite()
sqlite.sqlite.db:open()
local records = sqlite.sqlite.db:eval [[
select
f.path,
f.count,
datetime(strftime('%s', t.timestamp), 'unixepoch') datetime
from timestamps t
join files f
on f.id = t.file_id
order by path, datetime
]]
it("has converted into a valid DB", function()
assert.are.same({
{ path = "hoge1.txt", count = 4, datetime = "2023-08-21 00:03:00" },
{ path = "hoge1.txt", count = 4, datetime = "2023-08-21 00:04:00" },
{ path = "hoge2.txt", count = 3, datetime = "2023-08-21 00:06:00" },
{ path = "hoge2.txt", count = 3, datetime = "2023-08-21 00:07:00" },
{ path = "hoge3.txt", count = 2, datetime = "2023-08-21 00:08:00" },
{ path = "hoge3.txt", count = 2, datetime = "2023-08-21 00:09:00" },
{ path = "hoge4.txt", count = 1, datetime = "2023-08-21 00:10:00" },
}, records)
end)
end)
end)
end)

26
lua/frecency/tests/util.lua
View File

@ -1,17 +1,25 @@
local uv = vim.uv or vim.loop
local Path = require "plenary.path"
---@param entries string[]
---@return PlenaryPath the top dir of tree
---@return fun(): nil sweep all entries
local function make_tree(entries)
local dir = Path:new(Path.new(assert(uv.fs_mkdtemp "tests_XXXXXX")):absolute())
for _, entry in ipairs(entries) do
dir:joinpath(entry):touch { parents = true }
end
---@return PlenaryPath
---@return fun(): nil close swwp all entries
local function tmpdir()
local dir = Path:new(Path:new(assert(uv.fs_mkdtemp "tests_XXXXXX")):absolute())
return dir, function()
dir:rm { recursive = true }
end
end
return { make_tree = make_tree }
---@param entries string[]
---@return PlenaryPath dir the top dir of tree
---@return fun(): nil close sweep all entries
local function make_tree(entries)
local dir, close = tmpdir()
for _, entry in ipairs(entries) do
---@diagnostic disable-next-line: undefined-field
dir:joinpath(entry):touch { parents = true }
end
return dir, close
end
return { make_tree = make_tree, tmpdir = tmpdir }

50
lua/frecency/types.lua
View File

@ -1,4 +1,5 @@
---@diagnostic disable: unused-local
---@diagnostic disable: unused-local, missing-return
-- NOTE: types below are borrowed from sqlite.lua
---@class sqlite_db @Main sqlite.lua object.
@ -51,6 +52,7 @@
---@field filename string
---@field joinpath fun(self: PlenaryPath, ...): PlenaryPath
---@field make_relative fun(self: PlenaryPath, cwd: string): string
---@field parent PlenaryPath
---@field path { sep: string }
---@field rm fun(self: PlenaryPath, opts: { recursive: boolean }?): nil
@ -69,6 +71,8 @@
---@class PlenaryAsync
---@field control PlenaryAsyncControl
---@field util PlenaryAsyncUtil
---@field uv PlenaryAsyncUv
---@field void fun(f: fun(): nil): fun(): nil
local PlenaryAsync = {}
---@async
@ -96,6 +100,50 @@ function PlenaryAsyncControlChannelRx.recv() end
---@class PlenaryAsyncUtil
local PlenaryAsyncUtil = {}
---@class PlenaryAsyncUv
local PlenaryAsyncUv = {}
---@async
---@param path string
---@return string? err
---@return { mtime: integer, size: integer, type: "file"|"directory" }
function PlenaryAsyncUv.fs_stat(path) end
---@async
---@param path string
---@param flags string|integer
---@param mode integer
---@return string? err
---@return integer fd
function PlenaryAsyncUv.fs_open(path, flags, mode) end
---@async
---@param fd integer
---@param size integer
---@param offset integer?
---@return string? err
---@return string data
function PlenaryAsyncUv.fs_read(fd, size, offset) end
---@async
---@param fd integer
---@param data string
---@param offset integer?
---@return string? err
---@return integer bytes
function PlenaryAsyncUv.fs_write(fd, data, offset) end
---@async
---@param path string
---@return string? err
---@return boolean? success
function PlenaryAsyncUv.fs_unlink(path) end
---@async
---@param fd integer
---@return string? err
function PlenaryAsyncUv.fs_close(fd) end
---@async
---@param ms integer
---@return nil

54
lua/frecency/wait.lua Normal file
View File

@ -0,0 +1,54 @@
local async = require "plenary.async"
---@class FrecencyWait
---@field config FrecencyWaitConfig
local Wait = {}
---@class FrecencyWaitConfig
---@field time integer default: 5000
---@field interval integer default: 200
---@alias FrecencyWaitCallback fun(): nil
---@param f FrecencyWaitCallback
---@param opts FrecencyWaitConfig?
Wait.new = function(f, opts)
return setmetatable(
{ f = f, config = vim.tbl_extend("force", { time = 5000, interval = 200 }, opts or {}) },
{ __index = Wait }
)
end
---@async
---@private
Wait.f = function()
error "implement me"
end
---@return boolean ok
---@return nil|-1|-2 status
function Wait:run()
local done = false
async.void(function()
self.f()
done = true
end)()
return vim.wait(self.config.time, function()
return done
end, self.config.interval)
end
---@param f FrecencyWaitCallback
---@param opts FrecencyWaitConfig?
---@return nil
return function(f, opts)
local wait = Wait.new(f, opts)
local ok, status = wait:run()
if ok then
return
elseif status == -1 then
error "callback never returnes during the time"
elseif status == -2 then
error "callback is interrupted during the time"
end
end

5
lua/telescope/_extensions/frecency.lua
View File

@ -1,12 +1,13 @@
local frecency = require "frecency"
local sqlite = require "frecency.sqlite"
return require("telescope").register_extension {
setup = frecency.setup,
health = function()
if vim.F.npcall(require, "sqlite") then
if sqlite.can_use then
vim.health.ok "sqlite.lua installed."
else
vim.health.error "sqlite.lua is required for telescope-frecency.nvim to work."
vim.health.info "sqlite.lua is required when use_sqlite = true"
end
if vim.F.npcall(require, "nvim-web-devicons") then
vim.health.ok "nvim-web-devicons installed."