kristoferssolo/telescope-frecency.nvim
1
0
Fork 0
mirror of https://github.com/kristoferssolo/telescope-frecency.nvim.git synced 2025-10-21 20:10:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: add logic to store data by native code (#130)

Browse Source 
* refactor: make logic for Database be abstract
* feat: add logic for DB by string.dump
* fix: run with async.void to run synchronously
* test: add tests for native feature
* feat!: sort candidates by path when score is same
  This is needed because candidates from SQLite is sorted by id, but ones from native is sorted by path.
* chore: clean up types
* feat: add lock/unlock feature to access DB
* test: use async version of busted
  And disable benchmark tests (fix later)
* test: add tests for file_lock
* chore: use more explicit names
* chore: use plenary.log instead
* fix: wait async functions definitely
* feat: add migrator
* chore: fix logging
* fix: detect emptiness of the table
* fix: deal with buffer with no names
* test: loosen the condition temporarily
* test: add tests for migrator
* fix: return true when the table is not empty
* feat: load sqlite lazily not to require in start
* chore: add logging to calculate time for fetching
* feat: add converter from native code to SQLite
* feat: warn when sqlite.lua is not available
* feat: add FrecencyMigrateDB to migrate DB
* docs: add note for native code logic
* test: ignore type bug
This commit is contained in:
JINNOUCHI Yasushi 2023-08-27 18:51:16 +09:00 committed by GitHub
parent 5d1a01be63
commit 9037d696e6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 1283 additions and 426 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.luarc.json
View File

@ -1,6 +1,7 @@
{ {
"diagnostics": { "diagnostics": {
"globals": [ "globals": [
"a",
"describe", "describe",
"it", "it",
"vim" "vim"

49
README.md
View File

@ -31,7 +31,7 @@ The score is calculated using the age of the 10 most recent timestamps and the t
### Score calculation ### Score calculation
``` ```lua
score = frequency * recency_score / max_number_of_timestamps score = frequency * recency_score / max_number_of_timestamps
``` ```
## What about files that are neither 'frequent' _or_ 'recent' ? ## What about files that are neither 'frequent' _or_ 'recent' ?
@ -59,9 +59,11 @@ If the active buffer (prior to the finder being launched) is attached to an LSP
## Requirements ## Requirements
- [telescope.nvim](https://github.com/nvim-telescope/telescope.nvim) (required) - [telescope.nvim](https://github.com/nvim-telescope/telescope.nvim) (required)
- [sqlite.lua](https://github.com/kkharji/sqlite.lua) (required) - [sqlite.lua][] (required)
- [nvim-web-devicons](https://github.com/kyazdani42/nvim-web-devicons) (optional) - [nvim-web-devicons](https://github.com/kyazdani42/nvim-web-devicons) (optional)
[sqlite.lua]: https://github.com/kkharji/sqlite.lua
Timestamps and file records are stored in an [SQLite3](https://www.sqlite.org/index.html) database for persistence and speed. Timestamps and file records are stored in an [SQLite3](https://www.sqlite.org/index.html) database for persistence and speed.
This plugin uses `sqlite.lua` to perform the database transactions. This plugin uses `sqlite.lua` to perform the database transactions.
@ -73,9 +75,9 @@ This plugin uses `sqlite.lua` to perform the database transactions.
use { use {
"nvim-telescope/telescope-frecency.nvim", "nvim-telescope/telescope-frecency.nvim",
config = function() config = function()
require"telescope".load_extension("frecency") require("telescope").load_extension "frecency"
end, end,
requires = {"kkharji/sqlite.lua"} requires = { "kkharji/sqlite.lua" },
} }
``` ```
@ -85,9 +87,9 @@ use {
{ {
"nvim-telescope/telescope-frecency.nvim", "nvim-telescope/telescope-frecency.nvim",
config = function() config = function()
require"telescope".load_extension("frecency") require("telescope").load_extension "frecency"
end, end,
dependencies = {"kkharji/sqlite.lua"} dependencies = { "kkharji/sqlite.lua" },
} }
``` ```
@ -102,7 +104,7 @@ If no database is found when running Neovim with the plugin installed, a new one
or to map to a key: or to map to a key:
```lua ```lua
vim.api.nvim_set_keymap("n", "<leader><leader>", "<Cmd>lua require('telescope').extensions.frecency.frecency()<CR>", {noremap = true, silent = true}) vim.api.nvim_set_keymap("n", "<leader><leader>", "<Cmd>Telescope frecency<CR>")
``` ```
Use a specific workspace tag: Use a specific workspace tag:
@ -114,7 +116,7 @@ Use a specific workspace tag:
or or
```lua ```lua
vim.api.nvim_set_keymap("n", "<leader><leader>", "<Cmd>lua require('telescope').extensions.frecency.frecency({ workspace = 'CWD' })<CR>", {noremap = true, silent = true}) vim.api.nvim_set_keymap("n", "<leader><leader>", "<Cmd>Telescope frecency workspace=CWD<CR>")
``` ```
Filter tags are applied by typing the `:tag:` name (adding surrounding colons) in the finder query. Filter tags are applied by typing the `:tag:` name (adding surrounding colons) in the finder query.
@ -124,7 +126,7 @@ Entering `:<Tab>` will trigger omnicompletion for available tags.
See [default configuration](https://github.com/nvim-telescope/telescope.nvim#telescope-defaults) for full details on configuring Telescope. See [default configuration](https://github.com/nvim-telescope/telescope.nvim#telescope-defaults) for full details on configuring Telescope.
- `db_root` (default: `nil`) - `db_root` (default: `vim.fn.stdpath "data"`)
Path to parent directory of custom database location. Path to parent directory of custom database location.
Defaults to `$XDG_DATA_HOME/nvim` if unset. Defaults to `$XDG_DATA_HOME/nvim` if unset.
@ -133,7 +135,7 @@ See [default configuration](https://github.com/nvim-telescope/telescope.nvim#tel
Default workspace tag to filter by e.g. `'CWD'` to filter by default to the current directory. Can be overridden at query time by specifying another filter like `':*:'`. Default workspace tag to filter by e.g. `'CWD'` to filter by default to the current directory. Can be overridden at query time by specifying another filter like `':*:'`.
- `ignore_patterns` (default: `{"*.git/*", "*/tmp/*"}`) - `ignore_patterns` (default: `{ "*.git/*", "*/tmp/*", "term://*" }`)
Patterns in this table control which files are indexed (and subsequently which you'll see in the finder results). Patterns in this table control which files are indexed (and subsequently which you'll see in the finder results).
@ -141,7 +143,7 @@ See [default configuration](https://github.com/nvim-telescope/telescope.nvim#tel
To see the scores generated by the algorithm in the results, set this to `true`. To see the scores generated by the algorithm in the results, set this to `true`.
- `workspaces` (default: {}) - `workspaces` (default: `{}`)
This table contains mappings of `workspace_tag` -> `workspace_directory` This table contains mappings of `workspace_tag` -> `workspace_directory`
The key corresponds to the `:tag_name` used to select the filter in queries. The key corresponds to the `:tag_name` used to select the filter in queries.
@ -164,10 +166,13 @@ See [default configuration](https://github.com/nvim-telescope/telescope.nvim#tel
show_filter_column = { "LSP", "CWD", "FOO" } show_filter_column = { "LSP", "CWD", "FOO" }
``` ```
- `use_sqlite` (default: `true`) ***experimental feature***
Use [sqlite.lua] `true` or native code `false`. See [*Remove dependency for sqlite.lua*](#remove-dependency-for-sqlite.lua) for the detail.
### Example Configuration: ### Example Configuration:
``` ```lua
telescope.setup { telescope.setup {
extensions = { extensions = {
frecency = { frecency = {
@ -187,12 +192,14 @@ telescope.setup {
} }
``` ```
### SQL database location ## Note for Database
The default location for the sqlite3 database is `$XDG_DATA_HOME/nvim` (eg `~/.local/share/nvim/` on linux). ### Location
The default location for the database is `$XDG_DATA_HOME/nvim` (eg `~/.local/share/nvim/` on linux).
This can be configured with the `db_root` config option. This can be configured with the `db_root` config option.
### SQL database maintainance ### Maintainance
By default, frecency will prune files that no longer exist from the database. By default, frecency will prune files that no longer exist from the database.
In certain workflows, switching branches in a repository, that behaviour might not be desired. In certain workflows, switching branches in a repository, that behaviour might not be desired.
@ -210,7 +217,17 @@ The command `FrecencyValidate` can be used to clean the database when `auto_vali
:FrecencyValidate! :FrecencyValidate!
``` ```
### Highlight Groups ### Remove dependency for [sqlite.lua][]
***This is an experimental feature.***
In default, it uses SQLite3 library to access the DB. When `use_sqlite` option is set to `false`, it stores the whole data and saves them with encoding by `string.dump()` Lua function.
With this, we can remove the dependency for [sqlite.lua][] and obtain faster speed to open `:Telescope frecency`.
You can migrate from SQLite DB into native code by `:FrecencyMigrateDB` command. It converts data into native code, but does not delete the existent SQLite DB. You can use old SQLite logic by `use_sqlite = true` again.
## Highlight Groups
```vim ```vim
TelescopeBufferLoaded TelescopeBufferLoaded

149
lua/frecency/database.lua
View File

@ -1,136 +1,43 @@
local sqlite = require "sqlite" ---@diagnostic disable: missing-return, unused-local
local log = require "plenary.log"
---@class FrecencyDatabaseConfig ---@class FrecencyDatabaseConfig
---@field root string ---@field root string
---@class FrecencySqlite: sqlite_db
---@field files sqlite_tbl
---@field timestamps sqlite_tbl
---@class FrecencyFile
---@field count integer
---@field id integer
---@field path string
---@field score integer calculated from count and age
---@class FrecencyTimestamp
---@field age integer calculated from timestamp
---@field file_id integer
---@field id integer
---@field timestamp number
---@class FrecencyDatabaseGetFilesOptions ---@class FrecencyDatabaseGetFilesOptions
---@field path string? ---@field path string?
---@field workspace string? ---@field workspace string?
---@class FrecencyDatabase ---@class FrecencyDatabase
---@field config FrecencyDatabaseConfig ---@field config FrecencyDatabaseConfig
---@field private buf_registered_flag_name string ---@field has_entry fun(): boolean
---@field private fs FrecencyFS ---@field new fun(fs: FrecencyFS, config: FrecencyDatabaseConfig): FrecencyDatabase
---@field private sqlite FrecencySqlite ---@field protected fs FrecencyFS
local Database = {} local Database = {}
---@param fs FrecencyFS
---@param config FrecencyDatabaseConfig
---@return FrecencyDatabase
Database.new = function(fs, config)
local lib = sqlite.lib --[[@as sqlite_lib]]
local self = setmetatable(
{ config = config, buf_registered_flag_name = "telescope_frecency_registered", fs = fs },
{ __index = Database }
)
self.sqlite = sqlite {
uri = self.config.root .. "/file_frecency.sqlite3",
files = { id = true, count = { "integer", default = 1, required = true }, path = "string" },
timestamps = {
id = true,
file_id = { "integer", reference = "files.id", on_delete = "cascade" },
timestamp = { "real", default = lib.julianday "now" },
},
}
return self
end
---@return boolean
function Database:has_entry()
return self.sqlite.files:count() > 0
end
---@param paths string[] ---@param paths string[]
---@return integer ---@return nil
function Database:insert_files(paths) function Database:insert_files(paths) end
if #paths == 0 then
return 0 ---@return integer[]|string[]
end function Database:unlinked_entries() end
---@param files integer[]|string[]
---@return nil
function Database:remove_files(files) end
---@param path string ---@param path string
return self.sqlite.files:insert(vim.tbl_map(function(path) ---@param max_count integer
return { path = path, count = 0 } -- TODO: remove when sql.nvim#97 is closed ---@param datetime string?
end, paths)) ---@return nil
end function Database:update(path, max_count, datetime) end
---@async
---@class FrecencyDatabaseEntry
---@field ages number[]
---@field count integer
---@field path string
---@field score number
---@param workspace string? ---@param workspace string?
---@return FrecencyFile[] ---@param datetime string?
function Database:get_files(workspace) ---@return FrecencyDatabaseEntry[]
local query = workspace and { contains = { path = { workspace .. "/*" } } } or {} function Database:get_entries(workspace, datetime) end
log.debug { query = query }
return self.sqlite.files:get(query)
end
---@param datetime string? ISO8601 format string
---@return FrecencyTimestamp[]
function Database:get_timestamps(datetime)
local lib = sqlite.lib
local age = lib.cast((lib.julianday(datetime) - lib.julianday "timestamp") * 24 * 60, "integer")
return self.sqlite.timestamps:get { keys = { age = age, "id", "file_id" } }
end
---@param path string
---@return integer: id of the file entry
---@return boolean: whether the entry is inserted (true) or updated (false)
function Database:upsert_files(path)
local file = self.sqlite.files:get({ where = { path = path } })[1] --[[@as FrecencyFile?]]
if file then
self.sqlite.files:update { where = { id = file.id }, set = { count = file.count + 1 } }
return file.id, false
end
return self.sqlite.files:insert { path = path }, true
end
---@param file_id integer
---@param datetime string? ISO8601 format string
---@return integer
function Database:insert_timestamps(file_id, datetime)
return self.sqlite.timestamps:insert {
file_id = file_id,
timestamp = datetime and sqlite.lib.julianday(datetime) or nil,
}
end
---@param file_id integer
---@param max_count integer
function Database:trim_timestamps(file_id, max_count)
local timestamps = self.sqlite.timestamps:get { where = { file_id = file_id } } --[[@as FrecencyTimestamp[] ]]
local trim_at = timestamps[#timestamps - max_count + 1]
if trim_at then
self.sqlite.timestamps:remove { file_id = tostring(file_id), id = "<" .. tostring(trim_at.id) }
end
end
---@return integer[]
function Database:unlinked_entries()
---@param file FrecencyFile
return self.sqlite.files:map(function(file)
if not self.fs:is_valid_path(file.path) then
return file.id
end
end)
end
---@param ids integer[]
---@return nil
function Database:remove_files(ids)
self.sqlite.files:remove { id = ids }
end
return Database

173
lua/frecency/database/native.lua Normal file
View File

@ -0,0 +1,173 @@
local FileLock = require "frecency.file_lock"
local wait = require "frecency.wait"
local log = require "plenary.log"
local async = require "plenary.async" --[[@as PlenaryAsync]]
---@class FrecencyDatabaseNative: FrecencyDatabase
---@field version "v1"
---@field filename string
---@field file_lock FrecencyFileLock
---@field table FrecencyDatabaseNativeTable
local Native = {}
---@class FrecencyDatabaseNativeTable
---@field version string
---@field records table<string,FrecencyDatabaseNativeRecord>
---@class FrecencyDatabaseNativeRecord
---@field count integer
---@field timestamps integer[]
---@param fs FrecencyFS
---@param config FrecencyDatabaseConfig
---@return FrecencyDatabaseNative
Native.new = function(fs, config)
local version = "v1"
local self = setmetatable({
config = config,
fs = fs,
table = { version = version, records = {} },
version = version,
}, { __index = Native })
self.filename = self.config.root .. "/file_frecency.bin"
self.file_lock = FileLock.new(self.filename)
wait(function()
self:load()
end)
return self
end
---@return boolean
function Native:has_entry()
return not vim.tbl_isempty(self.table.records)
end
---@param paths string[]
---@return nil
function Native:insert_files(paths)
if #paths == 0 then
return
end
for _, path in ipairs(paths) do
self.table.records[path] = { count = 1, timestamps = { 0 } }
end
wait(function()
self:save()
end)
end
---@return string[]
function Native:unlinked_entries()
local paths = {}
for file in pairs(self.table.records) do
if not self.fs:is_valid_path(file) then
table.insert(paths, file)
end
end
return paths
end
---@param paths string[]
function Native:remove_files(paths)
for _, file in ipairs(paths) do
self.table.records[file] = nil
end
wait(function()
self:save()
end)
end
---@param path string
---@param max_count integer
---@param datetime string?
function Native:update(path, max_count, datetime)
local record = self.table.records[path] or { count = 0, timestamps = {} }
record.count = record.count + 1
local now = self:now(datetime)
table.insert(record.timestamps, now)
if #record.timestamps > max_count then
local new_table = {}
for i = #record.timestamps - max_count + 1, #record.timestamps do
table.insert(new_table, record.timestamps[i])
end
record.timestamps = new_table
end
self.table.records[path] = record
wait(function()
self:save()
end)
end
---@param workspace string?
---@param datetime string?
---@return FrecencyDatabaseEntry[]
function Native:get_entries(workspace, datetime)
-- TODO: check mtime of DB and reload it
-- self:load()
local now = self:now(datetime)
local items = {}
for path, record in pairs(self.table.records) do
if not workspace or path:find(workspace .. "/", 1, true) then
table.insert(items, {
path = path,
count = record.count,
ages = vim.tbl_map(function(v)
return (now - v) / 60
end, record.timestamps),
})
end
end
return items
end
---@private
---@param datetime string?
---@return integer
function Native:now(datetime)
return datetime and vim.fn.strptime("%FT%T%z", datetime) or os.time()
end
---@async
---@return nil
function Native:load()
local start = os.clock()
local err, data = self.file_lock:with(function()
local err, st = async.uv.fs_stat(self.filename)
if err then
return nil
end
local fd
err, fd = async.uv.fs_open(self.filename, "r", tonumber("644", 8))
assert(not err)
local data
err, data = async.uv.fs_read(fd, st.size)
assert(not err)
assert(not async.uv.fs_close(fd))
return data
end)
assert(not err, err)
local tbl = loadstring(data or "")() --[[@as FrecencyDatabaseNativeTable?]]
if tbl and tbl.version == self.version then
self.table = tbl
end
log.debug(("load() takes %f seconds"):format(os.clock() - start))
end
---@async
---@return nil
function Native:save()
local start = os.clock()
local err = self.file_lock:with(function()
local f = assert(load("return " .. vim.inspect(self.table)))
local data = string.dump(f)
local err, fd = async.uv.fs_open(self.filename, "w", tonumber("644", 8))
assert(not err)
assert(not async.uv.fs_write(fd, data))
assert(not async.uv.fs_close(fd))
return nil
end)
assert(not err, err)
log.debug(("save() takes %f seconds"):format(os.clock() - start))
end
return Native

135
lua/frecency/database/sqlite.lua Normal file
View File

@ -0,0 +1,135 @@
local sqlite = require "frecency.sqlite"
local log = require "plenary.log"
---@class FrecencySqliteDB: sqlite_db
---@field files sqlite_tbl
---@field timestamps sqlite_tbl
---@class FrecencyFile
---@field count integer
---@field id integer
---@field path string
---@field score integer calculated from count and age
---@class FrecencyTimestamp
---@field age integer calculated from timestamp
---@field file_id integer
---@field id integer
---@field timestamp number
---@class FrecencyDatabaseSqlite: FrecencyDatabase
---@field sqlite FrecencySqliteDB
local Sqlite = {}
---@param fs FrecencyFS
---@param config FrecencyDatabaseConfig
---@return FrecencyDatabaseSqlite
Sqlite.new = function(fs, config)
local lib = sqlite.lib
local self = setmetatable(
{ config = config, buf_registered_flag_name = "telescope_frecency_registered", fs = fs },
{ __index = Sqlite }
)
self.sqlite = sqlite {
uri = self.config.root .. "/file_frecency.sqlite3",
files = { id = true, count = { "integer", default = 1, required = true }, path = "string" },
timestamps = {
id = true,
file_id = { "integer", reference = "files.id", on_delete = "cascade" },
timestamp = { "real", default = lib.julianday "now" },
},
}
return self
end
---@return boolean
function Sqlite:has_entry()
return self.sqlite.files:count() > 0
end
---@param paths string[]
---@return integer
function Sqlite:insert_files(paths)
if #paths == 0 then
return 0
end
---@param path string
return self.sqlite.files:insert(vim.tbl_map(function(path)
return { path = path, count = 0 } -- TODO: remove when sql.nvim#97 is closed
end, paths))
end
---@param workspace string?
---@param datetime string?
---@return FrecencyDatabaseEntry[]
function Sqlite:get_entries(workspace, datetime)
local query = workspace and { contains = { path = { workspace .. "/*" } } } or {}
log.debug { query = query }
local files = self.sqlite.files:get(query) --[[@as FrecencyFile[] ]]
local lib = sqlite.lib
local age = lib.cast((lib.julianday(datetime) - lib.julianday "timestamp") * 24 * 60, "integer")
local timestamps = self.sqlite.timestamps:get { keys = { age = age, "id", "file_id" } } --[[@as FrecencyTimestamp[] ]]
---@type table<integer,number[]>
local age_map = {}
for _, timestamp in ipairs(timestamps) do
if not age_map[timestamp.file_id] then
age_map[timestamp.file_id] = {}
end
table.insert(age_map[timestamp.file_id], timestamp.age)
end
local items = {}
for _, file in ipairs(files) do
table.insert(items, { path = file.path, count = file.count, ages = age_map[file.id] })
end
return items
end
---@param datetime string? ISO8601 format string
---@return FrecencyTimestamp[]
function Sqlite:get_timestamps(datetime)
local lib = sqlite.lib
local age = lib.cast((lib.julianday(datetime) - lib.julianday "timestamp") * 24 * 60, "integer")
return self.sqlite.timestamps:get { keys = { age = age, "id", "file_id" } }
end
---@param path string
---@param count integer
---@param datetime string?
---@return nil
function Sqlite:update(path, count, datetime)
local file = self.sqlite.files:get({ where = { path = path } })[1] --[[@as FrecencyFile?]]
local file_id
if file then
self.sqlite.files:update { where = { id = file.id }, set = { count = file.count + 1 } }
file_id = file.id
else
file_id = self.sqlite.files:insert { path = path }
end
self.sqlite.timestamps:insert {
file_id = file_id,
timestamp = datetime and sqlite.lib.julianday(datetime) or nil,
}
local timestamps = self.sqlite.timestamps:get { where = { file_id = file_id } } --[[@as FrecencyTimestamp[] ]]
local trim_at = timestamps[#timestamps - count + 1]
if trim_at then
self.sqlite.timestamps:remove { file_id = tostring(file_id), id = "<" .. tostring(trim_at.id) }
end
end
---@return integer[]
function Sqlite:unlinked_entries()
---@param file FrecencyFile
return self.sqlite.files:map(function(file)
if not self.fs:is_valid_path(file.path) then
return file.id
end
end)
end
---@param ids integer[]
---@return nil
function Sqlite:remove_files(ids)
self.sqlite.files:remove { id = ids }
end
return Sqlite

5
lua/frecency/entry_maker.lua
View File

@ -24,7 +24,10 @@ EntryMaker.new = function(fs, web_devicons, config)
end, vim.api.nvim_list_bufs()) end, vim.api.nvim_list_bufs())
self.loaded = {} self.loaded = {}
for _, bufnr in ipairs(loaded_bufnrs) do for _, bufnr in ipairs(loaded_bufnrs) do
self.loaded[vim.api.nvim_buf_get_name(bufnr)] = true local bufname = vim.api.nvim_buf_get_name(bufnr)
if bufname then
self.loaded[bufname] = true
end
end end
return self return self
end end

84
lua/frecency/file_lock.lua Normal file
View File

@ -0,0 +1,84 @@
local async = require "plenary.async" --[[@as PlenaryAsync]]
local log = require "plenary.log"
---@class FrecencyFileLock
---@field base string
---@field config FrecencyFileLockConfig
---@field filename string
local FileLock = {}
---@class FrecencyFileLockConfig
---@field retry integer default: 5
---@field interval integer default: 500
---@param path string
---@param opts FrecencyFileLockConfig?
---@return FrecencyFileLock
FileLock.new = function(path, opts)
local config = vim.tbl_extend("force", { retry = 5, interval = 500 }, opts or {})
local self = setmetatable({ config = config }, { __index = FileLock })
self.filename = path .. ".lock"
return self
end
---@async
---@return string? err
function FileLock:get()
local count = 0
local err, fd
while true do
count = count + 1
err, fd = async.uv.fs_open(self.filename, "wx", tonumber("600", 8))
if not err then
break
end
async.util.sleep(self.config.interval)
if count == self.config.retry then
log.debug(("file_lock get() failed: retry count reached: %d"):format(count))
return "failed to get lock"
end
log.debug(("file_lock get() retry: %d"):format(count))
end
err = async.uv.fs_close(fd)
if err then
log.debug("file_lock get() failed: " .. err)
return err
end
end
---@async
---@return string? err
function FileLock:release()
local err = async.uv.fs_stat(self.filename)
if err then
log.debug("file_lock release() not found: " .. err)
return "lock not found"
end
err = async.uv.fs_unlink(self.filename)
if err then
log.debug("file_lock release() unlink failed: " .. err)
return err
end
end
---@async
---@generic T
---@param f fun(): T
---@return string? err
---@return T
function FileLock:with(f)
local err = self:get()
if err then
return err, nil
end
local ok, result_or_err = pcall(f)
err = self:release()
if err then
return err, nil
elseif ok then
return nil, result_or_err
end
return result_or_err, nil
end
return FileLock

77
lua/frecency/frecency.lua
View File

@ -1,10 +1,13 @@
local Database = require "frecency.database" local Sqlite = require "frecency.database.sqlite"
local Native = require "frecency.database.native"
local EntryMaker = require "frecency.entry_maker" local EntryMaker = require "frecency.entry_maker"
local FS = require "frecency.fs" local FS = require "frecency.fs"
local Finder = require "frecency.finder" local Finder = require "frecency.finder"
local Migrator = require "frecency.migrator"
local Picker = require "frecency.picker" local Picker = require "frecency.picker"
local Recency = require "frecency.recency" local Recency = require "frecency.recency"
local WebDevicons = require "frecency.web_devicons" local WebDevicons = require "frecency.web_devicons"
local sqlite_module = require "frecency.sqlite"
local log = require "plenary.log" local log = require "plenary.log"
---@class Frecency ---@class Frecency
@ -13,6 +16,7 @@ local log = require "plenary.log"
---@field private database FrecencyDatabase ---@field private database FrecencyDatabase
---@field private finder FrecencyFinder ---@field private finder FrecencyFinder
---@field private fs FrecencyFS ---@field private fs FrecencyFS
---@field private migrator FrecencyMigrator
---@field private picker FrecencyPicker ---@field private picker FrecencyPicker
---@field private recency FrecencyRecency ---@field private recency FrecencyRecency
local Frecency = {} local Frecency = {}
@ -29,6 +33,7 @@ local Frecency = {}
---@field show_filter_column boolean|string[]|nil default: true ---@field show_filter_column boolean|string[]|nil default: true
---@field show_scores boolean? default: false ---@field show_scores boolean? default: false
---@field show_unindexed boolean? default: true ---@field show_unindexed boolean? default: true
---@field use_sqlite boolean? default: true
---@field workspaces table<string, string>? default: {} ---@field workspaces table<string, string>? default: {}
---@param opts FrecencyConfig? ---@param opts FrecencyConfig?
@ -47,10 +52,21 @@ Frecency.new = function(opts)
show_filter_column = true, show_filter_column = true,
show_scores = false, show_scores = false,
show_unindexed = true, show_unindexed = true,
use_sqlite = true,
workspaces = {}, workspaces = {},
}, opts or {}) }, opts or {})
local self = setmetatable({ buf_registered = {}, config = config }, { __index = Frecency })--[[@as Frecency]] local self = setmetatable({ buf_registered = {}, config = config }, { __index = Frecency })--[[@as Frecency]]
self.fs = FS.new { ignore_patterns = config.ignore_patterns } self.fs = FS.new { ignore_patterns = config.ignore_patterns }
local Database
if not self.config.use_sqlite then
Database = Native
elseif not sqlite_module.can_use then
self:warn "use_sqlite = true, but sqlite module can not be found. It fallbacks to native code."
Database = Native
else
Database = Sqlite
end
self.database = Database.new(self.fs, { root = config.db_root }) self.database = Database.new(self.fs, { root = config.db_root })
local web_devicons = WebDevicons.new(not config.disable_devicons) local web_devicons = WebDevicons.new(not config.disable_devicons)
local entry_maker = EntryMaker.new(self.fs, web_devicons, { local entry_maker = EntryMaker.new(self.fs, web_devicons, {
@ -59,6 +75,7 @@ Frecency.new = function(opts)
}) })
self.finder = Finder.new(entry_maker, self.fs) self.finder = Finder.new(entry_maker, self.fs)
self.recency = Recency.new() self.recency = Recency.new()
self.migrator = Migrator.new(self.fs, self.recency, self.config.db_root)
return self return self
end end
@ -84,6 +101,10 @@ function Frecency:setup()
self:validate_database() self:validate_database()
end end
vim.api.nvim_create_user_command("FrecencyMigrateDB", function()
self:migrate_database()
end, { desc = "Migrate DB telescope-frecency to native code" })
local group = vim.api.nvim_create_augroup("TelescopeFrecency", {}) local group = vim.api.nvim_create_augroup("TelescopeFrecency", {})
vim.api.nvim_create_autocmd({ "BufWinEnter", "BufWritePost" }, { vim.api.nvim_create_autocmd({ "BufWinEnter", "BufWritePost" }, {
desc = "Update database for telescope-frecency", desc = "Update database for telescope-frecency",
@ -152,7 +173,6 @@ function Frecency:validate_database(force)
end) end)
end end
---@private
---@param bufnr integer ---@param bufnr integer
---@param datetime string? ISO8601 format string ---@param datetime string? ISO8601 format string
function Frecency:register(bufnr, datetime) function Frecency:register(bufnr, datetime)
@ -160,15 +180,40 @@ function Frecency:register(bufnr, datetime)
if self.buf_registered[bufnr] or not self.fs:is_valid_path(path) then if self.buf_registered[bufnr] or not self.fs:is_valid_path(path) then
return return
end end
local id, inserted = self.database:upsert_files(path) self.database:update(path, self.recency.config.max_count, datetime)
self.database:insert_timestamps(id, datetime)
self.database:trim_timestamps(id, self.recency.config.max_count)
if inserted and self.picker then
self.picker:discard_results()
end
self.buf_registered[bufnr] = true self.buf_registered[bufnr] = true
end end
---@param to_sqlite boolean?
---@return nil
function Frecency:migrate_database(to_sqlite)
local prompt = to_sqlite and "migrate the DB into SQLite from native code?"
or "migrate the DB into native code from SQLite?"
vim.ui.select({ "y", "n" }, {
prompt = prompt,
---@param item "y"|"n"
---@return string
format_item = function(item)
return item == "y" and "Yes, Migrate it." or "No. Do nothing."
end,
}, function(item)
if item == "n" then
self:notify "migration aborted"
return
elseif to_sqlite then
if sqlite_module.can_use then
self.migrator:to_sqlite()
else
self:error "sqlite.lua is unavailable"
return
end
else
self.migrator:to_v1()
end
self:notify "migration finished successfully"
end)
end
---@private ---@private
---@param fmt string ---@param fmt string
---@param ... any? ---@param ... any?
@ -185,4 +230,20 @@ function Frecency:notify(fmt, ...)
vim.notify(self:message(fmt, ...)) vim.notify(self:message(fmt, ...))
end end
---@private
---@param fmt string
---@param ... any?
---@return nil
function Frecency:warn(fmt, ...)
vim.notify(self:message(fmt, ...), vim.log.levels.WARN)
end
---@private
---@param fmt string
---@param ... any?
---@return nil
function Frecency:error(fmt, ...)
vim.notify(self:message(fmt, ...), vim.log.levels.ERROR)
end
return Frecency return Frecency

83
lua/frecency/migrator.lua Normal file
View File

@ -0,0 +1,83 @@
local Sqlite = require "frecency.database.sqlite"
local Native = require "frecency.database.native"
local wait = require "frecency.wait"
---@class FrecencyMigrator
---@field fs FrecencyFS
---@field recency FrecencyRecency
---@field root string
local Migrator = {}
---@param fs FrecencyFS
---@param recency FrecencyRecency
---@param root string
---@return FrecencyMigrator
Migrator.new = function(fs, recency, root)
return setmetatable({ fs = fs, recency = recency, root = root }, { __index = Migrator })
end
---@return nil
function Migrator:to_v1()
local native = Native.new(self.fs, { root = self.root })
native.table = self:v1_table_from_sqlite()
wait(function()
native:save()
end)
end
---@return nil
function Migrator:to_sqlite()
local sqlite = Sqlite.new(self.fs, { root = self.root })
local native = Native.new(self.fs, { root = self.root })
for path, record in pairs(native.table.records) do
local file_id = sqlite.sqlite.files:insert { path = path, count = record.count }
sqlite.sqlite.timestamps:insert(vim.tbl_map(function(timestamp)
return { file_id = file_id, timestamp = ('julianday(datetime(%d, "unixepoch"))'):format(timestamp) }
end, record.timestamps))
end
end
---@private
---@return FrecencyDatabaseNativeTable
function Migrator:v1_table_from_sqlite()
local sqlite = Sqlite.new(self.fs, { root = self.root })
---@type FrecencyDatabaseNativeTable
local tbl = { version = "v1", records = {} }
local files = sqlite.sqlite.files:get {} --[[@as FrecencyFile[] ]]
---@type table<integer,string>
local path_map = {}
for _, file in ipairs(files) do
tbl.records[file.path] = { count = file.count, timestamps = { 0 } }
path_map[file.id] = file.path
end
-- local timestamps = sqlite.sqlite.timestamps:get { keys = { "id", "file_id", epoch = "unixepoch(timestamp)" } } --[[@as FrecencyTimestamp[] ]]
local timestamps = sqlite.sqlite.timestamps:get {
keys = { "id", "file_id", epoch = "cast(strftime('%s', timestamp) as integer)" },
} --[[@as FrecencyTimestamp[] ]]
table.sort(timestamps, function(a, b)
return a.id < b.id
end)
for _, timestamp in ipairs(timestamps) do
local path = path_map[timestamp.file_id]
if path then
local record = tbl.records[path]
if record then
if #record.timestamps == 1 and record.timestamps[1] == 0 then
record.timestamps = {}
end
---@diagnostic disable-next-line: undefined-field
table.insert(record.timestamps, timestamp.epoch)
if #record.timestamps > self.recency.config.max_count then
local new_table = {}
for i = #record.timestamps - self.recency.config.max_count + 1, #record.timestamps do
table.insert(new_table, record.timestamps[i])
end
record.timestamps = new_table
end
end
end
end
return tbl
end
return Migrator

25
lua/frecency/picker.lua
View File

@ -184,28 +184,15 @@ end
---@return FrecencyFile[] ---@return FrecencyFile[]
function Picker:fetch_results(workspace, datetime) function Picker:fetch_results(workspace, datetime)
log.debug { workspace = workspace or "NONE" } log.debug { workspace = workspace or "NONE" }
local start_files = os.clock() local start_fetch = os.clock()
local files = self.database:get_files(workspace) local files = self.database:get_entries(workspace, datetime)
log.debug { files = #files } log.debug(("it takes %f seconds in fetching entries"):format(os.clock() - start_fetch))
log.debug(("it takes %f seconds in fetching files with workspace: %s"):format(os.clock() - start_files, workspace))
local start_timesatmps = os.clock()
local timestamps = self.database:get_timestamps(datetime)
log.debug { timestamps = #timestamps }
log.debug(("it takes %f seconds in fetching all timestamps"):format(os.clock() - start_timesatmps))
local start_results = os.clock() local start_results = os.clock()
local elapsed_recency = 0 local elapsed_recency = 0
---@type table<integer,number[]>
local age_map = {}
for _, timestamp in ipairs(timestamps) do
if not age_map[timestamp.file_id] then
age_map[timestamp.file_id] = {}
end
table.insert(age_map[timestamp.file_id], timestamp.age)
end
for _, file in ipairs(files) do for _, file in ipairs(files) do
local start_recency = os.clock() local start_recency = os.clock()
local ages = age_map[file.id] --[[@as number[]?]] file.score = file.ages and self.recency:calculate(file.count, file.ages) or 0
file.score = ages and self.recency:calculate(file.count, ages) or 0 file.ages = nil
elapsed_recency = elapsed_recency + (os.clock() - start_recency) elapsed_recency = elapsed_recency + (os.clock() - start_recency)
end end
log.debug(("it takes %f seconds in calculating recency"):format(elapsed_recency)) log.debug(("it takes %f seconds in calculating recency"):format(elapsed_recency))
@ -213,7 +200,7 @@ function Picker:fetch_results(workspace, datetime)
local start_sort = os.clock() local start_sort = os.clock()
table.sort(files, function(a, b) table.sort(files, function(a, b)
return a.score > b.score return a.score > b.score or (a.score == b.score and a.path > b.path)
end) end)
log.debug(("it takes %f seconds in sorting"):format(os.clock() - start_sort)) log.debug(("it takes %f seconds in sorting"):format(os.clock() - start_sort))
return files return files

17
lua/frecency/sqlite.lua Normal file
View File

@ -0,0 +1,17 @@
---@class FrecencySqlite
---@field can_use boolean
---@field lib sqlite_lib
---@overload fun(opts: table): FrecencySqliteDB
return setmetatable({}, {
__index = function(_, k)
if k == "lib" then
return require("sqlite").lib
elseif k == "can_use" then
return not not vim.F.npcall(require, "sqlite")
end
end,
__call = function(_, opts)
return require "sqlite"(opts)
end,
}) --[[@as FrecencySqlite]]

135
lua/frecency/tests/file_lock_spec.lua Normal file
View File

@ -0,0 +1,135 @@
local FileLock = require "frecency.file_lock"
local util = require "frecency.tests.util"
local async = require "plenary.async" --[[@as PlenaryAsync]]
require("plenary.async").tests.add_to_env()
local function with_dir(f)
local dir, close = util.make_tree {}
local filename = (dir / "file_lock_test").filename
f(filename)
close()
end
a.describe("file_lock", function()
a.describe("get()", function()
a.describe("when no lock file", function()
with_dir(function(filename)
local fl = FileLock.new(filename, { retry = 1, interval = 10 })
a.it("gets successfully", function()
assert.is.Nil(fl:get())
end)
end)
end)
a.describe("when with a lock file", function()
with_dir(function(filename)
local fl = FileLock.new(filename, { retry = 1, interval = 10 })
a.it("fails to get", function()
assert.is.Nil(async.uv.fs_open(fl.filename, "wx", tonumber("600", 8)))
assert.are.same("failed to get lock", fl:get())
end)
end)
end)
a.describe("when getting twice", function()
with_dir(function(filename)
local fl = FileLock.new(filename, { retry = 1, interval = 10 })
a.it("fails to get", function()
assert.is.Nil(fl:get())
assert.are.same("failed to get lock", fl:get())
end)
end)
end)
end)
a.describe("release()", function()
a.describe("when no lock file", function()
with_dir(function(filename)
local fl = FileLock.new(filename, { retry = 1, interval = 10 })
a.it("fails to release", function()
assert.are.same("lock not found", fl:release())
end)
end)
end)
a.describe("when with a lock file", function()
with_dir(function(filename)
local fl = FileLock.new(filename, { retry = 1, interval = 10 })
a.it("releases successfully", function()
assert.is.Nil(fl:get())
assert.is.Nil(fl:release())
end)
end)
end)
a.describe("when releasing twice", function()
with_dir(function(filename)
local fl = FileLock.new(filename, { retry = 1, interval = 10 })
a.it("fails to release", function()
assert.is.Nil(fl:get())
assert.is.Nil(fl:release())
assert.are.same("lock not found", fl:release())
end)
end)
end)
end)
a.describe("with()", function()
a.describe("when get() fails", function()
with_dir(function(filename)
local fl = FileLock.new(filename, { retry = 1, interval = 10 })
a.it("fails with a valid err", function()
assert.is.Nil(fl:get())
assert.are.same(
"failed to get lock",
fl:with(function()
return nil
end)
)
end)
end)
end)
a.describe("when release() fails", function()
with_dir(function(filename)
local fl = FileLock.new(filename, { retry = 1, interval = 10 })
a.it("fails with a valid err", function()
assert.are.same(
"lock not found",
fl:with(function()
assert.is.Nil(async.uv.fs_unlink(fl.filename))
return nil
end)
)
end)
end)
end)
a.describe("when f() fails", function()
with_dir(function(filename)
local fl = FileLock.new(filename, { retry = 1, interval = 10 })
a.it("fails with a valid err", function()
assert.has.match(
": error in hoge function$",
fl:with(function()
error "error in hoge function"
end)
)
end)
end)
end)
a.describe("when no errors", function()
with_dir(function(filename)
local fl = FileLock.new(filename, { retry = 1, interval = 10 })
a.it("run successfully and returns valid results", function()
local err, result = fl:with(function()
return "hogehogeo"
end)
assert.is.Nil(err)
assert.are.same("hogehogeo", result)
end)
end)
end)
end)
end)

74
lua/frecency/tests/frecency_spec.lua
View File

@ -2,15 +2,18 @@
local Frecency = require "frecency.frecency" local Frecency = require "frecency.frecency"
local Picker = require "frecency.picker" local Picker = require "frecency.picker"
local util = require "frecency.tests.util" local util = require "frecency.tests.util"
local Path = require "plenary.path"
local log = require "plenary.log" local log = require "plenary.log"
local Path = require "plenary.path"
local use_sqlite
---@param files string[] ---@param files string[]
---@param callback fun(frecency: Frecency, dir: PlenaryPath): nil ---@param callback fun(frecency: Frecency, dir: PlenaryPath): nil
---@return nil ---@return nil
local function with_files(files, callback) local function with_files(files, callback)
local dir, close = util.make_tree(files) local dir, close = util.make_tree(files)
local frecency = Frecency.new { db_root = dir.filename } log.debug { db_root = dir.filename }
local frecency = Frecency.new { db_root = dir.filename, use_sqlite = use_sqlite }
frecency.picker = Picker.new( frecency.picker = Picker.new(
frecency.database, frecency.database,
frecency.finder, frecency.finder,
@ -41,7 +44,6 @@ local function make_register(frecency, dir)
end end
end end
---comment
---@param frecency Frecency ---@param frecency Frecency
---@param dir PlenaryPath ---@param dir PlenaryPath
---@param callback fun(register: fun(file: string, datetime: string?): nil): nil ---@param callback fun(register: fun(file: string, datetime: string?): nil): nil
@ -86,6 +88,8 @@ local function with_fake_vim_ui_select(choice, callback)
end end
describe("frecency", function() describe("frecency", function()
local function test(db)
describe(db, function()
describe("register", function() describe("register", function()
describe("when opening files", function() describe("when opening files", function()
with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir) with_files({ "hoge1.txt", "hoge2.txt" }, function(frecency, dir)
@ -96,8 +100,8 @@ describe("frecency", function()
it("has valid records in DB", function() it("has valid records in DB", function()
local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00") local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00")
assert.are.same({ assert.are.same({
{ count = 1, id = 1, path = filepath(dir, "hoge1.txt"), score = 10 }, { count = 1, path = filepath(dir, "hoge2.txt"), score = 10 },
{ count = 1, id = 2, path = filepath(dir, "hoge2.txt"), score = 10 }, { count = 1, path = filepath(dir, "hoge1.txt"), score = 10 },
}, results) }, results)
end) end)
end) end)
@ -113,8 +117,8 @@ describe("frecency", function()
it("increases the score", function() it("increases the score", function()
local results = frecency.picker:fetch_results(nil, "2023-07-29T03:00:00+09:00") local results = frecency.picker:fetch_results(nil, "2023-07-29T03:00:00+09:00")
assert.are.same({ assert.are.same({
{ count = 2, id = 1, path = filepath(dir, "hoge1.txt"), score = 40 }, { count = 2, path = filepath(dir, "hoge1.txt"), score = 40 },
{ count = 1, id = 2, path = filepath(dir, "hoge2.txt"), score = 10 }, { count = 1, path = filepath(dir, "hoge2.txt"), score = 10 },
}, results) }, results)
end) end)
end) end)
@ -130,8 +134,8 @@ describe("frecency", function()
it("does not increase the score", function() it("does not increase the score", function()
local results = frecency.picker:fetch_results(nil, "2023-07-29T03:00:00+09:00") local results = frecency.picker:fetch_results(nil, "2023-07-29T03:00:00+09:00")
assert.are.same({ assert.are.same({
{ count = 1, id = 1, path = filepath(dir, "hoge1.txt"), score = 10 }, { count = 1, path = filepath(dir, "hoge2.txt"), score = 10 },
{ count = 1, id = 2, path = filepath(dir, "hoge2.txt"), score = 10 }, { count = 1, path = filepath(dir, "hoge1.txt"), score = 10 },
}, results) }, results)
end) end)
end) end)
@ -159,8 +163,8 @@ describe("frecency", function()
it("calculates score from the recent 10 times", function() it("calculates score from the recent 10 times", function()
local results = frecency.picker:fetch_results(nil, "2023-07-29T00:12:00+09:00") local results = frecency.picker:fetch_results(nil, "2023-07-29T00:12:00+09:00")
assert.are.same({ assert.are.same({
{ count = 12, id = 2, path = filepath(dir, "hoge2.txt"), score = 12 * (10 * 100) / 10 }, { count = 12, path = filepath(dir, "hoge2.txt"), score = 12 * (10 * 100) / 10 },
{ count = 2, id = 1, path = filepath(dir, "hoge1.txt"), score = 2 * (2 * 100) / 10 }, { count = 2, path = filepath(dir, "hoge1.txt"), score = 2 * (2 * 100) / 10 },
}, results) }, results)
end) end)
end) end)
@ -171,15 +175,18 @@ describe("frecency", function()
describe("after registered over >5000 files", function() describe("after registered over >5000 files", function()
with_files({}, function(frecency, dir) with_files({}, function(frecency, dir)
with_fake_register(frecency, dir, function(register) with_fake_register(frecency, dir, function(register)
local file_count = 6000 -- TODO: 6000 records is too many to use with native?
-- local file_count = 6000
local file_count = 600
if not os.getenv "CI" then if not os.getenv "CI" then
log.info "It works not on CI. Files is decreaed into 10 count." log.info "It works not on CI. Files is decreased into 10 count."
file_count = 10 file_count = 10
end end
local expected = {} local expected = {}
log.info(("making %d files and register them"):format(file_count))
for i = 1, file_count do for i = 1, file_count do
local file = ("hoge%08d.txt"):format(i) local file = ("hoge%08d.txt"):format(i)
table.insert(expected, { count = 1, id = i, path = filepath(dir, file), score = 10 }) table.insert(expected, { count = 1, path = filepath(dir, file), score = 10 })
register(file, "2023-07-29T00:00:00+09:00") register(file, "2023-07-29T00:00:00+09:00")
end end
local start = os.clock() local start = os.clock()
@ -212,8 +219,8 @@ describe("frecency", function()
it("removes no entries", function() it("removes no entries", function()
local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00") local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00")
assert.are.same({ assert.are.same({
{ count = 1, id = 1, path = filepath(dir, "hoge1.txt"), score = 10 }, { count = 1, path = filepath(dir, "hoge2.txt"), score = 10 },
{ count = 1, id = 2, path = filepath(dir, "hoge2.txt"), score = 10 }, { count = 1, path = filepath(dir, "hoge1.txt"), score = 10 },
}, results) }, results)
end) end)
end) end)
@ -239,11 +246,11 @@ describe("frecency", function()
return a.path < b.path return a.path < b.path
end) end)
assert.are.same({ assert.are.same({
{ count = 1, id = 1, path = filepath(dir, "hoge1.txt"), score = 10 }, { count = 1, path = filepath(dir, "hoge1.txt"), score = 10 },
{ count = 1, id = 2, path = filepath(dir, "hoge2.txt"), score = 10 }, { count = 1, path = filepath(dir, "hoge2.txt"), score = 10 },
{ count = 1, id = 3, path = filepath(dir, "hoge3.txt"), score = 10 }, { count = 1, path = filepath(dir, "hoge3.txt"), score = 10 },
{ count = 1, id = 4, path = filepath(dir, "hoge4.txt"), score = 10 }, { count = 1, path = filepath(dir, "hoge4.txt"), score = 10 },
{ count = 1, id = 5, path = filepath(dir, "hoge5.txt"), score = 10 }, { count = 1, path = filepath(dir, "hoge5.txt"), score = 10 },
}, results) }, results)
end) end)
end) end)
@ -277,8 +284,8 @@ describe("frecency", function()
return a.path < b.path return a.path < b.path
end) end)
assert.are.same({ assert.are.same({
{ count = 1, id = 4, path = filepath(dir, "hoge4.txt"), score = 10 }, { count = 1, path = filepath(dir, "hoge4.txt"), score = 10 },
{ count = 1, id = 5, path = filepath(dir, "hoge5.txt"), score = 10 }, { count = 1, path = filepath(dir, "hoge5.txt"), score = 10 },
}, results) }, results)
end) end)
end) end)
@ -311,11 +318,11 @@ describe("frecency", function()
return a.path < b.path return a.path < b.path
end) end)
assert.are.same({ assert.are.same({
{ count = 1, id = 1, path = filepath(dir, "hoge1.txt"), score = 10 }, { count = 1, path = filepath(dir, "hoge1.txt"), score = 10 },
{ count = 1, id = 2, path = filepath(dir, "hoge2.txt"), score = 10 }, { count = 1, path = filepath(dir, "hoge2.txt"), score = 10 },
{ count = 1, id = 3, path = filepath(dir, "hoge3.txt"), score = 10 }, { count = 1, path = filepath(dir, "hoge3.txt"), score = 10 },
{ count = 1, id = 4, path = filepath(dir, "hoge4.txt"), score = 10 }, { count = 1, path = filepath(dir, "hoge4.txt"), score = 10 },
{ count = 1, id = 5, path = filepath(dir, "hoge5.txt"), score = 10 }, { count = 1, path = filepath(dir, "hoge5.txt"), score = 10 },
}, results) }, results)
end) end)
end) end)
@ -342,7 +349,7 @@ describe("frecency", function()
it("needs confirmation for removing entries", function() it("needs confirmation for removing entries", function()
local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00") local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00")
assert.are.same({ assert.are.same({
{ count = 1, id = 2, path = filepath(dir, "hoge2.txt"), score = 10 }, { count = 1, path = filepath(dir, "hoge2.txt"), score = 10 },
}, results) }, results)
end) end)
end) end)
@ -367,7 +374,7 @@ describe("frecency", function()
it("needs no confirmation for removing entries", function() it("needs no confirmation for removing entries", function()
local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00") local results = frecency.picker:fetch_results(nil, "2023-07-29T02:00:00+09:00")
assert.are.same({ assert.are.same({
{ count = 1, id = 2, path = filepath(dir, "hoge2.txt"), score = 10 }, { count = 1, path = filepath(dir, "hoge2.txt"), score = 10 },
}, results) }, results)
end) end)
end) end)
@ -375,3 +382,10 @@ describe("frecency", function()
end) end)
end) end)
end) end)
end
use_sqlite = true
test "sqlite"
use_sqlite = false
test "native"
end)

129
lua/frecency/tests/migrator_spec.lua Normal file
View File

@ -0,0 +1,129 @@
---@diagnostic disable: undefined-field
local Migrator = require "frecency.migrator"
local FS = require "frecency.fs"
local Recency = require "frecency.recency"
local Sqlite = require "frecency.database.sqlite"
local Native = require "frecency.database.native"
local util = require "frecency.tests.util"
local wait = require "frecency.wait"
---@param callback fun(migrator: FrecencyMigrator, sqlite: FrecencyDatabase): nil
---@return nil
local function with(callback)
local dir, close = util.tmpdir()
local recency = Recency.new { max_count = 2 }
local fs = FS.new { ignore_patterns = {} }
local migrator = Migrator.new(fs, recency, dir.filename)
local sqlite = Sqlite.new(fs, { root = dir.filename })
callback(migrator, sqlite)
close()
end
---@param source table<string,{ count: integer, timestamps: string[] }>
local function v1_table(source)
local records = {}
for path, record in pairs(source) do
local timestamps = {}
for _, timestamp in ipairs(record.timestamps) do
table.insert(timestamps, vim.fn.strptime("%FT%T%z", timestamp))
end
records[path] = { count = record.count, timestamps = timestamps }
end
return { version = "v1", records = records }
end
describe("migrator", function()
describe("to_v1", function()
describe("when with simple database", function()
with(function(migrator, sqlite)
for _, path in ipairs { "hoge1.txt", "hoge2.txt" } do
sqlite:update(path, migrator.recency.config.max_count, "2023-08-21T00:00:00")
end
migrator:to_v1()
local native = Native.new(migrator.fs, { root = migrator.root })
it("has converted into a valid table", function()
assert.are.same(
v1_table {
["hoge1.txt"] = { count = 1, timestamps = { "2023-08-21T00:00:00+0000" } },
["hoge2.txt"] = { count = 1, timestamps = { "2023-08-21T00:00:00+0000" } },
},
native.table
)
end)
end)
end)
describe("when with more large database", function()
with(function(migrator, sqlite)
for i, path in ipairs {
"hoge1.txt",
"hoge1.txt",
"hoge1.txt",
"hoge1.txt",
"hoge2.txt",
"hoge2.txt",
"hoge2.txt",
"hoge3.txt",
"hoge3.txt",
"hoge4.txt",
} do
sqlite:update(path, migrator.recency.config.max_count, ("2023-08-21T00:%02d:00"):format(i))
end
migrator:to_v1()
local native = Native.new(migrator.fs, { root = migrator.root })
it("has converted into a valid table", function()
assert.are.same(
v1_table {
["hoge1.txt"] = { count = 4, timestamps = { "2023-08-21T00:03:00+0000", "2023-08-21T00:04:00+0000" } },
["hoge2.txt"] = { count = 3, timestamps = { "2023-08-21T00:06:00+0000", "2023-08-21T00:07:00+0000" } },
["hoge3.txt"] = { count = 2, timestamps = { "2023-08-21T00:08:00+0000", "2023-08-21T00:09:00+0000" } },
["hoge4.txt"] = { count = 1, timestamps = { "2023-08-21T00:10:00+0000" } },
},
native.table
)
end)
end)
end)
end)
describe("to_sqlite", function()
with(function(migrator, sqlite)
local native = Native.new(migrator.fs, { root = migrator.root })
native.table = v1_table {
["hoge1.txt"] = { count = 4, timestamps = { "2023-08-21T00:03:00+0000", "2023-08-21T00:04:00+0000" } },
["hoge2.txt"] = { count = 3, timestamps = { "2023-08-21T00:06:00+0000", "2023-08-21T00:07:00+0000" } },
["hoge3.txt"] = { count = 2, timestamps = { "2023-08-21T00:08:00+0000", "2023-08-21T00:09:00+0000" } },
["hoge4.txt"] = { count = 1, timestamps = { "2023-08-21T00:10:00+0000" } },
}
wait(function()
native:save()
end)
migrator:to_sqlite()
sqlite.sqlite.db:open()
local records = sqlite.sqlite.db:eval [[
select
f.path,
f.count,
datetime(strftime('%s', t.timestamp), 'unixepoch') datetime
from timestamps t
join files f
on f.id = t.file_id
order by path, datetime
]]
it("has converted into a valid DB", function()
assert.are.same({
{ path = "hoge1.txt", count = 4, datetime = "2023-08-21 00:03:00" },
{ path = "hoge1.txt", count = 4, datetime = "2023-08-21 00:04:00" },
{ path = "hoge2.txt", count = 3, datetime = "2023-08-21 00:06:00" },
{ path = "hoge2.txt", count = 3, datetime = "2023-08-21 00:07:00" },
{ path = "hoge3.txt", count = 2, datetime = "2023-08-21 00:08:00" },
{ path = "hoge3.txt", count = 2, datetime = "2023-08-21 00:09:00" },
{ path = "hoge4.txt", count = 1, datetime = "2023-08-21 00:10:00" },
}, records)
end)
end)
end)
end)

26
lua/frecency/tests/util.lua
View File

@ -1,17 +1,25 @@
local uv = vim.uv or vim.loop local uv = vim.uv or vim.loop
local Path = require "plenary.path" local Path = require "plenary.path"
---@param entries string[] ---@return PlenaryPath
---@return PlenaryPath the top dir of tree ---@return fun(): nil close swwp all entries
---@return fun(): nil sweep all entries local function tmpdir()
local function make_tree(entries) local dir = Path:new(Path:new(assert(uv.fs_mkdtemp "tests_XXXXXX")):absolute())
local dir = Path:new(Path.new(assert(uv.fs_mkdtemp "tests_XXXXXX")):absolute())
for _, entry in ipairs(entries) do
dir:joinpath(entry):touch { parents = true }
end
return dir, function() return dir, function()
dir:rm { recursive = true } dir:rm { recursive = true }
end end
end end
return { make_tree = make_tree } ---@param entries string[]
---@return PlenaryPath dir the top dir of tree
---@return fun(): nil close sweep all entries
local function make_tree(entries)
local dir, close = tmpdir()
for _, entry in ipairs(entries) do
---@diagnostic disable-next-line: undefined-field
dir:joinpath(entry):touch { parents = true }
end
return dir, close
end
return { make_tree = make_tree, tmpdir = tmpdir }

50
lua/frecency/types.lua
View File

@ -1,4 +1,5 @@
---@diagnostic disable: unused-local ---@diagnostic disable: unused-local, missing-return
-- NOTE: types below are borrowed from sqlite.lua -- NOTE: types below are borrowed from sqlite.lua
---@class sqlite_db @Main sqlite.lua object. ---@class sqlite_db @Main sqlite.lua object.
@ -51,6 +52,7 @@
---@field filename string ---@field filename string
---@field joinpath fun(self: PlenaryPath, ...): PlenaryPath ---@field joinpath fun(self: PlenaryPath, ...): PlenaryPath
---@field make_relative fun(self: PlenaryPath, cwd: string): string ---@field make_relative fun(self: PlenaryPath, cwd: string): string
---@field parent PlenaryPath
---@field path { sep: string } ---@field path { sep: string }
---@field rm fun(self: PlenaryPath, opts: { recursive: boolean }?): nil ---@field rm fun(self: PlenaryPath, opts: { recursive: boolean }?): nil
@ -69,6 +71,8 @@
---@class PlenaryAsync ---@class PlenaryAsync
---@field control PlenaryAsyncControl ---@field control PlenaryAsyncControl
---@field util PlenaryAsyncUtil ---@field util PlenaryAsyncUtil
---@field uv PlenaryAsyncUv
---@field void fun(f: fun(): nil): fun(): nil
local PlenaryAsync = {} local PlenaryAsync = {}
---@async ---@async
@ -96,6 +100,50 @@ function PlenaryAsyncControlChannelRx.recv() end
---@class PlenaryAsyncUtil ---@class PlenaryAsyncUtil
local PlenaryAsyncUtil = {} local PlenaryAsyncUtil = {}
---@class PlenaryAsyncUv
local PlenaryAsyncUv = {}
---@async
---@param path string
---@return string? err
---@return { mtime: integer, size: integer, type: "file"|"directory" }
function PlenaryAsyncUv.fs_stat(path) end
---@async
---@param path string
---@param flags string|integer
---@param mode integer
---@return string? err
---@return integer fd
function PlenaryAsyncUv.fs_open(path, flags, mode) end
---@async
---@param fd integer
---@param size integer
---@param offset integer?
---@return string? err
---@return string data
function PlenaryAsyncUv.fs_read(fd, size, offset) end
---@async
---@param fd integer
---@param data string
---@param offset integer?
---@return string? err
---@return integer bytes
function PlenaryAsyncUv.fs_write(fd, data, offset) end
---@async
---@param path string
---@return string? err
---@return boolean? success
function PlenaryAsyncUv.fs_unlink(path) end
---@async
---@param fd integer
---@return string? err
function PlenaryAsyncUv.fs_close(fd) end
---@async ---@async
---@param ms integer ---@param ms integer
---@return nil ---@return nil

54
lua/frecency/wait.lua Normal file
View File

@ -0,0 +1,54 @@
local async = require "plenary.async"
---@class FrecencyWait
---@field config FrecencyWaitConfig
local Wait = {}
---@class FrecencyWaitConfig
---@field time integer default: 5000
---@field interval integer default: 200
---@alias FrecencyWaitCallback fun(): nil
---@param f FrecencyWaitCallback
---@param opts FrecencyWaitConfig?
Wait.new = function(f, opts)
return setmetatable(
{ f = f, config = vim.tbl_extend("force", { time = 5000, interval = 200 }, opts or {}) },
{ __index = Wait }
)
end
---@async
---@private
Wait.f = function()
error "implement me"
end
---@return boolean ok
---@return nil|-1|-2 status
function Wait:run()
local done = false
async.void(function()
self.f()
done = true
end)()
return vim.wait(self.config.time, function()
return done
end, self.config.interval)
end
---@param f FrecencyWaitCallback
---@param opts FrecencyWaitConfig?
---@return nil
return function(f, opts)
local wait = Wait.new(f, opts)
local ok, status = wait:run()
if ok then
return
elseif status == -1 then
error "callback never returnes during the time"
elseif status == -2 then
error "callback is interrupted during the time"
end
end

5
lua/telescope/_extensions/frecency.lua
View File

@ -1,12 +1,13 @@
local frecency = require "frecency" local frecency = require "frecency"
local sqlite = require "frecency.sqlite"
return require("telescope").register_extension { return require("telescope").register_extension {
setup = frecency.setup, setup = frecency.setup,
health = function() health = function()
if vim.F.npcall(require, "sqlite") then if sqlite.can_use then
vim.health.ok "sqlite.lua installed." vim.health.ok "sqlite.lua installed."
else else
vim.health.error "sqlite.lua is required for telescope-frecency.nvim to work." vim.health.info "sqlite.lua is required when use_sqlite = true"
end end
if vim.F.npcall(require, "nvim-web-devicons") then if vim.F.npcall(require, "nvim-web-devicons") then
vim.health.ok "nvim-web-devicons installed." vim.health.ok "nvim-web-devicons installed."