From 7afdd3c32c222a359d6a906968dcbed5fbea2fb7 Mon Sep 17 00:00:00 2001 From: Senghan Bright Date: Thu, 28 Jan 2021 22:45:04 +0100 Subject: [PATCH] Feature: filtered workspaces * draft implementation of tags/filters * . * add filtering: - extended substring sorter to have modes: - when current string is prefixed by `:foo`, results are tag_names that come from tags/workspaces table. (if `:foo ` token is incomplete it is ignored) - when a complete workspace tag is matched ':foobar:', results are indexed_files filtered by if their parent_dir is a descendant of the workspace_dir - a recursive scan_dir() result is added to the :foobar: filter results; any non-indexed_files are given a score of zero, and are alphabetically sorted below the indexed_results - tab completion for tab_names in insert mode`:foo|` state: cycles through available options * add completion file * use attach_mappings for map * stop completion being enabled multiple times * improve keys * improve completion cancellation * add dynamic `lsp` tag * add dynamic `lsp` tag * fix empty lsp workspaces * remove hardcoded workspaces and allow config from ext_config * add filter highlight and some fixes * . * add workspace filters to readme * wip LSP workspace filter * merge ignore_patterns fix * change LSP_ROOT tagname to LSP * fix setting default values * . * update readme with filter instructions * remove debug message * improve relative paths * improve relative paths * WIP dynamic column sizes * WIP filter_column_width * fix keymaps * . * feat: persistent filters * refactor config creation * fix: filter directory column autosize * improve LSP workspace paths * . * remove workspace filter output * cache persistent filter results * fix cached results * . * remove results cache; sorting is the expensive part * respect ignore patterns for non-indexed files. * return table on on_input_filter_cb --- README.md | 49 +++- lua/telescope/_extensions/frecency.lua | 250 +++++++++++++++--- .../_extensions/frecency/db_client.lua | 63 +++-- .../_extensions/frecency/sql_wrapper.lua | 25 +- lua/telescope/_extensions/frecency/util.lua | 4 - plugin/frecency.vim | 36 +++ syntax/frecency.vim | 5 + 7 files changed, 356 insertions(+), 76 deletions(-) create mode 100644 plugin/frecency.vim create mode 100644 syntax/frecency.vim diff --git a/README.md b/README.md index d4b646d..f2a8412 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,21 @@ The score is calculated using the age of the 10 most recent timestamps and the t score = frequency * recency_score / max_number_of_timestamps ``` +### Workspace Filters + +The _Workspace filter_ feature enables you to select from user defined _filter tags_ that map to a directory or collection of directories. +Filters are applied by entering `:workspace_tag:` anywhere in the query. +Filter name completion is available by pressing `` after the first `:` character. + +When a filter is applied, results are reduced to entries whose path is a descendant of the workspace directory. +The indexed results are optionally augmented with a listing of _all_ files found in a recurssive search of the workspace directory. +Non-indexed files are given a score of zero and appear below the 'frecent' entries. +When a non-indexed file is opened, it gains a score value and is available in future 'frecent' search results. + +If the active buffer (prior to the finder being launched) is attached to an LSP server, an automatic `LSP` tag is available, which maps to the workspace directories provided by the language server. + +TODO: insert filter screenshot + ## Requirements - [telescope.nvim](https://github.com/nvim-telescope/telescope.nvim) (required) @@ -70,28 +85,46 @@ If no database is found when running Neovim with the plugin installed, a new one ```lua vim.api.nvim_set_keymap("n", "", "lua require('telescope').extensions.frecency.frecency()", {noremap = true, silent = true}) ``` +Filter tags are applied by typing the `:tag:` name (adding surrounding colons) in the finder query. +Entering `:` will trigger omnicompletion for available tags. ## Configuration See [default configuration](https://github.com/nvim-telescope/telescope.nvim#telescope-defaults) for full details on configuring Telescope. -- `ignore_patterns` +- `ignore_patterns` (default: `{"*.git/*", "*/tmp/*"}`) -This setting controls which files are indexed (and subsequently which you'll see in the finder results). + Patterns in this table control which files are indexed (and subsequently which you'll see in the finder results). -- `show_scores` +- `show_scores` (default : `false`) -To see the scores generated by the algorithm in the results, set this to `true`. + To see the scores generated by the algorithm in the results, set this to `true`. +- workspaces (default: {}) -If you've not configured the extension, the following values are used: + This table contains mappings of `workspace_tag` -> `workspace_directory` + The key corresponds to the `:tag_name` used to select the filter in queries. + The value corresponds to the top level directory by which results will be filtered. + +- `show_unindexed` (default: `true`) + +Determines if non-indexed files are included in workspace filter results. + +### Example Configuration: ``` telescope.setup { extensions = { frecency = { show_scores = false, + show_unindexed = true, ignore_patterns = {"*.git/*", "*/tmp/*"}, + workspaces = { + ["conf"] = "/home/my_username/.config", + ["data"] = "/home/my_username/.local/share", + ["project"] = "/home/my_username/projects", + ["wiki"] = "/home/my_username/wiki" + } } }, } @@ -100,10 +133,14 @@ telescope.setup { ### Highlight Groups ```vim -TelescopePathSeparator TelescopeBufferLoaded +TelescopePathSeparator +TelescopeFrecencyScores +TelescopeQueryFilter ``` +TODO: describe highlight groups + ## References - [Mozilla: Frecency algorithm](https://developer.mozilla.org/en-US/docs/Mozilla/Tech/Places/Frecency_algorithm) diff --git a/lua/telescope/_extensions/frecency.lua b/lua/telescope/_extensions/frecency.lua index a79ed48..dfc1d89 100644 --- a/lua/telescope/_extensions/frecency.lua +++ b/lua/telescope/_extensions/frecency.lua @@ -1,101 +1,263 @@ local has_telescope, telescope = pcall(require, "telescope") --- TODO: make dependency errors occur in a better way +-- TODO: make sure scandir unindexed have opts.ignore_patterns applied +-- TODO: make filters handle mulitple directories + if not has_telescope then error("This plugin requires telescope.nvim (https://github.com/nvim-telescope/telescope.nvim)") end --- finder code +local actions = require('telescope.actions') local conf = require('telescope.config').values local entry_display = require "telescope.pickers.entry_display" local finders = require "telescope.finders" +local mappings = require('telescope.mappings') local path = require('telescope.path') local pickers = require "telescope.pickers" local sorters = require "telescope.sorters" local utils = require('telescope.utils') +local db_client = require("telescope._extensions.frecency.db_client") local os_home = vim.loop.os_homedir() local os_path_sep = utils.get_separator() -local show_scores = false -local db_client + +local state = { + results = {}, + active_filter = nil, + active_filter_tag = nil, + last_filter = nil, + previous_buffer = nil, + cwd = nil, + show_scores = false, + user_workspaces = {}, + lsp_workspaces = {}, + picker = {} +} + +local function format_filepath(filename, opts) + local original_filename = filename + + if state.active_filter then + filename = path.make_relative(filename, state.active_filter) + else + filename = path.make_relative(filename, state.cwd) + -- check relative to home/current + if vim.startswith(filename, os_home) then + filename = "~/" .. path.make_relative(filename, os_home) + elseif filename ~= original_filename then + filename = "./" .. filename + end + end + + if opts.tail_path then + filename = utils.path_tail(filename) + elseif opts.shorten_path then + filename = utils.path_shorten(filename) + end + + return filename +end + +local function get_workspace_tags() + -- TODO: validate that workspaces are existing directories + local tags = {} + for k,_ in pairs(state.user_workspaces) do + table.insert(tags, k) + end + local lsp_workspaces = vim.api.nvim_buf_call(state.previous_buffer, vim.lsp.buf.list_workspace_folders) + + if not vim.tbl_isempty(lsp_workspaces) then + state.lsp_workspaces = lsp_workspaces + tags[#tags+1] = "LSP" + else + state.lsp_workspaces = {} + end + + -- print(vim.inspect(tags)) + -- TODO: sort tags - by collective frecency? + return tags +end local frecency = function(opts) opts = opts or {} - local cwd = vim.fn.expand(opts.cwd or vim.fn.getcwd()) - -- opts.lsp_workspace_filter = true - -- TODO: decide on how to handle cwd or lsp_workspace for pathname shorten? - local results = db_client.get_file_scores(opts) -- TODO: pass `filter_workspace` option + state.previous_buffer = vim.fn.bufnr('%') + state.cwd = vim.fn.expand(opts.cwd or vim.fn.getcwd()) - local display_cols = {} - display_cols[1] = show_scores and {width = 8} or nil - table.insert(display_cols, {remaining = true}) + local function get_display_cols() + local directory_col_width = 0 + if state.active_filter then + if state.active_filter_tag == "LSP" then + -- TODO: Only add +1 if opts.show_filter_thing is true, +1 is for the trailing slash + directory_col_width = #(utils.path_tail(state.active_filter)) + 1 + else + directory_col_width = #(path.make_relative(state.active_filter, os_home)) + 1 + end + end + local res = {} + res[1] = state.show_scores and {width = 8} or nil + if state.show_filter_column then + table.insert(res, {width = directory_col_width}) + end + table.insert(res, {remaining = true}) + return res + end local displayer = entry_display.create { separator = "", hl_chars = {[os_path_sep] = "TelescopePathSeparator"}, - items = display_cols + items = get_display_cols() } - -- TODO: look into why this gets called so much - local bufnr, buf_is_loaded, filename, hl_filename, display_items, original_filename - + local bufnr, buf_is_loaded, filename, hl_filename, display_items local make_display = function(entry) bufnr = vim.fn.bufnr buf_is_loaded = vim.api.nvim_buf_is_loaded - filename = entry.name + filename = entry.name hl_filename = buf_is_loaded(bufnr(filename)) and "TelescopeBufferLoaded" or "" + filename = format_filepath(filename, opts) - original_filename = filename + display_items = state.show_scores and {{entry.score, "TelescopeFrecencyScores"}} or {} - if opts.tail_path then - filename = utils.path_tail(filename) - elseif opts.shorten_path then - filename = utils.path_shorten(filename) - else -- check relative to home/current - filename = path.make_relative(filename, cwd) - if vim.startswith(filename, os_home) then - filename = "~/" .. path.make_relative(filename, os_home) - elseif filename ~= original_filename then - filename = "./" .. filename + -- TODO: store the column lengths here, rather than recalculating in get_display_cols() + -- TODO: only include filter_paths column if opts.show_filter_col is true + local filter_path = "" + if state.active_filter then + if state.active_filter_tag == "LSP" then + filter_path = utils.path_tail(state.active_filter) .. os_path_sep + else + filter_path = path.make_relative(state.active_filter, os_home) .. os_path_sep end end - - display_items = show_scores and {{entry.score, "Directory"}} or {} + table.insert(display_items, {filter_path, "Directory"}) table.insert(display_items, {filename, hl_filename}) return displayer(display_items) end - pickers.new(opts, { + local update_results = function(filter) + local filter_updated = false + + -- validate tag + local ws_dir = filter and state.user_workspaces[filter] + if filter == "LSP" and not vim.tbl_isempty(state.lsp_workspaces) then + ws_dir = state.lsp_workspaces[1] + end + + if ws_dir ~= state.active_filter then + filter_updated = true + state.active_filter = ws_dir + state.active_filter_tag = filter + end + + if vim.tbl_isempty(state.results) or filter_updated then + state.results = db_client.get_file_scores(state.show_unindexed, ws_dir) + end + return filter_updated + end + + -- populate initial results + update_results() + + local entry_maker = function(entry) + return { + value = entry.filename, + display = make_display, + ordinal = entry.filename, + name = entry.filename, + score = entry.score + } + end + + state.picker = pickers.new(opts, { prompt_title = "Frecency", - finder = finders.new_table { - results = results, - entry_maker = function(entry) - return { - value = entry.filename, - display = make_display, - ordinal = entry.filename, - name = entry.filename, - score = entry.score + on_input_filter_cb = function(query_text) + local delim = opts.filter_delimiter or ":" + -- check for :filter: in query text + local new_filter = query_text:gmatch(delim .. "%S+" .. delim)() + + if new_filter then + query_text = query_text:gsub(new_filter, "") + new_filter = new_filter:gsub(delim, "") + end + + local new_finder + local results_updated = update_results(new_filter) + if results_updated then + displayer = entry_display.create { + separator = "", + hl_chars = {[os_path_sep] = "TelescopePathSeparator"}, + items = get_display_cols() + } + + state.last_filter = new_filter + new_finder = finders.new_table { + results = state.results, + entry_maker = entry_maker } - end, + end + + return {prompt = query_text, updated_finder = new_finder} + end, + attach_mappings = function(prompt_bufnr) + actions.goto_file_selection_edit:replace(function() + local compinfo = vim.fn.complete_info() + if compinfo.pum_visible == 1 then + local keys = compinfo.selected == -1 and "" or ":" + local accept_completion = vim.api.nvim_replace_termcodes(keys, true, false, true) + vim.fn.nvim_feedkeys(accept_completion, "n", true) + else + actions._goto_file_selection(prompt_bufnr, "edit") + end + end) + + return true + end, + finder = finders.new_table { + results = state.results, + entry_maker = entry_maker }, previewer = conf.file_previewer(opts), sorter = sorters.get_substr_matcher(opts), - }):find() + }) + state.picker:find() + + -- restore last filter + if state.persistent_filter and state.last_filter then + vim.fn.nvim_feedkeys(":" .. state.last_filter .. ":", "n", true) + end + + local restore_vim_maps = {} + restore_vim_maps.i = { + [''] = "", + [''] = "" + } + mappings.apply_keymap(state.picker.prompt_bufnr, mappings.attach_mappings, restore_vim_maps) + + vim.api.nvim_buf_set_option(state.picker.prompt_bufnr, "filetype", "frecency") + vim.api.nvim_buf_set_option(state.picker.prompt_bufnr, "completefunc", "frecency#FrecencyComplete") + vim.api.nvim_buf_set_keymap(state.picker.prompt_bufnr, "i", "", "pumvisible() ? '' : ''", {expr = true, noremap = true}) +end + + +local function set_config_state(opt_name, value, default) + state[opt_name] = value == nil and default or value end return telescope.register_extension { setup = function(ext_config) - show_scores = ext_config.show_scores or false + set_config_state('show_scores', ext_config.show_scores, false) + set_config_state('show_unindexed', ext_config.show_unindexed, true) + set_config_state('show_filter_column', ext_config.show_filter_column, true) + set_config_state('persistent_filter', ext_config.persistent_filter, true) + set_config_state('user_workspaces', ext_config.workspaces, {}) -- start the database client - db_client = require("telescope._extensions.frecency.db_client") db_client.init(ext_config.ignore_patterns) end, exports = { - frecency = frecency, + frecency = frecency, + get_workspace_tags = get_workspace_tags, }, } diff --git a/lua/telescope/_extensions/frecency/db_client.lua b/lua/telescope/_extensions/frecency/db_client.lua index 676a70a..68bdb9f 100644 --- a/lua/telescope/_extensions/frecency/db_client.lua +++ b/lua/telescope/_extensions/frecency/db_client.lua @@ -1,5 +1,7 @@ local sqlwrap = require("telescope._extensions.frecency.sql_wrapper") +local scandir = require("plenary.scandir").scan_dir local util = require("telescope._extensions.frecency.util") + local MAX_TIMESTAMPS = 10 local DB_REMOVE_SAFETY_THRESHOLD = 10 @@ -119,38 +121,68 @@ local function filter_timestamps(timestamps, file_id) return res end -local function filter_workspace(filelist, workspace_path) +-- -- TODO: optimize this +-- local function find_in_table(tbl, target) +-- for _, entry in pairs(tbl) do +-- if entry.path == target then return true end +-- end +-- return false +-- end + +-- local function async_callback(result) +-- -- print(vim.inspect(result)) +-- end + +local function filter_workspace(workspace_path, show_unindexed) + local queries = sql_wrapper.queries + show_unindexed = show_unindexed == nil and true or show_unindexed + local res = {} - for _, entry in pairs(filelist) do - if vim.startwith(entry.path, workspace_path) then - table.insert(res, entry) + + res = sql_wrapper:do_transaction(queries.file_get_descendant_of, {path = workspace_path.."%"}) + if type(res) == "boolean" then res = {} end -- TODO: do this in sql_wrapper:transaction + + local scan_opts = { + respect_gitignore = true, + depth = 100, + hidden = true + } + + -- TODO: handle duplicate entries + if show_unindexed then + local unindexed_files = scandir(workspace_path, scan_opts) + for _, file in pairs(unindexed_files) do + if not file_is_ignored(file) then -- this causes some slowdown on large dirs + table.insert(res, { + id = 0, + path = file, + count = 0, + directory_id = 0 + }) + end end end + return res end -local function get_file_scores(opts) -- TODO: no need to pass in all opts here +local function get_file_scores(show_unindexed, workspace_path) if not sql_wrapper then return {} end local queries = sql_wrapper.queries - local scores = {} local files = sql_wrapper:do_transaction(queries.file_get_entries, {}) local timestamp_ages = sql_wrapper:do_transaction(queries.timestamp_get_all_entry_ages, {}) + local scores = {} if vim.tbl_isempty(files) then return scores end + files = workspace_path and filter_workspace(workspace_path, show_unindexed) or files - -- filter to LSP workspace directory - local buf_workspaces = opts.lsp_workspace_filter and vim.lsp.buf.list_workspace_folders() or {} - if not vim.tbl_isempty(buf_workspaces) then - for _, ws_path in pairs(buf_workspaces) do - files = filter_workspace(files, ws_path) - end - end - + local score for _, file_entry in ipairs(files) do + score = file_entry.count == 0 and 0 or calculate_file_score(file_entry.count, filter_timestamps(timestamp_ages, file_entry.id)) table.insert(scores, { filename = file_entry.path, - score = calculate_file_score(file_entry.count, filter_timestamps(timestamp_ages, file_entry.id)) + score = score }) end @@ -167,7 +199,6 @@ local function autocmd_handler(filepath) if not vim.b.telescope_frecency_registered then -- allow [noname] files to go unregistered until BufWritePost if not util.fs_stat(filepath).exists then return end - if file_is_ignored(filepath) then return end vim.b.telescope_frecency_registered = 1 diff --git a/lua/telescope/_extensions/frecency/sql_wrapper.lua b/lua/telescope/_extensions/frecency/sql_wrapper.lua index d704fe3..0e7b98b 100644 --- a/lua/telescope/_extensions/frecency/sql_wrapper.lua +++ b/lua/telescope/_extensions/frecency/sql_wrapper.lua @@ -1,5 +1,5 @@ local util = require("telescope._extensions.frecency.util") -local vim = vim +local vim = vim local has_sql, sql = pcall(require, "sql") if not has_sql then @@ -10,10 +10,19 @@ end local MAX_TIMESTAMPS = 10 local db_table = {} -db_table.files = "files" -db_table.timestamps = "timestamps" +db_table.files = "files" +db_table.timestamps = "timestamps" +db_table.workspaces = "workspaces" -- +-- TODO: NEXT! +-- extend substr sorter to have modes: +-- when current string is prefixed by `:foo`, results are tag_names that come from tags/workspaces table. (if `:foo ` token is incomplete it is ignored) +-- when a complete workspace tag is matched ':foobar:', results are indexed_files filtered by if their parent_dir is a descendant of the workspace_dir +-- a recursive scan_dir() result is added to the :foobar: filter results; any non-indexed_files are given a score of zero, and are alphabetically sorted below the indexed_results + +-- make tab completion for tab_names in insert mode`:foo|` state: cycles through available options + local M = {} function M:new() @@ -45,9 +54,9 @@ function M:bootstrap(opts) first_run = true -- create tables if they don't exist self.db:create(db_table.files, { - id = {"INTEGER", "PRIMARY", "KEY"}, - count = "INTEGER", - path = "TEXT" + id = {"INTEGER", "PRIMARY", "KEY"}, + count = "INTEGER", + path = "TEXT" }) self.db:create(db_table.timestamps, { id = {"INTEGER", "PRIMARY", "KEY"}, @@ -85,6 +94,10 @@ local cmd = { } local queries = { + file_get_descendant_of = { + cmd = cmd.eval, + cmd_data = "SELECT * FROM files WHERE path LIKE :path" + }, file_add_entry = { cmd = cmd.insert, cmd_data = db_table.files diff --git a/lua/telescope/_extensions/frecency/util.lua b/lua/telescope/_extensions/frecency/util.lua index 134d488..7feddfa 100644 --- a/lua/telescope/_extensions/frecency/util.lua +++ b/lua/telescope/_extensions/frecency/util.lua @@ -24,10 +24,6 @@ util.string_isempty = function(str) return str == nil or str == '' end -util.string_ends = function(str, token) - return str:sub(str:len() - token:len() + 1, -1) == token -end - util.split = function(str, delimiter) local result = {} for match in str:gmatch("[^" .. delimiter .. "]+") do diff --git a/plugin/frecency.vim b/plugin/frecency.vim new file mode 100644 index 0000000..60b743b --- /dev/null +++ b/plugin/frecency.vim @@ -0,0 +1,36 @@ +function! frecency#FrecencyComplete(findstart, base) + if a:findstart + let line = getline('.') + " don't complete if there's already a completed `:tag:` in line + if count(line, ":") >= 2 + return -3 + endif + + " locate the start of the tag + let start = col('.') - 1 + while start > 0 && line[start -1] != ':' + let start -= 1 + endwhile + + return start + else + if pumvisible() && !empty(v:completed_item) + return '' + end + + let l:workspace_tags = luaeval("require'telescope'.extensions.frecency.get_workspace_tags()") + let matches = [] + for ws_tag in l:workspace_tags + if ":" .. ws_tag =~ '^:' .. a:base + call add(matches, ws_tag) + endif + endfor + + return len(matches) != 0 ? matches : '' + end +endfunction + + + " lua require'telescope'.extensions.frecency.completefunc(action) + " lua require'telescope'.extensions.frecency.completefunc(res) + " require'telescope._extensions.frecency.db_client'.autocmd_handler(vim.fn.expand('')) diff --git a/syntax/frecency.vim b/syntax/frecency.vim new file mode 100644 index 0000000..92ad9e9 --- /dev/null +++ b/syntax/frecency.vim @@ -0,0 +1,5 @@ +if exists('b:current_syntax') | finish| endif + +syntax match WorkspaceFilter /:.\{-}:/ +hi def link WorkspaceFilter TelescopeQueryFilter +