Feature: filtered workspaces

* draft implementation of tags/filters

* .

* add filtering:

- extended substring sorter to have modes:
- when current string is prefixed by `:foo`, results are tag_names that
come from tags/workspaces table. (if `:foo ` token is incomplete it is
ignored)
- when a complete workspace tag is matched ':foobar:', results are
indexed_files filtered by if their parent_dir is a descendant of the
workspace_dir
- a recursive scan_dir() result is added to the  :foobar: filter
results; any non-indexed_files are given a score of zero, and are
alphabetically sorted below the indexed_results

- tab completion for tab_names in insert mode`:foo|` state: cycles
through available options

* add completion file

* use attach_mappings for <CR> map

* stop completion being enabled multiple times

* improve keys

* improve completion cancellation

* add dynamic `lsp` tag

* add dynamic `lsp` tag

* fix empty lsp workspaces

* remove hardcoded workspaces and allow config from ext_config

* add filter highlight and some fixes

* .

* add workspace filters to readme

* wip LSP workspace filter

* merge ignore_patterns fix

* change LSP_ROOT tagname to LSP

* fix setting default values

* .

* update readme with filter instructions

* remove debug message

* improve relative paths

* improve relative paths

* WIP dynamic column sizes

* WIP filter_column_width

* fix keymaps

* .

* feat: persistent filters

* refactor config creation

* fix: filter directory column autosize

* improve LSP workspace paths

* .

* remove workspace filter output

* cache persistent filter results

* fix cached results

* .

* remove results cache; sorting is the expensive part

* respect ignore patterns for non-indexed files.

* return table on on_input_filter_cb

lua/telescope/_extensions/frecency/db_client.lua

@@ -1,5 +1,7 @@
 local sqlwrap = require("telescope._extensions.frecency.sql_wrapper")
+local scandir = require("plenary.scandir").scan_dir
 local util    = require("telescope._extensions.frecency.util")
+
 local MAX_TIMESTAMPS = 10
 local DB_REMOVE_SAFETY_THRESHOLD = 10
 
@@ -119,38 +121,68 @@ local function filter_timestamps(timestamps, file_id)
   return res
 end
 
-local function filter_workspace(filelist, workspace_path)
+-- -- TODO: optimize this
+-- local function find_in_table(tbl, target)
+--   for _, entry in pairs(tbl) do
+--     if entry.path == target then return true end
+--   end
+--   return false
+-- end
+
+-- local function async_callback(result)
+--   -- print(vim.inspect(result))
+-- end
+
+local function filter_workspace(workspace_path, show_unindexed)
+  local queries = sql_wrapper.queries
+  show_unindexed = show_unindexed == nil and true or show_unindexed
+
   local res = {}
-  for _, entry in pairs(filelist) do
-    if vim.startwith(entry.path, workspace_path) then
-      table.insert(res, entry)
+
+  res = sql_wrapper:do_transaction(queries.file_get_descendant_of, {path = workspace_path.."%"})
+  if type(res) == "boolean" then res = {} end -- TODO: do this in sql_wrapper:transaction
+
+  local scan_opts = {
+    respect_gitignore = true,
+    depth             = 100,
+    hidden            = true
+  }
+
+ -- TODO: handle duplicate entries
+  if show_unindexed then
+    local unindexed_files = scandir(workspace_path, scan_opts)
+    for _, file in pairs(unindexed_files) do
+      if not file_is_ignored(file) then -- this causes some slowdown on large dirs
+        table.insert(res, {
+          id           = 0,
+          path         = file,
+          count        = 0,
+          directory_id = 0
+        })
+      end
     end
   end
+
   return res
 end
 
-local function get_file_scores(opts) -- TODO: no need to pass in all opts here
+local function get_file_scores(show_unindexed, workspace_path)
   if not sql_wrapper then return {} end
 
   local queries = sql_wrapper.queries
-  local scores = {}
   local files           = sql_wrapper:do_transaction(queries.file_get_entries, {})
   local timestamp_ages  = sql_wrapper:do_transaction(queries.timestamp_get_all_entry_ages, {})
 
+  local scores = {}
   if vim.tbl_isempty(files) then return scores end
+  files = workspace_path and filter_workspace(workspace_path, show_unindexed) or files
 
-  -- filter to LSP workspace directory
-  local buf_workspaces = opts.lsp_workspace_filter and vim.lsp.buf.list_workspace_folders() or {}
-  if not vim.tbl_isempty(buf_workspaces) then
-    for _, ws_path in pairs(buf_workspaces) do
-      files = filter_workspace(files, ws_path)
-    end
-  end
-
+  local score
   for _, file_entry in ipairs(files) do
+    score = file_entry.count == 0 and 0 or calculate_file_score(file_entry.count, filter_timestamps(timestamp_ages, file_entry.id))
     table.insert(scores, {
       filename = file_entry.path,
-      score    = calculate_file_score(file_entry.count, filter_timestamps(timestamp_ages, file_entry.id))
+      score    = score
     })
   end
 
@@ -167,7 +199,6 @@ local function autocmd_handler(filepath)
   if not vim.b.telescope_frecency_registered then
     -- allow [noname] files to go unregistered until BufWritePost
     if not util.fs_stat(filepath).exists then return end
-
     if file_is_ignored(filepath) then return end
 
     vim.b.telescope_frecency_registered = 1

lua/telescope/_extensions/frecency/sql_wrapper.lua

@@ -1,5 +1,5 @@
 local util    = require("telescope._extensions.frecency.util")
-local vim = vim
+local vim     = vim
 
 local has_sql, sql = pcall(require, "sql")
 if not has_sql then
@@ -10,10 +10,19 @@ end
 local MAX_TIMESTAMPS = 10
 
 local db_table = {}
-db_table.files      = "files"
-db_table.timestamps = "timestamps"
+db_table.files       = "files"
+db_table.timestamps  = "timestamps"
+db_table.workspaces  = "workspaces"
 --
 
+-- TODO: NEXT!
+-- extend substr sorter to have modes:
+-- when current string is prefixed by `:foo`, results are tag_names that come from tags/workspaces table. (if `:foo ` token is incomplete it is ignored)
+-- when a complete workspace tag is matched ':foobar:', results are indexed_files filtered by if their parent_dir is a descendant of the workspace_dir
+-- a recursive scan_dir() result is added to the  :foobar: filter results; any non-indexed_files are given a score of zero, and are alphabetically sorted below the indexed_results
+
+-- make tab completion for tab_names in insert mode`:foo|` state: cycles through available options
+
 local M = {}
 
 function M:new()
@@ -45,9 +54,9 @@ function M:bootstrap(opts)
     first_run = true
     -- create tables if they don't exist
     self.db:create(db_table.files, {
-      id        = {"INTEGER", "PRIMARY", "KEY"},
-      count     = "INTEGER",
-      path      = "TEXT"
+      id           = {"INTEGER", "PRIMARY", "KEY"},
+      count        = "INTEGER",
+      path         = "TEXT"
     })
     self.db:create(db_table.timestamps, {
       id        = {"INTEGER", "PRIMARY", "KEY"},
@@ -85,6 +94,10 @@ local cmd = {
 }
 
 local queries = {
+  file_get_descendant_of = {
+    cmd      = cmd.eval,
+    cmd_data = "SELECT * FROM files WHERE path LIKE :path"
+  },
   file_add_entry = {
     cmd      = cmd.insert,
     cmd_data = db_table.files

lua/telescope/_extensions/frecency/util.lua

@@ -24,10 +24,6 @@ util.string_isempty = function(str)
   return str == nil or str == ''
 end
 
-util.string_ends = function(str, token)
-  return str:sub(str:len() - token:len() + 1, -1) == token
-end
-
 util.split = function(str, delimiter)
   local result = {}
   for match in str:gmatch("[^" .. delimiter .. "]+") do