automatically remove stale entries from db

README.md

@@ -9,11 +9,11 @@ As the extension learns your editing habits over time, the sorting of the list i
 
 * _Scores shown in finder for illustration purposes only_
 
-## Frecency: sorting by "frequency" and "recency"
+## Frecency: Sorting by 'frequency' _and_ 'recency'
 
 'Frecency' is a score given to each unique file indexed in a file history database.
 
-A timestamp is recorded to once per session when a file is loaded into a buffer.
+A timestamp is recorded once per session when a file is loaded into a buffer.
 
 The score is calculated using the age of the 10 most recent timestamps and the total amount of times the file has been loaded:
 

lua/telescope/_extensions/frecency.lua

@@ -7,10 +7,7 @@ end
 
 -- start the database client
 local db_client = require("telescope._extensions.frecency.db_client")
--- vim.defer_fn(db_client.init, 100) -- TODO: this is a crappy attempt to lessen loadtime impact, use VimEnter?
 db_client.init()
-local os_path_sep = vim.loop.os_uname().sysname == "Windows" and "\\" or "/"
-
 
 -- finder code
 
@@ -19,11 +16,10 @@ local entry_display = require "telescope.pickers.entry_display"
 local finders       = require "telescope.finders"
 local pickers       = require "telescope.pickers"
 local previewers    = require "telescope.previewers"
--- local sorters       = require "telescope.sorters"
 local sorters       = require "telescope._extensions.frecency.sorter"
--- local conf          = require('telescope.config').values
 local path = require('telescope.path')
 local utils = require('telescope.utils')
+local os_path_sep = vim.loop.os_uname().sysname == "Windows" and "\\" or "/"
 
 local frecency = function(opts)
   opts = opts or {}
@@ -79,20 +75,12 @@ local frecency = function(opts)
       end,
     },
     -- previewer = conf.file_previewer(opts),
-    sorter    = sorters.get_substr_matcher(opts),
-    -- sorter    = conf.file_sorter(opts)
+    sorter = sorters.get_substr_matcher(opts),
   }):find()
 end
 
-local validate = function()
-  print("validate db")
-  db_client.validate()
-end
-
-
 return telescope.register_extension {
   exports = {
     frecency = frecency,
-    validate = validate,
   },
 }

lua/telescope/_extensions/frecency/db_client.lua

@@ -2,6 +2,7 @@ local sqlwrap = require("telescope._extensions.frecency.sql_wrapper")
 local util    = require("telescope._extensions.frecency.util")
 
 local MAX_TIMESTAMPS = 10
+local DB_REMOVE_SAFETY_THRESHOLD = 10
 
 -- modifier used as a weight in the recency_score calculation:
 local recency_modifier = {
@@ -40,11 +41,48 @@ local function file_is_ignored(filepath)
   return is_ignored
 end
 
+local function validate()
+  if not sql_wrapper then return {} end
+
+  local queries = sql_wrapper.queries
+  local files = sql_wrapper:do_transaction(queries.file_get_entries, {})
+  local pending_remove = {}
+  for _, entry in pairs(files) do
+    if not util.fs_stat(entry.path).exists -- file no longer exists
+      or file_is_ignored(entry.path) then -- cleanup entries that match the _current_ ignore list
+      table.insert(pending_remove, entry)
+    end
+  end
+
+   -- don't allow removal of >N values from DB without confirmation
+  local confirmed = false
+  if #pending_remove > DB_REMOVE_SAFETY_THRESHOLD then
+    if vim.fn.confirm("Telescope-Frecency: remove " .. #pending_remove .. " entries from SQLite3 database?", "&Yes\n&No", 2) then
+    confirmed = true
+    end
+  else
+    confirmed = true
+  end
+
+  if confirmed then
+    for _, entry in pairs(pending_remove) do
+      -- remove entries from file and timestamp tables
+      print("removing entry: " .. entry.path .. "[" .. entry.id .."]")
+      sql_wrapper:do_transaction(queries.file_delete_entry , {where = {id = entry.id }})
+      sql_wrapper:do_transaction(queries.timestamp_delete_entry, {where = {file_id = entry.id}})
+    end
+  else
+    print("TelescopeFrecency: validation aborted.")
+  end
+end
+
 local function init()
   if sql_wrapper then return end
 
   sql_wrapper = sqlwrap:new()
   local first_run = sql_wrapper:bootstrap()
+  validate()
+
   if first_run then
     -- TODO: this needs to be scheduled for after shada load
     vim.defer_fn(import_oldfiles, 100)
@@ -138,26 +176,8 @@ local function autocmd_handler(filepath)
   end
 end
 
-local function validate()
-  if not sql_wrapper then return {} end
-
-  local queries = sql_wrapper.queries
-  local files = sql_wrapper:do_transaction(queries.file_get_entries, {})
-  for _, entry in pairs(files) do
-    if not util.fs_stat(entry.path).exists
-      or file_is_ignored(entry.path) then -- cleanup entries that match the _current_ ignore list (DANGEROUS! bad pattern could wipe the database)
-
-      -- remove entries from file and timestamp tables
-      print("removing entry: " .. entry.path .. "[" .. entry.id .."]")
-      sql_wrapper:do_transaction(queries.file_delete_entry , {where = {id = entry.id }})
-      sql_wrapper:do_transaction(queries.timestamp_delete_entry, {where = {file_id = entry.id}})
-    end
-  end
-end
-
 return {
   init            = init,
   get_file_scores = get_file_scores,
   autocmd_handler = autocmd_handler,
-  validate        = validate,
 }

lua/telescope/_extensions/frecency/sorter.lua

@@ -7,7 +7,7 @@ local substr_highlighter = function(_, prompt, display)
   local highlights = {}
   display = display:lower()
 
-  local search_terms = util.split(prompt, " ")
+  local search_terms = util.split(prompt, "%s")
   local hl_start, hl_end
 
   for _, word in pairs(search_terms) do