refactor: reach the master (#93)

* refactor(db): use new sql.nvim api

* refactor(*): create const, algo and add more util

* refactor(picker): try to cleanup and make it easy to read

FIXME: it seems that I didn't refactor the entry maker right.

* refactor(*): move util to lua/frecency

* misc(db): flag possible bug

* refactor(db): reflect changes introduced in https://github.com/tami5/sql.nvim/pull/96

* refactor(db): move set_config to frecency.lua

* new(db): ignore term and octo paths

This may possibly fix the issue with entry.count being nil.

* misc(picker): move health out of export

* refactor(util)

* refactor(db): general

* misc(*): nothing major

* refactor(db): abbreviate table namespace access

* refactor(picker): working-ish

* fix(*): general

* refactor(picker): move fd function to the end of the file

* refactor(*): remove the need for db.init

* new(db): use foreign keys

* sync with sqlite.lua@#105

* feat: add settings for StyLua

* fix: detect the valid module in healthcheck

See #65

* style: fix by StyLua

* fix: detect CWD tag to cut paths

See #66

* fix: show icon before directory

See #67

* fix: deal with show_filter_column option validly

* feat: support opts.workspace (#68)

* doc(readme): update config example (#33)

remove comma causing error.

* doc(readme): fix packer install instructions (#34)

Co-authored-by: tami5 <65782666+tami5@users.noreply.github.com>

* doc: follow sqlite new release (#40)

* refactor(sql_wrapper): follow sqlite new release

* update readme

* refactor: follow telescope's interface changes

See #46

* feat: add default_workspace tag

See #43

* fix: fetch workspaces in addition to completing

See #72

* Update url for sqlite dependency (#64)

The old repository on github redirects to this one.

* fix: use vim.notify not to block outputs

See #75

* feat: opts.path_display to customize

See #76

* Enable to specify tags to show the tails (#77)

* Enable to specify tags to show the tails

* Add doc for show_filter_column

* feat: use more reasonable matcher to sort

See #73

* Fix broken Frecency Algorithm link in README.md (#82)

MDN appear to have removed the Frecency Algorithm page linked in the README document. Updating link to use archived version.

* fix: set the telescope default filetype in prompt

See #81

* feat: use the newer API to define autocmds

See #79

* refactor: use new API and add check for devicons

* feat: detect entries when it has added new ones

See #87

* fix: use valid widths to show entries

* fix: use substr matcher to use sorting by scores

See #94

* Revert "fix: set the telescope default filetype in prompt"

This reverts commit 4937f7045438412e31a77374f91230bdbcbeb34d.

* fix: enable to filter by valid paths

* refactor: do nothing until calling setup()

See #80

* fix: fix typo for `workspaces`

* fix: show `0` score for unindexed instead of `nil`

* style: fix by StyLua

* fix: return a valid entry with get()

It have used where() to get the entry and where() uses get() to fetch
from DB. But this table has been customized by overwriting get(), so it
has a bug to return the same entry every time it calls. This fixes it.

* refactor: get the buffer name explicitly

* fix: clean up logic to validate DB

* feat: enable to work db_root option

* fix: show no msg when no need to validate in auto

---------

Co-authored-by: Tami <65782666+tami5@users.noreply.github.com>
Co-authored-by: Munif Tanjim <hello@muniftanjim.dev>
Co-authored-by: premell <65544203+premell@users.noreply.github.com>
Co-authored-by: Anshuman Medhi <amedhi@connect.ust.hk>
Co-authored-by: Lucas Hoffmann <lucc@users.noreply.github.com>
Co-authored-by: Rohan Orton <rohan.orton@gmail.com>

lua/frecency/algo.lua

@@ -0,0 +1,21 @@
+local const = require "frecency.const"
+local algo = {}
+
+algo.calculate_file_score = function(file)
+  if not file.count or file.count == 0 then
+    return 0
+  end
+  local recency_score = 0
+  for _, ts in pairs(file.timestamps) do
+    for _, rank in ipairs(const.recency_modifier) do
+      if ts.age <= rank.age then
+        recency_score = recency_score + rank.value
+        goto continue
+      end
+    end
+    ::continue::
+  end
+  return file.count * recency_score / const.max_timestamps
+end
+
+return algo

lua/frecency/const.lua

@@ -0,0 +1,18 @@
+return {
+  max_timestamps = 10,
+  db_remove_safety_threshold = 10,
+  -- modifier used as a weight in the recency_score calculation:
+  recency_modifier = {
+    [1] = { age = 240, value = 100 }, -- past 4 hours
+    [2] = { age = 1440, value = 80 }, -- past day
+    [3] = { age = 4320, value = 60 }, -- past 3 days
+    [4] = { age = 10080, value = 40 }, -- past week
+    [5] = { age = 43200, value = 20 }, -- past month
+    [6] = { age = 129600, value = 10 }, -- past 90 days
+  },
+  ignore_patterns = {
+    "*.git/*",
+    "*/tmp/*",
+    "term://*",
+  },
+}

lua/frecency/db.lua

@@ -0,0 +1,192 @@
+local util = require "frecency.util"
+local const = require "frecency.const"
+local algo = require "frecency.algo"
+local sqlite = require "sqlite"
+local p = require "plenary.path"
+local s = sqlite.lib
+
+---@class FrecencySqlite: sqlite_db
+---@field files sqlite_tbl
+---@field timestamps sqlite_tbl
+
+---@class FrecencyDBConfig
+---@field db_root string: default "${stdpath.data}"
+---@field ignore_patterns table: extra ignore patterns: default empty
+---@field safe_mode boolean: When enabled, the user will be prompted when entries > 10, default true
+---@field auto_validate boolean: When this to false, stale entries will never be automatically removed, default true
+
+---@class FrecencyDB
+---@field sqlite FrecencySqlite
+---@field config FrecencyConfig
+local db = {
+  config = {
+    db_root = vim.fn.stdpath "data",
+    ignore_patterns = {},
+    db_safe_mode = true,
+    auto_validate = true,
+  },
+}
+
+---Set database configuration
+---@param config FrecencyDBConfig
+function db.set_config(config)
+  db.config = vim.tbl_extend("keep", config, db.config)
+  db.sqlite = sqlite {
+    uri = db.config.db_root .. "/file_frecency.sqlite3",
+    files = {
+      id = true,
+      count = { "integer", default = 0, required = true },
+      path = "string",
+    },
+    timestamps = {
+      id = true,
+      timestamp = { "real", default = s.julianday "now" },
+      file_id = { "integer", reference = "files.id", on_delete = "cascade" },
+    },
+  }
+end
+
+---Get timestamps with a computed filed called age.
+---If file_id is nil, then get all timestamps.
+---@param opts table
+---- { file_id } number: id file_id corresponding to `files.id`. return all if { file_id } is nil
+---- { with_age } boolean: whether to include age, default false.
+---@return table { id, file_id, age }
+---@overload func()
+function db.get_timestamps(opts)
+  opts = opts or {}
+  local where = opts.file_id and { file_id = opts.file_id } or nil
+  local compute_age = opts.with_age and s.cast((s.julianday() - s.julianday "timestamp") * 24 * 60, "integer") or nil
+  return db.sqlite.timestamps:__get { where = where, keys = { age = compute_age, "id", "file_id" } }
+end
+
+---Trim database entries
+---@param file_id any
+function db.trim_timestamps(file_id)
+  local timestamps = db.get_timestamps { file_id = file_id, with_age = true }
+  local trim_at = timestamps[(#timestamps - const.max_timestamps) + 1]
+  if trim_at then
+    db.sqlite.timestamps:remove { file_id = file_id, id = "<" .. trim_at.id }
+  end
+end
+
+---Get file entries
+---@param opts table:
+---- { ws_path } string: get files with matching workspace path.
+---- { show_unindexed } boolean: whether to include unindexed files, false if no ws_path is given.
+---- { with_score } boolean: whether to include score in the result and sort the files by score.
+---@overload func()
+---@return table[]: files entries
+function db.get_files(opts)
+  opts = opts or {}
+  local query = {}
+  if opts.ws_path then
+    query.contains = { path = { opts.ws_path .. "*" } }
+  elseif opts.path then
+    query.where = { path = opts.path }
+  end
+  local files = db.sqlite.files:__get(query)
+
+  if vim.F.if_nil(opts.with_score, true) then
+    ---NOTE: this might get slower with big db, it might be better to query with db.get_timestamp.
+    ---TODO: test the above assumption
+    local timestamps = db.get_timestamps { with_age = true }
+    for _, file in ipairs(files) do
+      file.timestamps = util.tbl_match("file_id", file.id, timestamps)
+      file.score = algo.calculate_file_score(file)
+    end
+
+    table.sort(files, function(a, b)
+      return a.score > b.score
+    end)
+  end
+
+  if opts.ws_path and opts.show_unindexed then
+    util.include_unindexed(files, opts.ws_path)
+  end
+
+  return files
+end
+---Insert or update a given path
+---@param path string
+---@return number: row id
+---@return boolean: true if it has inserted
+function db.insert_or_update_files(path)
+  local entry = (db.get_files({ path = path })[1] or {})
+  local file_id = entry.id
+  local has_added_entry = not file_id
+
+  if file_id then
+    db.sqlite.files:update { where = { id = file_id }, set = { count = entry.count + 1 } }
+  else
+    file_id = db.sqlite.files:insert { path = path }
+  end
+  return file_id, has_added_entry
+end
+
+---Add or update file path
+---@param path string|nil: path to file or use current
+---@return boolean: true if it has added an entry
+---@overload func()
+function db.update(path)
+  path = path or vim.fn.expand "%:p"
+  if vim.b.telescope_frecency_registered or util.path_invalid(path, db.ignore_patterns) then
+    -- print "ignoring autocmd"
+    return
+  else
+    vim.b.telescope_frecency_registered = 1
+  end
+  --- Insert or update path
+  local file_id, has_added_entry = db.insert_or_update_files(path)
+  --- Register timestamp for this update.
+  db.sqlite.timestamps:insert { file_id = file_id }
+  --- Trim timestamps to max_timestamps per file
+  db.trim_timestamps(file_id)
+  return has_added_entry
+end
+
+---Remove unlinked file entries, along with timestamps linking to it.
+---@param entries table[]|table|nil: if nil it will remove all entries
+---@param silent boolean: whether to notify user on changes made, default false
+function db.remove(entries, silent)
+  if type(entries) == "nil" then
+    local count = db.sqlite.files:count()
+    db.sqlite.files:remove()
+    if not vim.F.if_nil(silent, false) then
+      vim.notify(("Telescope-frecency: removed all entries. number of entries removed %d ."):format(count))
+    end
+    return
+  end
+
+  entries = (entries[1] and entries[1].id) and entries or { entries }
+
+  for _, entry in pairs(entries) do
+    db.sqlite.files:remove { id = entry.id }
+  end
+
+  if not vim.F.if_nil(silent, false) then
+    vim.notify(("Telescope-frecency: removed %d missing entries."):format(#entries))
+  end
+end
+
+---Remove file entries that no longer exists.
+function db.validate(opts)
+  opts = opts or {}
+
+  -- print "running validate"
+  local threshold = const.db_remove_safety_threshold
+  local unlinked = db.sqlite.files:map(function(entry)
+    local invalid = (not util.path_exists(entry.path) or util.path_is_ignored(entry.path, db.ignore_patterns))
+    return invalid and entry or nil
+  end)
+
+  if #unlinked > 0 then
+    if opts.force or not db.config.db_safe_mode or (#unlinked > threshold and util.confirm_deletion(#unlinked)) then
+      db.remove(unlinked)
+    elseif not opts.auto then
+      util.abort_remove_unlinked_files()
+    end
+  end
+end
+
+return db

lua/frecency/picker.lua

@@ -0,0 +1,336 @@
+local has_devicons, devicons = pcall(require, "nvim-web-devicons")
+local p = require "plenary.path"
+local util = require "frecency.util"
+local os_home = vim.loop.os_homedir()
+local os_path_sep = p.path.sep
+local actions = require "telescope.actions"
+local conf = require("telescope.config").values
+local entry_display = require "telescope.pickers.entry_display"
+local finders = require "telescope.finders"
+local pickers = require "telescope.pickers"
+local sorters = require "telescope.sorters"
+local ts_util = require "telescope.utils"
+local db = require "frecency.db"
+
+---TODO: Describe FrecencyPicker fields
+
+---@class FrecencyPicker
+---@field db FrecencyDB: where the files will be stored
+---@field results table
+---@field active_filter string
+---@field active_filter_tag string
+---@field previous_buffer string
+---@field cwd string
+---@field lsp_workspaces table
+---@field picker table
+---@field updated boolean: true if a new entry is added into DB
+local m = {
+  results = {},
+  active_filter = nil,
+  active_filter_tag = nil,
+  last_filter = nil,
+  previous_buffer = nil,
+  cwd = nil,
+  lsp_workspaces = {},
+  picker = {},
+  updated = false,
+}
+
+m.__index = m
+
+---@class FrecencyConfig
+---@field show_unindexed boolean: default true
+---@field show_filter_column boolean|string[]: default true
+---@field workspaces table: default {}
+---@field disable_devicons boolean: default false
+---@field default_workspace string: default nil
+m.config = {
+  show_scores = true,
+  show_unindexed = true,
+  show_filter_column = true,
+  workspaces = {},
+  disable_devicons = false,
+  default_workspace = nil,
+}
+
+---Setup frecency picker
+m.set_prompt_options = function(buffer)
+  vim.bo[buffer].filetype = "frecency"
+  vim.bo[buffer].completefunc = "v:lua.require'telescope'.extensions.frecency.complete"
+  vim.keymap.set("i", "<Tab>", "pumvisible() ? '<C-n>' : '<C-x><C-u>'", { buffer = buffer, expr = true })
+  vim.keymap.set("i", "<S-Tab>", "pumvisible() ? '<C-p>' : ''", { buffer = buffer, expr = true })
+end
+
+---returns `true` if workspaces exit
+---@param bufnr integer
+---@param force boolean?
+---@return boolean workspaces_exist
+m.fetch_lsp_workspaces = function(bufnr, force)
+  if not vim.tbl_isempty(m.lsp_workspaces) and not force then
+    return true
+  end
+
+  local lsp_workspaces = vim.api.nvim_buf_call(bufnr, vim.lsp.buf.list_workspace_folders)
+  if not vim.tbl_isempty(lsp_workspaces) then
+    m.lsp_workspaces = lsp_workspaces
+    return true
+  end
+
+  m.lsp_workspaces = {}
+  return false
+end
+
+---Update Frecency Picker result
+---@param filter string
+---@return boolean
+m.update = function(filter)
+  local filter_updated = false
+  local ws_dir = filter and m.config.workspaces[filter] or nil
+
+  if filter == "LSP" and not vim.tbl_isempty(m.lsp_workspaces) then
+    ws_dir = m.lsp_workspaces[1]
+  elseif filter == "CWD" then
+    ws_dir = m.cwd
+  end
+
+  if ws_dir ~= m.active_filter then
+    filter_updated = true
+    m.active_filter, m.active_filter_tag = ws_dir, filter
+  end
+
+  m.results = (vim.tbl_isempty(m.results) or m.updated or filter_updated)
+      and db.get_files { ws_path = ws_dir, show_unindexed = m.config.show_unindexed }
+    or m.results
+
+  return filter_updated
+end
+
+---@param opts table telescope picker table
+---@return fun(filename: string): string
+m.filepath_formatter = function(opts)
+  local path_opts = {}
+  for k, v in pairs(opts) do
+    path_opts[k] = v
+  end
+
+  return function(filename)
+    path_opts.cwd = m.active_filter or m.cwd
+    return ts_util.transform_path(path_opts, filename)
+  end
+end
+
+m.should_show_tail = function()
+  local filters = type(m.config.show_filter_column) == "table" and m.config.show_filter_column or { "LSP", "CWD" }
+  return vim.tbl_contains(filters, m.active_filter_tag)
+end
+
+---Create entry maker function.
+---@param entry table
+---@return function
+m.maker = function(entry)
+  local filter_column_width = (function()
+    if m.active_filter then
+      if m.should_show_tail() then
+        -- TODO: Only add +1 if m.show_filter_thing is true, +1 is for the trailing slash
+        return #(ts_util.path_tail(m.active_filter)) + 1
+      end
+      return #(p:new(m.active_filter):make_relative(os_home)) + 1
+    end
+    return 0
+  end)()
+
+  local displayer = entry_display.create {
+    separator = "",
+    hl_chars = { [os_path_sep] = "TelescopePathSeparator" },
+    items = (function()
+      local i = m.config.show_scores and { { width = 8 } } or {}
+      if has_devicons and not m.config.disable_devicons then
+        table.insert(i, { width = 2 })
+      end
+      if m.config.show_filter_column then
+        table.insert(i, { width = filter_column_width })
+      end
+      table.insert(i, { remaining = true })
+      return i
+    end)(),
+  }
+
+  local filter_path = (function()
+    if m.config.show_filter_column and m.active_filter then
+      return m.should_show_tail() and ts_util.path_tail(m.active_filter) .. os_path_sep
+        or p:new(m.active_filter):make_relative(os_home) .. os_path_sep
+    end
+    return ""
+  end)()
+
+  local formatter = m.filepath_formatter(m.opts)
+
+  return {
+    filename = entry.path,
+    ordinal = entry.path,
+    name = entry.path,
+    score = entry.score,
+    display = function(e)
+      return displayer((function()
+        local i = m.config.show_scores and { { entry.score, "TelescopeFrecencyScores" } } or {}
+        if has_devicons and not m.config.disable_devicons then
+          table.insert(i, { devicons.get_icon(e.name, string.match(e.name, "%a+$"), { default = true }) })
+        end
+        table.insert(i, { filter_path, "Directory" })
+        table.insert(i, {
+          formatter(e.name),
+          util.buf_is_loaded(e.name) and "TelescopeBufferLoaded" or "",
+        })
+        return i
+      end)())
+    end,
+  }
+end
+
+---Find files
+---@param opts table: telescope picker opts
+m.fd = function(opts)
+  opts = opts or {}
+
+  if not opts.path_display then
+    opts.path_display = function(path_opts, filename)
+      local original_filename = filename
+
+      filename = p:new(filename):make_relative(path_opts.cwd)
+      if not m.active_filter then
+        if vim.startswith(filename, os_home) then
+          filename = "~/" .. p:new(filename):make_relative(os_home)
+        elseif filename ~= original_filename then
+          filename = "./" .. filename
+        end
+      end
+
+      return filename
+    end
+  end
+
+  m.previous_buffer, m.cwd, m.opts = vim.fn.bufnr "%", vim.fn.expand(opts.cwd or vim.loop.cwd()), opts
+  -- TODO: should we update this every time it calls frecency on other buffers?
+  m.fetch_lsp_workspaces(m.previous_buffer)
+  m.update()
+
+  local picker_opts = {
+    prompt_title = "Frecency",
+    finder = finders.new_table { results = m.results, entry_maker = m.maker },
+    previewer = conf.file_previewer(opts),
+    sorter = sorters.get_substr_matcher(opts),
+  }
+
+  picker_opts.on_input_filter_cb = function(query_text)
+    local o = {}
+    local delim = m.config.filter_delimiter or ":" -- check for :filter: in query text
+    local matched, new_filter = query_text:match("^%s*(" .. delim .. "(%S+)" .. delim .. ")")
+    new_filter = new_filter or opts.workspace or m.config.default_workspace
+
+    o.prompt = matched and query_text:sub(matched:len() + 1) or query_text
+    if m.update(new_filter) then
+      m.last_filter = new_filter
+      o.updated_finder = finders.new_table { results = m.results, entry_maker = m.maker }
+    end
+
+    return o
+  end
+
+  picker_opts.attach_mappings = function(prompt_bufnr)
+    actions.select_default:replace_if(function()
+      return vim.fn.complete_info().pum_visible == 1
+    end, function()
+      local keys = vim.fn.complete_info().selected == -1 and "<C-e><Bs><Right>" or "<C-y><Right>:"
+      local accept_completion = vim.api.nvim_replace_termcodes(keys, true, false, true)
+      vim.api.nvim_feedkeys(accept_completion, "n", true)
+    end)
+    return true
+  end
+
+  m.picker = pickers.new(opts, picker_opts)
+  m.picker:find()
+  m.set_prompt_options(m.picker.prompt_bufnr)
+end
+
+---TODO: this seems to be forgotten and just exported in old implementation.
+---@return table
+m.workspace_tags = function()
+  -- Add user config workspaces.
+  -- TODO: validate that workspaces are existing directories
+  local tags = {}
+  for k, _ in pairs(m.config.workspaces) do
+    table.insert(tags, k)
+  end
+
+  -- Add CWD filter
+  --  NOTE: hmmm :cwd::lsp: is easier to write.
+  table.insert(tags, "CWD")
+
+  -- Add LSP workpace(s)
+  if m.fetch_lsp_workspaces(m.previous_buffer, true) then
+    table.insert(tags, "LSP")
+  end
+
+  -- TODO: sort tags - by collective frecency? (?????? is this still relevant)
+  return tags
+end
+
+m.complete = function(findstart, base)
+  if findstart == 1 then
+    local line = vim.api.nvim_get_current_line()
+    local start = line:find ":"
+    -- don't complete if there's already a completed `:tag:` in line
+    if not start or line:find(":", start + 1) then
+      return -3
+    end
+    return start
+  else
+    if vim.fn.pumvisible() == 1 and #vim.v.completed_item > 0 then
+      return ""
+    end
+
+    local matches = vim.tbl_filter(function(v)
+      return vim.startswith(v, base)
+    end, m.workspace_tags())
+
+    return #matches > 0 and matches or ""
+  end
+end
+
+---Setup Frecency Picker
+---@param db FrecencyDB
+---@param config FrecencyConfig
+m.setup = function(config)
+  m.config = vim.tbl_extend("keep", config, m.config)
+  db.set_config(config)
+
+  --- Seed files table with oldfiles when it's empty.
+  if db.sqlite.files:count() == 0 then
+    -- TODO: this needs to be scheduled for after shada load??
+    for _, path in ipairs(vim.v.oldfiles) do
+      db.sqlite.files:insert { path = path, count = 0 } -- TODO: remove when sql.nvim#97 is closed
+    end
+    vim.notify(("Telescope-Frecency: Imported %d entries from oldfiles."):format(#vim.v.oldfiles))
+  end
+
+  -- TODO: perhaps ignore buffer without file path here?
+  local group = vim.api.nvim_create_augroup("TelescopeFrecency", {})
+  vim.api.nvim_create_autocmd({ "BufWinEnter", "BufWritePost" }, {
+    group = group,
+    callback = function(args)
+      local path = vim.api.nvim_buf_get_name(args.buf)
+      local has_added_entry = db.update(path)
+      m.updated = m.updated or has_added_entry
+    end,
+  })
+
+  vim.api.nvim_create_user_command("FrecencyValidate", function(cmd_info)
+    db.validate { force = cmd_info.bang }
+  end, { bang = true, desc = "Clean up DB for telescope-frecency" })
+
+  if db.config.auto_validate then
+    db.validate { auto = true }
+  end
+end
+
+return m

lua/frecency/util.lua

@@ -0,0 +1,96 @@
+local uv = vim.loop
+local const = require "frecency.const"
+local Path = require "plenary.path"
+
+local util = {}
+
+-- stolen from penlight
+
+---escape any Lua 'magic' characters in a string
+util.escape = function(str)
+  return (str:gsub("[%-%.%+%[%]%(%)%$%^%%%?%*]", "%%%1"))
+end
+
+util.string_isempty = function(str)
+  return str == nil or str == ""
+end
+
+util.filemask = function(mask)
+  mask = util.escape(mask)
+  return "^" .. mask:gsub("%%%*", ".*"):gsub("%%%?", ".") .. "$"
+end
+
+util.path_is_ignored = function(filepath, ignore_patters)
+  local i = ignore_patters and vim.tbl_flatten { ignore_patters, const.ignore_patterns } or const.ignore_patterns
+  local is_ignored = false
+  for _, pattern in ipairs(i) do
+    if filepath:find(util.filemask(pattern)) ~= nil then
+      is_ignored = true
+      goto continue
+    end
+  end
+
+  ::continue::
+  return is_ignored
+end
+
+util.path_exists = function(path)
+  return Path:new(path):exists()
+end
+
+util.path_invalid = function(path, ignore_patterns)
+  local p = Path:new(path)
+  if
+    util.string_isempty(path)
+    or (not p:is_file())
+    or (not p:exists())
+    or util.path_is_ignored(path, ignore_patterns)
+  then
+    return true
+  else
+    return false
+  end
+end
+
+util.confirm_deletion = function(num_of_entries)
+  local question = "Telescope-Frecency: remove %d entries from SQLite3 database?"
+  return vim.fn.confirm(question:format(num_of_entries), "&Yes\n&No", 2) == 1
+end
+
+util.abort_remove_unlinked_files = function()
+  ---TODO: refactor all messages to a lua file. alarts.lua?
+  vim.notify "TelescopeFrecency: validation aborted."
+end
+
+util.tbl_match = function(field, val, tbl)
+  return vim.tbl_filter(function(t)
+    return t[field] == val
+  end, tbl)
+end
+
+---Wrappe around Path:new():make_relative
+---@return string
+util.path_relative = function(path, cwd)
+  return Path:new(path):make_relative(cwd)
+end
+
+---Given a filename, check if there's a buffer with the given name.
+---@return boolean
+util.buf_is_loaded = function(filename)
+  return vim.api.nvim_buf_is_loaded(vim.fn.bufnr(filename))
+end
+
+util.include_unindexed = function(files, ws_path)
+  local scan_opts = { respect_gitignore = true, depth = 100, hidden = true }
+
+  -- TODO: make sure scandir unindexed have opts.ignore_patterns applied
+  -- TODO: make filters handle mulitple directories
+  local unindexed_files = require("plenary.scandir").scan_dir(ws_path, scan_opts)
+  for _, file in pairs(unindexed_files) do
+    if not util.path_is_ignored(file) then -- this causes some slowdown on large dirs
+      table.insert(files, { id = 0, path = file, count = 0, directory_id = 0, score = 0 })
+    end
+  end
+end
+
+return util