'live_grep' freezes on large text files (#211)

lua/fzf-lua/actions.lua

@@ -213,6 +213,15 @@ end
 M.run_builtin = function(selected)
   local method = selected[1]
   vim.cmd(string.format("lua require'fzf-lua'.%s()", method))
+  -- not sure what is causing this, tested with
+  -- 'NVIM v0.6.0-dev+575-g2ef9d2a66'
+  -- vim.cmd("startinsert") doesn't start INSERT mode
+  -- 'mode' returns { blocking = false, mode = "t" }
+  -- manually input 'i' seems to workaround this issue
+  local mode = vim.api.nvim_get_mode()
+  if mode.mode and mode.mode ~= 'i' then
+    vim.cmd[[noautocmd lua vim.api.nvim_feedkeys('i', 'n', true)]]
+  end
 end
 
 M.ex_run = function(selected)

lua/fzf-lua/config.lua

@@ -238,7 +238,8 @@ M.globals.grep = {
     actions             = M.globals.files.actions,
     -- live_grep_glob options
     glob_flag           = "--iglob",  -- for case sensitive globs use '--glob'
-    glob_separator      = "%s%-%-"    -- query separator pattern (lua): ' --'
+    glob_separator      = "%s%-%-",   -- query separator pattern (lua): ' --'
+    data_limit          = 128 * 1024, -- 'live_grep' libuv chunk data limit
   }
 M.globals.args = {
     previewer           = M._default_previewer_fn,
@@ -546,7 +547,10 @@ function M.normalize_opts(opts, defaults)
 
   -- Merge required tables from globals
   for _, k in ipairs({ 'winopts', 'keymap', 'fzf_opts', 'previewers' }) do
-    opts[k] = vim.tbl_deep_extend("keep", opts[k] or {}, M.globals[k] or {})
+    opts[k] = vim.tbl_deep_extend("keep",
+      -- must clone or map will be saved as reference
+      -- and then overwritten if found in 'backward_compat'
+      opts[k] or {}, utils.tbl_deep_clone(M.globals[k]) or {})
   end
 
   -- backward compatibility, rhs overrides lhs

lua/fzf-lua/core.lua

@@ -6,6 +6,7 @@ local config = require "fzf-lua.config"
 local actions = require "fzf-lua.actions"
 local win = require "fzf-lua.win"
 local libuv = require "fzf-lua.libuv"
+local shell = require "fzf-lua.shell"
 
 local M = {}
 
@@ -370,7 +371,7 @@ M.set_fzf_interactive_cb = function(opts)
   local placeholder = utils._if(opts._is_skim, '"{}"', '{q}')
 
   local uv = vim.loop
-  local raw_async_act = require("fzf.actions").raw_async_action(function(pipe, args)
+  local raw_async_act = shell.raw_async_action(function(pipe, args)
 
     coroutine.wrap(function()
 

lua/fzf-lua/init.lua

@@ -148,6 +148,7 @@ local _modules = {
   'path',
   'utils',
   'libuv',
+  'shell',
   'config',
   'actions',
 }

lua/fzf-lua/libuv.lua

@@ -1,6 +1,5 @@
+local shell = require "fzf-lua.shell"
 local utils = require "fzf-lua.utils"
-local async_action = require("fzf.actions").async_action
-local raw_async_action = require("fzf.actions").raw_async_action
 local uv = vim.loop
 
 local M = {}
@@ -17,6 +16,14 @@ local function find_last_newline(str)
   end
 end
 
+local function find_next_newline(str, start_idx)
+  for i=start_idx or 1,#str do
+    if string_byte(str, i) == 10 then
+        return i
+    end
+  end
+end
+
 local function coroutine_callback(fn)
   local co = coroutine.running()
   local callback = function(...)
@@ -49,11 +56,10 @@ M.spawn = function(opts, fn_transform, fn_done)
   local error_pipe = uv.new_pipe(false)
   local write_cb_count = 0
   local prev_line_content = nil
+  local num_lines = 0
 
   if opts.fn_transform then fn_transform = opts.fn_transform end
 
-  local shell = vim.env.SHELL or "sh"
-
   local finish = function(sig, pid)
     output_pipe:shutdown()
     error_pipe:shutdown()
@@ -68,7 +74,7 @@ M.spawn = function(opts, fn_transform, fn_done)
 
   -- https://github.com/luvit/luv/blob/master/docs.md
   -- uv.spawn returns tuple: handle, pid
-  local _, pid = uv.spawn(shell, {
+  local _, pid = uv.spawn(vim.env.SHELL or "sh", {
     args = { "-c", opts.cmd },
     stdio = { nil, output_pipe, error_pipe },
     cwd = opts.cwd
@@ -105,13 +111,43 @@ M.spawn = function(opts, fn_transform, fn_done)
     end)
   end
 
-  local function process_lines(str)
+  local function process_lines(data)
-    write_cb(str:gsub("[^\n]+",
+    if opts.data_limit and opts.data_limit > 0 and #data>opts.data_limit then
+      vim.defer_fn(function()
+        utils.warn(("received large data chunk (%db), consider adding '--max-columns=512' to ripgrep flags\nDATA: '%s'")
+          :format(#data, utils.strip_ansi_coloring(data):sub(1,80)))
+      end, 0)
+    end
+    write_cb(data:gsub("[^\n]+",
       function(x)
         return fn_transform(x)
       end))
   end
 
+  --[[ local function process_lines(data)
+    if opts.data_limit and opts.data_limit > 0 and #data>opts.data_limit then
+      vim.defer_fn(function()
+        utils.warn(("received large data chunk (%db, consider adding '--max-columns=512' to ripgrep flags\nDATA: '%s'")
+          :format(#data, utils.strip_ansi_coloring(data):sub(1,80)))
+      end, 0)
+    end
+    local start_idx = 1
+    repeat
+      num_lines = num_lines + 1
+      local nl_idx = find_next_newline(data, start_idx)
+      local line = data:sub(start_idx, nl_idx)
+      if #line > 1024 then
+        vim.defer_fn(function()
+          utils.warn(("long line %d bytes, '%s'")
+            :format(#line, utils.strip_ansi_coloring(line):sub(1,60)))
+        end, 0)
+        line = line:sub(1,512) .. '\n'
+      end
+      write_cb(fn_transform(line))
+      start_idx = nl_idx + 1
+    until start_idx >= #data
+  end --]]
+
   local read_cb = function(err, data)
 
     if err then
@@ -134,7 +170,11 @@ M.spawn = function(opts, fn_transform, fn_done)
     else
       local nl_index = find_last_newline(data)
       if not nl_index then
-        prev_line_content = data
+        -- chunk size is 64K, limit previous line length to 1K
+        -- max line length is therefor 1K + 64K (leftover + full chunk)
+        -- without this we can memory fault on extremely long lines (#185)
+        -- or have UI freezes (#211)
+        prev_line_content = data:sub(1, 1024)
       else
         prev_line_content = string_sub(data, nl_index + 1)
         local stripped_with_newline = string_sub(data, 1, nl_index)
@@ -194,7 +234,7 @@ end
 
 M.spawn_nvim_fzf_action = function(fn, fzf_field_expression)
 
-  return async_action(function(pipe, ...)
+  return shell.async_action(function(pipe, ...)
 
     local function on_finish(_, _)
       if pipe and not uv.is_closing(pipe) then
@@ -224,7 +264,7 @@ M.spawn_reload_cmd_action = function(opts, fzf_field_expression)
 
   local _pid = nil
 
-  return raw_async_action(function(pipe, args)
+  return shell.raw_async_action(function(pipe, args)
 
     local function on_pid(pid)
       _pid = pid
@@ -258,7 +298,10 @@ M.spawn_reload_cmd_action = function(opts, fzf_field_expression)
         cb_finish = on_finish,
         cb_write = on_write,
         cb_pid = on_pid,
-      }, opts._fn_transform)
+        data_limit = opts.data_limit,
+        -- must send false, 'coroutinify' adds callback as last argument
+        -- which will conflict with the 'fn_transform' argument
+      }, opts._fn_transform or false)
 
   end, fzf_field_expression)
 end

lua/fzf-lua/previewer/builtin.lua

@@ -1,7 +1,7 @@
 local path = require "fzf-lua.path"
+local shell = require "fzf-lua.shell"
 local utils = require "fzf-lua.utils"
 local previewer_base = require "fzf-lua.previewer"
-local raw_action = require("fzf.actions").raw_action
 
 local api = vim.api
 local fn = vim.fn
@@ -169,7 +169,7 @@ function Previewer.base:display_entry(entry_str)
 end
 
 function Previewer.base:action(_)
-  local act = raw_action(function (items, _, _)
+  local act = shell.raw_action(function (items, _, _)
     self:display_entry(items[1])
     return ""
   end, "{}")

lua/fzf-lua/previewer/fzf.lua

@@ -1,8 +1,8 @@
 local path = require "fzf-lua.path"
+local shell = require "fzf-lua.shell"
 local utils = require "fzf-lua.utils"
 local libuv = require "fzf-lua.libuv"
 local previewer_base = require "fzf-lua.previewer"
-local raw_action = require("fzf.actions").raw_action
 
 local Previewer = {}
 Previewer.base = {}
@@ -60,7 +60,7 @@ function Previewer.cmd:action(o)
   if self.opts._line_placeholder then
     filespec = "{1}"
   end
-  local act = raw_action(function (items, fzf_lines, _)
+  local act = shell.raw_action(function (items, fzf_lines, _)
     -- only preview first item
     local file = path.entry_to_file(items[1], not self.relative and self.opts.cwd)
     return file.path

lua/fzf-lua/providers/buffers.lua

@@ -2,9 +2,9 @@ if not pcall(require, "fzf") then
   return
 end
 
-local action = require("fzf.actions").action
 local core = require "fzf-lua.core"
 local path = require "fzf-lua.path"
+local shell = require "fzf-lua.shell"
 local utils = require "fzf-lua.utils"
 local config = require "fzf-lua.config"
 local actions = require "fzf-lua.actions"
@@ -153,7 +153,7 @@ M.buffers = function(opts)
   opts = config.normalize_opts(opts, config.globals.buffers)
   if not opts then return end
 
-    local act = action(function (items, fzf_lines, _)
+    local act = shell.action(function (items, fzf_lines, _)
       -- only preview first item
       local item = items[1]
       local buf = getbufnumber(item)

lua/fzf-lua/providers/colorschemes.lua

@@ -2,9 +2,8 @@ if not pcall(require, "fzf") then
   return
 end
 
-local action = require("fzf.actions").action
 local core = require "fzf-lua.core"
-local utils = require "fzf-lua.utils"
+local shell = require "fzf-lua.shell"
 local config = require "fzf-lua.config"
 local actions = require "fzf-lua.actions"
 
@@ -24,7 +23,7 @@ M.colorschemes = function(opts)
   if not opts then return end
 
   coroutine.wrap(function ()
-    local prev_act = action(function (args)
+    local prev_act = shell.action(function (args)
       if opts.live_preview and args then
         local colorscheme = args[1]
         vim.cmd("colorscheme " .. colorscheme)

lua/fzf-lua/providers/files.lua

@@ -4,6 +4,7 @@ end
 
 local core = require "fzf-lua.core"
 local utils = require "fzf-lua.utils"
+local shell = require "fzf-lua.shell"
 local config = require "fzf-lua.config"
 local libuv = require "fzf-lua.libuv"
 
@@ -63,7 +64,7 @@ M.files_resume = function(opts)
     return
   end
 
-  local raw_act = require("fzf.actions").raw_action(function(args)
+  local raw_act = shell.raw_action(function(args)
     last_query = args[1]
   end, "{q}")
 

lua/fzf-lua/providers/lsp.lua

@@ -2,13 +2,10 @@ if not pcall(require, "fzf") then
   return
 end
 
-local raw_action = require("fzf.actions").raw_action
-local raw_async_action = require("fzf.actions").raw_async_action
 local core = require "fzf-lua.core"
 local utils = require "fzf-lua.utils"
 local config = require "fzf-lua.config"
 local actions = require "fzf-lua.actions"
-local uv = vim.loop
 
 local M = {}
 

lua/fzf-lua/providers/module.lua

@@ -2,8 +2,8 @@ if not pcall(require, "fzf") then
   return
 end
 
-local action = require("fzf.actions").action
 local core = require "fzf-lua.core"
+local shell = require "fzf-lua.shell"
 local config = require "fzf-lua.config"
 local actions = require "fzf-lua.actions"
 
@@ -18,7 +18,7 @@ M.metatable = function(opts)
 
   coroutine.wrap(function ()
 
-    local prev_act = action(function (args)
+    local prev_act = shell.action(function (args)
       -- TODO: retreive method help
       local help = ''
       return string.format("%s:%s", args[1], help)

lua/fzf-lua/providers/nvim.lua

@@ -2,9 +2,9 @@ if not pcall(require, "fzf") then
   return
 end
 
-local action = require("fzf.actions").action
 local core = require "fzf-lua.core"
 local utils = require "fzf-lua.utils"
+local shell = require "fzf-lua.shell"
 local config = require "fzf-lua.config"
 local actions = require "fzf-lua.actions"
 
@@ -19,7 +19,7 @@ M.commands = function(opts)
 
     local commands = vim.api.nvim_get_commands {}
 
-    local prev_act = action(function (args)
+    local prev_act = shell.action(function (args)
       local cmd = args[1]
       if commands[cmd] then
         cmd = vim.inspect(commands[cmd])
@@ -100,7 +100,7 @@ M.marks = function(opts)
     local marks = vim.fn.execute("marks")
     marks = vim.split(marks, "\n")
 
-    local prev_act = action(function (args, fzf_lines, _)
+    local prev_act = shell.action(function (args, fzf_lines, _)
       local mark = args[1]:match("[^ ]+")
       local bufnr, lnum, _, _ = unpack(vim.fn.getpos("'"..mark))
       if vim.api.nvim_buf_is_loaded(bufnr) then
@@ -154,7 +154,7 @@ M.registers = function(opts)
       table.insert(registers, string.char(i))
     end
 
-    local prev_act = action(function (args)
+    local prev_act = shell.action(function (args)
       local r = args[1]:match("%[(.*)%] ")
       local _, contents = pcall(vim.fn.getreg, r)
       return contents or args[1]
@@ -216,7 +216,7 @@ M.keymaps = function(opts)
       end
     end
 
-    local prev_act = action(function (args)
+    local prev_act = shell.action(function (args)
       local k = args[1]:match("(%[.*%]) ")
       local v = keymaps[k]
       if v then

lua/fzf-lua/shell.lua

@@ -0,0 +1,112 @@
+-- for testing, copied from:
+-- https://github.com/vijaymarupudi/nvim-fzf/blob/master/lua/fzf/actions.lua
+local uv = vim.loop
+local path = require "fzf-lua.path"
+
+local M = {}
+
+local _counter = 0
+local _registry = {}
+
+function M.register_func(fn)
+  _counter = _counter + 1
+  _registry[_counter] = fn
+  return _counter
+end
+
+function M.get_func(counter)
+  return _registry[counter]
+end
+
+-- creates a new address to listen to messages from actions. This is important,
+-- if the user is using a custom fixed $NVIM_LISTEN_ADDRESS. Different neovim
+-- instances will then use the same path as the address and it causes a mess,
+-- i.e. actions stop working on the old instance. So we create our own (random
+-- path) RPC server for this instance if it hasn't been started already.
+local action_server_address = nil
+
+function M.raw_async_action(fn, fzf_field_expression)
+
+  if not fzf_field_expression then
+    fzf_field_expression = "{+}"
+  end
+
+  local receiving_function = function(pipe_path, ...)
+    local pipe = uv.new_pipe(false)
+    local args = {...}
+    uv.pipe_connect(pipe, pipe_path, function(err)
+      vim.schedule(function ()
+        fn(pipe, unpack(args))
+      end)
+    end)
+  end
+
+  if not action_server_address then
+    action_server_address = vim.fn.serverstart()
+  end
+
+  local id = M.register_func(receiving_function)
+
+  -- this is for windows WSL and AppImage users, their nvim path isn't just
+  -- 'nvim', it can be something else
+  local nvim_command = vim.v.argv[1]
+
+  local action_string = string.format("%s --headless --clean --cmd %s %s %s %s",
+    vim.fn.shellescape(nvim_command),
+    vim.fn.shellescape("luafile " .. path.join{vim.g.fzf_lua_directory, "shell_helper.lua"}),
+    vim.fn.shellescape(action_server_address),
+    id,
+    fzf_field_expression)
+  return action_string, id
+end
+
+function M.async_action(fn, fzf_field_expression)
+  local action_string, id = M.raw_async_action(fn, fzf_field_expression)
+  return vim.fn.shellescape(action_string), id
+end
+
+function M.raw_action(fn, fzf_field_expression)
+
+  local receiving_function = function(pipe, ...)
+    local ret = fn(...)
+
+    local on_complete = function(err)
+      -- We are NOT asserting, in case fzf closes the pipe before we can send
+      -- the preview.
+      -- assert(not err)
+      uv.close(pipe)
+    end
+
+    if type(ret) == "string" then
+      uv.write(pipe, ret, on_complete)
+    elseif type(ret) == nil then
+      on_complete()
+    elseif type(ret) == "table" then
+      if not vim.tbl_isempty(ret) then
+        uv.write(pipe, vim.tbl_map(function(x) return x.."\n" end, ret), on_complete)
+      else
+        on_complete()
+      end
+    else
+      uv.write(pipe, tostring(ret) .. "\n", on_complete)
+    end
+  end
+
+  return M.raw_async_action(receiving_function, fzf_field_expression)
+end
+
+function M.action(fn, fzf_field_expression)
+  local action_string, id = M.raw_action(fn, fzf_field_expression)
+  return vim.fn.shellescape(action_string), id
+end
+
+-- set to 'true' to use 'nvim-fzf'
+-- set to 'false' for debugging using the local version
+if false then
+  M.action = require("fzf.actions").action
+  M.raw_action = require("fzf.actions").raw_action
+  M.async_action = require("fzf.actions").async_action
+  M.raw_async_action = require("fzf.actions").raw_async_action
+end
+
+return M

lua/fzf-lua/shell_helper.lua

@@ -0,0 +1,69 @@
+-- for testing, copied from:
+-- https://github.com/vijaymarupudi/nvim-fzf/blob/master/action_helper.lua
+local uv = vim.loop
+
+local function get_preview_socket()
+  local tmp = vim.fn.tempname()
+  local socket = uv.new_pipe(false)
+  uv.pipe_bind(socket, tmp)
+  return socket, tmp
+end
+
+local preview_socket, preview_socket_path = get_preview_socket()
+
+uv.listen(preview_socket, 100, function(err)
+  local preview_receive_socket = uv.new_pipe(false)
+  -- start listening
+  uv.accept(preview_socket, preview_receive_socket)
+  preview_receive_socket:read_start(function(err, data)
+    assert(not err)
+    if not data then
+      uv.close(preview_receive_socket)
+      uv.close(preview_socket)
+      vim.schedule(function()
+        vim.cmd[[qall]]
+      end)
+      return
+    end
+    io.write(data)
+  end)
+end)
+
+
+local function_id = tonumber(vim.fn.argv(1))
+local success, errmsg = pcall(function ()
+  local nargs = vim.fn.argc()
+  local args = {}
+  -- this is guaranteed to be 2 or more, we are interested in those greater than 2
+  for i=3,nargs do
+    -- vim uses zero indexing
+    table.insert(args, vim.fn.argv(i - 1))
+  end
+  local environ = vim.fn.environ()
+  local chan_id = vim.fn.sockconnect("pipe", vim.fn.argv(0), { rpc = true })
+  -- for skim compatibility
+  local preview_lines = environ.FZF_PREVIEW_LINES or environ.LINES
+  local preview_cols = environ.FZF_PREVIEW_COLUMNS or environ.COLUMNS
+  vim.rpcrequest(chan_id, "nvim_exec_lua", [[
+    local luaargs = {...}
+    local function_id = luaargs[1]
+    local preview_socket_path = luaargs[2]
+    local fzf_selection = luaargs[3]
+    local fzf_lines = luaargs[4]
+    local fzf_columns = luaargs[5]
+    local usr_func = require"fzf-lua.shell".get_func(function_id)
+    return usr_func(preview_socket_path, fzf_selection, fzf_lines, fzf_columns)
+  ]], {
+    function_id,
+    preview_socket_path,
+    args,
+    tonumber(preview_lines),
+    tonumber(preview_cols)
+  })
+  vim.fn.chanclose(chan_id)
+end)
+
+if not success then
+  io.write("FzfLua Error:\n\n" .. errmsg .. "\n")
+  vim.cmd [[qall]]
+end