treesitter: add standard &rtp/parser/ search path for parsers

This commit is contained in:
Björn Linse 2020-01-25 13:43:41 +01:00
parent 64807303df
commit 00c57c98df
8 changed files with 100 additions and 102 deletions

View File

@ -123,22 +123,6 @@ if (-not $NoTests) {
npm.cmd install -g neovim
Get-Command -CommandType Application neovim-node-host.cmd
npm.cmd link neovim
$env:TREE_SITTER_DIR = $env:USERPROFILE + "\tree-sitter-build"
mkdir "$env:TREE_SITTER_DIR\bin"
$xbits = if ($bits -eq '32') {'x86'} else {'x64'}
Invoke-WebRequest -UseBasicParsing -Uri "https://github.com/tree-sitter/tree-sitter/releases/download/0.15.9/tree-sitter-windows-$xbits.gz" -OutFile tree-sitter.exe.gz
C:\msys64\usr\bin\gzip -d tree-sitter.exe.gz
Invoke-WebRequest -UseBasicParsing -Uri "https://codeload.github.com/tree-sitter/tree-sitter-c/zip/v0.15.2" -OutFile tree_sitter_c.zip
Expand-Archive .\tree_sitter_c.zip -DestinationPath .
cd tree-sitter-c-0.15.2
..\tree-sitter.exe test
if (-Not (Test-Path -PathType Leaf "$env:TREE_SITTER_DIR\bin\c.dll")) {
exit 1
}
}
if ($compiler -eq 'MSVC') {

View File

@ -27,29 +27,5 @@ nvm use 10
npm install -g neovim
npm link neovim
echo "Install tree-sitter npm package"
# FIXME
# https://github.com/tree-sitter/tree-sitter/commit/e14e285a1087264a8c74a7c62fcaecc49db9d904
# If queries added to tree-sitter-c, we can use latest tree-sitter-cli
npm install -g tree-sitter-cli@v0.15.9
echo "Install tree-sitter c parser"
curl "https://codeload.github.com/tree-sitter/tree-sitter-c/tar.gz/v0.15.2" -o tree_sitter_c.tar.gz
tar xf tree_sitter_c.tar.gz
cd tree-sitter-c-0.15.2
export TREE_SITTER_DIR=$HOME/tree-sitter-build/
mkdir -p "$TREE_SITTER_DIR/bin"
if [[ "$BUILD_32BIT" != "ON" ]]; then
# builds c parser in $HOME/tree-sitter-build/bin/c.(so|dylib)
tree-sitter test
else
# no tree-sitter binary for 32bit linux, so fake it (no tree-sitter unit tests)
cd src/
gcc -m32 -o "$TREE_SITTER_DIR/bin/c.so" -shared parser.c -I.
fi
test -f "$TREE_SITTER_DIR/bin/c.so"
sudo cpanm -n Neovim::Ext || cat "$HOME/.cpanm/build.log"
perl -W -e 'use Neovim::Ext; print $Neovim::Ext::VERSION'

View File

@ -19,7 +19,7 @@ exit_suite --continue
source ~/.nvm/nvm.sh
nvm use 10
export TREE_SITTER_DIR=$HOME/tree-sitter-build/
enter_suite tests

View File

@ -494,10 +494,12 @@ VIM.TREESITTER *lua-treesitter*
Nvim integrates the tree-sitter library for incremental parsing of buffers.
Currently Nvim does not provide the tree-sitter parsers, instead these must
be built separately, for instance using the tree-sitter utility.
The parser is loaded into nvim using >
be built separately, for instance using the tree-sitter utility. The only
exception is a C parser being included in official builds for testing
purposes. Parsers are searched for as `parser/{lang}.*` in any 'runtimepath'
directory. A parser can also be loaded manually using a full path: >
vim.treesitter.add_language("/path/to/c_parser.so", "c")
vim.treesitter.require_language("python", "/path/to/python.so")
<Create a parser for a buffer and a given language (if another plugin uses the
same buffer/language combination, it will be safely reused). Use >

View File

@ -31,8 +31,6 @@ function Parser:_on_lines(bufnr, _, start_row, old_stop_row, stop_row, old_byte_
end
local M = {
add_language=vim._ts_add_language,
inspect_language=vim._ts_inspect_language,
parse_query = vim._ts_parse_query,
}
@ -45,12 +43,34 @@ setmetatable(M, {
end
})
function M.create_parser(bufnr, ft, id)
function M.require_language(lang, path)
if vim._ts_has_language(lang) then
return true
end
if path == nil then
local fname = 'parser/' .. lang .. '.*'
local paths = a.nvim_get_runtime_file(fname, false)
if #paths == 0 then
-- TODO(bfredl): help tag?
error("no parser for '"..lang.."' language")
end
path = paths[1]
end
vim._ts_add_language(path, lang)
end
function M.inspect_language(lang)
M.require_language(lang)
return vim._ts_inspect_language(lang)
end
function M.create_parser(bufnr, lang, id)
M.require_language(lang)
if bufnr == 0 then
bufnr = a.nvim_get_current_buf()
end
local self = setmetatable({bufnr=bufnr, lang=ft, valid=false}, Parser)
self._parser = vim._create_ts_parser(ft)
local self = setmetatable({bufnr=bufnr, lang=lang, valid=false}, Parser)
self._parser = vim._create_ts_parser(lang)
self.change_cbs = {}
self:parse()
-- TODO(bfredl): use weakref to self, so that the parser is free'd is no plugin is
@ -94,6 +114,7 @@ local Query = {}
Query.__index = Query
function M.parse_query(lang, query)
M.require_language(lang)
local self = setmetatable({}, Query)
self.query = vim._ts_parse_query(lang, query)
self.info = self.query:inspect()

View File

@ -1025,9 +1025,12 @@ static void nlua_add_treesitter(lua_State *const lstate) FUNC_ATTR_NONNULL_ALL
lua_pushcfunction(lstate, create_tslua_parser);
lua_setfield(lstate, -2, "_create_ts_parser");
lua_pushcfunction(lstate, tslua_register_lang);
lua_pushcfunction(lstate, tslua_add_language);
lua_setfield(lstate, -2, "_ts_add_language");
lua_pushcfunction(lstate, tslua_has_language);
lua_setfield(lstate, -2, "_ts_has_language");
lua_pushcfunction(lstate, tslua_inspect_lang);
lua_setfield(lstate, -2, "_ts_inspect_language");

View File

@ -119,7 +119,14 @@ void tslua_init(lua_State *L)
build_meta(L, "treesitter_querycursor", querycursor_meta);
}
int tslua_register_lang(lua_State *L)
int tslua_has_language(lua_State *L)
{
const char *lang_name = luaL_checkstring(L, 1);
lua_pushboolean(L, pmap_has(cstr_t)(langs, lang_name));
return 1;
}
int tslua_add_language(lua_State *L)
{
if (lua_gettop(L) < 2 || !lua_isstring(L, 1) || !lua_isstring(L, 2)) {
return luaL_error(L, "string expected");

View File

@ -6,7 +6,6 @@ local clear = helpers.clear
local eq = helpers.eq
local insert = helpers.insert
local exec_lua = helpers.exec_lua
local iswin = helpers.iswin
local feed = helpers.feed
local pcall_err = helpers.pcall_err
local matches = helpers.matches
@ -16,37 +15,35 @@ before_each(clear)
describe('treesitter API', function()
-- error tests not requiring a parser library
it('handles missing language', function()
eq('Error executing lua: .../treesitter.lua: no such language: borklang',
eq("Error executing lua: .../treesitter.lua: no parser for 'borklang' language",
pcall_err(exec_lua, "parser = vim.treesitter.create_parser(0, 'borklang')"))
-- actual message depends on platform
matches('Error executing lua: Failed to load parser: uv_dlopen: .+',
pcall_err(exec_lua, "parser = vim.treesitter.add_language('borkbork.so', 'borklang')"))
matches("Error executing lua: Failed to load parser: uv_dlopen: .+",
pcall_err(exec_lua, "parser = vim.treesitter.require_language('borklang', 'borkbork.so')"))
eq('Error executing lua: [string "<nvim>"]:1: no such language: borklang',
eq("Error executing lua: .../treesitter.lua: no parser for 'borklang' language",
pcall_err(exec_lua, "parser = vim.treesitter.inspect_language('borklang')"))
end)
end)
describe('treesitter API with C parser', function()
local ts_path = os.getenv("TREE_SITTER_DIR")
-- The tests after this requires an actual parser
if ts_path == nil then
it("works", function() pending("TREE_SITTER_PATH not set, skipping treesitter parser tests") end)
return
local function check_parser()
local status, msg = unpack(exec_lua([[ return {pcall(vim.treesitter.require_language, 'c')} ]]))
if not status then
if helpers.isCI() then
error("treesitter C parser not found, required on CI: " .. msg)
else
pending('no C parser, skipping')
end
end
return status
end
before_each(function()
local path = ts_path .. '/bin/c'..(iswin() and '.dll' or '.so')
exec_lua([[
local path = ...
vim.treesitter.add_language(path,'c')
]], path)
end)
it('parses buffer', function()
if not check_parser() then return end
insert([[
int main() {
int x = 3;
@ -138,6 +135,8 @@ void ui_refresh(void)
]]
it('support query and iter by capture', function()
if not check_parser() then return end
insert(test_text)
local res = exec_lua([[
@ -167,6 +166,8 @@ void ui_refresh(void)
end)
it('support query and iter by match', function()
if not check_parser() then return end
insert(test_text)
local res = exec_lua([[
@ -198,6 +199,8 @@ void ui_refresh(void)
end)
it('supports highlighting', function()
if not check_parser() then return end
local hl_text = [[
/// Schedule Lua callback on main loop's event queue
static int nlua_schedule(lua_State *const lstate)
@ -357,41 +360,43 @@ static int nlua_schedule(lua_State *const lstate)
end)
it('inspects language', function()
local keys, fields, symbols = unpack(exec_lua([[
local lang = vim.treesitter.inspect_language('c')
local keys, symbols = {}, {}
for k,_ in pairs(lang) do
keys[k] = true
end
if not check_parser() then return end
-- symbols array can have "holes" and is thus not a valid msgpack array
-- but we don't care about the numbers here (checked in the parser test)
for _, v in pairs(lang.symbols) do
table.insert(symbols, v)
end
return {keys, lang.fields, symbols}
]]))
eq({fields=true, symbols=true}, keys)
local fset = {}
for _,f in pairs(fields) do
eq("string", type(f))
fset[f] = true
local keys, fields, symbols = unpack(exec_lua([[
local lang = vim.treesitter.inspect_language('c')
local keys, symbols = {}, {}
for k,_ in pairs(lang) do
keys[k] = true
end
eq(true, fset["directive"])
eq(true, fset["initializer"])
local has_named, has_anonymous
for _,s in pairs(symbols) do
eq("string", type(s[1]))
eq("boolean", type(s[2]))
if s[1] == "for_statement" and s[2] == true then
has_named = true
elseif s[1] == "|=" and s[2] == false then
has_anonymous = true
end
-- symbols array can have "holes" and is thus not a valid msgpack array
-- but we don't care about the numbers here (checked in the parser test)
for _, v in pairs(lang.symbols) do
table.insert(symbols, v)
end
eq({true,true}, {has_named,has_anonymous})
return {keys, lang.fields, symbols}
]]))
eq({fields=true, symbols=true}, keys)
local fset = {}
for _,f in pairs(fields) do
eq("string", type(f))
fset[f] = true
end
eq(true, fset["directive"])
eq(true, fset["initializer"])
local has_named, has_anonymous
for _,s in pairs(symbols) do
eq("string", type(s[1]))
eq("boolean", type(s[2]))
if s[1] == "for_statement" and s[2] == true then
has_named = true
elseif s[1] == "|=" and s[2] == false then
has_anonymous = true
end
end
eq({true,true}, {has_named,has_anonymous})
end)
end)