Browse Source

tree-sitter: vendor tree-sitter runtime

tree-sitter/tree-sitter commit 7685b7861c

Included files are:
lib/include/tree-sitter/*.h
lib/src/*.[ch]
LICENSE
tags/nightly
Björn Linse 5 months ago
parent
commit
3bddf05023

+ 3
- 0
codecov.yml View File

@@ -25,3 +25,6 @@ coverage:
changes: no

comment: off

ignore:
- "src/tree_sitter"

+ 21
- 0
src/tree_sitter/LICENSE View File

@@ -0,0 +1,21 @@
The MIT License (MIT)

Copyright (c) 2018 Max Brunsfeld

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

+ 81
- 0
src/tree_sitter/alloc.h View File

@@ -0,0 +1,81 @@
#ifndef TREE_SITTER_ALLOC_H_
#define TREE_SITTER_ALLOC_H_

#ifdef __cplusplus
extern "C" {
#endif

#include <stdlib.h>
#include <stdbool.h>
#include <stdio.h>

#if defined(TREE_SITTER_TEST)

void *ts_record_malloc(size_t);
void *ts_record_calloc(size_t, size_t);
void *ts_record_realloc(void *, size_t);
void ts_record_free(void *);
bool ts_toggle_allocation_recording(bool);

static inline void *ts_malloc(size_t size) {
return ts_record_malloc(size);
}

static inline void *ts_calloc(size_t count, size_t size) {
return ts_record_calloc(count, size);
}

static inline void *ts_realloc(void *buffer, size_t size) {
return ts_record_realloc(buffer, size);
}

static inline void ts_free(void *buffer) {
ts_record_free(buffer);
}

#else

#include <stdlib.h>

static inline bool ts_toggle_allocation_recording(bool value) {
return false;
}

static inline void *ts_malloc(size_t size) {
void *result = malloc(size);
if (size > 0 && !result) {
fprintf(stderr, "tree-sitter failed to allocate %lu bytes", size);
exit(1);
}
return result;
}

static inline void *ts_calloc(size_t count, size_t size) {
void *result = calloc(count, size);
if (count > 0 && !result) {
fprintf(stderr, "tree-sitter failed to allocate %lu bytes", count * size);
exit(1);
}
return result;
}

static inline void *ts_realloc(void *buffer, size_t size) {
void *result = realloc(buffer, size);
if (size > 0 && !result) {
fprintf(stderr, "tree-sitter failed to reallocate %lu bytes", size);
exit(1);
}
return result;
}

static inline void ts_free(void *buffer) {
free(buffer);
}

#endif

#ifdef __cplusplus
}
#endif

#endif // TREE_SITTER_ALLOC_H_

+ 660
- 0
src/tree_sitter/api.h View File

@@ -0,0 +1,660 @@
#ifndef TREE_SITTER_API_H_
#define TREE_SITTER_API_H_

#ifdef __cplusplus
extern "C" {
#endif

#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <stdbool.h>

/****************************/
/* Section - ABI Versioning */
/****************************/

#define TREE_SITTER_LANGUAGE_VERSION 11
#define TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION 9

/*******************/
/* Section - Types */
/*******************/

typedef uint16_t TSSymbol;
typedef uint16_t TSFieldId;
typedef struct TSLanguage TSLanguage;
typedef struct TSParser TSParser;
typedef struct TSTree TSTree;

typedef enum {
TSInputEncodingUTF8,
TSInputEncodingUTF16,
} TSInputEncoding;

typedef enum {
TSSymbolTypeRegular,
TSSymbolTypeAnonymous,
TSSymbolTypeAuxiliary,
} TSSymbolType;

typedef struct {
uint32_t row;
uint32_t column;
} TSPoint;

typedef struct {
TSPoint start_point;
TSPoint end_point;
uint32_t start_byte;
uint32_t end_byte;
} TSRange;

typedef struct {
void *payload;
const char *(*read)(void *payload, uint32_t byte_index, TSPoint position, uint32_t *bytes_read);
TSInputEncoding encoding;
} TSInput;

typedef enum {
TSLogTypeParse,
TSLogTypeLex,
} TSLogType;

typedef struct {
void *payload;
void (*log)(void *payload, TSLogType, const char *);
} TSLogger;

typedef struct {
uint32_t start_byte;
uint32_t old_end_byte;
uint32_t new_end_byte;
TSPoint start_point;
TSPoint old_end_point;
TSPoint new_end_point;
} TSInputEdit;

typedef struct {
uint32_t context[4];
const void *id;
const TSTree *tree;
} TSNode;

typedef struct {
const void *tree;
const void *id;
uint32_t context[2];
} TSTreeCursor;

/********************/
/* Section - Parser */
/********************/

/**
* Create a new parser.
*/
TSParser *ts_parser_new(void);

/**
* Delete the parser, freeing all of the memory that it used.
*/
void ts_parser_delete(TSParser *parser);

/**
* Set the language that the parser should use for parsing.
*
* Returns a boolean indicating whether or not the language was successfully
* assigned. True means assignment succeeded. False means there was a version
* mismatch: the language was generated with an incompatible version of the
* Tree-sitter CLI. Check the language's version using `ts_language_version`
* and compare it to this library's `TREE_SITTER_LANGUAGE_VERSION` and
* `TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION` constants.
*/
bool ts_parser_set_language(TSParser *self, const TSLanguage *language);

/**
* Get the parser's current language.
*/
const TSLanguage *ts_parser_language(const TSParser *self);

/**
* Set the spans of text that the parser should include when parsing.
*
* By default, the parser will always include entire documents. This function
* allows you to parse only a *portion* of a document but still return a syntax
* tree whose ranges match up with the document as a whole. You can also pass
* multiple disjoint ranges.
*
* The second and third parameters specify the location and length of an array
* of ranges. The parser does *not* take ownership of these ranges; it copies
* the data, so it doesn't matter how these ranges are allocated.
*/
void ts_parser_set_included_ranges(
TSParser *self,
const TSRange *ranges,
uint32_t length
);

/**
* Get the ranges of text that the parser will include when parsing.
*
* The returned pointer is owned by the parser. The caller should not free it
* or write to it. The length of the array will be written to the given
* `length` pointer.
*/
const TSRange *ts_parser_included_ranges(
const TSParser *self,
uint32_t *length
);

/**
* Use the parser to parse some source code and create a syntax tree.
*
* If you are parsing this document for the first time, pass `NULL` for the
* `old_tree` parameter. Otherwise, if you have already parsed an earlier
* version of this document and the document has since been edited, pass the
* previous syntax tree so that the unchanged parts of it can be reused.
* This will save time and memory. For this to work correctly, you must have
* already edited the old syntax tree using the `ts_tree_edit` function in a
* way that exactly matches the source code changes.
*
* The `TSInput` parameter lets you specify how to read the text. It has the
* following three fields:
* 1. `read`: A function to retrieve a chunk of text at a given byte offset
* and (row, column) position. The function should return a pointer to the
* text and write its length to the the `bytes_read` pointer. The parser
* does not take ownership of this buffer; it just borrows it until it has
* finished reading it. The function should write a zero value to the
* `bytes_read` pointer to indicate the end of the document.
* 2. `payload`: An arbitrary pointer that will be passed to each invocation
* of the `read` function.
* 3. `encoding`: An indication of how the text is encoded. Either
* `TSInputEncodingUTF8` or `TSInputEncodingUTF16`.
*
* This function returns a syntax tree on success, and `NULL` on failure. There
* are three possible reasons for failure:
* 1. The parser does not have a language assigned. Check for this using the
`ts_parser_language` function.
* 2. Parsing was cancelled due to a timeout that was set by an earlier call to
* the `ts_parser_set_timeout_micros` function. You can resume parsing from
* where the parser left out by calling `ts_parser_parse` again with the
* same arguments. Or you can start parsing from scratch by first calling
* `ts_parser_reset`.
* 3. Parsing was cancelled using a cancellation flag that was set by an
* earlier call to `ts_parser_set_cancellation_flag`. You can resume parsing
* from where the parser left out by calling `ts_parser_parse` again with
* the same arguments.
*/
TSTree *ts_parser_parse(
TSParser *self,
const TSTree *old_tree,
TSInput input
);

/**
* Use the parser to parse some source code stored in one contiguous buffer.
* The first two parameters are the same as in the `ts_parser_parse` function
* above. The second two parameters indicate the location of the buffer and its
* length in bytes.
*/
TSTree *ts_parser_parse_string(
TSParser *self,
const TSTree *old_tree,
const char *string,
uint32_t length
);

/**
* Use the parser to parse some source code stored in one contiguous buffer with
* a given encoding. The first four parameters work the same as in the
* `ts_parser_parse_string` method above. The final parameter indicates whether
* the text is encoded as UTF8 or UTF16.
*/
TSTree *ts_parser_parse_string_encoding(
TSParser *self,
const TSTree *old_tree,
const char *string,
uint32_t length,
TSInputEncoding encoding
);

/**
* Instruct the parser to start the next parse from the beginning.
*
* If the parser previously failed because of a timeout or a cancellation, then
* by default, it will resume where it left off on the next call to
* `ts_parser_parse` or other parsing functions. If you don't want to resume,
* and instead intend to use this parser to parse some other document, you must
* call this `ts_parser_reset` first.
*/
void ts_parser_reset(TSParser *self);

/**
* Set the maximum duration in microseconds that parsing should be allowed to
* take before halting. If parsing takes longer than this, it will halt early,
* returning NULL. See `ts_parser_parse` for more information.
*/
void ts_parser_set_timeout_micros(TSParser *self, uint64_t timeout);

/**
* Get the duration in microseconds that parsing is allowed to take.
*/
uint64_t ts_parser_timeout_micros(const TSParser *self);

/**
* Set the parser's current cancellation flag pointer. If a non-null pointer is
* assigned, then the parser will periodically read from this pointer during
* parsing. If it reads a non-zero value, it will halt early, returning NULL.
* See `ts_parser_parse` for more information.
*/
void ts_parser_set_cancellation_flag(TSParser *self, const size_t *flag);

/**
* Get the parser's current cancellation flag pointer.
*/
const size_t *ts_parser_cancellation_flag(const TSParser *self);

/**
* Set the logger that a parser should use during parsing.
*
* The parser does not take ownership over the logger payload. If a logger was
* previously assigned, the caller is responsible for releasing any memory
* owned by the previous logger.
*/
void ts_parser_set_logger(TSParser *self, TSLogger logger);

/**
* Get the parser's current logger.
*/
TSLogger ts_parser_logger(const TSParser *self);

/**
* Set the file descriptor to which the parser should write debugging graphs
* during parsing. The graphs are formatted in the DOT language. You may want
* to pipe these graphs directly to a `dot(1)` process in order to generate
* SVG output. You can turn off this logging by passing a negative number.
*/
void ts_parser_print_dot_graphs(TSParser *self, int file);

/**
* Set whether or not the parser should halt immediately upon detecting an
* error. This will generally result in a syntax tree with an error at the
* root, and one or more partial syntax trees within the error. This behavior
* may not be supported long-term.
*/
void ts_parser_halt_on_error(TSParser *self, bool halt);

/******************/
/* Section - Tree */
/******************/

/**
* Create a shallow copy of the syntax tree. This is very fast.
*
* You need to copy a syntax tree in order to use it on more than one thread at
* a time, as syntax trees are not thread safe.
*/
TSTree *ts_tree_copy(const TSTree *self);

/**
* Delete the syntax tree, freeing all of the memory that it used.
*/
void ts_tree_delete(TSTree *self);

/**
* Get the root node of the syntax tree.
*/
TSNode ts_tree_root_node(const TSTree *self);

/**
* Get the language that was used to parse the syntax tree.
*/
const TSLanguage *ts_tree_language(const TSTree *);

/**
* Edit the syntax tree to keep it in sync with source code that has been
* edited.
*
* You must describe the edit both in terms of byte offsets and in terms of
* (row, column) coordinates.
*/
void ts_tree_edit(TSTree *self, const TSInputEdit *edit);

/**
* Compare a new syntax tree to a previous syntax tree representing the same
* document, returning an array of ranges whose syntactic structure has changed.
*
* For this to work correctly, the old syntax tree must have been edited such
* that its ranges match up to the new tree. Generally, you'll want to call
* this function right after calling one of the `ts_parser_parse` functions,
* passing in the new tree that was returned from `ts_parser_parse` and the old
* tree that was passed as a parameter.
*
* The returned array is allocated using `malloc` and the caller is responsible
* for freeing it using `free`. The length of the array will be written to the
* given `length` pointer.
*/
TSRange *ts_tree_get_changed_ranges(
const TSTree *self,
const TSTree *old_tree,
uint32_t *length
);

/**
* Write a DOT graph describing the syntax tree to the given file.
*/
void ts_tree_print_dot_graph(const TSTree *, FILE *);

/******************/
/* Section - Node */
/******************/

/**
* Get the node's type as a null-terminated string.
*/
const char *ts_node_type(TSNode);

/**
* Get the node's type as a numerical id.
*/
TSSymbol ts_node_symbol(TSNode);

/**
* Get the node's start byte.
*/
uint32_t ts_node_start_byte(TSNode);

/**
* Get the node's start position in terms of rows and columns.
*/
TSPoint ts_node_start_point(TSNode);

/**
* Get the node's end byte.
*/
uint32_t ts_node_end_byte(TSNode);

/**
* Get the node's end position in terms of rows and columns.
*/
TSPoint ts_node_end_point(TSNode);

/**
* Get an S-expression representing the node as a string.
*
* This string is allocated with `malloc` and the caller is responsible for
* freeing it using `free`.
*/
char *ts_node_string(TSNode);

/**
* Check if the node is null. Functions like `ts_node_child` and
* `ts_node_next_sibling` will return a null node to indicate that no such node
* was found.
*/
bool ts_node_is_null(TSNode);

/**
* Check if the node is *named*. Named nodes correspond to named rules in the
* grammar, whereas *anonymous* nodes correspond to string literals in the
* grammar.
*/
bool ts_node_is_named(TSNode);

/**
* Check if the node is *missing*. Missing nodes are inserted by the parser in
* order to recover from certain kinds of syntax errors.
*/
bool ts_node_is_missing(TSNode);

/**
* Check if the node is *missing*. Missing nodes are inserted by the parser in
* order to recover from certain kinds of syntax errors.
*/
bool ts_node_is_extra(TSNode);

/**
* Check if a syntax node has been edited.
*/
bool ts_node_has_changes(TSNode);

/**
* Check if the node is a syntax error or contains any syntax errors.
*/
bool ts_node_has_error(TSNode);

/**
* Get the node's immediate parent.
*/
TSNode ts_node_parent(TSNode);

/**
* Get the node's child at the given index, where zero represents the first
* child.
*/
TSNode ts_node_child(TSNode, uint32_t);

/**
* Get the node's number of children.
*/
uint32_t ts_node_child_count(TSNode);

/**
* Get the node's *named* child at the given index.
*
* See also `ts_node_is_named`.
*/
TSNode ts_node_named_child(TSNode, uint32_t);

/**
* Get the node's number of *named* children.
*
* See also `ts_node_is_named`.
*/
uint32_t ts_node_named_child_count(TSNode);

/**
* Get the node's child with the given field name.
*/
TSNode ts_node_child_by_field_name(
TSNode self,
const char *field_name,
uint32_t field_name_length
);

/**
* Get the node's child with the given numerical field id.
*
* You can convert a field name to an id using the
* `ts_language_field_id_for_name` function.
*/
TSNode ts_node_child_by_field_id(TSNode, TSFieldId);

/**
* Get the node's next / previous sibling.
*/
TSNode ts_node_next_sibling(TSNode);
TSNode ts_node_prev_sibling(TSNode);

/**
* Get the node's next / previous *named* sibling.
*/
TSNode ts_node_next_named_sibling(TSNode);
TSNode ts_node_prev_named_sibling(TSNode);

/**
* Get the node's first child that extends beyond the given byte offset.
*/
TSNode ts_node_first_child_for_byte(TSNode, uint32_t);

/**
* Get the node's first named child that extends beyond the given byte offset.
*/
TSNode ts_node_first_named_child_for_byte(TSNode, uint32_t);

/**
* Get the smallest node within this node that spans the given range of bytes
* or (row, column) positions.
*/
TSNode ts_node_descendant_for_byte_range(TSNode, uint32_t, uint32_t);
TSNode ts_node_descendant_for_point_range(TSNode, TSPoint, TSPoint);

/**
* Get the smallest named node within this node that spans the given range of
* bytes or (row, column) positions.
*/
TSNode ts_node_named_descendant_for_byte_range(TSNode, uint32_t, uint32_t);
TSNode ts_node_named_descendant_for_point_range(TSNode, TSPoint, TSPoint);

/**
* Edit the node to keep it in-sync with source code that has been edited.
*
* This function is only rarely needed. When you edit a syntax tree with the
* `ts_tree_edit` function, all of the nodes that you retrieve from the tree
* afterward will already reflect the edit. You only need to use `ts_node_edit`
* when you have a `TSNode` instance that you want to keep and continue to use
* after an edit.
*/
void ts_node_edit(TSNode *, const TSInputEdit *);

/**
* Check if two nodes are identical.
*/
bool ts_node_eq(TSNode, TSNode);

/************************/
/* Section - TreeCursor */
/************************/

/**
* Create a new tree cursor starting from the given node.
*
* A tree cursor allows you to walk a syntax tree more efficiently than is
* possible using the `TSNode` functions. It is a mutable object that is always
* on a certain syntax node, and can be moved imperatively to different nodes.
*/
TSTreeCursor ts_tree_cursor_new(TSNode);

/**
* Delete a tree cursor, freeing all of the memory that it used.
*/
void ts_tree_cursor_delete(TSTreeCursor *);

/**
* Re-initialize a tree cursor to start at a different ndoe.
*/
void ts_tree_cursor_reset(TSTreeCursor *, TSNode);

/**
* Get the tree cursor's current node.
*/
TSNode ts_tree_cursor_current_node(const TSTreeCursor *);

/**
* Get the field name of the tree cursor's current node.
*
* This returns `NULL` if the current node doesn't have a field.
* See also `ts_node_child_by_field_name`.
*/
const char *ts_tree_cursor_current_field_name(const TSTreeCursor *);

/**
* Get the field name of the tree cursor's current node.
*
* This returns zero if the current node doesn't have a field.
* See also `ts_node_child_by_field_id`, `ts_language_field_id_for_name`.
*/
TSFieldId ts_tree_cursor_current_field_id(const TSTreeCursor *);

/**
* Move the cursor to the parent of its current node.
*
* This returns `true` if the cursor successfully moved, and returns `false`
* if there was no parent node (the cursor was already on the root node).
*/
bool ts_tree_cursor_goto_parent(TSTreeCursor *);

/**
* Move the cursor to the next sibling of its current node.
*
* This returns `true` if the cursor successfully moved, and returns `false`
* if there was no next sibling node.
*/
bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *);

/**
* Move the cursor to the first schild of its current node.
*
* This returns `true` if the cursor successfully moved, and returns `false`
* if there were no children.
*/
bool ts_tree_cursor_goto_first_child(TSTreeCursor *);

/**
* Move the cursor to the first schild of its current node that extends beyond
* the given byte offset.
*
* This returns the index of the child node if one was found, and returns -1
* if no such child was found.
*/
int64_t ts_tree_cursor_goto_first_child_for_byte(TSTreeCursor *, uint32_t);

TSTreeCursor ts_tree_cursor_copy(const TSTreeCursor *);

/**********************/
/* Section - Language */
/**********************/

/**
* Get the number of distinct node types in the language.
*/
uint32_t ts_language_symbol_count(const TSLanguage *);

/**
* Get a node type string for the given numerical id.
*/
const char *ts_language_symbol_name(const TSLanguage *, TSSymbol);

/**
* Get the numerical id for the given node type string.
*/
TSSymbol ts_language_symbol_for_name(const TSLanguage *, const char *);

/**
* Get the number of distinct field names in the language.
*/
uint32_t ts_language_field_count(const TSLanguage *);

/**
* Get the field name string for the given numerical id.
*/
const char *ts_language_field_name_for_id(const TSLanguage *, TSFieldId);

/**
* Get the numerical id for the given field name string.
*/
TSFieldId ts_language_field_id_for_name(const TSLanguage *, const char *, uint32_t);

/**
* Check whether the given node type id belongs to named nodes, anonymous nodes,
* or a hidden nodes.
*
* See also `ts_node_is_named`. Hidden nodes are never returned from the API.
*/
TSSymbolType ts_language_symbol_type(const TSLanguage *, TSSymbol);

/**
* Get the ABI version number for this language. This version number is used
* to ensure that languages were generated by a compatible version of
* Tree-sitter.
*
* See also `ts_parser_set_language`.
*/
uint32_t ts_language_version(const TSLanguage *);

#ifdef __cplusplus
}
#endif

#endif // TREE_SITTER_API_H_

+ 142
- 0
src/tree_sitter/array.h View File

@@ -0,0 +1,142 @@
#ifndef TREE_SITTER_ARRAY_H_
#define TREE_SITTER_ARRAY_H_

#ifdef __cplusplus
extern "C" {
#endif

#include <string.h>
#include <stdlib.h>
#include <stdint.h>
#include <assert.h>
#include <stdbool.h>
#include "./alloc.h"

#define Array(T) \
struct { \
T *contents; \
uint32_t size; \
uint32_t capacity; \
}

#define array_init(self) \
((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL)

#define array_new() \
{ NULL, 0, 0 }

#define array_get(self, index) \
(assert((uint32_t)index < (self)->size), &(self)->contents[index])

#define array_front(self) array_get(self, 0)

#define array_back(self) array_get(self, (self)->size - 1)

#define array_clear(self) ((self)->size = 0)

#define array_reserve(self, new_capacity) \
array__reserve((VoidArray *)(self), array__elem_size(self), new_capacity)

#define array_erase(self, index) \
array__erase((VoidArray *)(self), array__elem_size(self), index)

#define array_delete(self) array__delete((VoidArray *)self)

#define array_push(self, element) \
(array__grow((VoidArray *)(self), 1, array__elem_size(self)), \
(self)->contents[(self)->size++] = (element))

#define array_grow_by(self, count) \
(array__grow((VoidArray *)(self), count, array__elem_size(self)), \
memset((self)->contents + (self)->size, 0, (count) * array__elem_size(self)), \
(self)->size += (count))

#define array_push_all(self, other) \
array_splice((self), (self)->size, 0, (other)->size, (other)->contents)

#define array_splice(self, index, old_count, new_count, new_contents) \
array__splice((VoidArray *)(self), array__elem_size(self), index, old_count, \
new_count, new_contents)

#define array_insert(self, index, element) \
array__splice((VoidArray *)(self), array__elem_size(self), index, 0, 1, &element)

#define array_pop(self) ((self)->contents[--(self)->size])

#define array_assign(self, other) \
array__assign((VoidArray *)(self), (const VoidArray *)(other), array__elem_size(self))

// Private

typedef Array(void) VoidArray;

#define array__elem_size(self) sizeof(*(self)->contents)

static inline void array__delete(VoidArray *self) {
ts_free(self->contents);
self->contents = NULL;
self->size = 0;
self->capacity = 0;
}

static inline void array__erase(VoidArray *self, size_t element_size,
uint32_t index) {
assert(index < self->size);
char *contents = (char *)self->contents;
memmove(contents + index * element_size, contents + (index + 1) * element_size,
(self->size - index - 1) * element_size);
self->size--;
}

static inline void array__reserve(VoidArray *self, size_t element_size, uint32_t new_capacity) {
if (new_capacity > self->capacity) {
if (self->contents) {
self->contents = ts_realloc(self->contents, new_capacity * element_size);
} else {
self->contents = ts_calloc(new_capacity, element_size);
}
self->capacity = new_capacity;
}
}

static inline void array__assign(VoidArray *self, const VoidArray *other, size_t element_size) {
array__reserve(self, element_size, other->size);
self->size = other->size;
memcpy(self->contents, other->contents, self->size * element_size);
}

static inline void array__grow(VoidArray *self, size_t count, size_t element_size) {
size_t new_size = self->size + count;
if (new_size > self->capacity) {
size_t new_capacity = self->capacity * 2;
if (new_capacity < 8) new_capacity = 8;
if (new_capacity < new_size) new_capacity = new_size;
array__reserve(self, element_size, new_capacity);
}
}

static inline void array__splice(VoidArray *self, size_t element_size,
uint32_t index, uint32_t old_count,
uint32_t new_count, const void *elements) {
uint32_t new_size = self->size + new_count - old_count;
uint32_t old_end = index + old_count;
uint32_t new_end = index + new_count;
assert(old_end <= self->size);

array__reserve(self, element_size, new_size);

char *contents = (char *)self->contents;
if (self->size > old_end)
memmove(contents + new_end * element_size, contents + old_end * element_size,
(self->size - old_end) * element_size);
if (new_count > 0)
memcpy((contents + index * element_size), elements,
new_count * element_size);
self->size += new_count - old_count;
}

#ifdef __cplusplus
}
#endif

#endif // TREE_SITTER_ARRAY_H_

+ 42
- 0
src/tree_sitter/atomic.h View File

@@ -0,0 +1,42 @@
#ifndef TREE_SITTER_ATOMIC_H_
#define TREE_SITTER_ATOMIC_H_

#include <stdint.h>

#ifdef _WIN32

#include <windows.h>

static inline size_t atomic_load(const volatile size_t *p) {
return *p;
}

static inline uint32_t atomic_inc(volatile uint32_t *p) {
return InterlockedIncrement(p);
}

static inline uint32_t atomic_dec(volatile uint32_t *p) {
return InterlockedDecrement(p);
}

#else

static inline size_t atomic_load(const volatile size_t *p) {
#ifdef __ATOMIC_RELAXED
return __atomic_load_n(p, __ATOMIC_RELAXED);
#else
return __sync_fetch_and_add((volatile size_t *)p, 0);
#endif
}

static inline uint32_t atomic_inc(volatile uint32_t *p) {
return __sync_add_and_fetch(p, 1u);
}

static inline uint32_t atomic_dec(volatile uint32_t *p) {
return __sync_sub_and_fetch(p, 1u);
}

#endif

#endif // TREE_SITTER_ATOMIC_H_

+ 141
- 0
src/tree_sitter/clock.h View File

@@ -0,0 +1,141 @@
#ifndef TREE_SITTER_CLOCK_H_
#define TREE_SITTER_CLOCK_H_

#include <stdint.h>

typedef uint64_t TSDuration;

#ifdef _WIN32

// Windows:
// * Represent a time as a performance counter value.
// * Represent a duration as a number of performance counter ticks.

#include <windows.h>
typedef uint64_t TSClock;

static inline TSDuration duration_from_micros(uint64_t micros) {
LARGE_INTEGER frequency;
QueryPerformanceFrequency(&frequency);
return micros * (uint64_t)frequency.QuadPart / 1000000;
}

static inline uint64_t duration_to_micros(TSDuration self) {
LARGE_INTEGER frequency;
QueryPerformanceFrequency(&frequency);
return self * 1000000 / (uint64_t)frequency.QuadPart;
}

static inline TSClock clock_null(void) {
return 0;
}

static inline TSClock clock_now(void) {
LARGE_INTEGER result;
QueryPerformanceCounter(&result);
return (uint64_t)result.QuadPart;
}

static inline TSClock clock_after(TSClock base, TSDuration duration) {
return base + duration;
}

static inline bool clock_is_null(TSClock self) {
return !self;
}

static inline bool clock_is_gt(TSClock self, TSClock other) {
return self > other;
}

#elif defined(CLOCK_MONOTONIC) && !defined(__APPLE__)

// POSIX with monotonic clock support (Linux)
// * Represent a time as a monotonic (seconds, nanoseconds) pair.
// * Represent a duration as a number of microseconds.
//
// On these platforms, parse timeouts will correspond accurately to
// real time, regardless of what other processes are running.

#include <time.h>
typedef struct timespec TSClock;

static inline TSDuration duration_from_micros(uint64_t micros) {
return micros;
}

static inline uint64_t duration_to_micros(TSDuration self) {
return self;
}

static inline TSClock clock_now(void) {
TSClock result;
clock_gettime(CLOCK_MONOTONIC, &result);
return result;
}

static inline TSClock clock_null(void) {
return (TSClock) {0, 0};
}

static inline TSClock clock_after(TSClock base, TSDuration duration) {
TSClock result = base;
result.tv_sec += duration / 1000000;
result.tv_nsec += (duration % 1000000) * 1000;
return result;
}

static inline bool clock_is_null(TSClock self) {
return !self.tv_sec;
}

static inline bool clock_is_gt(TSClock self, TSClock other) {
if (self.tv_sec > other.tv_sec) return true;
if (self.tv_sec < other.tv_sec) return false;
return self.tv_nsec > other.tv_nsec;
}

#else

// macOS or POSIX without monotonic clock support
// * Represent a time as a process clock value.
// * Represent a duration as a number of process clock ticks.
//
// On these platforms, parse timeouts may be affected by other processes,
// which is not ideal, but is better than using a non-monotonic time API
// like `gettimeofday`.

#include <time.h>
typedef uint64_t TSClock;

static inline TSDuration duration_from_micros(uint64_t micros) {
return micros * (uint64_t)CLOCKS_PER_SEC / 1000000;
}

static inline uint64_t duration_to_micros(TSDuration self) {
return self * 1000000 / (uint64_t)CLOCKS_PER_SEC;
}

static inline TSClock clock_null(void) {
return 0;
}

static inline TSClock clock_now(void) {
return (uint64_t)clock();
}

static inline TSClock clock_after(TSClock base, TSDuration duration) {
return base + duration;
}

static inline bool clock_is_null(TSClock self) {
return !self;
}

static inline bool clock_is_gt(TSClock self, TSClock other) {
return self > other;
}

#endif

#endif // TREE_SITTER_CLOCK_H_

+ 11
- 0
src/tree_sitter/error_costs.h View File

@@ -0,0 +1,11 @@
#ifndef TREE_SITTER_ERROR_COSTS_H_
#define TREE_SITTER_ERROR_COSTS_H_

#define ERROR_STATE 0
#define ERROR_COST_PER_RECOVERY 500
#define ERROR_COST_PER_MISSING_TREE 110
#define ERROR_COST_PER_SKIPPED_TREE 100
#define ERROR_COST_PER_SKIPPED_LINE 30
#define ERROR_COST_PER_SKIPPED_CHAR 1

#endif

+ 482
- 0
src/tree_sitter/get_changed_ranges.c View File

@@ -0,0 +1,482 @@
#include "./get_changed_ranges.h"
#include "./subtree.h"
#include "./language.h"
#include "./error_costs.h"
#include "./tree_cursor.h"
#include <assert.h>

// #define DEBUG_GET_CHANGED_RANGES

static void ts_range_array_add(TSRangeArray *self, Length start, Length end) {
if (self->size > 0) {
TSRange *last_range = array_back(self);
if (start.bytes <= last_range->end_byte) {
last_range->end_byte = end.bytes;
last_range->end_point = end.extent;
return;
}
}

if (start.bytes < end.bytes) {
TSRange range = { start.extent, end.extent, start.bytes, end.bytes };
array_push(self, range);
}
}

bool ts_range_array_intersects(const TSRangeArray *self, unsigned start_index,
uint32_t start_byte, uint32_t end_byte) {
for (unsigned i = start_index; i < self->size; i++) {
TSRange *range = &self->contents[i];
if (range->end_byte > start_byte) {
if (range->start_byte >= end_byte) break;
return true;
}
}
return false;
}

void ts_range_array_get_changed_ranges(
const TSRange *old_ranges, unsigned old_range_count,
const TSRange *new_ranges, unsigned new_range_count,
TSRangeArray *differences
) {
unsigned new_index = 0;
unsigned old_index = 0;
Length current_position = length_zero();
bool in_old_range = false;
bool in_new_range = false;

while (old_index < old_range_count || new_index < new_range_count) {
const TSRange *old_range = &old_ranges[old_index];
const TSRange *new_range = &new_ranges[new_index];

Length next_old_position;
if (in_old_range) {
next_old_position = (Length) {old_range->end_byte, old_range->end_point};
} else if (old_index < old_range_count) {
next_old_position = (Length) {old_range->start_byte, old_range->start_point};
} else {
next_old_position = LENGTH_MAX;
}

Length next_new_position;
if (in_new_range) {
next_new_position = (Length) {new_range->end_byte, new_range->end_point};
} else if (new_index < new_range_count) {
next_new_position = (Length) {new_range->start_byte, new_range->start_point};
} else {
next_new_position = LENGTH_MAX;
}

if (next_old_position.bytes < next_new_position.bytes) {
if (in_old_range != in_new_range) {
ts_range_array_add(differences, current_position, next_old_position);
}
if (in_old_range) old_index++;
current_position = next_old_position;
in_old_range = !in_old_range;
} else if (next_new_position.bytes < next_old_position.bytes) {
if (in_old_range != in_new_range) {
ts_range_array_add(differences, current_position, next_new_position);
}
if (in_new_range) new_index++;
current_position = next_new_position;
in_new_range = !in_new_range;
} else {
if (in_old_range != in_new_range) {
ts_range_array_add(differences, current_position, next_new_position);
}
if (in_old_range) old_index++;
if (in_new_range) new_index++;
in_old_range = !in_old_range;
in_new_range = !in_new_range;
current_position = next_new_position;
}
}
}

typedef struct {
TreeCursor cursor;
const TSLanguage *language;
unsigned visible_depth;
bool in_padding;
} Iterator;

static Iterator iterator_new(TreeCursor *cursor, const Subtree *tree, const TSLanguage *language) {
array_clear(&cursor->stack);
array_push(&cursor->stack, ((TreeCursorEntry){
.subtree = tree,
.position = length_zero(),
.child_index = 0,
.structural_child_index = 0,
}));
return (Iterator) {
.cursor = *cursor,
.language = language,
.visible_depth = 1,
.in_padding = false,
};
}

static bool iterator_done(Iterator *self) {
return self->cursor.stack.size == 0;
}

static Length iterator_start_position(Iterator *self) {
TreeCursorEntry entry = *array_back(&self->cursor.stack);
if (self->in_padding) {
return entry.position;
} else {
return length_add(entry.position, ts_subtree_padding(*entry.subtree));
}
}

static Length iterator_end_position(Iterator *self) {
TreeCursorEntry entry = *array_back(&self->cursor.stack);
Length result = length_add(entry.position, ts_subtree_padding(*entry.subtree));
if (self->in_padding) {
return result;
} else {
return length_add(result, ts_subtree_size(*entry.subtree));
}
}

static bool iterator_tree_is_visible(const Iterator *self) {
TreeCursorEntry entry = *array_back(&self->cursor.stack);
if (ts_subtree_visible(*entry.subtree)) return true;
if (self->cursor.stack.size > 1) {
Subtree parent = *self->cursor.stack.contents[self->cursor.stack.size - 2].subtree;
const TSSymbol *alias_sequence = ts_language_alias_sequence(
self->language,
parent.ptr->production_id
);
return alias_sequence && alias_sequence[entry.structural_child_index] != 0;
}
return false;
}

static void iterator_get_visible_state(const Iterator *self, Subtree *tree,
TSSymbol *alias_symbol, uint32_t *start_byte) {
uint32_t i = self->cursor.stack.size - 1;

if (self->in_padding) {
if (i == 0) return;
i--;
}

for (; i + 1 > 0; i--) {
TreeCursorEntry entry = self->cursor.stack.contents[i];

if (i > 0) {
const Subtree *parent = self->cursor.stack.contents[i - 1].subtree;
const TSSymbol *alias_sequence = ts_language_alias_sequence(
self->language,
parent->ptr->production_id
);
if (alias_sequence) {
*alias_symbol = alias_sequence[entry.structural_child_index];
}
}

if (ts_subtree_visible(*entry.subtree) || *alias_symbol) {
*tree = *entry.subtree;
*start_byte = entry.position.bytes;
break;
}
}
}

static void iterator_ascend(Iterator *self) {
if (iterator_done(self)) return;
if (iterator_tree_is_visible(self) && !self->in_padding) self->visible_depth--;
if (array_back(&self->cursor.stack)->child_index > 0) self->in_padding = false;
self->cursor.stack.size--;
}

static bool iterator_descend(Iterator *self, uint32_t goal_position) {
if (self->in_padding) return false;

bool did_descend;
do {
did_descend = false;
TreeCursorEntry entry = *array_back(&self->cursor.stack);
Length position = entry.position;
uint32_t structural_child_index = 0;
for (uint32_t i = 0, n = ts_subtree_child_count(*entry.subtree); i < n; i++) {
const Subtree *child = &entry.subtree->ptr->children[i];
Length child_left = length_add(position, ts_subtree_padding(*child));
Length child_right = length_add(child_left, ts_subtree_size(*child));

if (child_right.bytes > goal_position) {
array_push(&self->cursor.stack, ((TreeCursorEntry){
.subtree = child,
.position = position,
.child_index = i,
.structural_child_index = structural_child_index,
}));

if (iterator_tree_is_visible(self)) {
if (child_left.bytes > goal_position) {
self->in_padding = true;
} else {
self->visible_depth++;
}
return true;
}

did_descend = true;
break;
}

position = child_right;
if (!ts_subtree_extra(*child)) structural_child_index++;
}
} while (did_descend);

return false;
}

static void iterator_advance(Iterator *self) {
if (self->in_padding) {
self->in_padding = false;
if (iterator_tree_is_visible(self)) {
self->visible_depth++;
} else {
iterator_descend(self, 0);
}
return;
}

for (;;) {
if (iterator_tree_is_visible(self)) self->visible_depth--;
TreeCursorEntry entry = array_pop(&self->cursor.stack);
if (iterator_done(self)) return;

const Subtree *parent = array_back(&self->cursor.stack)->subtree;
uint32_t child_index = entry.child_index + 1;
if (ts_subtree_child_count(*parent) > child_index) {
Length position = length_add(entry.position, ts_subtree_total_size(*entry.subtree));
uint32_t structural_child_index = entry.structural_child_index;
if (!ts_subtree_extra(*entry.subtree)) structural_child_index++;
const Subtree *next_child = &parent->ptr->children[child_index];

array_push(&self->cursor.stack, ((TreeCursorEntry){
.subtree = next_child,
.position = position,
.child_index = child_index,
.structural_child_index = structural_child_index,
}));

if (iterator_tree_is_visible(self)) {
if (ts_subtree_padding(*next_child).bytes > 0) {
self->in_padding = true;
} else {
self->visible_depth++;
}
} else {
iterator_descend(self, 0);
}
break;
}
}
}

typedef enum {
IteratorDiffers,
IteratorMayDiffer,
IteratorMatches,
} IteratorComparison;

static IteratorComparison iterator_compare(const Iterator *old_iter, const Iterator *new_iter) {
Subtree old_tree = NULL_SUBTREE;
Subtree new_tree = NULL_SUBTREE;
uint32_t old_start = 0;
uint32_t new_start = 0;
TSSymbol old_alias_symbol = 0;
TSSymbol new_alias_symbol = 0;
iterator_get_visible_state(old_iter, &old_tree, &old_alias_symbol, &old_start);
iterator_get_visible_state(new_iter, &new_tree, &new_alias_symbol, &new_start);

if (!old_tree.ptr && !new_tree.ptr) return IteratorMatches;
if (!old_tree.ptr || !new_tree.ptr) return IteratorDiffers;

if (
old_alias_symbol == new_alias_symbol &&
ts_subtree_symbol(old_tree) == ts_subtree_symbol(new_tree)
) {
if (old_start == new_start &&
!ts_subtree_has_changes(old_tree) &&
ts_subtree_symbol(old_tree) != ts_builtin_sym_error &&
ts_subtree_size(old_tree).bytes == ts_subtree_size(new_tree).bytes &&
ts_subtree_parse_state(old_tree) != TS_TREE_STATE_NONE &&
ts_subtree_parse_state(new_tree) != TS_TREE_STATE_NONE &&
(ts_subtree_parse_state(old_tree) == ERROR_STATE) ==
(ts_subtree_parse_state(new_tree) == ERROR_STATE)) {
return IteratorMatches;
} else {
return IteratorMayDiffer;
}
}

return IteratorDiffers;
}

#ifdef DEBUG_GET_CHANGED_RANGES
static inline void iterator_print_state(Iterator *self) {
TreeCursorEntry entry = *array_back(&self->cursor.stack);
TSPoint start = iterator_start_position(self).extent;
TSPoint end = iterator_end_position(self).extent;
const char *name = ts_language_symbol_name(self->language, ts_subtree_symbol(*entry.subtree));
printf(
"(%-25s %s\t depth:%u [%u, %u] - [%u, %u])",
name, self->in_padding ? "(p)" : " ",
self->visible_depth,
start.row + 1, start.column,
end.row + 1, end.column
);
}
#endif

unsigned ts_subtree_get_changed_ranges(const Subtree *old_tree, const Subtree *new_tree,
TreeCursor *cursor1, TreeCursor *cursor2,
const TSLanguage *language,
const TSRangeArray *included_range_differences,
TSRange **ranges) {
TSRangeArray results = array_new();

Iterator old_iter = iterator_new(cursor1, old_tree, language);
Iterator new_iter = iterator_new(cursor2, new_tree, language);

unsigned included_range_difference_index = 0;

Length position = iterator_start_position(&old_iter);
Length next_position = iterator_start_position(&new_iter);
if (position.bytes < next_position.bytes) {
ts_range_array_add(&results, position, next_position);
position = next_position;
} else if (position.bytes > next_position.bytes) {
ts_range_array_add(&results, next_position, position);
next_position = position;
}

do {
#ifdef DEBUG_GET_CHANGED_RANGES
printf("At [%-2u, %-2u] Compare ", position.extent.row + 1, position.extent.column);
iterator_print_state(&old_iter);
printf("\tvs\t");
iterator_print_state(&new_iter);
puts("");
#endif

// Compare the old and new subtrees.
IteratorComparison comparison = iterator_compare(&old_iter, &new_iter);

// Even if the two subtrees appear to be identical, they could differ
// internally if they contain a range of text that was previously
// excluded from the parse, and is now included, or vice-versa.
if (comparison == IteratorMatches && ts_range_array_intersects(
included_range_differences,
included_range_difference_index,
position.bytes,
iterator_end_position(&old_iter).bytes
)) {
comparison = IteratorMayDiffer;
}

bool is_changed = false;
switch (comparison) {
// If the subtrees are definitely identical, move to the end
// of both subtrees.
case IteratorMatches:
next_position = iterator_end_position(&old_iter);
break;

// If the subtrees might differ internally, descend into both
// subtrees, finding the first child that spans the current position.
case IteratorMayDiffer:
if (iterator_descend(&old_iter, position.bytes)) {
if (!iterator_descend(&new_iter, position.bytes)) {
is_changed = true;
next_position = iterator_end_position(&old_iter);
}
} else if (iterator_descend(&new_iter, position.bytes)) {
is_changed = true;
next_position = iterator_end_position(&new_iter);
} else {
next_position = length_min(
iterator_end_position(&old_iter),
iterator_end_position(&new_iter)
);
}
break;

// If the subtrees are different, record a change and then move
// to the end of both subtrees.
case IteratorDiffers:
is_changed = true;
next_position = length_min(
iterator_end_position(&old_iter),
iterator_end_position(&new_iter)
);
break;
}

// Ensure that both iterators are caught up to the current position.
while (
!iterator_done(&old_iter) &&
iterator_end_position(&old_iter).bytes <= next_position.bytes
) iterator_advance(&old_iter);
while (
!iterator_done(&new_iter) &&
iterator_end_position(&new_iter).bytes <= next_position.bytes
) iterator_advance(&new_iter);

// Ensure that both iterators are at the same depth in the tree.
while (old_iter.visible_depth > new_iter.visible_depth) {
iterator_ascend(&old_iter);
}
while (new_iter.visible_depth > old_iter.visible_depth) {
iterator_ascend(&new_iter);
}

if (is_changed) {
#ifdef DEBUG_GET_CHANGED_RANGES
printf(
" change: [[%u, %u] - [%u, %u]]\n",
position.extent.row + 1, position.extent.column,
next_position.extent.row + 1, next_position.extent.column
);
#endif

ts_range_array_add(&results, position, next_position);
}

position = next_position;

// Keep track of the current position in the included range differences
// array in order to avoid scanning the entire array on each iteration.
while (included_range_difference_index < included_range_differences->size) {
const TSRange *range = &included_range_differences->contents[
included_range_difference_index
];
if (range->end_byte <= position.bytes) {
included_range_difference_index++;
} else {
break;
}
}
} while (!iterator_done(&old_iter) && !iterator_done(&new_iter));

Length old_size = ts_subtree_total_size(*old_tree);
Length new_size = ts_subtree_total_size(*new_tree);
if (old_size.bytes < new_size.bytes) {
ts_range_array_add(&results, old_size, new_size);
} else if (new_size.bytes < old_size.bytes) {
ts_range_array_add(&results, new_size, old_size);
}

*cursor1 = old_iter.cursor;
*cursor2 = new_iter.cursor;
*ranges = results.contents;
return results.size;
}

+ 36
- 0
src/tree_sitter/get_changed_ranges.h View File

@@ -0,0 +1,36 @@
#ifndef TREE_SITTER_GET_CHANGED_RANGES_H_
#define TREE_SITTER_GET_CHANGED_RANGES_H_

#ifdef __cplusplus
extern "C" {
#endif

#include "./tree_cursor.h"
#include "./subtree.h"

typedef Array(TSRange) TSRangeArray;

void ts_range_array_get_changed_ranges(
const TSRange *old_ranges, unsigned old_range_count,
const TSRange *new_ranges, unsigned new_range_count,
TSRangeArray *differences
);

bool ts_range_array_intersects(
const TSRangeArray *self, unsigned start_index,
uint32_t start_byte, uint32_t end_byte
);

unsigned ts_subtree_get_changed_ranges(
const Subtree *old_tree, const Subtree *new_tree,
TreeCursor *cursor1, TreeCursor *cursor2,
const TSLanguage *language,
const TSRangeArray *included_range_differences,
TSRange **ranges
);

#ifdef __cplusplus
}
#endif

#endif // TREE_SITTER_GET_CHANGED_RANGES_H_

+ 107
- 0
src/tree_sitter/language.c View File

@@ -0,0 +1,107 @@
#include "./language.h"
#include "./subtree.h"
#include "./error_costs.h"
#include <string.h>

void ts_language_table_entry(const TSLanguage *self, TSStateId state,
TSSymbol symbol, TableEntry *result) {
if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) {
result->action_count = 0;
result->is_reusable = false;
result->actions = NULL;
} else {
assert(symbol < self->token_count);
uint32_t action_index = ts_language_lookup(self, state, symbol);
const TSParseActionEntry *entry = &self->parse_actions[action_index];
result->action_count = entry->count;
result->is_reusable = entry->reusable;
result->actions = (const TSParseAction *)(entry + 1);
}
}

uint32_t ts_language_symbol_count(const TSLanguage *language) {
return language->symbol_count + language->alias_count;
}

uint32_t ts_language_version(const TSLanguage *language) {
return language->version;
}

TSSymbolMetadata ts_language_symbol_metadata(const TSLanguage *language, TSSymbol symbol) {
if (symbol == ts_builtin_sym_error) {
return (TSSymbolMetadata){.visible = true, .named = true};
} else if (symbol == ts_builtin_sym_error_repeat) {
return (TSSymbolMetadata){.visible = false, .named = false};
} else {
return language->symbol_metadata[symbol];
}
}

const char *ts_language_symbol_name(const TSLanguage *language, TSSymbol symbol) {
if (symbol == ts_builtin_sym_error) {
return "ERROR";
} else if (symbol == ts_builtin_sym_error_repeat) {
return "_ERROR";
} else {
return language->symbol_names[symbol];
}
}

TSSymbol ts_language_symbol_for_name(const TSLanguage *self, const char *name) {
if (!strcmp(name, "ERROR")) return ts_builtin_sym_error;

uint32_t count = ts_language_symbol_count(self);
for (TSSymbol i = 0; i < count; i++) {
if (!strcmp(self->symbol_names[i], name)) {
return i;
}
}
return 0;
}

TSSymbolType ts_language_symbol_type(const TSLanguage *language, TSSymbol symbol) {
TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol);
if (metadata.named) {
return TSSymbolTypeRegular;
} else if (metadata.visible) {
return TSSymbolTypeAnonymous;
} else {
return TSSymbolTypeAuxiliary;
}
}

uint32_t ts_language_field_count(const TSLanguage *self) {
if (self->version >= TREE_SITTER_LANGUAGE_VERSION_WITH_FIELDS) {
return self->field_count;
} else {
return 0;
}
}

const char *ts_language_field_name_for_id(const TSLanguage *self, TSFieldId id) {
uint32_t count = ts_language_field_count(self);
if (count) {
return self->field_names[id];
} else {
return NULL;
}
}

TSFieldId ts_language_field_id_for_name(
const TSLanguage *self,
const char *name,
uint32_t name_length
) {
uint32_t count = ts_language_field_count(self);
for (TSSymbol i = 1; i < count + 1; i++) {
switch (strncmp(name, self->field_names[i], name_length)) {
case 0:
return i;
case -1:
return 0;
default:
break;
}
}
return 0;
}

+ 138
- 0
src/tree_sitter/language.h View File

@@ -0,0 +1,138 @@
#ifndef TREE_SITTER_LANGUAGE_H_
#define TREE_SITTER_LANGUAGE_H_

#ifdef __cplusplus
extern "C" {
#endif

#include "./subtree.h"
#include "tree_sitter/parser.h"

#define ts_builtin_sym_error_repeat (ts_builtin_sym_error - 1)
#define TREE_SITTER_LANGUAGE_VERSION_WITH_FIELDS 10
#define TREE_SITTER_LANGUAGE_VERSION_WITH_SMALL_STATES 11

typedef struct {
const TSParseAction *actions;
uint32_t action_count;
bool is_reusable;
} TableEntry;

void ts_language_table_entry(const TSLanguage *, TSStateId, TSSymbol, TableEntry *);

TSSymbolMetadata ts_language_symbol_metadata(const TSLanguage *, TSSymbol);

static inline bool ts_language_is_symbol_external(const TSLanguage *self, TSSymbol symbol) {
return 0 < symbol && symbol < self->external_token_count + 1;
}

static inline const TSParseAction *ts_language_actions(const TSLanguage *self,
TSStateId state,
TSSymbol symbol,
uint32_t *count) {
TableEntry entry;
ts_language_table_entry(self, state, symbol, &entry);
*count = entry.action_count;
return entry.actions;
}

static inline bool ts_language_has_actions(const TSLanguage *self,
TSStateId state,
TSSymbol symbol) {
TableEntry entry;
ts_language_table_entry(self, state, symbol, &entry);
return entry.action_count > 0;
}

static inline bool ts_language_has_reduce_action(const TSLanguage *self,
TSStateId state,
TSSymbol symbol) {
TableEntry entry;
ts_language_table_entry(self, state, symbol, &entry);
return entry.action_count > 0 && entry.actions[0].type == TSParseActionTypeReduce;
}

static inline uint16_t ts_language_lookup(
const TSLanguage *self,
TSStateId state,
TSSymbol symbol
) {
if (
self->version >= TREE_SITTER_LANGUAGE_VERSION_WITH_SMALL_STATES &&
state >= self->large_state_count
) {
uint32_t index = self->small_parse_table_map[state - self->large_state_count];
const uint16_t *data = &self->small_parse_table[index];
uint16_t section_count = *(data++);
for (unsigned i = 0; i < section_count; i++) {
uint16_t section_value = *(data++);
uint16_t symbol_count = *(data++);
for (unsigned i = 0; i < symbol_count; i++) {
if (*(data++) == symbol) return section_value;
}
}
return 0;
} else {
return self->parse_table[state * self->symbol_count + symbol];
}
}

static inline TSStateId ts_language_next_state(const TSLanguage *self,
TSStateId state,
TSSymbol symbol) {
if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) {
return 0;
} else if (symbol < self->token_count) {
uint32_t count;
const TSParseAction *actions = ts_language_actions(self, state, symbol, &count);
if (count > 0) {
TSParseAction action = actions[count - 1];
if (action.type == TSParseActionTypeShift || action.type == TSParseActionTypeRecover) {
return action.params.state;
}
}
return 0;
} else {
return ts_language_lookup(self, state, symbol);
}
}

static inline const bool *
ts_language_enabled_external_tokens(const TSLanguage *self,
unsigned external_scanner_state) {
if (external_scanner_state == 0) {
return NULL;
} else {
return self->external_scanner.states + self->external_token_count * external_scanner_state;
}
}

static inline const TSSymbol *
ts_language_alias_sequence(const TSLanguage *self, uint32_t production_id) {
return production_id > 0 ?
self->alias_sequences + production_id * self->max_alias_sequence_length :
NULL;
}

static inline void ts_language_field_map(
const TSLanguage *self,
uint32_t production_id,
const TSFieldMapEntry **start,
const TSFieldMapEntry **end
) {
if (self->version < TREE_SITTER_LANGUAGE_VERSION_WITH_FIELDS || self->field_count == 0) {
*start = NULL;
*end = NULL;
return;
}

TSFieldMapSlice slice = self->field_map_slices[production_id];
*start = &self->field_map_entries[slice.index];
*end = &self->field_map_entries[slice.index] + slice.length;
}

#ifdef __cplusplus
}
#endif

#endif // TREE_SITTER_LANGUAGE_H_

+ 44
- 0
src/tree_sitter/length.h View File

@@ -0,0 +1,44 @@
#ifndef TREE_SITTER_LENGTH_H_
#define TREE_SITTER_LENGTH_H_

#include <stdlib.h>
#include <stdbool.h>
#include "./point.h"
#include "tree_sitter/api.h"

typedef struct {
uint32_t bytes;
TSPoint extent;
} Length;

static const Length LENGTH_UNDEFINED = {0, {0, 1}};
static const Length LENGTH_MAX = {UINT32_MAX, {UINT32_MAX, UINT32_MAX}};

static inline bool length_is_undefined(Length length) {
return length.bytes == 0 && length.extent.column != 0;
}

static inline Length length_min(Length len1, Length len2) {
return (len1.bytes < len2.bytes) ? len1 : len2;
}

static inline Length length_add(Length len1, Length len2) {
Length result;
result.bytes = len1.bytes + len2.bytes;
result.extent = point_add(len1.extent, len2.extent);
return result;
}

static inline Length length_sub(Length len1, Length len2) {
Length result;
result.bytes = len1.bytes - len2.bytes;
result.extent = point_sub(len1.extent, len2.extent);
return result;
}

static inline Length length_zero(void) {
Length result = {0, {0, 0}};
return result;
}

#endif

+ 322
- 0
src/tree_sitter/lexer.c View File

@@ -0,0 +1,322 @@
#include <stdio.h>
#include "./lexer.h"
#include "./subtree.h"
#include "./length.h"
#include "./utf16.h"
#include "utf8proc.h"

#define LOG(...) \
if (self->logger.log) { \
snprintf(self->debug_buffer, TREE_SITTER_SERIALIZATION_BUFFER_SIZE, __VA_ARGS__); \
self->logger.log(self->logger.payload, TSLogTypeLex, self->debug_buffer); \
}

#define LOG_CHARACTER(message, character) \
LOG( \
32 <= character && character < 127 ? \
message " character:'%c'" : \
message " character:%d", character \
)

static const char empty_chunk[3] = { 0, 0 };

static const int32_t BYTE_ORDER_MARK = 0xFEFF;

static void ts_lexer__get_chunk(Lexer *self) {
self->chunk_start = self->current_position.bytes;
self->chunk = self->input.read(
self->input.payload,
self->current_position.bytes,
self->current_position.extent,
&self->chunk_size
);
if (!self->chunk_size) self->chunk = empty_chunk;
}

typedef utf8proc_ssize_t (*DecodeFunction)(
const utf8proc_uint8_t *,
utf8proc_ssize_t,
utf8proc_int32_t *
);

static void ts_lexer__get_lookahead(Lexer *self) {
uint32_t position_in_chunk = self->current_position.bytes - self->chunk_start;
const uint8_t *chunk = (const uint8_t *)self->chunk + position_in_chunk;
uint32_t size = self->chunk_size - position_in_chunk;

if (size == 0) {
self->lookahead_size = 1;
self->data.lookahead = '\0';
return;
}

DecodeFunction decode =
self->input.encoding == TSInputEncodingUTF8 ? utf8proc_iterate : utf16_iterate;

self->lookahead_size = decode(chunk, size, &self->data.lookahead);

// If this chunk ended in the middle of a multi-byte character,
// try again with a fresh chunk.
if (self->data.lookahead == -1 && size < 4) {
ts_lexer__get_chunk(self);
chunk = (const uint8_t *)self->chunk;
size = self->chunk_size;
self->lookahead_size = decode(chunk, size, &self->data.lookahead);
}

if (self->data.lookahead == -1) {
self->lookahead_size = 1;
}
}

static void ts_lexer__advance(TSLexer *payload, bool skip) {
Lexer *self = (Lexer *)payload;
if (self->chunk == empty_chunk)
return;

if (self->lookahead_size) {
self->current_position.bytes += self->lookahead_size;
if (self->data.lookahead == '\n') {
self->current_position.extent.row++;
self->current_position.extent.column = 0;
} else {
self->current_position.extent.column += self->lookahead_size;
}
}

TSRange *current_range = &self->included_ranges[self->current_included_range_index];
if (self->current_position.bytes == current_range->end_byte) {
self->current_included_range_index++;
if (self->current_included_range_index == self->included_range_count) {
self->data.lookahead = '\0';
self->lookahead_size = 1;
return;
} else {
current_range++;
self->current_position = (Length) {
current_range->start_byte,
current_range->start_point,
};
}
}

if (skip) {
LOG_CHARACTER("skip", self->data.lookahead);
self->token_start_position = self->current_position;
} else {
LOG_CHARACTER("consume", self->data.lookahead);
}

if (self->current_position.bytes >= self->chunk_start + self->chunk_size) {
ts_lexer__get_chunk(self);
}

ts_lexer__get_lookahead(self);
}

static void ts_lexer__mark_end(TSLexer *payload) {
Lexer *self = (Lexer *)payload;
TSRange *current_included_range = &self->included_ranges[self->current_included_range_index];
if (self->current_included_range_index > 0 &&
self->current_position.bytes == current_included_range->start_byte) {
TSRange *previous_included_range = current_included_range - 1;
self->token_end_position = (Length) {
previous_included_range->end_byte,
previous_included_range->end_point,
};
} else {
self->token_end_position = self->current_position;
}
}

static uint32_t ts_lexer__get_column(TSLexer *payload) {
Lexer *self = (Lexer *)payload;
uint32_t goal_byte = self->current_position.bytes;

self->current_position.bytes -= self->current_position.extent.column;
self->current_position.extent.column = 0;

if (self->current_position.bytes < self->chunk_start) {
ts_lexer__get_chunk(self);
}

uint32_t result = 0;
while (self->current_position.bytes < goal_byte) {
ts_lexer__advance(payload, false);
result++;
}

return result;
}

static bool ts_lexer__is_at_included_range_start(TSLexer *payload) {
const Lexer *self = (const Lexer *)payload;
TSRange *current_range = &self->included_ranges[self->current_included_range_index];
return self->current_position.bytes == current_range->start_byte;
}

// The lexer's methods are stored as a struct field so that generated
// parsers can call them without needing to be linked against this library.

void ts_lexer_init(Lexer *self) {
*self = (Lexer) {
.data = {
.advance = ts_lexer__advance,
.mark_end = ts_lexer__mark_end,
.get_column = ts_lexer__get_column,
.is_at_included_range_start = ts_lexer__is_at_included_range_start,
.lookahead = 0,
.result_symbol = 0,
},
.chunk = NULL,
.chunk_start = 0,
.current_position = {UINT32_MAX, {0, 0}},
.logger = {
.payload = NULL,
.log = NULL
},
.current_included_range_index = 0,
};

self->included_ranges = NULL;
ts_lexer_set_included_ranges(self, NULL, 0);
ts_lexer_reset(self, length_zero());
}

void ts_lexer_delete(Lexer *self) {
ts_free(self->included_ranges);
}

void ts_lexer_set_input(Lexer *self, TSInput input) {
self->input = input;
self->data.lookahead = 0;
self->lookahead_size = 0;
self->chunk = 0;
self->chunk_start = 0;
self->chunk_size = 0;
}

static void ts_lexer_goto(Lexer *self, Length position) {
bool found_included_range = false;
for (unsigned i = 0; i < self->included_range_count; i++) {
TSRange *included_range = &self->included_ranges[i];
if (included_range->end_byte > position.bytes) {
if (included_range->start_byte > position.bytes) {
position = (Length) {
.bytes = included_range->start_byte,
.extent = included_range->start_point,
};
}

self->current_included_range_index = i;
found_included_range = true;
break;
}
}

if (!found_included_range) {
TSRange *last_included_range = &self->included_ranges[self->included_range_count - 1];
position = (Length) {
.bytes = last_included_range->end_byte,
.extent = last_included_range->end_point,
};
self->chunk = empty_chunk;
self->chunk_start = position.bytes;
self->chunk_size = 2;
}

self->token_start_position = position;
self->token_end_position = LENGTH_UNDEFINED;
self->current_position = position;

if (self->chunk && (position.bytes < self->chunk_start ||
position.bytes >= self->chu