Atom/src/tree-sitter-language-mode.js

1511 lines
42 KiB
JavaScript

const Parser = require('tree-sitter');
const { Point, Range, spliceArray } = require('text-buffer');
const { Patch } = require('superstring');
const { Emitter } = require('event-kit');
const ScopeDescriptor = require('./scope-descriptor');
const Token = require('./token');
const TokenizedLine = require('./tokenized-line');
const TextMateLanguageMode = require('./text-mate-language-mode');
const { matcherForSelector } = require('./selectors');
let nextId = 0;
const MAX_RANGE = new Range(Point.ZERO, Point.INFINITY).freeze();
const PARSER_POOL = [];
const WORD_REGEX = /\w/;
class TreeSitterLanguageMode {
static _patchSyntaxNode() {
if (!Parser.SyntaxNode.prototype.hasOwnProperty('range')) {
Object.defineProperty(Parser.SyntaxNode.prototype, 'range', {
get() {
return rangeForNode(this);
}
});
}
}
constructor({ buffer, grammar, config, grammars, syncTimeoutMicros }) {
TreeSitterLanguageMode._patchSyntaxNode();
this.id = nextId++;
this.buffer = buffer;
this.grammar = grammar;
this.config = config;
this.grammarRegistry = grammars;
this.parser = new Parser();
this.rootLanguageLayer = new LanguageLayer(this, grammar, 0);
this.injectionsMarkerLayer = buffer.addMarkerLayer();
if (syncTimeoutMicros != null) {
this.syncTimeoutMicros = syncTimeoutMicros;
}
this.rootScopeDescriptor = new ScopeDescriptor({
scopes: [this.grammar.scopeName]
});
this.emitter = new Emitter();
this.isFoldableCache = [];
this.hasQueuedParse = false;
this.grammarForLanguageString = this.grammarForLanguageString.bind(this);
this.rootLanguageLayer
.update(null)
.then(() => this.emitter.emit('did-tokenize'));
// TODO: Remove this once TreeSitterLanguageMode implements its own auto-indentation system. This
// is temporarily needed in order to delegate to the TextMateLanguageMode's auto-indent system.
this.regexesByPattern = {};
}
async parseCompletePromise() {
let done = false;
while (!done) {
if (this.rootLanguageLayer.currentParsePromise) {
await this.rootLanguageLayer.currentParsePromises;
} else {
done = true;
for (const marker of this.injectionsMarkerLayer.getMarkers()) {
if (marker.languageLayer.currentParsePromise) {
done = false;
await marker.languageLayer.currentParsePromise;
break;
}
}
}
await new Promise(resolve => setTimeout(resolve, 0));
}
}
destroy() {
this.injectionsMarkerLayer.destroy();
this.rootLanguageLayer = null;
this.parser = null;
}
getLanguageId() {
return this.grammar.scopeName;
}
bufferDidChange({ oldRange, newRange, oldText, newText }) {
const edit = this.rootLanguageLayer._treeEditForBufferChange(
oldRange.start,
oldRange.end,
newRange.end,
oldText,
newText
);
this.rootLanguageLayer.handleTextChange(edit, oldText, newText);
for (const marker of this.injectionsMarkerLayer.getMarkers()) {
marker.languageLayer.handleTextChange(edit, oldText, newText);
}
}
bufferDidFinishTransaction({ changes }) {
for (let i = 0, { length } = changes; i < length; i++) {
const { oldRange, newRange } = changes[i];
spliceArray(
this.isFoldableCache,
newRange.start.row,
oldRange.end.row - oldRange.start.row,
{ length: newRange.end.row - newRange.start.row }
);
}
this.rootLanguageLayer.update(null);
}
parse(language, oldTree, ranges) {
const parser = PARSER_POOL.pop() || new Parser();
parser.setLanguage(language);
const result = parser.parseTextBuffer(this.buffer.buffer, oldTree, {
syncTimeoutMicros: this.syncTimeoutMicros,
includedRanges: ranges
});
if (result.then) {
return result.then(tree => {
PARSER_POOL.push(parser);
return tree;
});
} else {
PARSER_POOL.push(parser);
return result;
}
}
get tree() {
return this.rootLanguageLayer.tree;
}
updateForInjection(grammar) {
this.rootLanguageLayer.updateInjections(grammar);
}
/*
Section - Highlighting
*/
buildHighlightIterator() {
if (!this.rootLanguageLayer) return new NullHighlightIterator();
return new HighlightIterator(this);
}
onDidTokenize(callback) {
return this.emitter.on('did-tokenize', callback);
}
onDidChangeHighlighting(callback) {
return this.emitter.on('did-change-highlighting', callback);
}
classNameForScopeId(scopeId) {
return this.grammar.classNameForScopeId(scopeId);
}
/*
Section - Commenting
*/
commentStringsForPosition(position) {
const range =
this.firstNonWhitespaceRange(position.row) ||
new Range(position, position);
const { grammar } = this.getSyntaxNodeAndGrammarContainingRange(range);
return grammar.commentStrings;
}
isRowCommented(row) {
const range = this.firstNonWhitespaceRange(row);
if (range) {
const firstNode = this.getSyntaxNodeContainingRange(range);
if (firstNode) return firstNode.type.includes('comment');
}
return false;
}
/*
Section - Indentation
*/
suggestedIndentForLineAtBufferRow(row, line, tabLength) {
return this._suggestedIndentForLineWithScopeAtBufferRow(
row,
line,
this.rootScopeDescriptor,
tabLength
);
}
suggestedIndentForBufferRow(row, tabLength, options) {
return this._suggestedIndentForLineWithScopeAtBufferRow(
row,
this.buffer.lineForRow(row),
this.rootScopeDescriptor,
tabLength,
options
);
}
indentLevelForLine(line, tabLength) {
let indentLength = 0;
for (let i = 0, { length } = line; i < length; i++) {
const char = line[i];
if (char === '\t') {
indentLength += tabLength - (indentLength % tabLength);
} else if (char === ' ') {
indentLength++;
} else {
break;
}
}
return indentLength / tabLength;
}
/*
Section - Folding
*/
isFoldableAtRow(row) {
if (this.isFoldableCache[row] != null) return this.isFoldableCache[row];
const result =
this.getFoldableRangeContainingPoint(Point(row, Infinity), 0, true) !=
null;
this.isFoldableCache[row] = result;
return result;
}
getFoldableRanges() {
return this.getFoldableRangesAtIndentLevel(null);
}
/**
* TODO: Make this method generate folds for nested languages (currently,
* folds are only generated for the root language layer).
*/
getFoldableRangesAtIndentLevel(goalLevel) {
let result = [];
let stack = [{ node: this.tree.rootNode, level: 0 }];
while (stack.length > 0) {
const { node, level } = stack.pop();
const range = this.getFoldableRangeForNode(node, this.grammar);
if (range) {
if (goalLevel == null || level === goalLevel) {
let updatedExistingRange = false;
for (let i = 0, { length } = result; i < length; i++) {
if (
result[i].start.row === range.start.row &&
result[i].end.row === range.end.row
) {
result[i] = range;
updatedExistingRange = true;
break;
}
}
if (!updatedExistingRange) result.push(range);
}
}
const parentStartRow = node.startPosition.row;
const parentEndRow = node.endPosition.row;
for (
let children = node.namedChildren, i = 0, { length } = children;
i < length;
i++
) {
const child = children[i];
const { startPosition: childStart, endPosition: childEnd } = child;
if (childEnd.row > childStart.row) {
if (
childStart.row === parentStartRow &&
childEnd.row === parentEndRow
) {
stack.push({ node: child, level: level });
} else {
const childLevel =
range &&
range.containsPoint(childStart) &&
range.containsPoint(childEnd)
? level + 1
: level;
if (childLevel <= goalLevel || goalLevel == null) {
stack.push({ node: child, level: childLevel });
}
}
}
}
}
return result.sort((a, b) => a.start.row - b.start.row);
}
getFoldableRangeContainingPoint(point, tabLength, existenceOnly = false) {
if (!this.tree) return null;
let smallestRange;
this._forEachTreeWithRange(new Range(point, point), (tree, grammar) => {
let node = tree.rootNode.descendantForPosition(
this.buffer.clipPosition(point)
);
while (node) {
if (existenceOnly && node.startPosition.row < point.row) return;
if (node.endPosition.row > point.row) {
const range = this.getFoldableRangeForNode(node, grammar);
if (range && rangeIsSmaller(range, smallestRange)) {
smallestRange = range;
return;
}
}
node = node.parent;
}
});
return existenceOnly
? smallestRange && smallestRange.start.row === point.row
: smallestRange;
}
_forEachTreeWithRange(range, callback) {
if (this.rootLanguageLayer.tree) {
callback(this.rootLanguageLayer.tree, this.rootLanguageLayer.grammar);
}
const injectionMarkers = this.injectionsMarkerLayer.findMarkers({
intersectsRange: range
});
for (const injectionMarker of injectionMarkers) {
const { tree, grammar } = injectionMarker.languageLayer;
if (tree) callback(tree, grammar);
}
}
getFoldableRangeForNode(node, grammar, existenceOnly) {
const { children } = node;
const childCount = children.length;
for (var i = 0, { length } = grammar.folds; i < length; i++) {
const foldSpec = grammar.folds[i];
if (foldSpec.matchers && !hasMatchingFoldSpec(foldSpec.matchers, node))
continue;
let foldStart;
const startEntry = foldSpec.start;
if (startEntry) {
let foldStartNode;
if (startEntry.index != null) {
foldStartNode = children[startEntry.index];
if (
!foldStartNode ||
(startEntry.matchers &&
!hasMatchingFoldSpec(startEntry.matchers, foldStartNode))
)
continue;
} else {
foldStartNode = children.find(child =>
hasMatchingFoldSpec(startEntry.matchers, child)
);
if (!foldStartNode) continue;
}
foldStart = new Point(foldStartNode.endPosition.row, Infinity);
} else {
foldStart = new Point(node.startPosition.row, Infinity);
}
let foldEnd;
const endEntry = foldSpec.end;
if (endEntry) {
let foldEndNode;
if (endEntry.index != null) {
const index =
endEntry.index < 0 ? childCount + endEntry.index : endEntry.index;
foldEndNode = children[index];
if (
!foldEndNode ||
(endEntry.type && endEntry.type !== foldEndNode.type)
)
continue;
} else {
foldEndNode = children.find(child =>
hasMatchingFoldSpec(endEntry.matchers, child)
);
if (!foldEndNode) continue;
}
if (foldEndNode.startPosition.row <= foldStart.row) continue;
foldEnd = foldEndNode.startPosition;
if (
this.buffer.findInRangeSync(
WORD_REGEX,
new Range(foldEnd, new Point(foldEnd.row, Infinity))
)
) {
foldEnd = new Point(foldEnd.row - 1, Infinity);
}
} else {
const { endPosition } = node;
if (endPosition.column === 0) {
foldEnd = Point(endPosition.row - 1, Infinity);
} else if (childCount > 0) {
foldEnd = endPosition;
} else {
foldEnd = Point(endPosition.row, 0);
}
}
return existenceOnly ? true : new Range(foldStart, foldEnd);
}
}
/*
Section - Syntax Tree APIs
*/
getSyntaxNodeContainingRange(range, where = _ => true) {
return this.getSyntaxNodeAndGrammarContainingRange(range, where).node;
}
getSyntaxNodeAndGrammarContainingRange(range, where = _ => true) {
const startIndex = this.buffer.characterIndexForPosition(range.start);
const endIndex = this.buffer.characterIndexForPosition(range.end);
const searchEndIndex = Math.max(0, endIndex - 1);
let smallestNode = null;
let smallestNodeGrammar = this.grammar;
this._forEachTreeWithRange(range, (tree, grammar) => {
let node = tree.rootNode.descendantForIndex(startIndex, searchEndIndex);
while (node) {
if (
nodeContainsIndices(node, startIndex, endIndex) &&
where(node, grammar)
) {
if (nodeIsSmaller(node, smallestNode)) {
smallestNode = node;
smallestNodeGrammar = grammar;
}
break;
}
node = node.parent;
}
});
return { node: smallestNode, grammar: smallestNodeGrammar };
}
getRangeForSyntaxNodeContainingRange(range, where) {
const node = this.getSyntaxNodeContainingRange(range, where);
return node && node.range;
}
getSyntaxNodeAtPosition(position, where) {
return this.getSyntaxNodeContainingRange(
new Range(position, position),
where
);
}
bufferRangeForScopeAtPosition(selector, position) {
const nodeCursorAdapter = new NodeCursorAdaptor();
if (typeof selector === 'string') {
const match = matcherForSelector(selector);
selector = (node, grammar) => {
const rules = grammar.scopeMap.get([node.type], [0], node.named);
nodeCursorAdapter.node = node;
const scopeName = applyLeafRules(rules, nodeCursorAdapter);
if (scopeName != null) {
return match(scopeName);
}
};
}
if (selector === null) selector = undefined;
const node = this.getSyntaxNodeAtPosition(position, selector);
return node && node.range;
}
/*
Section - Backward compatibility shims
*/
tokenizedLineForRow(row) {
const lineText = this.buffer.lineForRow(row);
const tokens = [];
const iterator = this.buildHighlightIterator();
let start = { row, column: 0 };
const scopes = iterator.seek(start, row);
while (true) {
const end = iterator.getPosition();
if (end.row > row) {
end.row = row;
end.column = lineText.length;
}
if (end.column > start.column) {
tokens.push(
new Token({
value: lineText.substring(start.column, end.column),
scopes: scopes.map(s => this.grammar.scopeNameForScopeId(s))
})
);
}
if (end.column < lineText.length) {
const closeScopeCount = iterator.getCloseScopeIds().length;
for (let i = 0; i < closeScopeCount; i++) {
scopes.pop();
}
scopes.push(...iterator.getOpenScopeIds());
start = end;
iterator.moveToSuccessor();
} else {
break;
}
}
return new TokenizedLine({
openScopes: [],
text: lineText,
tokens,
tags: [],
ruleStack: [],
lineEnding: this.buffer.lineEndingForRow(row),
tokenIterator: null,
grammar: this.grammar
});
}
syntaxTreeScopeDescriptorForPosition(point) {
const nodes = [];
point = this.buffer.clipPosition(Point.fromObject(point));
// If the position is the end of a line, get node of left character instead of newline
// This is to match TextMate behaviour, see https://github.com/atom/atom/issues/18463
if (
point.column > 0 &&
point.column === this.buffer.lineLengthForRow(point.row)
) {
point = point.copy();
point.column--;
}
this._forEachTreeWithRange(new Range(point, point), tree => {
let node = tree.rootNode.descendantForPosition(point);
while (node) {
nodes.push(node);
node = node.parent;
}
});
// The nodes are mostly already sorted from smallest to largest,
// but for files with multiple syntax trees (e.g. ERB), each tree's
// nodes are separate. Sort the nodes from largest to smallest.
nodes.reverse();
nodes.sort(
(a, b) => a.startIndex - b.startIndex || b.endIndex - a.endIndex
);
const nodeTypes = nodes.map(node => node.type);
nodeTypes.unshift(this.grammar.scopeName);
return new ScopeDescriptor({ scopes: nodeTypes });
}
scopeDescriptorForPosition(point) {
point = this.buffer.clipPosition(Point.fromObject(point));
// If the position is the end of a line, get scope of left character instead of newline
// This is to match TextMate behaviour, see https://github.com/atom/atom/issues/18463
if (
point.column > 0 &&
point.column === this.buffer.lineLengthForRow(point.row)
) {
point = point.copy();
point.column--;
}
const iterator = this.buildHighlightIterator();
const scopes = [];
for (const scope of iterator.seek(point, point.row + 1)) {
scopes.push(this.grammar.scopeNameForScopeId(scope));
}
if (point.isEqual(iterator.getPosition())) {
for (const scope of iterator.getOpenScopeIds()) {
scopes.push(this.grammar.scopeNameForScopeId(scope));
}
}
if (scopes.length === 0 || scopes[0] !== this.grammar.scopeName) {
scopes.unshift(this.grammar.scopeName);
}
return new ScopeDescriptor({ scopes });
}
tokenForPosition(point) {
const node = this.getSyntaxNodeAtPosition(point);
const scopes = this.scopeDescriptorForPosition(point).getScopesArray();
return new Token({ value: node.text, scopes });
}
getGrammar() {
return this.grammar;
}
/*
Section - Private
*/
firstNonWhitespaceRange(row) {
return this.buffer.findInRangeSync(
/\S/,
new Range(new Point(row, 0), new Point(row, Infinity))
);
}
grammarForLanguageString(languageString) {
return this.grammarRegistry.treeSitterGrammarForLanguageString(
languageString
);
}
emitRangeUpdate(range) {
const startRow = range.start.row;
const endRow = range.end.row;
for (let row = startRow; row < endRow; row++) {
this.isFoldableCache[row] = undefined;
}
this.emitter.emit('did-change-highlighting', range);
}
}
class LanguageLayer {
constructor(languageMode, grammar, depth) {
this.languageMode = languageMode;
this.grammar = grammar;
this.tree = null;
this.currentParsePromise = null;
this.patchSinceCurrentParseStarted = null;
this.depth = depth;
}
buildHighlightIterator() {
if (this.tree) {
return new LayerHighlightIterator(this, this.tree.walk());
} else {
return new NullHighlightIterator();
}
}
handleTextChange(edit, oldText, newText) {
const { startPosition, oldEndPosition, newEndPosition } = edit;
if (this.tree) {
this.tree.edit(edit);
if (this.editedRange) {
if (startPosition.isLessThan(this.editedRange.start)) {
this.editedRange.start = startPosition;
}
if (oldEndPosition.isLessThan(this.editedRange.end)) {
this.editedRange.end = newEndPosition.traverse(
this.editedRange.end.traversalFrom(oldEndPosition)
);
} else {
this.editedRange.end = newEndPosition;
}
} else {
this.editedRange = new Range(startPosition, newEndPosition);
}
}
if (this.patchSinceCurrentParseStarted) {
this.patchSinceCurrentParseStarted.splice(
startPosition,
oldEndPosition.traversalFrom(startPosition),
newEndPosition.traversalFrom(startPosition),
oldText,
newText
);
}
}
destroy() {
for (const marker of this.languageMode.injectionsMarkerLayer.getMarkers()) {
if (marker.parentLanguageLayer === this) {
marker.languageLayer.destroy();
marker.destroy();
}
}
}
async update(nodeRangeSet) {
if (!this.currentParsePromise) {
while (
!this.destroyed &&
(!this.tree || this.tree.rootNode.hasChanges())
) {
const params = { async: false };
this.currentParsePromise = this._performUpdate(nodeRangeSet, params);
if (!params.async) break;
await this.currentParsePromise;
}
this.currentParsePromise = null;
}
}
updateInjections(grammar) {
if (grammar.injectionRegex) {
if (!this.currentParsePromise)
this.currentParsePromise = Promise.resolve();
this.currentParsePromise = this.currentParsePromise.then(async () => {
await this._populateInjections(MAX_RANGE, null);
this.currentParsePromise = null;
});
}
}
async _performUpdate(nodeRangeSet, params) {
let includedRanges = null;
if (nodeRangeSet) {
includedRanges = nodeRangeSet.getRanges(this.languageMode.buffer);
if (includedRanges.length === 0) {
this.tree = null;
this.destroyed = true;
return;
}
}
let affectedRange = this.editedRange;
this.editedRange = null;
this.patchSinceCurrentParseStarted = new Patch();
let tree = this.languageMode.parse(
this.grammar.languageModule,
this.tree,
includedRanges
);
if (tree.then) {
params.async = true;
tree = await tree;
}
const changes = this.patchSinceCurrentParseStarted.getChanges();
this.patchSinceCurrentParseStarted = null;
for (const {
oldStart,
newStart,
oldEnd,
newEnd,
oldText,
newText
} of changes) {
const newExtent = Point.fromObject(newEnd).traversalFrom(newStart);
tree.edit(
this._treeEditForBufferChange(
newStart,
oldEnd,
Point.fromObject(oldStart).traverse(newExtent),
oldText,
newText
)
);
}
if (this.tree) {
const rangesWithSyntaxChanges = this.tree.getChangedRanges(tree);
this.tree = tree;
if (rangesWithSyntaxChanges.length > 0) {
for (const range of rangesWithSyntaxChanges) {
this.languageMode.emitRangeUpdate(rangeForNode(range));
}
const combinedRangeWithSyntaxChange = new Range(
rangesWithSyntaxChanges[0].startPosition,
last(rangesWithSyntaxChanges).endPosition
);
if (affectedRange) {
this.languageMode.emitRangeUpdate(affectedRange);
affectedRange = affectedRange.union(combinedRangeWithSyntaxChange);
} else {
affectedRange = combinedRangeWithSyntaxChange;
}
}
} else {
this.tree = tree;
this.languageMode.emitRangeUpdate(rangeForNode(tree.rootNode));
if (includedRanges) {
affectedRange = new Range(
includedRanges[0].startPosition,
last(includedRanges).endPosition
);
} else {
affectedRange = MAX_RANGE;
}
}
if (affectedRange) {
const injectionPromise = this._populateInjections(
affectedRange,
nodeRangeSet
);
if (injectionPromise) {
params.async = true;
return injectionPromise;
}
}
}
_populateInjections(range, nodeRangeSet) {
const existingInjectionMarkers = this.languageMode.injectionsMarkerLayer
.findMarkers({ intersectsRange: range })
.filter(marker => marker.parentLanguageLayer === this);
if (existingInjectionMarkers.length > 0) {
range = range.union(
new Range(
existingInjectionMarkers[0].getRange().start,
last(existingInjectionMarkers).getRange().end
)
);
}
const markersToUpdate = new Map();
const nodes = this.tree.rootNode.descendantsOfType(
Object.keys(this.grammar.injectionPointsByType),
range.start,
range.end
);
let existingInjectionMarkerIndex = 0;
for (const node of nodes) {
for (const injectionPoint of this.grammar.injectionPointsByType[
node.type
]) {
const languageName = injectionPoint.language(node);
if (!languageName) continue;
const grammar = this.languageMode.grammarForLanguageString(
languageName
);
if (!grammar) continue;
const contentNodes = injectionPoint.content(node);
if (!contentNodes) continue;
const injectionNodes = [].concat(contentNodes);
if (!injectionNodes.length) continue;
const injectionRange = rangeForNode(node);
let marker;
for (
let i = existingInjectionMarkerIndex,
n = existingInjectionMarkers.length;
i < n;
i++
) {
const existingMarker = existingInjectionMarkers[i];
const comparison = existingMarker.getRange().compare(injectionRange);
if (comparison > 0) {
break;
} else if (comparison === 0) {
existingInjectionMarkerIndex = i;
if (existingMarker.languageLayer.grammar === grammar) {
marker = existingMarker;
break;
}
} else {
existingInjectionMarkerIndex = i;
}
}
if (!marker) {
marker = this.languageMode.injectionsMarkerLayer.markRange(
injectionRange
);
marker.languageLayer = new LanguageLayer(
this.languageMode,
grammar,
this.depth + 1
);
marker.parentLanguageLayer = this;
}
markersToUpdate.set(
marker,
new NodeRangeSet(
nodeRangeSet,
injectionNodes,
injectionPoint.newlinesBetween,
injectionPoint.includeChildren
)
);
}
}
for (const marker of existingInjectionMarkers) {
if (!markersToUpdate.has(marker)) {
marker.languageLayer.destroy();
this.languageMode.emitRangeUpdate(marker.getRange());
marker.destroy();
}
}
if (markersToUpdate.size > 0) {
const promises = [];
for (const [marker, nodeRangeSet] of markersToUpdate) {
promises.push(marker.languageLayer.update(nodeRangeSet));
}
return Promise.all(promises);
}
}
_treeEditForBufferChange(start, oldEnd, newEnd, oldText, newText) {
const startIndex = this.languageMode.buffer.characterIndexForPosition(
start
);
return {
startIndex,
oldEndIndex: startIndex + oldText.length,
newEndIndex: startIndex + newText.length,
startPosition: start,
oldEndPosition: oldEnd,
newEndPosition: newEnd
};
}
}
class HighlightIterator {
constructor(languageMode) {
this.languageMode = languageMode;
this.iterators = null;
}
seek(targetPosition, endRow) {
const injectionMarkers = this.languageMode.injectionsMarkerLayer.findMarkers(
{
intersectsRange: new Range(targetPosition, new Point(endRow + 1, 0))
}
);
const containingTags = [];
const containingTagStartIndices = [];
const targetIndex = this.languageMode.buffer.characterIndexForPosition(
targetPosition
);
this.iterators = [];
const iterator = this.languageMode.rootLanguageLayer.buildHighlightIterator();
if (iterator.seek(targetIndex, containingTags, containingTagStartIndices)) {
this.iterators.push(iterator);
}
// Populate the iterators array with all of the iterators whose syntax
// trees span the given position.
for (const marker of injectionMarkers) {
const iterator = marker.languageLayer.buildHighlightIterator();
if (
iterator.seek(targetIndex, containingTags, containingTagStartIndices)
) {
this.iterators.push(iterator);
}
}
// Sort the iterators so that the last one in the array is the earliest
// in the document, and represents the current position.
this.iterators.sort((a, b) => b.compare(a));
this.detectCoveredScope();
return containingTags;
}
moveToSuccessor() {
// Advance the earliest layer iterator to its next scope boundary.
let leader = last(this.iterators);
// Maintain the sorting of the iterators by their position in the document.
if (leader.moveToSuccessor()) {
const leaderIndex = this.iterators.length - 1;
let i = leaderIndex;
while (i > 0 && this.iterators[i - 1].compare(leader) < 0) i--;
if (i < leaderIndex) {
this.iterators.splice(i, 0, this.iterators.pop());
}
} else {
// If the layer iterator was at the end of its syntax tree, then remove
// it from the array.
this.iterators.pop();
}
this.detectCoveredScope();
}
// Detect whether or not another more deeply-nested language layer has a
// scope boundary at this same position. If so, the current language layer's
// scope boundary should not be reported.
detectCoveredScope() {
const layerCount = this.iterators.length;
if (layerCount > 1) {
const first = this.iterators[layerCount - 1];
const next = this.iterators[layerCount - 2];
if (
next.offset === first.offset &&
next.atEnd === first.atEnd &&
next.depth > first.depth
) {
this.currentScopeIsCovered = true;
return;
}
}
this.currentScopeIsCovered = false;
}
getPosition() {
const iterator = last(this.iterators);
if (iterator) {
return iterator.getPosition();
} else {
return Point.INFINITY;
}
}
getCloseScopeIds() {
const iterator = last(this.iterators);
if (iterator && !this.currentScopeIsCovered) {
return iterator.getCloseScopeIds();
}
return [];
}
getOpenScopeIds() {
const iterator = last(this.iterators);
if (iterator && !this.currentScopeIsCovered) {
return iterator.getOpenScopeIds();
}
return [];
}
logState() {
const iterator = last(this.iterators);
if (iterator && iterator.treeCursor) {
console.log(
iterator.getPosition(),
iterator.treeCursor.nodeType,
`depth=${iterator.languageLayer.depth}`,
new Range(
iterator.languageLayer.tree.rootNode.startPosition,
iterator.languageLayer.tree.rootNode.endPosition
).toString()
);
if (this.currentScopeIsCovered) {
console.log('covered');
} else {
console.log(
'close',
iterator.closeTags.map(id =>
this.languageMode.grammar.scopeNameForScopeId(id)
)
);
console.log(
'open',
iterator.openTags.map(id =>
this.languageMode.grammar.scopeNameForScopeId(id)
)
);
}
}
}
}
class LayerHighlightIterator {
constructor(languageLayer, treeCursor) {
this.languageLayer = languageLayer;
this.depth = this.languageLayer.depth;
// The iterator is always positioned at either the start or the end of some node
// in the syntax tree.
this.atEnd = false;
this.treeCursor = treeCursor;
this.offset = 0;
// In order to determine which selectors match its current node, the iterator maintains
// a list of the current node's ancestors. Because the selectors can use the `:nth-child`
// pseudo-class, each node's child index is also stored.
this.containingNodeTypes = [];
this.containingNodeChildIndices = [];
this.containingNodeEndIndices = [];
// At any given position, the iterator exposes the list of class names that should be
// *ended* at its current position and the list of class names that should be *started*
// at its current position.
this.closeTags = [];
this.openTags = [];
}
seek(targetIndex, containingTags, containingTagStartIndices) {
while (this.treeCursor.gotoParent()) {}
this.atEnd = true;
this.closeTags.length = 0;
this.openTags.length = 0;
this.containingNodeTypes.length = 0;
this.containingNodeChildIndices.length = 0;
this.containingNodeEndIndices.length = 0;
const containingTagEndIndices = [];
if (targetIndex >= this.treeCursor.endIndex) {
return false;
}
let childIndex = -1;
for (;;) {
this.containingNodeTypes.push(this.treeCursor.nodeType);
this.containingNodeChildIndices.push(childIndex);
this.containingNodeEndIndices.push(this.treeCursor.endIndex);
const scopeId = this._currentScopeId();
if (scopeId) {
if (this.treeCursor.startIndex < targetIndex) {
insertContainingTag(
scopeId,
this.treeCursor.startIndex,
containingTags,
containingTagStartIndices
);
containingTagEndIndices.push(this.treeCursor.endIndex);
} else {
this.atEnd = false;
this.openTags.push(scopeId);
this._moveDown();
break;
}
}
childIndex = this.treeCursor.gotoFirstChildForIndex(targetIndex);
if (childIndex === null) break;
if (this.treeCursor.startIndex >= targetIndex) this.atEnd = false;
}
if (this.atEnd) {
this.offset = this.treeCursor.endIndex;
for (let i = 0, { length } = containingTags; i < length; i++) {
if (containingTagEndIndices[i] === this.offset) {
this.closeTags.push(containingTags[i]);
}
}
} else {
this.offset = this.treeCursor.startIndex;
}
return true;
}
moveToSuccessor() {
this.closeTags.length = 0;
this.openTags.length = 0;
while (!this.closeTags.length && !this.openTags.length) {
if (this.atEnd) {
if (this._moveRight()) {
const scopeId = this._currentScopeId();
if (scopeId) this.openTags.push(scopeId);
this.atEnd = false;
this._moveDown();
} else if (this._moveUp(true)) {
this.atEnd = true;
} else {
return false;
}
} else if (!this._moveDown()) {
const scopeId = this._currentScopeId();
if (scopeId) this.closeTags.push(scopeId);
this.atEnd = true;
this._moveUp(false);
}
}
if (this.atEnd) {
this.offset = this.treeCursor.endIndex;
} else {
this.offset = this.treeCursor.startIndex;
}
return true;
}
getPosition() {
if (this.atEnd) {
return this.treeCursor.endPosition;
} else {
return this.treeCursor.startPosition;
}
}
compare(other) {
const result = this.offset - other.offset;
if (result !== 0) return result;
if (this.atEnd && !other.atEnd) return -1;
if (other.atEnd && !this.atEnd) return 1;
return this.languageLayer.depth - other.languageLayer.depth;
}
getCloseScopeIds() {
return this.closeTags.slice();
}
getOpenScopeIds() {
return this.openTags.slice();
}
// Private methods
_moveUp(atLastChild) {
let result = false;
const { endIndex } = this.treeCursor;
let depth = this.containingNodeEndIndices.length;
// The iterator should not move up until it has visited all of the children of this node.
while (
depth > 1 &&
(atLastChild || this.containingNodeEndIndices[depth - 2] === endIndex)
) {
atLastChild = false;
result = true;
this.treeCursor.gotoParent();
this.containingNodeTypes.pop();
this.containingNodeChildIndices.pop();
this.containingNodeEndIndices.pop();
--depth;
const scopeId = this._currentScopeId();
if (scopeId) this.closeTags.push(scopeId);
}
return result;
}
_moveDown() {
let result = false;
const { startIndex } = this.treeCursor;
// Once the iterator has found a scope boundary, it needs to stay at the same
// position, so it should not move down if the first child node starts later than the
// current node.
while (this.treeCursor.gotoFirstChild()) {
if (
(this.closeTags.length || this.openTags.length) &&
this.treeCursor.startIndex > startIndex
) {
this.treeCursor.gotoParent();
break;
}
result = true;
this.containingNodeTypes.push(this.treeCursor.nodeType);
this.containingNodeChildIndices.push(0);
this.containingNodeEndIndices.push(this.treeCursor.endIndex);
const scopeId = this._currentScopeId();
if (scopeId) this.openTags.push(scopeId);
}
return result;
}
_moveRight() {
if (this.treeCursor.gotoNextSibling()) {
const depth = this.containingNodeTypes.length;
this.containingNodeTypes[depth - 1] = this.treeCursor.nodeType;
this.containingNodeChildIndices[depth - 1]++;
this.containingNodeEndIndices[depth - 1] = this.treeCursor.endIndex;
return true;
}
}
_currentScopeId() {
const value = this.languageLayer.grammar.scopeMap.get(
this.containingNodeTypes,
this.containingNodeChildIndices,
this.treeCursor.nodeIsNamed
);
const scopeName = applyLeafRules(value, this.treeCursor);
if (scopeName) {
return this.languageLayer.languageMode.grammar.idForScope(scopeName);
}
}
}
const applyLeafRules = (rules, cursor) => {
if (!rules || typeof rules === 'string') return rules;
if (Array.isArray(rules)) {
for (let i = 0, { length } = rules; i !== length; ++i) {
const result = applyLeafRules(rules[i], cursor);
if (result) return result;
}
return undefined;
}
if (typeof rules === 'object') {
if (rules.exact) {
return cursor.nodeText === rules.exact
? applyLeafRules(rules.scopes, cursor)
: undefined;
}
if (rules.match) {
return rules.match.test(cursor.nodeText)
? applyLeafRules(rules.scopes, cursor)
: undefined;
}
}
};
class NodeCursorAdaptor {
get nodeText() {
return this.node.text;
}
}
class NullHighlightIterator {
seek() {
return [];
}
compare() {
return 1;
}
moveToSuccessor() {}
getPosition() {
return Point.INFINITY;
}
getOpenScopeIds() {
return [];
}
getCloseScopeIds() {
return [];
}
}
class NodeRangeSet {
constructor(previous, nodes, newlinesBetween, includeChildren) {
this.previous = previous;
this.nodes = nodes;
this.newlinesBetween = newlinesBetween;
this.includeChildren = includeChildren;
}
getRanges(buffer) {
const previousRanges = this.previous && this.previous.getRanges(buffer);
const result = [];
for (const node of this.nodes) {
let position = node.startPosition;
let index = node.startIndex;
if (!this.includeChildren) {
for (const child of node.children) {
const nextIndex = child.startIndex;
if (nextIndex > index) {
this._pushRange(buffer, previousRanges, result, {
startIndex: index,
endIndex: nextIndex,
startPosition: position,
endPosition: child.startPosition
});
}
position = child.endPosition;
index = child.endIndex;
}
}
if (node.endIndex > index) {
this._pushRange(buffer, previousRanges, result, {
startIndex: index,
endIndex: node.endIndex,
startPosition: position,
endPosition: node.endPosition
});
}
}
return result;
}
_pushRange(buffer, previousRanges, newRanges, newRange) {
if (!previousRanges) {
if (this.newlinesBetween) {
const { startIndex, startPosition } = newRange;
this._ensureNewline(buffer, newRanges, startIndex, startPosition);
}
newRanges.push(newRange);
return;
}
for (const previousRange of previousRanges) {
if (previousRange.endIndex <= newRange.startIndex) continue;
if (previousRange.startIndex >= newRange.endIndex) break;
const startIndex = Math.max(
previousRange.startIndex,
newRange.startIndex
);
const endIndex = Math.min(previousRange.endIndex, newRange.endIndex);
const startPosition = Point.max(
previousRange.startPosition,
newRange.startPosition
);
const endPosition = Point.min(
previousRange.endPosition,
newRange.endPosition
);
if (this.newlinesBetween) {
this._ensureNewline(buffer, newRanges, startIndex, startPosition);
}
newRanges.push({ startIndex, endIndex, startPosition, endPosition });
}
}
// For injection points with `newlinesBetween` enabled, ensure that a
// newline is included between each disjoint range.
_ensureNewline(buffer, newRanges, startIndex, startPosition) {
const lastRange = newRanges[newRanges.length - 1];
if (lastRange && lastRange.endPosition.row < startPosition.row) {
newRanges.push({
startPosition: new Point(
startPosition.row - 1,
buffer.lineLengthForRow(startPosition.row - 1)
),
endPosition: new Point(startPosition.row, 0),
startIndex: startIndex - startPosition.column - 1,
endIndex: startIndex - startPosition.column
});
}
}
}
function insertContainingTag(tag, index, tags, indices) {
const i = indices.findIndex(existingIndex => existingIndex > index);
if (i === -1) {
tags.push(tag);
indices.push(index);
} else {
tags.splice(i, 0, tag);
indices.splice(i, 0, index);
}
}
// Return true iff `mouse` is smaller than `house`. Only correct if
// mouse and house overlap.
//
// * `mouse` {Range}
// * `house` {Range}
function rangeIsSmaller(mouse, house) {
if (!house) return true;
const mvec = vecFromRange(mouse);
const hvec = vecFromRange(house);
return Point.min(mvec, hvec) === mvec;
}
function vecFromRange({ start, end }) {
return end.translate(start.negate());
}
function rangeForNode(node) {
return new Range(node.startPosition, node.endPosition);
}
function nodeContainsIndices(node, start, end) {
if (node.startIndex < start) return node.endIndex >= end;
if (node.startIndex === start) return node.endIndex > end;
return false;
}
function nodeIsSmaller(left, right) {
if (!left) return false;
if (!right) return true;
return left.endIndex - left.startIndex < right.endIndex - right.startIndex;
}
function last(array) {
return array[array.length - 1];
}
function hasMatchingFoldSpec(specs, node) {
return specs.some(
({ type, named }) => type === node.type && named === node.isNamed
);
}
// TODO: Remove this once TreeSitterLanguageMode implements its own auto-indent system.
[
'_suggestedIndentForLineWithScopeAtBufferRow',
'suggestedIndentForEditedBufferRow',
'increaseIndentRegexForScopeDescriptor',
'decreaseIndentRegexForScopeDescriptor',
'decreaseNextIndentRegexForScopeDescriptor',
'regexForPattern',
'getNonWordCharacters'
].forEach(methodName => {
TreeSitterLanguageMode.prototype[methodName] =
TextMateLanguageMode.prototype[methodName];
});
TreeSitterLanguageMode.LanguageLayer = LanguageLayer;
TreeSitterLanguageMode.prototype.syncTimeoutMicros = 1000;
module.exports = TreeSitterLanguageMode;