From 50ad1e06f9ee627bf3e8d80a1b5d0e67b0219d90 Mon Sep 17 00:00:00 2001
From: Anthony Sottile <asottile@umich.edu>
Date: Fri, 27 Mar 2020 17:59:35 -0700
Subject: [PATCH] Add demo for showing vs code's tokenization

---
 testing/vsc_test/.gitignore   |  2 ++
 testing/vsc_test/package.json |  5 ++++
 testing/vsc_test/vsc.js       | 51 +++++++++++++++++++++++++++++++++++
 3 files changed, 58 insertions(+)
 create mode 100644 testing/vsc_test/.gitignore
 create mode 100644 testing/vsc_test/package.json
 create mode 100644 testing/vsc_test/vsc.js

diff --git a/testing/vsc_test/.gitignore b/testing/vsc_test/.gitignore
new file mode 100644
index 0000000..d502512
--- /dev/null
+++ b/testing/vsc_test/.gitignore
@@ -0,0 +1,2 @@
+/node_modules
+/package-lock.json
diff --git a/testing/vsc_test/package.json b/testing/vsc_test/package.json
new file mode 100644
index 0000000..f343256
--- /dev/null
+++ b/testing/vsc_test/package.json
@@ -0,0 +1,5 @@
+{
+  "dependencies": [
+    "vscode-textmate"
+  ]
+}
diff --git a/testing/vsc_test/vsc.js b/testing/vsc_test/vsc.js
new file mode 100644
index 0000000..1413fb9
--- /dev/null
+++ b/testing/vsc_test/vsc.js
@@ -0,0 +1,51 @@
+const fs = require('fs');
+const vsctm = require('vscode-textmate');
+
+if (process.argv.length < 4) {
+    console.log('usage: t.js GRAMMAR FILE');
+    process.exit(1);
+}
+
+const grammar = process.argv[2];
+const file = process.argv[3];
+
+const scope = JSON.parse(fs.readFileSync(grammar, {encoding: 'UTF-8'})).scopeName;
+
+/**
+ * Utility to read a file as a promise
+ */
+function readFile(path) {
+    return new Promise((resolve, reject) => {
+        fs.readFile(path, (error, data) => error ? reject(error) : resolve(data));
+    })
+}
+
+// Create a registry that can create a grammar from a scope name.
+const registry = new vsctm.Registry({
+    loadGrammar: (scopeName) => {
+        if (scopeName === scope) {
+            return readFile(grammar).then(data => vsctm.parseRawGrammar(data.toString(), grammar))
+        }
+        console.log(`Unknown scope name: ${scopeName}`);
+        return null;
+    }
+});
+
+// Load the JavaScript grammar and any other grammars included by it async.
+registry.loadGrammar(scope).then(grammar => {
+    const text = fs.readFileSync(file, {encoding: 'UTF-8'}).trimEnd('\n').split(/\n/);
+    let ruleStack = vsctm.INITIAL;
+    for (let i = 0; i < text.length; i++) {
+        const line = text[i];
+        const lineTokens = grammar.tokenizeLine(line, ruleStack);
+        console.log(`\nTokenizing line: ${line}`);
+        for (let j = 0; j < lineTokens.tokens.length; j++) {
+            const token = lineTokens.tokens[j];
+            console.log(` - token from ${token.startIndex} to ${token.endIndex} ` +
+              `(${line.substring(token.startIndex, token.endIndex)}) ` +
+              `with scopes ${token.scopes.join(', ')}`
+            );
+        }
+        ruleStack = lineTokens.ruleStack;
+    }
+});