From 6525269386a5425339d9710f7da97a75c055490b Mon Sep 17 00:00:00 2001
From: rxi <rxi@users.noreply.github.com>
Date: Thu, 14 May 2020 10:08:12 +0100
Subject: [PATCH] Made tokenizer skip parsing process on plain-text files

This, along with the earlier rencache changes should resolve #64
---
 data/core/tokenizer.lua | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/data/core/tokenizer.lua b/data/core/tokenizer.lua
index d6ec38e..98aafc7 100644
--- a/data/core/tokenizer.lua
+++ b/data/core/tokenizer.lua
@@ -42,6 +42,10 @@ function tokenizer.tokenize(syntax, text, state)
   local res = {}
   local i = 1
 
+  if #syntax.patterns == 0 then
+    return { "normal", text }
+  end
+
   while i <= #text do
     -- continue trying to match the end pattern of a pair if we have a state set
     if state then