Moved highlighter code from DocView to Doc

* Only one highlighter state is kept per-document as opposed
  to one per-docview
* Fixes a bug with retaining older highlighter state as a
  DocView wasn't able to detect lines changing above it's viewport
* Renames `highlighter` module to more descriptive `tokenizer`

data/core/doc/highlighter.lua

@@ -0,0 +1,85 @@
+local core = require "core"
+local syntax = require "core.syntax"
+local config = require "core.config"
+local tokenizer = require "core.tokenizer"
+local Object = require "core.object"
+
+
+local Highlighter = Object:extend()
+
+
+function Highlighter:new(doc)
+  self.doc = doc
+  self:reset_syntax()
+
+  -- init incremental syntax highlighting
+  core.add_thread(function()
+    while true do
+      if self.last_valid_line > self.max_wanted_line then
+        self.max_wanted_line = 0
+        coroutine.yield(1 / config.fps)
+
+      else
+        local max = math.min(self.last_valid_line + 40, self.max_wanted_line)
+
+        for i = self.last_valid_line, max do
+          local state = (i > 1) and self.lines[i - 1].state
+          local line = self.lines[i]
+          if not (line and line.init_state == state) then
+            self.lines[i] = self:tokenize_line(i, state)
+          end
+        end
+
+        self.last_valid_line = max + 1
+        core.redraw = true
+        coroutine.yield()
+      end
+    end
+  end, self)
+end
+
+
+function Highlighter:reset_syntax()
+  local syn = syntax.get(self.doc.filename or "")
+  if self.syntax ~= syn then
+    self.syntax = syn
+    self.lines = {}
+    self.last_valid_line = 1
+    self.max_wanted_line = 0
+  end
+end
+
+
+function Highlighter:invalidate(idx)
+  self.last_valid_line = idx
+end
+
+
+function Highlighter:tokenize_line(idx, state)
+  local line = {}
+  line.init_state = state
+  line.text = self.doc.lines[idx]
+  line.tokens, line.state = tokenizer.tokenize(self.syntax, line.text, state)
+  return line
+end
+
+
+function Highlighter:get_line(idx)
+  local line = self.lines[idx]
+  if not line or line.text ~= self.doc.lines[idx] then
+    local prev = self.lines[idx - 1]
+    line = self:tokenize_line(idx, prev and prev.state)
+    self.lines[idx] = line
+    self.last_valid_line = math.min(self.last_valid_line, idx)
+  end
+  self.max_wanted_line = math.max(self.max_wanted_line, idx)
+  return line
+end
+
+
+function Highlighter:each_token(idx)
+  return tokenizer.each_token(self:get_line(idx).tokens)
+end
+
+
+return Highlighter

data/core/doc/init.lua

@@ -1,4 +1,5 @@
 local Object = require "core.object"
+local Highlighter = require "core.doc.highlighter"
 local config = require "core.config"
 local common = require "core.common"
 
@@ -19,7 +20,6 @@ local function splice(t, at, remove, insert)
   insert = insert or {}
   local offset = #insert - remove
   local old_len = #t
-  local new_len = old_len + offset
   if offset < 0 then
     for i = at - offset, old_len - offset do
       t[i + offset] = t[i]
@@ -49,6 +49,7 @@ function Doc:reset()
   self.undo_stack = { idx = 1 }
   self.redo_stack = { idx = 1 }
   self.clean_change_id = 1
+  self.highlighter = Highlighter(self)
 end
 
 
@@ -68,6 +69,7 @@ function Doc:load(filename)
     table.insert(self.lines, "\n")
   end
   fp:close()
+  self.highlighter:reset_syntax()
 end
 
 
@@ -80,6 +82,7 @@ function Doc:save(filename)
   end
   fp:close()
   self.filename = filename or self.filename
+  self.highlighter:reset_syntax()
   self:clean()
 end
 
@@ -233,6 +236,9 @@ local function insert(self, undo_stack, time, line, col, text)
   local line2, col2 = self:position_offset(line, col, #text)
   push_undo(self, undo_stack, time, "selection", self:get_selection())
   push_undo(self, undo_stack, time, "remove", line, col, line2, col2)
+
+  -- update highlighter
+  self.highlighter:invalidate(line)
 end
 
 
@@ -252,6 +258,9 @@ local function remove(self, undo_stack, time, line1, col1, line2, col2)
 
   -- splice line into line array
   splice(self.lines, line1, line2 - line1 + 1, { before .. after })
+
+  -- update highlighter
+  self.highlighter:invalidate(line1)
 end
 
 

data/core/docview.lua

@@ -5,7 +5,6 @@ local style = require "core.style"
 local syntax = require "core.syntax"
 local translate = require "core.doc.translate"
 local View = require "core.view"
-local highlighter = require "core.highlighter"
 
 
 local DocView = View:extend()
@@ -51,15 +50,6 @@ DocView.translate = {
 local blink_period = 0.8
 
 
-local function reset_syntax(self)
-  local syn = syntax.get(self.doc.filename or "")
-  if self.syntax ~= syn then
-    self.syntax = syn
-    self.cache = { last_valid = 1 }
-  end
-end
-
-
 function DocView:new(doc)
   DocView.super.new(self)
   self.cursor = "ibeam"
@@ -68,32 +58,6 @@ function DocView:new(doc)
   self.font = "code_font"
   self.last_x_offset = {}
   self.blink_timer = 0
-  reset_syntax(self)
-
-  -- init thread for incremental highlighting
-  self.updated_highlighting = false
-  core.add_thread(function()
-    while true do
-      local _, max = self:get_visible_line_range()
-
-      if self.cache.last_valid > max then
-        coroutine.yield(1 / config.fps)
-
-      else
-        max = math.min(self.cache.last_valid + 20, max)
-        for i = self.cache.last_valid, max do
-          local state = (i > 1) and self.cache[i - 1].state
-          local cl = self.cache[i]
-          if not (cl and cl.init_state == state) then
-            self.cache[i] = self:tokenize_line(i, state)
-          end
-        end
-        self.cache.last_valid = max + 1
-        self.updated_highlighting = true
-        coroutine.yield()
-      end
-    end
-  end, self)
 end
 
 
@@ -131,27 +95,6 @@ function DocView:get_scrollable_size()
 end
 
 
-function DocView:tokenize_line(idx, state)
-  local cl = {}
-  cl.init_state = state
-  cl.text = self.doc.lines[idx]
-  cl.tokens, cl.state = highlighter.tokenize(self.syntax, cl.text, state)
-  return cl
-end
-
-
-function DocView:get_cached_line(idx)
-  local cl = self.cache[idx]
-  if not cl or cl.text ~= self.doc.lines[idx] then
-    local prev = self.cache[idx-1]
-    cl = self:tokenize_line(idx, prev and prev.state)
-    self.cache[idx] = cl
-    self.cache.last_valid = math.min(self.cache.last_valid, idx)
-  end
-  return cl
-end
-
-
 function DocView:get_font()
   return style[self.font]
 end
@@ -308,16 +251,6 @@ function DocView:update()
     self.last_line, self.last_col = line, col
   end
 
-  if self.updated_highlighting then
-    self.updated_highlighting = false
-    core.redraw = true
-  end
-
-  if self.doc.filename ~= self.last_filename then
-    reset_syntax(self)
-    self.last_filename = self.doc.filename
-  end
-
   -- update blink timer
   if self == core.active_view and not self.mouse_selecting then
     local n = blink_period / 2
@@ -339,10 +272,9 @@ end
 
 
 function DocView:draw_line_text(idx, x, y)
-  local cl = self:get_cached_line(idx)
   local tx, ty = x, y + self:get_line_text_y_offset()
   local font = self:get_font()
-  for _, type, text in highlighter.each_token(cl.tokens) do
+  for _, type, text in self.doc.highlighter:each_token(idx) do
     local color = style.syntax[type]
     tx = renderer.draw_text(font, text, tx, ty, color)
   end
@@ -355,9 +287,9 @@ function DocView:draw_line_body(idx, x, y)
   -- draw selection if it overlaps this line
   local line1, col1, line2, col2 = self.doc:get_selection(true)
   if idx >= line1 and idx <= line2 then
-    local cl = self:get_cached_line(idx)
+    local text = self.doc.lines[idx]
     if line1 ~= idx then col1 = 1 end
-    if line2 ~= idx then col2 = #cl.text + 1 end
+    if line2 ~= idx then col2 = #text + 1 end
     local x1 = x + self:get_col_x_offset(idx, col1)
     local x2 = x + self:get_col_x_offset(idx, col2)
     local lh = self:get_line_height()

data/core/tokenizer.lua

@@ -1,4 +1,4 @@
-local highlighter = {}
+local tokenizer = {}
 
 
 local function push_token(t, type, text)
@@ -38,7 +38,7 @@ local function find_non_escaped(text, pattern, offset, esc)
 end
 
 
-function highlighter.tokenize(syntax, text, state)
+function tokenizer.tokenize(syntax, text, state)
   local res = {}
   local i = 1
 
@@ -100,9 +100,9 @@ local function iter(t, i)
   end
 end
 
-function highlighter.each_token(t)
+function tokenizer.each_token(t)
   return iter, t, -1
 end
 
 
-return highlighter
+return tokenizer