wip

for example: ruby

README.md

@@ -6,6 +6,10 @@ babi
 
 a text editor, eventually...
 
+### installation
+
+`pip install babi`
+
 ### why is it called babi?
 
 I usually use the text editor `nano`, frequently I typo this.  on a qwerty

babi/buf.py

@@ -0,0 +1,314 @@
+import bisect
+import contextlib
+from typing import Callable
+from typing import Generator
+from typing import Iterator
+from typing import List
+from typing import NamedTuple
+from typing import Optional
+from typing import Tuple
+
+from babi._types import Protocol
+from babi.horizontal_scrolling import line_x
+from babi.horizontal_scrolling import scrolled_line
+from babi.horizontal_scrolling import wcwidth
+from babi.margin import Margin
+
+SetCallback = Callable[['Buf', int, str], None]
+DelCallback = Callable[['Buf', int, str], None]
+InsCallback = Callable[['Buf', int], None]
+
+
+def _offsets(s: str) -> Tuple[int, ...]:
+    ret = [0]
+    for c in s:
+        if c == '\t':
+            ret.append(ret[-1] + (4 - ret[-1] % 4))
+        else:
+            ret.append(ret[-1] + wcwidth(c))
+    return tuple(ret)
+
+
+class Modification(Protocol):
+    def __call__(self, buf: 'Buf') -> None: ...
+
+
+class SetModification(NamedTuple):
+    idx: int
+    s: str
+
+    def __call__(self, buf: 'Buf') -> None:
+        buf[self.idx] = self.s
+
+
+class InsModification(NamedTuple):
+    idx: int
+    s: str
+
+    def __call__(self, buf: 'Buf') -> None:
+        buf.insert(self.idx, self.s)
+
+
+class DelModification(NamedTuple):
+    idx: int
+
+    def __call__(self, buf: 'Buf') -> None:
+        del buf[self.idx]
+
+
+class Buf:
+    def __init__(self, lines: List[str]) -> None:
+        self._lines = lines
+        self.file_y = self.y = self._x = self._x_hint = 0
+
+        self._set_callbacks: List[SetCallback] = [self._set_cb]
+        self._del_callbacks: List[DelCallback] = [self._del_cb]
+        self._ins_callbacks: List[InsCallback] = [self._ins_cb]
+
+        self._positions: List[Optional[Tuple[int, ...]]] = []
+
+    # read only interface
+
+    def __repr__(self) -> str:
+        return (
+            f'{type(self).__name__}('
+            f'{self._lines!r}, x={self.x}, y={self.y}, file_y={self.file_y}'
+            f')'
+        )
+
+    def __bool__(self) -> bool:
+        return bool(self._lines)
+
+    def __getitem__(self, idx: int) -> str:
+        return self._lines[idx]
+
+    def __iter__(self) -> Iterator[str]:
+        yield from self._lines
+
+    def __len__(self) -> int:
+        return len(self._lines)
+
+    # mutators
+
+    def __setitem__(self, idx: int, val: str) -> None:
+        if idx < 0:
+            idx %= len(self)
+        victim = self._lines[idx]
+
+        self._lines[idx] = val
+
+        for set_callback in self._set_callbacks:
+            set_callback(self, idx, victim)
+
+    def __delitem__(self, idx: int) -> None:
+        if idx < 0:
+            idx %= len(self)
+        victim = self._lines[idx]
+
+        del self._lines[idx]
+
+        for del_callback in self._del_callbacks:
+            del_callback(self, idx, victim)
+
+    def insert(self, idx: int, val: str) -> None:
+        if idx < 0:
+            idx %= len(self)
+
+        self._lines.insert(idx, val)
+
+        for ins_callback in self._ins_callbacks:
+            ins_callback(self, idx)
+
+    # also mutators, but implemented using above functions
+
+    def append(self, val: str) -> None:
+        self.insert(len(self), val)
+
+    def pop(self, idx: int = -1) -> str:
+        victim = self[idx]
+        del self[idx]
+        return victim
+
+    def restore_eof_invariant(self) -> None:
+        """the file lines will always contain a blank empty string at the end
+        to simplify rendering.  call this whenever the last line may change
+        """
+        if self[-1] != '':
+            self.append('')
+
+    # event handling
+
+    def add_set_callback(self, cb: SetCallback) -> None:
+        self._set_callbacks.append(cb)
+
+    def remove_set_callback(self, cb: SetCallback) -> None:
+        self._set_callbacks.remove(cb)
+
+    def add_del_callback(self, cb: DelCallback) -> None:
+        self._del_callbacks.append(cb)
+
+    def remove_del_callback(self, cb: DelCallback) -> None:
+        self._del_callbacks.remove(cb)
+
+    def add_ins_callback(self, cb: InsCallback) -> None:
+        self._ins_callbacks.append(cb)
+
+    def remove_ins_callback(self, cb: InsCallback) -> None:
+        self._ins_callbacks.remove(cb)
+
+    @contextlib.contextmanager
+    def record(self) -> Generator[List[Modification], None, None]:
+        modifications: List[Modification] = []
+
+        def set_cb(buf: 'Buf', idx: int, victim: str) -> None:
+            modifications.append(SetModification(idx, victim))
+
+        def del_cb(buf: 'Buf', idx: int, victim: str) -> None:
+            modifications.append(InsModification(idx, victim))
+
+        def ins_cb(buf: 'Buf', idx: int) -> None:
+            modifications.append(DelModification(idx))
+
+        self.add_set_callback(set_cb)
+        self.add_del_callback(del_cb)
+        self.add_ins_callback(ins_cb)
+        try:
+            yield modifications
+        finally:
+            self.remove_ins_callback(ins_cb)
+            self.remove_del_callback(del_cb)
+            self.remove_set_callback(set_cb)
+
+    def apply(self, modifications: List[Modification]) -> List[Modification]:
+        with self.record() as ret_modifications:
+            for modification in reversed(modifications):
+                modification(self)
+        return ret_modifications
+
+    # position properties
+
+    @property
+    def displayable_count(self) -> int:
+        return len(self._lines) - self.file_y
+
+    @property
+    def x(self) -> int:
+        return self._x
+
+    @x.setter
+    def x(self, x: int) -> None:
+        self._x = x
+        self._x_hint = self._cursor_x
+
+    def _extend_positions(self, idx: int) -> None:
+        self._positions.extend([None] * (1 + idx - len(self._positions)))
+
+    def _set_cb(self, buf: 'Buf', idx: int, victim: str) -> None:
+        self._extend_positions(idx)
+        self._positions[idx] = None
+
+    def _del_cb(self, buf: 'Buf', idx: int, victim: str) -> None:
+        self._extend_positions(idx)
+        del self._positions[idx]
+
+    def _ins_cb(self, buf: 'Buf', idx: int) -> None:
+        self._extend_positions(idx)
+        self._positions.insert(idx, None)
+
+    def line_positions(self, idx: int) -> Tuple[int, ...]:
+        self._extend_positions(idx)
+        value = self._positions[idx]
+        if value is None:
+            value = self._positions[idx] = _offsets(self._lines[idx])
+        return value
+
+    def line_x(self, margin: Margin) -> int:
+        return line_x(self._cursor_x, margin.cols)
+
+    @property
+    def _cursor_x(self) -> int:
+        return self.line_positions(self.y)[self.x]
+
+    def cursor_position(self, margin: Margin) -> Tuple[int, int]:
+        y = self.y - self.file_y + margin.header
+        x = self._cursor_x - self.line_x(margin)
+        return y, x
+
+    # rendered lines
+
+    def rendered_line(self, idx: int, margin: Margin) -> str:
+        line = self._lines[idx]
+        positions = self.line_positions(idx)
+        cursor_x = self._cursor_x if idx == self.y else 0
+        return scrolled_line(line, positions, cursor_x, margin.cols)
+
+    # movement
+
+    def scroll_screen_if_needed(self, margin: Margin) -> None:
+        # if the `y` is not on screen, make it so
+        if not (self.file_y <= self.y < self.file_y + margin.body_lines):
+            self.file_y = max(self.y - margin.body_lines // 2, 0)
+
+    def _set_x_after_vertical_movement(self) -> None:
+        positions = self.line_positions(self.y)
+        x = bisect.bisect_left(positions, self._x_hint)
+        x = min(len(self._lines[self.y]), x)
+        if positions[x] > self._x_hint:
+            x -= 1
+        self._x = x
+
+    def up(self, margin: Margin) -> None:
+        if self.y > 0:
+            self.y -= 1
+            if self.y < self.file_y:
+                self.file_y = max(self.file_y - margin.scroll_amount, 0)
+            self._set_x_after_vertical_movement()
+
+    def down(self, margin: Margin) -> None:
+        if self.y < len(self._lines) - 1:
+            self.y += 1
+            if self.y >= self.file_y + margin.body_lines:
+                self.file_y += margin.scroll_amount
+            self._set_x_after_vertical_movement()
+
+    def right(self, margin: Margin) -> None:
+        if self.x >= len(self._lines[self.y]):
+            if self.y < len(self._lines) - 1:
+                self.down(margin)
+                self.home()
+        else:
+            self.x += 1
+
+    def left(self, margin: Margin) -> None:
+        if self.x == 0:
+            if self.y > 0:
+                self.up(margin)
+                self.end()
+        else:
+            self.x -= 1
+
+    def home(self) -> None:
+        self.x = 0
+
+    def end(self) -> None:
+        self.x = len(self._lines[self.y])
+
+    # screen movement
+
+    def file_up(self, margin: Margin) -> None:
+        if self.file_y > 0:
+            self.file_y -= 1
+            if self.y > self.file_y + margin.body_lines - 1:
+                self.up(margin)
+
+    def file_down(self, margin: Margin) -> None:
+        if self.file_y < len(self._lines) - 1:
+            self.file_y += 1
+            if self.y < self.file_y:
+                self.down(margin)
+
+    # key input
+
+    def c(self, s: str) -> None:
+        self[self.y] = self[self.y][:self.x] + s + self[self.y][self.x:]
+        self.x += len(s)

babi/fdict.py

@@ -3,8 +3,10 @@ from typing import Iterable
 from typing import Mapping
 from typing import TypeVar
 
-TKey = TypeVar('TKey')
+from babi._types import Protocol
-TValue = TypeVar('TValue')
+
+TKey = TypeVar('TKey', contravariant=True)
+TValue = TypeVar('TValue', covariant=True)
 
 
 class FDict(Generic[TKey, TValue]):
@@ -22,3 +24,21 @@ class FDict(Generic[TKey, TValue]):
 
     def values(self) -> Iterable[TValue]:
         return self._dct.values()
+
+
+class Indexable(Generic[TKey, TValue], Protocol):
+    def __getitem__(self, key: TKey) -> TValue: ...
+
+
+class FChainMap(Generic[TKey, TValue]):
+    def __init__(self, *mappings: Indexable[TKey, TValue]) -> None:
+        self._mappings = mappings
+
+    def __getitem__(self, key: TKey) -> TValue:
+        for mapping in reversed(self._mappings):
+            try:
+                return mapping[key]
+            except KeyError:
+                pass
+        else:
+            raise KeyError(key)

babi/file.py

@@ -21,16 +21,14 @@ from typing import TYPE_CHECKING
 from typing import TypeVar
 from typing import Union
 
+from babi.buf import Buf
+from babi.buf import Modification
 from babi.color_manager import ColorManager
 from babi.hl.interface import FileHL
 from babi.hl.interface import HLFactory
 from babi.hl.replace import Replace
 from babi.hl.selection import Selection
 from babi.hl.trailing_whitespace import TrailingWhitespace
-from babi.horizontal_scrolling import line_x
-from babi.horizontal_scrolling import scrolled_line
-from babi.list_spy import ListSpy
-from babi.list_spy import MutableSequenceNoSlice
 from babi.margin import Margin
 from babi.prompt import PromptResult
 from babi.status import Status
@@ -41,15 +39,6 @@ if TYPE_CHECKING:
 TCallable = TypeVar('TCallable', bound=Callable[..., Any])
 
 
-def _restore_lines_eof_invariant(lines: MutableSequenceNoSlice) -> None:
-    """The file lines will always contain a blank empty string at the end to
-    simplify rendering.  This should be called whenever the end of the file
-    might change.
-    """
-    if not lines or lines[-1] != '':
-        lines.append('')
-
-
 def get_lines(sio: IO[str]) -> Tuple[List[str], str, bool, str]:
     sha256 = hashlib.sha256()
     lines = []
@@ -63,7 +52,8 @@ def get_lines(sio: IO[str]) -> Tuple[List[str], str, bool, str]:
                 break
         else:
             lines.append(line)
-    _restore_lines_eof_invariant(lines)
+    # always make sure we end in a newline
+    lines.append('')
     (nl, _), = newlines.most_common(1)
     mixed = len({k for k, v in newlines.items() if v}) > 1
     return lines, nl, mixed, sha256.hexdigest()
@@ -71,13 +61,13 @@ def get_lines(sio: IO[str]) -> Tuple[List[str], str, bool, str]:
 
 class Action:
     def __init__(
-            self, *, name: str, spy: ListSpy,
+            self, *, name: str, modifications: List[Modification],
             start_x: int, start_y: int, start_modified: bool,
             end_x: int, end_y: int, end_modified: bool,
             final: bool,
     ):
         self.name = name
-        self.spy = spy
+        self.modifications = modifications
         self.start_x = start_x
         self.start_y = start_y
         self.start_modified = start_modified
@@ -87,9 +77,8 @@ class Action:
         self.final = final
 
     def apply(self, file: 'File') -> 'Action':
-        spy = ListSpy(file.lines)
         action = Action(
-            name=self.name, spy=spy,
+            name=self.name, modifications=file.buf.apply(self.modifications),
             start_x=self.end_x, start_y=self.end_y,
             start_modified=self.end_modified,
             end_x=self.start_x, end_y=self.start_y,
@@ -97,11 +86,9 @@ class Action:
             final=True,
         )
 
-        self.spy.undo(spy)
+        file.buf.y = self.start_y
-        file.x = self.start_x
+        file.buf.x = self.start_x
-        file.y = self.start_y
         file.modified = self.start_modified
-        file.touch(spy.min_line_touched)
 
         return action
 
@@ -162,8 +149,8 @@ class _SearchIter:
         self.reg = reg
         self.offset = offset
         self.wrapped = False
-        self._start_x = file.x + offset
+        self._start_x = file.buf.x + offset
-        self._start_y = file.y
+        self._start_y = file.buf.y
 
     def __iter__(self) -> '_SearchIter':
         return self
@@ -179,28 +166,28 @@ class _SearchIter:
         return Found(y, match)
 
     def __next__(self) -> Tuple[int, Match[str]]:
-        x = self.file.x + self.offset
+        x = self.file.buf.x + self.offset
-        y = self.file.y
+        y = self.file.buf.y
 
-        match = self.reg.search(self.file.lines[y], x)
+        match = self.reg.search(self.file.buf[y], x)
         if match:
             return self._stop_if_past_original(y, match)
 
         if self.wrapped:
             for line_y in range(y + 1, self._start_y + 1):
-                match = self.reg.search(self.file.lines[line_y])
+                match = self.reg.search(self.file.buf[line_y])
                 if match:
                     return self._stop_if_past_original(line_y, match)
         else:
-            for line_y in range(y + 1, len(self.file.lines)):
+            for line_y in range(y + 1, len(self.file.buf)):
-                match = self.reg.search(self.file.lines[line_y])
+                match = self.reg.search(self.file.buf[line_y])
                 if match:
                     return self._stop_if_past_original(line_y, match)
 
             self.wrapped = True
 
             for line_y in range(0, self._start_y + 1):
-                match = self.reg.search(self.file.lines[line_y])
+                match = self.reg.search(self.file.buf[line_y])
                 if match:
                     return self._stop_if_past_original(line_y, match)
 
@@ -216,10 +203,10 @@ class File:
     ) -> None:
         self.filename = filename
         self.modified = False
-        self.lines: MutableSequenceNoSlice = []
+        self.buf = Buf([])
         self.nl = '\n'
-        self.file_y = self.y = self.x = self.x_hint = 0
         self.sha256: Optional[str] = None
+        self._in_edit_action = False
         self.undo_stack: List[Action] = []
         self.redo_stack: List[Action] = []
         self._hl_factories = hl_factories
@@ -229,7 +216,7 @@ class File:
         self._file_hls: Tuple[FileHL, ...] = ()
 
     def ensure_loaded(self, status: Status, stdin: str) -> None:
-        if self.lines:
+        if self.buf:
             return
 
         if self.filename == '-':
@@ -237,10 +224,10 @@ class File:
             self.filename = None
             self.modified = True
             sio = io.StringIO(stdin)
-            self.lines, self.nl, mixed, self.sha256 = get_lines(sio)
+            lines, self.nl, mixed, self.sha256 = get_lines(sio)
         elif self.filename is not None and os.path.isfile(self.filename):
             with open(self.filename, newline='') as f:
-                self.lines, self.nl, mixed, self.sha256 = get_lines(f)
+                lines, self.nl, mixed, self.sha256 = get_lines(f)
         else:
             if self.filename is not None:
                 if os.path.lexists(self.filename):
@@ -248,8 +235,9 @@ class File:
                     self.filename = None
                 else:
                     status.update('(new file)')
-            sio = io.StringIO('')
+            lines, self.nl, mixed, self.sha256 = get_lines(io.StringIO(''))
-            self.lines, self.nl, mixed, self.sha256 = get_lines(sio)
+
+        self.buf = Buf(lines)
 
         if mixed:
             status.update(f'mixed newlines will be converted to {self.nl!r}')
@@ -258,7 +246,7 @@ class File:
         file_hls = []
         for factory in self._hl_factories:
             if self.filename is not None:
-                hl = factory.file_highlighter(self.filename, self.lines[0])
+                hl = factory.file_highlighter(self.filename, self.buf[0])
                 file_hls.append(hl)
             else:
                 file_hls.append(factory.blank_file_highlighter())
@@ -266,156 +254,109 @@ class File:
             *file_hls,
             self._trailing_whitespace, self._replace_hl, self.selection,
         )
+        for file_hl in self._file_hls:
+            file_hl.register_callbacks(self.buf)
 
     def __repr__(self) -> str:
         return f'<{type(self).__name__} {self.filename!r}>'
 
     # movement
 
-    def scroll_screen_if_needed(self, margin: Margin) -> None:
-        # if the `y` is not on screen, make it so
-        if self.file_y <= self.y < self.file_y + margin.body_lines:
-            return
-
-        self.file_y = max(self.y - margin.body_lines // 2, 0)
-
-    def _scroll_amount(self) -> int:
-        return int(curses.LINES / 2 + .5)
-
-    def _set_x_after_vertical_movement(self) -> None:
-        self.x = min(len(self.lines[self.y]), self.x_hint)
-
-    def _increment_y(self, margin: Margin) -> None:
-        self.y += 1
-        if self.y >= self.file_y + margin.body_lines:
-            self.file_y += self._scroll_amount()
-
-    def _decrement_y(self) -> None:
-        self.y -= 1
-        if self.y < self.file_y:
-            self.file_y -= self._scroll_amount()
-            self.file_y = max(self.file_y, 0)
-
     @action
     def up(self, margin: Margin) -> None:
-        if self.y > 0:
+        self.buf.up(margin)
-            self._decrement_y()
-            self._set_x_after_vertical_movement()
 
     @action
     def down(self, margin: Margin) -> None:
-        if self.y < len(self.lines) - 1:
+        self.buf.down(margin)
-            self._increment_y(margin)
-            self._set_x_after_vertical_movement()
 
     @action
     def right(self, margin: Margin) -> None:
-        if self.x >= len(self.lines[self.y]):
+        self.buf.right(margin)
-            if self.y < len(self.lines) - 1:
-                self.x = 0
-                self._increment_y(margin)
-        else:
-            self.x += 1
-        self.x_hint = self.x
 
     @action
     def left(self, margin: Margin) -> None:
-        if self.x == 0:
+        self.buf.left(margin)
-            if self.y > 0:
-                self._decrement_y()
-                self.x = len(self.lines[self.y])
-        else:
-            self.x -= 1
-        self.x_hint = self.x
 
     @action
     def home(self, margin: Margin) -> None:
-        self.x = self.x_hint = 0
+        self.buf.home()
 
     @action
     def end(self, margin: Margin) -> None:
-        self.x = self.x_hint = len(self.lines[self.y])
+        self.buf.end()
 
     @action
     def ctrl_up(self, margin: Margin) -> None:
-        self.file_y = max(0, self.file_y - 1)
+        self.buf.file_up(margin)
-        self.y = min(self.y, self.file_y + margin.body_lines - 1)
-        self._set_x_after_vertical_movement()
 
     @action
     def ctrl_down(self, margin: Margin) -> None:
-        self.file_y = min(len(self.lines) - 1, self.file_y + 1)
+        self.buf.file_down(margin)
-        self.y = max(self.y, self.file_y)
-        self._set_x_after_vertical_movement()
 
     @action
     def ctrl_right(self, margin: Margin) -> None:
-        line = self.lines[self.y]
+        line = self.buf[self.buf.y]
         # if we're at the second to last character, jump to end of line
-        if self.x == len(line) - 1:
+        if self.buf.x == len(line) - 1:
-            self.x = self.x_hint = self.x + 1
+            self.buf.right(margin)
         # if we're at the end of the line, jump forward to the next non-ws
-        elif self.x == len(line):
+        elif self.buf.x == len(line):
             while (
-                    self.y < len(self.lines) - 1 and (
+                    self.buf.y < len(self.buf) - 1 and (
-                        self.x == len(self.lines[self.y]) or
+                        self.buf.x == len(self.buf[self.buf.y]) or
-                        self.lines[self.y][self.x].isspace()
+                        self.buf[self.buf.y][self.buf.x].isspace()
                     )
             ):
-                if self.x == len(self.lines[self.y]):
+                self.buf.right(margin)
-                    self._increment_y(margin)
-                    self.x = self.x_hint = 0
-                else:
-                    self.x = self.x_hint = self.x + 1
         # if we're inside the line, jump to next position that's not our type
         else:
-            self.x = self.x_hint = self.x + 1
+            self.buf.right(margin)
-            tp = line[self.x].isalnum()
+            tp = line[self.buf.x].isalnum()
-            while self.x < len(line) and tp == line[self.x].isalnum():
+            while self.buf.x < len(line) and tp == line[self.buf.x].isalnum():
-                self.x = self.x_hint = self.x + 1
+                self.buf.right(margin)
 
     @action
     def ctrl_left(self, margin: Margin) -> None:
-        line = self.lines[self.y]
+        line = self.buf[self.buf.y]
         # if we're at position 1 and it's not a space, go to the beginning
-        if self.x == 1 and not line[:self.x].isspace():
+        if self.buf.x == 1 and not line[:self.buf.x].isspace():
-            self.x = self.x_hint = 0
+            self.buf.left(margin)
         # if we're at the beginning or it's all space up to here jump to the
         # end of the previous non-space line
-        elif self.x == 0 or line[:self.x].isspace():
+        elif self.buf.x == 0 or line[:self.buf.x].isspace():
-            self.x = self.x_hint = 0
+            self.buf.x = 0
-            while self.y > 0 and (self.x == 0 or not self.lines[self.y]):
+            while self.buf.y > 0 and self.buf.x == 0:
-                self._decrement_y()
+                self.buf.left(margin)
-                self.x = self.x_hint = len(self.lines[self.y])
         else:
-            self.x = self.x_hint = self.x - 1
+            self.buf.left(margin)
-            tp = line[self.x - 1].isalnum()
+            tp = line[self.buf.x - 1].isalnum()
-            while self.x > 0 and tp == line[self.x - 1].isalnum():
+            while self.buf.x > 0 and tp == line[self.buf.x - 1].isalnum():
-                self.x = self.x_hint = self.x - 1
+                self.buf.left(margin)
 
     @action
     def ctrl_home(self, margin: Margin) -> None:
-        self.x = self.x_hint = 0
+        self.buf.x = 0
-        self.y = self.file_y = 0
+        self.buf.y = self.buf.file_y = 0
 
     @action
     def ctrl_end(self, margin: Margin) -> None:
-        self.x = self.x_hint = 0
+        self.buf.x = 0
-        self.y = len(self.lines) - 1
+        self.buf.y = len(self.buf) - 1
-        self.scroll_screen_if_needed(margin)
+        self.buf.scroll_screen_if_needed(margin)
 
     @action
     def go_to_line(self, lineno: int, margin: Margin) -> None:
-        self.x = self.x_hint = 0
+        self.buf.x = 0
         if lineno == 0:
-            self.y = 0
+            self.buf.y = 0
-        elif lineno > len(self.lines):
+        elif lineno > len(self.buf):
-            self.y = len(self.lines) - 1
+            self.buf.y = len(self.buf) - 1
         elif lineno < 0:
-            self.y = max(0, lineno + len(self.lines))
+            self.buf.y = max(0, lineno + len(self.buf))
         else:
-            self.y = lineno - 1
+            self.buf.y = lineno - 1
-        self.scroll_screen_if_needed(margin)
+        self.buf.scroll_screen_if_needed(margin)
 
     @action
     def search(
@@ -430,14 +371,14 @@ class File:
         except StopIteration:
             status.update('no matches')
         else:
-            if line_y == self.y and match.start() == self.x:
+            if line_y == self.buf.y and match.start() == self.buf.x:
                 status.update('this is the only occurrence')
             else:
                 if search.wrapped:
                     status.update('search wrapped')
-                self.y = line_y
+                self.buf.y = line_y
-                self.x = self.x_hint = match.start()
+                self.buf.x = match.start()
-                self.scroll_screen_if_needed(margin)
+                self.buf.scroll_screen_if_needed(margin)
 
     @clear_selection
     def replace(
@@ -452,38 +393,36 @@ class File:
         res: Union[str, PromptResult] = ''
         search = _SearchIter(self, reg, offset=0)
         for line_y, match in search:
-            self.y = line_y
+            end = match.end()
-            self.x = self.x_hint = match.start()
+            self.buf.y = line_y
-            self.scroll_screen_if_needed(screen.margin)
+            self.buf.x = match.start()
+            self.buf.scroll_screen_if_needed(screen.margin)
             if res != 'a':  # make `a` replace the rest of them
-                with self._replace_hl.region(self.y, self.x, match.end()):
+                with self._replace_hl.region(self.buf.y, self.buf.x, end):
                     screen.draw()
                     res = screen.quick_prompt('replace', ('yes', 'no', 'all'))
             if res in {'y', 'a'}:
                 count += 1
                 with self.edit_action_context('replace', final=True):
                     replaced = match.expand(replace)
-                    line = screen.file.lines[line_y]
+                    line = screen.file.buf[line_y]
                     if '\n' in replaced:
                         replaced_lines = replaced.split('\n')
-                        self.lines[line_y] = (
+                        self.buf[line_y] = (
                             f'{line[:match.start()]}{replaced_lines[0]}'
                         )
                         for i, ins_line in enumerate(replaced_lines[1:-1], 1):
-                            self.lines.insert(line_y + i, ins_line)
+                            self.buf.insert(line_y + i, ins_line)
                         last_insert = line_y + len(replaced_lines) - 1
-                        self.lines.insert(
+                        self.buf.insert(
-                            last_insert,
+                            last_insert, f'{replaced_lines[-1]}{line[end:]}',
-                            f'{replaced_lines[-1]}{line[match.end():]}',
                         )
-                        self.y = last_insert
+                        self.buf.y = last_insert
-                        self.x = self.x_hint = 0
+                        self.buf.x = 0
                         search.offset = len(replaced_lines[-1])
                     else:
-                        self.lines[line_y] = (
+                        self.buf[line_y] = (
-                            f'{line[:match.start()]}'
+                            f'{line[:match.start()]}{replaced}{line[end:]}'
-                            f'{replaced}'
-                            f'{line[match.end():]}'
                         )
                         search.offset = len(replaced)
             elif res == 'n':
@@ -500,21 +439,21 @@ class File:
 
     @action
     def page_up(self, margin: Margin) -> None:
-        if self.y < margin.body_lines:
+        if self.buf.y < margin.body_lines:
-            self.y = self.file_y = 0
+            self.buf.y = self.buf.file_y = 0
         else:
-            pos = max(self.file_y - margin.page_size, 0)
+            pos = max(self.buf.file_y - margin.page_size, 0)
-            self.y = self.file_y = pos
+            self.buf.y = self.buf.file_y = pos
-        self.x = self.x_hint = 0
+        self.buf.x = 0
 
     @action
     def page_down(self, margin: Margin) -> None:
-        if self.file_y + margin.body_lines >= len(self.lines):
+        if self.buf.file_y + margin.body_lines >= len(self.buf):
-            self.y = len(self.lines) - 1
+            self.buf.y = len(self.buf) - 1
         else:
-            pos = self.file_y + margin.page_size
+            pos = self.buf.file_y + margin.page_size
-            self.y = self.file_y = pos
+            self.buf.y = self.buf.file_y = pos
-        self.x = self.x_hint = 0
+        self.buf.x = 0
 
     # editing
 
@@ -522,47 +461,51 @@ class File:
     @clear_selection
     def backspace(self, margin: Margin) -> None:
         # backspace at the beginning of the file does nothing
-        if self.y == 0 and self.x == 0:
+        if self.buf.y == 0 and self.buf.x == 0:
             pass
         # backspace at the end of the file does not change the contents
-        elif self.y == len(self.lines) - 1:
+        elif self.buf.y == len(self.buf) - 1:
-            self._decrement_y()
+            self.buf.left(margin)
-            self.x = self.x_hint = len(self.lines[self.y])
         # at the beginning of the line, we join the current line and
         # the previous line
-        elif self.x == 0:
+        elif self.buf.x == 0:
-            victim = self.lines.pop(self.y)
+            y, victim = self.buf.y, self.buf.pop(self.buf.y)
-            new_x = len(self.lines[self.y - 1])
+            self.buf.left(margin)
-            self.lines[self.y - 1] += victim
+            self.buf[y - 1] += victim
-            self._decrement_y()
-            self.x = self.x_hint = new_x
         else:
-            s = self.lines[self.y]
+            s = self.buf[self.buf.y]
-            self.lines[self.y] = s[:self.x - 1] + s[self.x:]
+            self.buf[self.buf.y] = s[:self.buf.x - 1] + s[self.buf.x:]
-            self.x = self.x_hint = self.x - 1
+            self.buf.left(margin)
 
     @edit_action('delete text', final=False)
     @clear_selection
     def delete(self, margin: Margin) -> None:
+        if (
             # noop at end of the file
-        if self.y == len(self.lines) - 1:
+            self.buf.y == len(self.buf) - 1 or
+            # noop at end of last real line
+            (
+                self.buf.y == len(self.buf) - 2 and
+                self.buf.x == len(self.buf[self.buf.y])
+            )
+        ):
             pass
         # if we're at the end of the line, collapse the line afterwards
-        elif self.x == len(self.lines[self.y]):
+        elif self.buf.x == len(self.buf[self.buf.y]):
-            victim = self.lines.pop(self.y + 1)
+            victim = self.buf.pop(self.buf.y + 1)
-            self.lines[self.y] += victim
+            self.buf[self.buf.y] += victim
         else:
-            s = self.lines[self.y]
+            s = self.buf[self.buf.y]
-            self.lines[self.y] = s[:self.x] + s[self.x + 1:]
+            self.buf[self.buf.y] = s[:self.buf.x] + s[self.buf.x + 1:]
 
     @edit_action('line break', final=False)
     @clear_selection
     def enter(self, margin: Margin) -> None:
-        s = self.lines[self.y]
+        s = self.buf[self.buf.y]
-        self.lines[self.y] = s[:self.x]
+        self.buf[self.buf.y] = s[:self.buf.x]
-        self.lines.insert(self.y + 1, s[self.x:])
+        self.buf.insert(self.buf.y + 1, s[self.buf.x:])
-        self._increment_y(margin)
+        self.buf.down(margin)
-        self.x = self.x_hint = 0
+        self.buf.x = 0
 
     @edit_action('indent selection', final=True)
     def _indent_selection(self, margin: Margin) -> None:
@@ -570,21 +513,21 @@ class File:
         sel_y, sel_x = self.selection.start
         (s_y, _), (e_y, _) = self.selection.get()
         for l_y in range(s_y, e_y + 1):
-            if self.lines[l_y]:
+            if self.buf[l_y]:
-                self.lines[l_y] = ' ' * 4 + self.lines[l_y]
+                self.buf[l_y] = ' ' * 4 + self.buf[l_y]
-                if l_y == self.y:
+                if l_y == self.buf.y:
-                    self.x = self.x_hint = self.x + 4
+                    self.buf.x += 4
                 if l_y == sel_y and sel_x != 0:
                     sel_x += 4
-        self.selection.set(sel_y, sel_x, self.y, self.x)
+        self.selection.set(sel_y, sel_x, self.buf.y, self.buf.x)
 
     @edit_action('insert tab', final=False)
     def _tab(self, margin: Margin) -> None:
-        n = 4 - self.x % 4
+        n = 4 - self.buf.x % 4
-        line = self.lines[self.y]
+        line = self.buf[self.buf.y]
-        self.lines[self.y] = line[:self.x] + n * ' ' + line[self.x:]
+        self.buf[self.buf.y] = line[:self.buf.x] + n * ' ' + line[self.buf.x:]
-        self.x = self.x_hint = self.x + n
+        self.buf.x += n
-        _restore_lines_eof_invariant(self.lines)
+        self.buf.restore_eof_invariant()
 
     def tab(self, margin: Margin) -> None:
         if self.selection.start is not None:
@@ -606,21 +549,21 @@ class File:
         sel_y, sel_x = self.selection.start
         (s_y, _), (e_y, _) = self.selection.get()
         for l_y in range(s_y, e_y + 1):
-            n = self._dedent_line(self.lines[l_y])
+            n = self._dedent_line(self.buf[l_y])
             if n:
-                self.lines[l_y] = self.lines[l_y][n:]
+                self.buf[l_y] = self.buf[l_y][n:]
-                if l_y == self.y:
+                if l_y == self.buf.y:
-                    self.x = self.x_hint = max(self.x - n, 0)
+                    self.buf.x = max(self.buf.x - n, 0)
                 if l_y == sel_y:
                     sel_x = max(sel_x - n, 0)
-        self.selection.set(sel_y, sel_x, self.y, self.x)
+        self.selection.set(sel_y, sel_x, self.buf.y, self.buf.x)
 
     @edit_action('dedent', final=True)
     def _dedent(self, margin: Margin) -> None:
-        n = self._dedent_line(self.lines[self.y])
+        n = self._dedent_line(self.buf[self.buf.y])
         if n:
-            self.lines[self.y] = self.lines[self.y][n:]
+            self.buf[self.buf.y] = self.buf[self.buf.y][n:]
-            self.x = self.x_hint = max(self.x - n, 0)
+            self.buf.x = max(self.buf.x - n, 0)
 
     def shift_tab(self, margin: Margin) -> None:
         if self.selection.start is not None:
@@ -634,20 +577,20 @@ class File:
         ret = []
         (s_y, s_x), (e_y, e_x) = self.selection.get()
         if s_y == e_y:
-            ret.append(self.lines[s_y][s_x:e_x])
+            ret.append(self.buf[s_y][s_x:e_x])
-            self.lines[s_y] = self.lines[s_y][:s_x] + self.lines[s_y][e_x:]
+            self.buf[s_y] = self.buf[s_y][:s_x] + self.buf[s_y][e_x:]
         else:
-            ret.append(self.lines[s_y][s_x:])
+            ret.append(self.buf[s_y][s_x:])
             for l_y in range(s_y + 1, e_y):
-                ret.append(self.lines[l_y])
+                ret.append(self.buf[l_y])
-            ret.append(self.lines[e_y][:e_x])
+            ret.append(self.buf[e_y][:e_x])
 
-            self.lines[s_y] = self.lines[s_y][:s_x] + self.lines[e_y][e_x:]
+            self.buf[s_y] = self.buf[s_y][:s_x] + self.buf[e_y][e_x:]
             for _ in range(s_y + 1, e_y + 1):
-                self.lines.pop(s_y + 1)
+                self.buf.pop(s_y + 1)
-        self.y = s_y
+        self.buf.y = s_y
-        self.x = self.x_hint = s_x
+        self.buf.x = s_x
-        self.scroll_screen_if_needed(margin)
+        self.buf.scroll_screen_if_needed(margin)
         return tuple(ret)
 
     def cut(self, cut_buffer: Tuple[str, ...]) -> Tuple[str, ...]:
@@ -656,21 +599,21 @@ class File:
             cut_buffer = ()
 
         with self.edit_action_context('cut', final=False):
-            if self.y == len(self.lines) - 1:
+            if self.buf.y == len(self.buf) - 1:
                 return cut_buffer
             else:
-                victim = self.lines.pop(self.y)
+                victim = self.buf.pop(self.buf.y)
-                self.x = self.x_hint = 0
+                self.buf.x = 0
                 return cut_buffer + (victim,)
 
     def _uncut(self, cut_buffer: Tuple[str, ...], margin: Margin) -> None:
         for cut_line in cut_buffer:
-            line = self.lines[self.y]
+            line = self.buf[self.buf.y]
-            before, after = line[:self.x], line[self.x:]
+            before, after = line[:self.buf.x], line[self.buf.x:]
-            self.lines[self.y] = before + cut_line
+            self.buf[self.buf.y] = before + cut_line
-            self.lines.insert(self.y + 1, after)
+            self.buf.insert(self.buf.y + 1, after)
-            self._increment_y(margin)
+            self.buf.down(margin)
-            self.x = self.x_hint = 0
+            self.buf.x = 0
 
     @edit_action('uncut', final=True)
     @clear_selection
@@ -684,30 +627,31 @@ class File:
             cut_buffer: Tuple[str, ...], margin: Margin,
     ) -> None:
         self._uncut(cut_buffer, margin)
-        self._decrement_y()
+        self.buf.up(margin)
-        self.x = self.x_hint = len(self.lines[self.y])
+        self.buf.x = len(self.buf[self.buf.y])
-        self.lines[self.y] += self.lines.pop(self.y + 1)
+        self.buf[self.buf.y] += self.buf.pop(self.buf.y + 1)
+        self.buf.restore_eof_invariant()
 
     def _sort(self, margin: Margin, s_y: int, e_y: int) -> None:
-        # self.lines intentionally does not support slicing so we use islice
+        # self.buf intentionally does not support slicing so we use islice
-        lines = sorted(itertools.islice(self.lines, s_y, e_y))
+        lines = sorted(itertools.islice(self.buf, s_y, e_y))
         for i, line in zip(range(s_y, e_y), lines):
-            self.lines[i] = line
+            self.buf[i] = line
 
-        self.y = s_y
+        self.buf.y = s_y
-        self.x = self.x_hint = 0
+        self.buf.x = 0
-        self.scroll_screen_if_needed(margin)
+        self.buf.scroll_screen_if_needed(margin)
 
     @edit_action('sort', final=True)
     def sort(self, margin: Margin) -> None:
-        self._sort(margin, 0, len(self.lines) - 1)
+        self._sort(margin, 0, len(self.buf) - 1)
 
     @edit_action('sort selection', final=True)
     @clear_selection
     def sort_selection(self, margin: Margin) -> None:
         (s_y, _), (e_y, _) = self.selection.get()
-        e_y = min(e_y + 1, len(self.lines) - 1)
+        e_y = min(e_y + 1, len(self.buf) - 1)
-        if self.lines[e_y - 1] == '':
+        if self.buf[e_y - 1] == '':
             e_y -= 1
         self._sort(margin, s_y, e_y)
 
@@ -733,7 +677,6 @@ class File:
         b'kEND5': ctrl_end,
         # editing
         b'KEY_BACKSPACE': backspace,
-        b'^H': backspace,  # ^Backspace
         b'KEY_DC': delete,
         b'^M': enter,
         b'^I': tab,
@@ -755,14 +698,12 @@ class File:
 
     @edit_action('text', final=False)
     @clear_selection
-    def c(self, wch: str, margin: Margin) -> None:
+    def c(self, wch: str) -> None:
-        s = self.lines[self.y]
+        self.buf.c(wch)
-        self.lines[self.y] = s[:self.x] + wch + s[self.x:]
+        self.buf.restore_eof_invariant()
-        self.x = self.x_hint = self.x + len(wch)
-        _restore_lines_eof_invariant(self.lines)
 
     def finalize_previous_action(self) -> None:
-        assert not isinstance(self.lines, ListSpy), 'nested edit/movement'
+        assert not self._in_edit_action, 'nested edit/movement'
         self.selection.clear()
         if self.undo_stack:
             self.undo_stack[-1].final = True
@@ -781,106 +722,100 @@ class File:
             final: bool,
     ) -> Generator[None, None, None]:
         continue_last = self._continue_last_action(name)
-        if continue_last:
+        if not continue_last and self.undo_stack:
-            spy = self.undo_stack[-1].spy
-        else:
-            if self.undo_stack:
             self.undo_stack[-1].final = True
-            spy = ListSpy(self.lines)
 
-        before_x, before_line = self.x, self.y
+        before_x, before_line = self.buf.x, self.buf.y
         before_modified = self.modified
-        assert not isinstance(self.lines, ListSpy), 'recursive action?'
+        assert not self._in_edit_action, f'recursive action? {name}'
-        orig, self.lines = self.lines, spy
+        self._in_edit_action = True
         try:
+            with self.buf.record() as modifications:
                 yield
         finally:
-            self.lines = orig
+            self._in_edit_action = False
             self.redo_stack.clear()
             if continue_last:
-                self.undo_stack[-1].end_x = self.x
+                self.undo_stack[-1].end_x = self.buf.x
-                self.undo_stack[-1].end_y = self.y
+                self.undo_stack[-1].end_y = self.buf.y
-                self.touch(spy.min_line_touched)
+                self.undo_stack[-1].modifications.extend(modifications)
-            elif spy.has_modifications:
+            elif modifications:
                 self.modified = True
                 action = Action(
-                    name=name, spy=spy,
+                    name=name, modifications=modifications,
                     start_x=before_x, start_y=before_line,
                     start_modified=before_modified,
-                    end_x=self.x, end_y=self.y,
+                    end_x=self.buf.x, end_y=self.buf.y,
                     end_modified=True,
                     final=final,
                 )
                 self.undo_stack.append(action)
-                self.touch(spy.min_line_touched)
 
     @contextlib.contextmanager
     def select(self) -> Generator[None, None, None]:
         if self.selection.start is None:
-            start = (self.y, self.x)
+            start = (self.buf.y, self.buf.x)
         else:
             start = self.selection.start
         try:
             yield
         finally:
-            self.selection.set(*start, self.y, self.x)
+            self.selection.set(*start, self.buf.y, self.buf.x)
 
     # positioning
 
-    def rendered_y(self, margin: Margin) -> int:
-        return self.y - self.file_y + margin.header
-
-    def rendered_x(self) -> int:
-        return self.x - line_x(self.x, curses.COLS)
-
     def move_cursor(
             self,
             stdscr: 'curses._CursesWindow',
             margin: Margin,
     ) -> None:
-        stdscr.move(self.rendered_y(margin), self.rendered_x())
+        stdscr.move(*self.buf.cursor_position(margin))
-
-    def touch(self, lineno: int) -> None:
-        for file_hl in self._file_hls:
-            file_hl.touch(lineno)
 
     def draw(self, stdscr: 'curses._CursesWindow', margin: Margin) -> None:
-        to_display = min(len(self.lines) - self.file_y, margin.body_lines)
+        to_display = min(self.buf.displayable_count, margin.body_lines)
 
         for file_hl in self._file_hls:
-            file_hl.highlight_until(self.lines, self.file_y + to_display)
+            # XXX: this will go away?
+            file_hl.highlight_until(self.buf, self.buf.file_y + to_display)
 
         for i in range(to_display):
             draw_y = i + margin.header
-            l_y = self.file_y + i
+            l_y = self.buf.file_y + i
-            x = self.x if l_y == self.y else 0
+            stdscr.insstr(draw_y, 0, self.buf.rendered_line(l_y, margin))
-            line = scrolled_line(self.lines[l_y], x, curses.COLS)
-            stdscr.insstr(draw_y, 0, line)
 
-            l_x = line_x(x, curses.COLS)
+            l_x = self.buf.line_x(margin) if l_y == self.buf.y else 0
-            l_x_max = l_x + curses.COLS
+            l_x_max = l_x + margin.cols
             for file_hl in self._file_hls:
                 for region in file_hl.regions[l_y]:
-                    if region.x >= l_x_max:
+                    l_positions = self.buf.line_positions(l_y)
+                    r_x = l_positions[region.x]
+                    # the selection highlight intentionally extends one past
+                    # the end of the line, which won't have a position
+                    if region.end == len(l_positions):
+                        r_end = l_positions[-1] + 1
+                    else:
+                        r_end = l_positions[region.end]
+
+                    if r_x >= l_x_max:
                         break
-                    elif region.end <= l_x:
+                    elif r_end <= l_x:
                         continue
 
-                    if l_x and region.x <= l_x:
+                    if l_x and r_x <= l_x:
                         if file_hl.include_edge:
                             h_s_x = 0
                         else:
                             h_s_x = 1
                     else:
-                        h_s_x = region.x - l_x
+                        h_s_x = r_x - l_x
 
-                    if region.end >= l_x_max:
+                    if r_end >= l_x_max and l_x_max < l_positions[-1]:
                         if file_hl.include_edge:
-                            h_e_x = curses.COLS
+                            h_e_x = margin.cols
                         else:
-                            h_e_x = curses.COLS - 1
+                            h_e_x = margin.cols - 1
                     else:
-                        h_e_x = region.end - l_x
+                        h_e_x = r_end - l_x
 
                     stdscr.chgat(draw_y, h_s_x, h_e_x - h_s_x, region.attr)
 

babi/highlight.py

@@ -14,7 +14,7 @@ from typing import TypeVar
 from identify.identify import tags_from_filename
 
 from babi._types import Protocol
-from babi.fdict import FDict
+from babi.fdict import FChainMap
 from babi.reg import _Reg
 from babi.reg import _RegSet
 from babi.reg import ERR_REG
@@ -67,6 +67,8 @@ class _Rule(Protocol):
     def include(self) -> Optional[str]: ...
     @property
     def patterns(self) -> 'Tuple[_Rule, ...]': ...
+    @property
+    def repository(self) -> 'FChainMap[str, _Rule]': ...
 
 
 @uniquely_constructed
@@ -83,9 +85,24 @@ class Rule(NamedTuple):
     while_captures: Captures
     include: Optional[str]
     patterns: Tuple[_Rule, ...]
+    repository: FChainMap[str, _Rule]
 
     @classmethod
-    def from_dct(cls, dct: Dict[str, Any]) -> _Rule:
+    def make(
+            cls,
+            dct: Dict[str, Any],
+            parent_repository: FChainMap[str, _Rule],
+    ) -> _Rule:
+        if 'repository' in dct:
+            # this looks odd, but it's so we can have a self-referential
+            # immutable-after-construction chain map
+            repository_dct: Dict[str, _Rule] = {}
+            repository = FChainMap(parent_repository, repository_dct)
+            for k, sub_dct in dct['repository'].items():
+                repository_dct[k] = Rule.make(sub_dct, repository)
+        else:
+            repository = parent_repository
+
         name = _split_name(dct.get('name'))
         match = dct.get('match')
         begin = dct.get('begin')
@@ -95,7 +112,7 @@ class Rule(NamedTuple):
 
         if 'captures' in dct:
             captures = tuple(
-                (int(k), Rule.from_dct(v))
+                (int(k), Rule.make(v, repository))
                 for k, v in dct['captures'].items()
             )
         else:
@@ -103,7 +120,7 @@ class Rule(NamedTuple):
 
         if 'beginCaptures' in dct:
             begin_captures = tuple(
-                (int(k), Rule.from_dct(v))
+                (int(k), Rule.make(v, repository))
                 for k, v in dct['beginCaptures'].items()
             )
         else:
@@ -111,7 +128,7 @@ class Rule(NamedTuple):
 
         if 'endCaptures' in dct:
             end_captures = tuple(
-                (int(k), Rule.from_dct(v))
+                (int(k), Rule.make(v, repository))
                 for k, v in dct['endCaptures'].items()
             )
         else:
@@ -119,7 +136,7 @@ class Rule(NamedTuple):
 
         if 'whileCaptures' in dct:
             while_captures = tuple(
-                (int(k), Rule.from_dct(v))
+                (int(k), Rule.make(v, repository))
                 for k, v in dct['whileCaptures'].items()
             )
         else:
@@ -141,7 +158,7 @@ class Rule(NamedTuple):
         include = dct.get('include')
 
         if 'patterns' in dct:
-            patterns = tuple(Rule.from_dct(d) for d in dct['patterns'])
+            patterns = tuple(Rule.make(d, repository) for d in dct['patterns'])
         else:
             patterns = ()
 
@@ -158,29 +175,33 @@ class Rule(NamedTuple):
             while_captures=while_captures,
             include=include,
             patterns=patterns,
+            repository=repository,
         )
 
 
 @uniquely_constructed
 class Grammar(NamedTuple):
     scope_name: str
+    repository: FChainMap[str, _Rule]
     patterns: Tuple[_Rule, ...]
-    repository: FDict[str, _Rule]
 
     @classmethod
-    def from_data(cls, data: Dict[str, Any]) -> 'Grammar':
+    def make(cls, data: Dict[str, Any]) -> 'Grammar':
         scope_name = data['scopeName']
-        patterns = tuple(Rule.from_dct(dct) for dct in data['patterns'])
         if 'repository' in data:
-            repository = FDict({
+            # this looks odd, but it's so we can have a self-referential
-                k: Rule.from_dct(dct) for k, dct in data['repository'].items()
+            # immutable-after-construction chain map
-            })
+            repository_dct: Dict[str, _Rule] = {}
+            repository = FChainMap(repository_dct)
+            for k, dct in data['repository'].items():
+                repository_dct[k] = Rule.make(dct, repository)
         else:
-            repository = FDict({})
+            repository = FChainMap()
+        patterns = tuple(Rule.make(d, repository) for d in data['patterns'])
         return cls(
             scope_name=scope_name,
-            patterns=patterns,
             repository=repository,
+            patterns=patterns,
         )
 
 
@@ -530,22 +551,23 @@ class Compiler:
     def _include(
             self,
             grammar: Grammar,
+            repository: FChainMap[str, _Rule],
             s: str,
     ) -> Tuple[List[str], Tuple[_Rule, ...]]:
         if s == '$self':
             return self._patterns(grammar, grammar.patterns)
         elif s == '$base':
             grammar = self._grammars.grammar_for_scope(self._root_scope)
-            return self._include(grammar, '$self')
+            return self._include(grammar, grammar.repository, '$self')
         elif s.startswith('#'):
-            return self._patterns(grammar, (grammar.repository[s[1:]],))
+            return self._patterns(grammar, (repository[s[1:]],))
         elif '#' not in s:
             grammar = self._grammars.grammar_for_scope(s)
-            return self._include(grammar, '$self')
+            return self._include(grammar, grammar.repository, '$self')
         else:
             scope, _, s = s.partition('#')
             grammar = self._grammars.grammar_for_scope(scope)
-            return self._include(grammar, f'#{s}')
+            return self._include(grammar, grammar.repository, f'#{s}')
 
     @functools.lru_cache(maxsize=None)
     def _patterns(
@@ -557,7 +579,9 @@ class Compiler:
         ret_rules: List[_Rule] = []
         for rule in rules:
             if rule.include is not None:
-                tmp_regs, tmp_rules = self._include(grammar, rule.include)
+                tmp_regs, tmp_rules = self._include(
+                    grammar, rule.repository, rule.include,
+                )
                 ret_regs.extend(tmp_regs)
                 ret_rules.extend(tmp_rules)
             elif rule.match is None and rule.begin is None and rule.patterns:
@@ -633,7 +657,7 @@ class Grammars:
             os.path.splitext(filename)[0]: os.path.join(directory, filename)
             for directory in directories
             if os.path.exists(directory)
-            for filename in os.listdir(directory)
+            for filename in sorted(os.listdir(directory))
             if filename.endswith('.json')
         }
 
@@ -669,7 +693,7 @@ class Grammars:
             pass
 
         raw = self._raw_for_scope(scope)
-        ret = self._parsed[scope] = Grammar.from_data(raw)
+        ret = self._parsed[scope] = Grammar.make(raw)
         return ret
 
     def compiler_for_scope(self, scope: str) -> Compiler:

babi/hl/interface.py

@@ -2,7 +2,7 @@ from typing import NamedTuple
 from typing import Tuple
 
 from babi._types import Protocol
-from babi.list_spy import SequenceNoSlice
+from babi.buf import Buf
 
 
 class HL(NamedTuple):
@@ -23,8 +23,8 @@ class FileHL(Protocol):
     def include_edge(self) -> bool: ...
     @property
     def regions(self) -> RegionsMapping: ...
-    def highlight_until(self, lines: SequenceNoSlice, idx: int) -> None: ...
+    def highlight_until(self, lines: Buf, idx: int) -> None: ...
-    def touch(self, lineno: int) -> None: ...
+    def register_callbacks(self, buf: Buf) -> None: ...
 
 
 class HLFactory(Protocol):

babi/hl/replace.py

@@ -4,9 +4,9 @@ import curses
 from typing import Dict
 from typing import Generator
 
+from babi.buf import Buf
 from babi.hl.interface import HL
 from babi.hl.interface import HLs
-from babi.list_spy import SequenceNoSlice
 
 
 class Replace:
@@ -15,10 +15,10 @@ class Replace:
     def __init__(self) -> None:
         self.regions: Dict[int, HLs] = collections.defaultdict(tuple)
 
-    def highlight_until(self, lines: SequenceNoSlice, idx: int) -> None:
+    def highlight_until(self, lines: Buf, idx: int) -> None:
         """our highlight regions are populated in other ways"""
 
-    def touch(self, lineno: int) -> None:
+    def register_callbacks(self, buf: Buf) -> None:
         """our highlight regions are populated in other ways"""
 
     @contextlib.contextmanager

babi/hl/selection.py

@@ -4,9 +4,9 @@ from typing import Dict
 from typing import Optional
 from typing import Tuple
 
+from babi.buf import Buf
 from babi.hl.interface import HL
 from babi.hl.interface import HLs
-from babi.list_spy import SequenceNoSlice
 
 
 class Selection:
@@ -17,7 +17,10 @@ class Selection:
         self.start: Optional[Tuple[int, int]] = None
         self.end: Optional[Tuple[int, int]] = None
 
-    def highlight_until(self, lines: SequenceNoSlice, idx: int) -> None:
+    def register_callbacks(self, buf: Buf) -> None:
+        """our highlight regions are populated in other ways"""
+
+    def highlight_until(self, lines: Buf, idx: int) -> None:
         if self.start is None or self.end is None:
             return
 
@@ -36,9 +39,6 @@ class Selection:
                 )
             self.regions[e_y] = (HL(x=0, end=e_x, attr=attr),)
 
-    def touch(self, lineno: int) -> None:
-        """our highlight regions are populated in other ways"""
-
     def get(self) -> Tuple[Tuple[int, int], Tuple[int, int]]:
         assert self.start is not None and self.end is not None
         if self.start < self.end:

babi/hl/syntax.py

@@ -7,6 +7,7 @@ from typing import NamedTuple
 from typing import Optional
 from typing import Tuple
 
+from babi.buf import Buf
 from babi.color_manager import ColorManager
 from babi.highlight import Compiler
 from babi.highlight import Grammars
@@ -14,7 +15,6 @@ from babi.highlight import highlight_line
 from babi.highlight import State
 from babi.hl.interface import HL
 from babi.hl.interface import HLs
-from babi.list_spy import SequenceNoSlice
 from babi.theme import Style
 from babi.theme import Theme
 from babi.user_data import prefix_data
@@ -86,7 +86,24 @@ class FileSyntax:
 
         return new_state, tuple(regs)
 
-    def highlight_until(self, lines: SequenceNoSlice, idx: int) -> None:
+    def _set_cb(self, lines: Buf, idx: int, victim: str) -> None:
+        del self.regions[idx:]
+        del self._states[idx:]
+
+    def _del_cb(self, lines: Buf, idx: int, victim: str) -> None:
+        del self.regions[idx:]
+        del self._states[idx:]
+
+    def _ins_cb(self, lines: Buf, idx: int) -> None:
+        del self.regions[idx:]
+        del self._states[idx:]
+
+    def register_callbacks(self, buf: Buf) -> None:
+        buf.add_set_callback(self._set_cb)
+        buf.add_del_callback(self._del_cb)
+        buf.add_ins_callback(self._ins_cb)
+
+    def highlight_until(self, lines: Buf, idx: int) -> None:
         if self._hl is None:
             # the docs claim better performance with power of two sizing
             size = max(4096, 2 ** (int(math.log(len(lines), 2)) + 2))
@@ -103,10 +120,6 @@ class FileSyntax:
             self._states.append(state)
             self.regions.append(regions)
 
-    def touch(self, lineno: int) -> None:
-        del self._states[lineno:]
-        del self.regions[lineno:]
-
 
 class Syntax(NamedTuple):
     grammars: Grammars

babi/hl/trailing_whitespace.py

@@ -1,10 +1,10 @@
 import curses
 from typing import List
 
+from babi.buf import Buf
 from babi.color_manager import ColorManager
 from babi.hl.interface import HL
 from babi.hl.interface import HLs
-from babi.list_spy import SequenceNoSlice
 
 
 class TrailingWhitespace:
@@ -30,9 +30,23 @@ class TrailingWhitespace:
             attr = curses.color_pair(pair)
             return (HL(x=i, end=len(line), attr=attr),)
 
-    def highlight_until(self, lines: SequenceNoSlice, idx: int) -> None:
+    def _set_cb(self, lines: Buf, idx: int, victim: str) -> None:
+        if idx < len(self.regions):
+            self.regions[idx] = self._trailing_ws(lines[idx])
+
+    def _del_cb(self, lines: Buf, idx: int, victim: str) -> None:
+        if idx < len(self.regions):
+            del self.regions[idx]
+
+    def _ins_cb(self, lines: Buf, idx: int) -> None:
+        if idx < len(self.regions):
+            self.regions.insert(idx, self._trailing_ws(lines[idx]))
+
+    def register_callbacks(self, buf: Buf) -> None:
+        buf.add_set_callback(self._set_cb)
+        buf.add_del_callback(self._del_cb)
+        buf.add_ins_callback(self._ins_cb)
+
+    def highlight_until(self, lines: Buf, idx: int) -> None:
         for i in range(len(self.regions), idx):
             self.regions.append(self._trailing_ws(lines[i]))
-
-    def touch(self, lineno: int) -> None:
-        del self.regions[lineno:]

babi/horizontal_scrolling.py

@@ -1,3 +1,10 @@
+import bisect
+import curses
+from typing import Tuple
+
+from babi.cached_property import cached_property
+
+
 def line_x(x: int, width: int) -> int:
     if x + 1 < width:
         return 0
@@ -13,15 +20,48 @@ def line_x(x: int, width: int) -> int:
         )
 
 
-def scrolled_line(s: str, x: int, width: int) -> str:
+def scrolled_line(
-    l_x = line_x(x, width)
+        s: str,
+        positions: Tuple[int, ...],
+        cursor_x: int,
+        width: int,
+) -> str:
+    l_x = line_x(cursor_x, width)
     if l_x:
-        s = f'«{s[l_x + 1:]}'
+        l_x_min = l_x + 1
-        if len(s) > width:
+        start = bisect.bisect_left(positions, l_x_min)
-            return f'{s[:width - 1]}»'
+        pad_left = '«' * (positions[start] - l_x)
+
+        l_x_max = l_x + width
+        if positions[-1] > l_x_max:
+            end_max = l_x_max - 1
+            end = bisect.bisect_left(positions, end_max)
+            if positions[end] > end_max:
+                end -= 1
+            pad_right = '»' * (l_x_max - positions[end])
+            return f'{pad_left}{s[start:end].expandtabs(4)}{pad_right}'
         else:
-            return s.ljust(width)
+            return f'{pad_left}{s[start:]}'.ljust(width)
-    elif len(s) > width:
+    elif positions[-1] > width:
-        return f'{s[:width - 1]}»'
+        end_max = width - 1
+        end = bisect.bisect_left(positions, end_max)
+        if positions[end] > end_max:
+            end -= 1
+        pad_right = '»' * (width - positions[end])
+        return f'{s[:end].expandtabs(4)}{pad_right}'
     else:
-        return s.ljust(width)
+        return s.expandtabs(4).ljust(width)
+
+
+class _CalcWidth:
+    @cached_property
+    def _window(self) -> 'curses._CursesWindow':
+        return curses.newwin(1, 10)
+
+    def wcwidth(self, c: str) -> int:
+        self._window.addstr(0, 0, c)
+        return self._window.getyx()[1]
+
+
+wcwidth = _CalcWidth().wcwidth
+del _CalcWidth

babi/list_spy.py

@@ -1,85 +0,0 @@
-import functools
-import sys
-from typing import Callable
-from typing import Iterator
-from typing import List
-
-from babi._types import Protocol
-
-
-class SequenceNoSlice(Protocol):
-    def __len__(self) -> int: ...
-    def __getitem__(self, idx: int) -> str: ...
-
-    def __iter__(self) -> Iterator[str]:
-        for i in range(len(self)):
-            yield self[i]
-
-
-class MutableSequenceNoSlice(SequenceNoSlice, Protocol):
-    def __setitem__(self, idx: int, val: str) -> None: ...
-    def __delitem__(self, idx: int) -> None: ...
-    def insert(self, idx: int, val: str) -> None: ...
-
-    def append(self, val: str) -> None:
-        self.insert(len(self), val)
-
-    def pop(self, idx: int = -1) -> str:
-        victim = self[idx]
-        del self[idx]
-        return victim
-
-
-def _del(lst: MutableSequenceNoSlice, *, idx: int) -> None:
-    del lst[idx]
-
-
-def _set(lst: MutableSequenceNoSlice, *, idx: int, val: str) -> None:
-    lst[idx] = val
-
-
-def _ins(lst: MutableSequenceNoSlice, *, idx: int, val: str) -> None:
-    lst.insert(idx, val)
-
-
-class ListSpy(MutableSequenceNoSlice):
-    def __init__(self, lst: MutableSequenceNoSlice) -> None:
-        self._lst = lst
-        self._undo: List[Callable[[MutableSequenceNoSlice], None]] = []
-        self.min_line_touched = sys.maxsize
-
-    def __repr__(self) -> str:
-        return f'{type(self).__name__}({self._lst})'
-
-    def __len__(self) -> int:
-        return len(self._lst)
-
-    def __getitem__(self, idx: int) -> str:
-        return self._lst[idx]
-
-    def __setitem__(self, idx: int, val: str) -> None:
-        self._undo.append(functools.partial(_set, idx=idx, val=self._lst[idx]))
-        self.min_line_touched = min(idx, self.min_line_touched)
-        self._lst[idx] = val
-
-    def __delitem__(self, idx: int) -> None:
-        if idx < 0:
-            idx %= len(self)
-        self._undo.append(functools.partial(_ins, idx=idx, val=self._lst[idx]))
-        self.min_line_touched = min(idx, self.min_line_touched)
-        del self._lst[idx]
-
-    def insert(self, idx: int, val: str) -> None:
-        if idx < 0:
-            idx %= len(self)
-        self._undo.append(functools.partial(_del, idx=idx))
-        self.min_line_touched = min(idx, self.min_line_touched)
-        self._lst.insert(idx, val)
-
-    def undo(self, lst: MutableSequenceNoSlice) -> None:
-        for fn in reversed(self._undo):
-            fn(lst)
-
-    @property
-    def has_modifications(self) -> bool:
-        return bool(self._undo)

babi/main.py

@@ -5,6 +5,7 @@ import sys
 from typing import Optional
 from typing import Sequence
 
+from babi.buf import Buf
 from babi.file import File
 from babi.perf import Perf
 from babi.perf import perf_log
@@ -32,7 +33,7 @@ def _edit(screen: Screen, stdin: str) -> EditResult:
                 return ret
         elif key.keyname == b'STRING':
             assert isinstance(key.wch, str), key.wch
-            screen.file.c(key.wch, screen.margin)
+            screen.file.c(key.wch)
         else:
             screen.status.update(f'unknown key: {key}')
 
@@ -68,7 +69,7 @@ def c_main(
 
 def _key_debug(stdscr: 'curses._CursesWindow') -> int:
     screen = Screen(stdscr, ['<<key debug>>'], Perf())
-    screen.file.lines = ['']
+    screen.file.buf = Buf([''])
 
     while True:
         screen.status.update('press q to quit')
@@ -76,7 +77,7 @@ def _key_debug(stdscr: 'curses._CursesWindow') -> int:
         screen.file.move_cursor(screen.stdscr, screen.margin)
 
         key = screen.get_char()
-        screen.file.lines.insert(-1, f'{key.wch!r} {key.keyname.decode()!r}')
+        screen.file.buf.insert(-1, f'{key.wch!r} {key.keyname.decode()!r}')
         screen.file.down(screen.margin)
         if key.wch == curses.KEY_RESIZE:
             screen.resize()

babi/margin.py

@@ -3,12 +3,20 @@ from typing import NamedTuple
 
 
 class Margin(NamedTuple):
-    header: bool
+    lines: int
-    footer: bool
+    cols: int
+
+    @property
+    def header(self) -> bool:
+        return self.lines > 2
+
+    @property
+    def footer(self) -> bool:
+        return self.lines > 1
 
     @property
     def body_lines(self) -> int:
-        return curses.LINES - self.header - self.footer
+        return self.lines - self.header - self.footer
 
     @property
     def page_size(self) -> int:
@@ -17,11 +25,11 @@ class Margin(NamedTuple):
         else:
             return self.body_lines - 2
 
+    @property
+    def scroll_amount(self) -> int:
+        # integer round up without banker's rounding (so 1/2 => 1 instead of 0)
+        return int(self.lines / 2 + .5)
+
     @classmethod
     def from_current_screen(cls) -> 'Margin':
-        if curses.LINES == 1:
+        return cls(curses.LINES, curses.COLS)
-            return cls(header=False, footer=False)
-        elif curses.LINES == 2:
-            return cls(header=False, footer=True)
-        else:
-            return cls(header=True, footer=True)

babi/prompt.py

@@ -6,8 +6,7 @@ from typing import Tuple
 from typing import TYPE_CHECKING
 from typing import Union
 
-from babi.horizontal_scrolling import line_x
+from babi.buf import Buf
-from babi.horizontal_scrolling import scrolled_line
 
 if TYPE_CHECKING:
     from babi.main import Screen  # XXX: circular
@@ -19,43 +18,54 @@ class Prompt:
     def __init__(self, screen: 'Screen', prompt: str, lst: List[str]) -> None:
         self._screen = screen
         self._prompt = prompt
-        self._lst = lst
+        self._buf = Buf(lst)
-        self._y = len(lst) - 1
+        self._buf.y = self._buf.file_y = len(lst) - 1
         self._x = len(self._s)
 
+    @property
+    def _x(self) -> int:
+        return self._buf.x
+
+    @_x.setter
+    def _x(self, x: int) -> None:
+        self._buf.x = x
+
     @property
     def _s(self) -> str:
-        return self._lst[self._y]
+        return self._buf[self._buf.y]
 
     @_s.setter
     def _s(self, s: str) -> None:
-        self._lst[self._y] = s
+        self._buf[self._buf.y] = s
 
     def _render_prompt(self, *, base: Optional[str] = None) -> None:
         base = base or self._prompt
-        if not base or curses.COLS < 7:
+        if not base or self._screen.margin.cols < 7:
             prompt_s = ''
-        elif len(base) > curses.COLS - 6:
+        elif len(base) > self._screen.margin.cols - 6:
-            prompt_s = f'{base[:curses.COLS - 7]}…: '
+            prompt_s = f'{base[:self._screen.margin.cols - 7]}…: '
         else:
             prompt_s = f'{base}: '
-        width = curses.COLS - len(prompt_s)
+
-        line = scrolled_line(self._s, self._x, width)
+        width = self._screen.margin.cols - len(prompt_s)
-        cmd = f'{prompt_s}{line}'
+        margin = self._screen.margin._replace(cols=width)
-        self._screen.stdscr.insstr(curses.LINES - 1, 0, cmd, curses.A_REVERSE)
+        cmd = f'{prompt_s}{self._buf.rendered_line(self._buf.y, margin)}'
-        x = len(prompt_s) + self._x - line_x(self._x, width)
+        prompt_line = self._screen.margin.lines - 1
-        self._screen.stdscr.move(curses.LINES - 1, x)
+        self._screen.stdscr.insstr(prompt_line, 0, cmd, curses.A_REVERSE)
+
+        _, x_off = self._buf.cursor_position(margin)
+        self._screen.stdscr.move(prompt_line, len(prompt_s) + x_off)
 
     def _up(self) -> None:
-        self._y = max(0, self._y - 1)
+        self._buf.up(self._screen.margin)
-        self._x = len(self._s)
+        self._x = len(self._buf[self._buf.y])
 
     def _down(self) -> None:
-        self._y = min(len(self._lst) - 1, self._y + 1)
+        self._buf.down(self._screen.margin)
-        self._x = len(self._s)
+        self._x = len(self._buf[self._buf.y])
 
     def _right(self) -> None:
-        self._x = min(len(self._s), self._x + 1)
+        self._x = min(len(self._buf[self._buf.y]), self._x + 1)
 
     def _left(self) -> None:
         self._x = max(0, self._x - 1)
@@ -64,11 +74,11 @@ class Prompt:
         self._x = 0
 
     def _end(self) -> None:
-        self._x = len(self._s)
+        self._x = len(self._buf[self._buf.y])
 
     def _ctrl_left(self) -> None:
         if self._x <= 1:
-            self._x = 0
+            self._buf.home()
         else:
             self._x -= 1
             tp = self._s[self._x - 1].isalnum()
@@ -77,7 +87,7 @@ class Prompt:
 
     def _ctrl_right(self) -> None:
         if self._x >= len(self._s) - 1:
-            self._x = len(self._s)
+            self._buf.end()
         else:
             self._x += 1
             tp = self._s[self._x].isalnum()
@@ -102,9 +112,9 @@ class Prompt:
     def _check_failed(self, idx: int, s: str) -> Tuple[bool, int]:
         failed = False
         for search_idx in range(idx, -1, -1):
-            if s in self._lst[search_idx]:
+            if s in self._buf[search_idx]:
-                idx = self._y = search_idx
+                idx = self._buf.y = search_idx
-                self._x = self._lst[search_idx].index(s)
+                self._x = self._buf[search_idx].index(s)
                 break
         else:
             failed = True
@@ -112,7 +122,7 @@ class Prompt:
 
     def _reverse_search(self) -> Union[None, str, PromptResult]:
         reverse_s = ''
-        idx = self._y
+        idx = self._buf.y
         while True:
             fail, idx = self._check_failed(idx, reverse_s)
 
@@ -126,7 +136,7 @@ class Prompt:
             key = self._screen.get_char()
             if key.keyname == b'KEY_RESIZE':
                 self._screen.resize()
-            elif key.keyname == b'KEY_BACKSPACE' or key.keyname == b'^H':
+            elif key.keyname == b'KEY_BACKSPACE':
                 reverse_s = reverse_s[:-1]
             elif key.keyname == b'^R':
                 idx = max(0, idx - 1)
@@ -163,7 +173,6 @@ class Prompt:
         b'kLFT5': _ctrl_left,
         # editing
         b'KEY_BACKSPACE': _backspace,
-        b'^H': _backspace,  # ^Backspace
         b'KEY_DC': _delete,
         b'^K': _cut_to_end,
         # misc
@@ -174,8 +183,7 @@ class Prompt:
     }
 
     def _c(self, c: str) -> None:
-        self._s = self._s[:self._x] + c + self._s[self._x:]
+        self._buf.c(c)
-        self._x += len(c)
 
     def run(self) -> Union[PromptResult, str]:
         while True:

babi/reg.py

@@ -43,26 +43,26 @@ def _replace_esc(s: str, chars: str) -> str:
 
 class _Reg:
     def __init__(self, s: str) -> None:
-        self._pattern = _replace_esc(s, 'z')
+        self._pattern = s
 
     def __repr__(self) -> str:
         return f'{type(self).__name__}({self._pattern!r})'
 
     @cached_property
     def _reg(self) -> onigurumacffi._Pattern:
-        return onigurumacffi.compile(self._pattern)
+        return onigurumacffi.compile(_replace_esc(self._pattern, 'z'))
 
     @cached_property
     def _reg_no_A(self) -> onigurumacffi._Pattern:
-        return onigurumacffi.compile(_replace_esc(self._pattern, 'A'))
+        return onigurumacffi.compile(_replace_esc(self._pattern, 'Az'))
 
     @cached_property
     def _reg_no_G(self) -> onigurumacffi._Pattern:
-        return onigurumacffi.compile(_replace_esc(self._pattern, 'G'))
+        return onigurumacffi.compile(_replace_esc(self._pattern, 'Gz'))
 
     @cached_property
     def _reg_no_A_no_G(self) -> onigurumacffi._Pattern:
-        return onigurumacffi.compile(_replace_esc(self._pattern, 'AG'))
+        return onigurumacffi.compile(_replace_esc(self._pattern, 'AGz'))
 
     def _get_reg(
             self,

babi/screen.py

@@ -61,6 +61,36 @@ SEQUENCE_KEYNAME = {
     '\x1b[1;6H': b'kHOM6',  # Shift + ^Home
     '\x1b[1;6F': b'kEND6',  # Shift + ^End
 }
+KEYNAME_REWRITE = {
+    # windows-curses: numeric pad arrow keys
+    # - some overlay keyboards pick these as well
+    # - in xterm it seems these are mapped automatically
+    b'KEY_A2': b'KEY_UP',
+    b'KEY_C2': b'KEY_DOWN',
+    b'KEY_B3': b'KEY_RIGHT',
+    b'KEY_B1': b'KEY_LEFT',
+    b'PADSTOP': b'KEY_DC',
+    b'KEY_A3': b'KEY_PPAGE',
+    b'KEY_C3': b'KEY_NPAGE',
+    b'KEY_A1': b'KEY_HOME',
+    b'KEY_C1': b'KEY_END',
+    # windows-curses: map to our M- names
+    b'ALT_U': b'M-u',
+    # windows-curses: arguably these names are better than the xterm names
+    b'CTL_UP': b'kUP5',
+    b'CTL_DOWN': b'kDN5',
+    b'CTL_RIGHT': b'kRIT5',
+    b'CTL_LEFT': b'kLFT5',
+    b'ALT_RIGHT': b'kRIT3',
+    b'ALT_LEFT': b'kLFT3',
+    # windows-curses: idk why these are different
+    b'KEY_SUP': b'KEY_SR',
+    b'KEY_SDOWN': b'KEY_SF',
+    # macos: (sends this for backspace key, others interpret this as well)
+    b'^?': b'KEY_BACKSPACE',
+    # linux, perhaps others
+    b'^H': b'KEY_BACKSPACE',  # ^Backspace on my keyboard
+}
 
 
 class Key(NamedTuple):
@@ -105,7 +135,7 @@ class Screen:
         else:
             files = ''
             version_width = len(VERSION_STR) + 2
-        centered = filename.center(curses.COLS)[version_width:]
+        centered = filename.center(self.margin.cols)[version_width:]
         s = f' {VERSION_STR} {files}{centered}{files}'
         self.stdscr.insstr(0, 0, s, curses.A_REVERSE)
 
@@ -202,12 +232,10 @@ class Screen:
         elif isinstance(wch, str) and wch.isprintable():
             wch = self._get_string(wch)
             return Key(wch, b'STRING')
-        elif wch == '\x7f':  # pragma: no cover (macos)
-            keyname = curses.keyname(curses.KEY_BACKSPACE)
-            return Key(wch, keyname)
 
         key = wch if isinstance(wch, int) else ord(wch)
         keyname = curses.keyname(key)
+        keyname = KEYNAME_REWRITE.get(keyname, keyname)
         return Key(wch, keyname)
 
     def get_char(self) -> Key:
@@ -225,7 +253,7 @@ class Screen:
     def resize(self) -> None:
         curses.update_lines_cols()
         self.margin = Margin.from_current_screen()
-        self.file.scroll_screen_if_needed(self.margin)
+        self.file.buf.scroll_screen_if_needed(self.margin)
         self.draw()
 
     def quick_prompt(
@@ -236,13 +264,14 @@ class Screen:
         opts = [opt[0] for opt in opt_strs]
         while True:
             x = 0
+            prompt_line = self.margin.lines - 1
 
             def _write(s: str, *, attr: int = curses.A_REVERSE) -> None:
                 nonlocal x
 
-                if x >= curses.COLS:
+                if x >= self.margin.cols:
                     return
-                self.stdscr.insstr(curses.LINES - 1, x, s, attr)
+                self.stdscr.insstr(prompt_line, x, s, attr)
                 x += len(s)
 
             _write(prompt)
@@ -254,15 +283,15 @@ class Screen:
                     _write(', ')
             _write(']?')
 
-            if x < curses.COLS - 1:
+            if x < self.margin.cols - 1:
-                s = ' ' * (curses.COLS - x)
+                s = ' ' * (self.margin.cols - x)
-                self.stdscr.insstr(curses.LINES - 1, x, s, curses.A_REVERSE)
+                self.stdscr.insstr(prompt_line, x, s, curses.A_REVERSE)
                 x += 1
             else:
-                x = curses.COLS - 1
+                x = self.margin.cols - 1
-                self.stdscr.insstr(curses.LINES - 1, x, '…', curses.A_REVERSE)
+                self.stdscr.insstr(prompt_line, x, '…', curses.A_REVERSE)
 
-            self.stdscr.move(curses.LINES - 1, x)
+            self.stdscr.move(prompt_line, x)
 
             key = self.get_char()
             if key.keyname == b'KEY_RESIZE':
@@ -317,9 +346,9 @@ class Screen:
                 self.file.go_to_line(lineno, self.margin)
 
     def current_position(self) -> None:
-        line = f'line {self.file.y + 1}'
+        line = f'line {self.file.buf.y + 1}'
-        col = f'col {self.file.x + 1}'
+        col = f'col {self.file.buf.x + 1}'
-        line_count = max(len(self.file.lines) - 1, 1)
+        line_count = max(len(self.file.buf) - 1, 1)
         lines_word = 'line' if line_count == 1 else 'lines'
         self.status.update(f'{line}, {col} (of {line_count} {lines_word})')
 
@@ -358,7 +387,7 @@ class Screen:
         else:
             action = from_stack.pop()
             to_stack.append(action.apply(self.file))
-            self.file.scroll_screen_if_needed(self.margin)
+            self.file.buf.scroll_screen_if_needed(self.margin)
             self.status.update(f'{op}: {action.name}')
             self.file.selection.clear()
 
@@ -385,6 +414,8 @@ class Screen:
     def command(self) -> Optional[EditResult]:
         response = self.prompt('', history='command')
         if response == ':q':
+            return self.quit_save_modified()
+        elif response == ':q!':
             return EditResult.EXIT
         elif response == ':w':
             self.save()
@@ -421,7 +452,7 @@ class Screen:
         else:
             sha256 = hashlib.sha256(b'').hexdigest()
 
-        contents = self.file.nl.join(self.file.lines)
+        contents = self.file.nl.join(self.file.buf)
         sha256_to_save = hashlib.sha256(contents.encode()).hexdigest()
 
         # the file on disk is the same as when we opened it
@@ -429,12 +460,12 @@ class Screen:
             self.status.update('(file changed on disk, not implemented)')
             return PromptResult.CANCELLED
 
-        with open(self.file.filename, 'w') as f:
+        with open(self.file.filename, 'w', newline='') as f:
             f.write(contents)
 
         self.file.modified = False
         self.file.sha256 = sha256_to_save
-        num_lines = len(self.file.lines) - 1
+        num_lines = len(self.file.buf) - 1
         lines = 'lines' if num_lines != 1 else 'line'
         self.status.update(f'saved! ({num_lines} {lines} written)')
 

babi/status.py

@@ -18,14 +18,14 @@ class Status:
 
     def draw(self, stdscr: 'curses._CursesWindow', margin: Margin) -> None:
         if margin.footer or self._status:
-            stdscr.insstr(curses.LINES - 1, 0, ' ' * curses.COLS)
+            stdscr.insstr(margin.lines - 1, 0, ' ' * margin.cols)
             if self._status:
                 status = f' {self._status} '
-                x = (curses.COLS - len(status)) // 2
+                x = (margin.cols - len(status)) // 2
                 if x < 0:
                     x = 0
                     status = status.strip()
-                stdscr.insstr(curses.LINES - 1, x, status, curses.A_REVERSE)
+                stdscr.insstr(margin.lines - 1, x, status, curses.A_REVERSE)
 
     def tick(self, margin: Margin) -> None:
         # when the window is only 1-tall, hide the status quicker

bin/download-theme

@@ -39,7 +39,6 @@ def json_with_comments(s: bytes) -> Any:
         idx = match.end()
         match = TOKEN.search(s, idx)
 
-    print(bio.getvalue())
     bio.seek(0)
     return json.load(bio)
 

requirements-dev.txt

@@ -3,3 +3,4 @@ coverage
 git+https://github.com/asottile/hecate@875567f
 pytest
 remote-pdb
+wcwidth

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = babi
-version = 0.0.4
+version = 0.0.7
 description = a text editor
 long_description = file: README.md
 long_description_content_type = text/markdown

tests/buf_test.py

@@ -0,0 +1,178 @@
+import pytest
+
+from babi.buf import Buf
+
+
+def test_buf_repr():
+    ret = repr(Buf(['a', 'b', 'c']))
+    assert ret == "Buf(['a', 'b', 'c'], x=0, y=0, file_y=0)"
+
+
+def test_buf_item_retrieval():
+    buf = Buf(['a', 'b', 'c'])
+    assert buf[1] == 'b'
+    assert buf[-1] == 'c'
+    with pytest.raises(IndexError):
+        buf[3]
+
+
+def test_buf_del():
+    lst = ['a', 'b', 'c']
+
+    buf = Buf(lst)
+
+    with buf.record() as modifications:
+        del buf[1]
+
+    assert lst == ['a', 'c']
+
+    buf.apply(modifications)
+
+    assert lst == ['a', 'b', 'c']
+
+
+def test_buf_del_with_negative():
+    lst = ['a', 'b', 'c']
+
+    buf = Buf(lst)
+
+    with buf.record() as modifications:
+        del buf[-1]
+
+    assert lst == ['a', 'b']
+
+    buf.apply(modifications)
+
+    assert lst == ['a', 'b', 'c']
+
+
+def test_buf_insert():
+    lst = ['a', 'b', 'c']
+
+    buf = Buf(lst)
+
+    with buf.record() as modifications:
+        buf.insert(1, 'q')
+
+    assert lst == ['a', 'q', 'b', 'c']
+
+    buf.apply(modifications)
+
+    assert lst == ['a', 'b', 'c']
+
+
+def test_buf_insert_with_negative():
+    lst = ['a', 'b', 'c']
+
+    buf = Buf(lst)
+
+    with buf.record() as modifications:
+        buf.insert(-1, 'q')
+
+    assert lst == ['a', 'b', 'q', 'c']
+
+    buf.apply(modifications)
+
+    assert lst == ['a', 'b', 'c']
+
+
+def test_buf_set_value():
+    lst = ['a', 'b', 'c']
+
+    buf = Buf(lst)
+
+    with buf.record() as modifications:
+        buf[1] = 'hello'
+
+    assert lst == ['a', 'hello', 'c']
+
+    buf.apply(modifications)
+
+    assert lst == ['a', 'b', 'c']
+
+
+def test_buf_set_value_idx_negative():
+    lst = ['a', 'b', 'c']
+
+    buf = Buf(lst)
+
+    with buf.record() as modifications:
+        buf[-1] = 'hello'
+
+    assert lst == ['a', 'b', 'hello']
+
+    buf.apply(modifications)
+
+    assert lst == ['a', 'b', 'c']
+
+
+def test_buf_multiple_modifications():
+    lst = ['a', 'b', 'c']
+
+    buf = Buf(lst)
+
+    with buf.record() as modifications:
+        buf[1] = 'hello'
+        buf.insert(1, 'ohai')
+        del buf[0]
+
+    assert lst == ['ohai', 'hello', 'c']
+
+    buf.apply(modifications)
+
+    assert lst == ['a', 'b', 'c']
+
+
+def test_buf_iter():
+    buf = Buf(['a', 'b', 'c'])
+    buf_iter = iter(buf)
+    assert next(buf_iter) == 'a'
+    assert next(buf_iter) == 'b'
+    assert next(buf_iter) == 'c'
+    with pytest.raises(StopIteration):
+        next(buf_iter)
+
+
+def test_buf_append():
+    lst = ['a', 'b', 'c']
+
+    buf = Buf(lst)
+
+    with buf.record() as modifications:
+        buf.append('q')
+
+    assert lst == ['a', 'b', 'c', 'q']
+
+    buf.apply(modifications)
+
+    assert lst == ['a', 'b', 'c']
+
+
+def test_buf_pop_default():
+    lst = ['a', 'b', 'c']
+
+    buf = Buf(lst)
+
+    with buf.record() as modifications:
+        buf.pop()
+
+    assert lst == ['a', 'b']
+
+    buf.apply(modifications)
+
+    assert lst == ['a', 'b', 'c']
+
+
+def test_buf_pop_idx():
+    lst = ['a', 'b', 'c']
+
+    buf = Buf(lst)
+
+    with buf.record() as modifications:
+        buf.pop(1)
+
+    assert lst == ['a', 'c']
+
+    buf.apply(modifications)
+
+    assert lst == ['a', 'b', 'c']

tests/fdict_test.py

@@ -1,3 +1,6 @@
+import pytest
+
+from babi.fdict import FChainMap
 from babi.fdict import FDict
 
 
@@ -5,3 +8,21 @@ def test_fdict_repr():
     # mostly because this shouldn't get hit elsewhere but is uesful for
     # debugging purposes
     assert repr(FDict({1: 2, 3: 4})) == 'FDict({1: 2, 3: 4})'
+
+
+def test_f_chain_map():
+    chain_map = FChainMap({1: 2}, {3: 4}, FDict({1: 5}))
+    assert chain_map[1] == 5
+    assert chain_map[3] == 4
+
+    with pytest.raises(KeyError) as excinfo:
+        chain_map[2]
+    k, = excinfo.value.args
+    assert k == 2
+
+
+def test_f_chain_map_extend():
+    chain_map = FChainMap({1: 2})
+    assert chain_map[1] == 2
+    chain_map = FChainMap(chain_map, {1: 5})
+    assert chain_map[1] == 5

tests/features/conftest.py

@@ -9,6 +9,7 @@ from typing import Union
 from unittest import mock
 
 import pytest
+import wcwidth
 
 from babi._types import Protocol
 from babi.main import main
@@ -46,7 +47,6 @@ def ten_lines(tmpdir):
 
 class Screen:
     def __init__(self, width, height):
-        self.disabled = True
         self.nodelay = False
         self.width = width
         self.height = height
@@ -64,6 +64,16 @@ class Screen:
             self._prev_screenshot = ret
         return ret
 
+    def addstr(self, y, x, s, attr):
+        self.lines[y] = self.lines[y][:x] + s + self.lines[y][x + len(s):]
+
+        line_attr = self.attrs[y]
+        new = [attr] * len(s)
+        self.attrs[y] = line_attr[:x] + new + line_attr[x + len(s):]
+
+        self.y = y
+        self.x = x + wcwidth.wcswidth(s)
+
     def insstr(self, y, x, s, attr):
         line = self.lines[y]
         self.lines[y] = (line[:x] + s + line[x:])[:self.width]
@@ -174,7 +184,8 @@ class CursesError(NamedTuple):
 
 
 class CursesScreen:
-    def __init__(self, runner):
+    def __init__(self, screen, runner):
+        self._screen = screen
         self._runner = runner
         self._bkgd_attr = (-1, -1, 0)
 
@@ -198,20 +209,26 @@ class CursesScreen:
         pass
 
     def nodelay(self, val):
-        self._runner.screen.nodelay = val
+        self._screen.nodelay = val
+
+    def addstr(self, y, x, s, attr=0):
+        self._screen.addstr(y, x, s, self._to_attr(attr))
 
     def insstr(self, y, x, s, attr=0):
-        self._runner.screen.insstr(y, x, s, self._to_attr(attr))
+        self._screen.insstr(y, x, s, self._to_attr(attr))
 
     def clrtoeol(self):
-        s = self._runner.screen.width * ' '
+        s = self._screen.width * ' '
-        self.insstr(self._runner.screen.y, self._runner.screen.x, s)
+        self.insstr(self._screen.y, self._screen.x, s)
 
     def chgat(self, y, x, n, attr):
-        self._runner.screen.chgat(y, x, n, self._to_attr(attr))
+        self._screen.chgat(y, x, n, self._to_attr(attr))
 
     def move(self, y, x):
-        self._runner.screen.move(y, x)
+        self._screen.move(y, x)
+
+    def getyx(self):
+        return self._screen.y, self._screen.x
 
     def get_wch(self):
         return self._runner._get_wch()
@@ -373,8 +390,8 @@ class DeferredRunner:
     def _curses__noop(self, *_, **__):
         pass
 
-    _curses_cbreak = _curses_noecho = _curses_nonl = _curses__noop
+    _curses_cbreak = _curses_endwin = _curses_noecho = _curses__noop
-    _curses_raw = _curses_use_default_colors = _curses__noop
+    _curses_nonl = _curses_raw = _curses_use_default_colors = _curses__noop
 
     _curses_error = curses.error  # so we don't mock the exception
 
@@ -400,11 +417,10 @@ class DeferredRunner:
 
     def _curses_initscr(self):
         self._curses_update_lines_cols()
-        self.screen.disabled = False
+        return CursesScreen(self.screen, self)
-        return CursesScreen(self)
 
-    def _curses_endwin(self):
+    def _curses_newwin(self, height, width):
-        self.screen.disabled = True
+        return CursesScreen(Screen(width, height), self)
 
     def _curses_not_implemented(self, fn):
         def fn_inner(*args, **kwargs):

tests/features/cut_uncut_test.py

@@ -132,3 +132,21 @@ def test_selection_cut_uncut_selection_offscreen_x(run):
         h.await_text_missing('hello')
         h.press('^K')
         h.await_text('hello\n')
+
+
+def test_selection_cut_uncut_at_end_of_file(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('S-Down')
+        h.press('S-Right')
+        h.press('^K')
+        h.await_text_missing('line_0')
+        h.await_text_missing('line_1')
+        h.await_text('ine_1')
+
+        h.press('^End')
+        h.press('^U')
+        h.await_text('line_0\nl\n')
+        h.await_cursor_position(x=1, y=11)
+
+        h.press('Down')
+        h.await_cursor_position(x=0, y=12)

tests/features/movement_test.py

@@ -411,3 +411,78 @@ def test_sequence_handling(run_only_fake):
         h.press(' test7')
         h.await_text('test1 test2 test3 test4 test5 test6 test7')
         h.await_text(r'\x1b[1;')
+
+
+def test_indentation_using_tabs(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write(
+        f'123456789\n'
+        f'\t12\t{"x" * 20}\n'
+        f'\tnot long\n',
+    )
+
+    with run(str(f), width=20) as h, and_exit(h):
+        h.await_text(
+            '123456789\n'
+            '    12  xxxxxxxxxxx»\n'
+            '    not long\n',
+        )
+
+        h.press('Down')
+        h.await_cursor_position(x=0, y=2)
+        h.press('Up')
+        h.await_cursor_position(x=0, y=1)
+
+        h.press('Right')
+        h.await_cursor_position(x=1, y=1)
+        h.press('Down')
+        h.await_cursor_position(x=0, y=2)
+        h.press('Up')
+        h.await_cursor_position(x=1, y=1)
+
+        h.press('Down')
+        h.await_cursor_position(x=0, y=2)
+        h.press('Right')
+        h.await_cursor_position(x=4, y=2)
+        h.press('Up')
+        h.await_cursor_position(x=4, y=1)
+
+
+def test_movement_with_wide_characters(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write(
+        f'{"🙃" * 20}\n'
+        f'a{"🙃" * 20}\n',
+    )
+
+    with run(str(f), width=20) as h, and_exit(h):
+        h.await_text(
+            '🙃🙃🙃🙃🙃🙃🙃🙃🙃»»\n'
+            'a🙃🙃🙃🙃🙃🙃🙃🙃🙃»\n',
+        )
+
+        for _ in range(10):
+            h.press('Right')
+        h.await_text(
+            '««🙃🙃🙃🙃🙃🙃🙃🙃»»\n'
+            'a🙃🙃🙃🙃🙃🙃🙃🙃🙃»\n',
+        )
+
+        for _ in range(6):
+            h.press('Right')
+        h.await_text(
+            '««🙃🙃🙃🙃🙃🙃🙃\n'
+            'a🙃🙃🙃🙃🙃🙃🙃🙃🙃»\n',
+        )
+
+        h.press('Down')
+        h.await_text(
+            '🙃🙃🙃🙃🙃🙃🙃🙃🙃»»\n'
+            '«🙃🙃🙃🙃🙃🙃🙃🙃\n',
+        )
+
+        h.press('Left')
+        h.await_text(
+            '🙃🙃🙃🙃🙃🙃🙃🙃🙃»»\n'
+            '«🙃🙃🙃🙃🙃🙃🙃🙃🙃»\n',
+        )

tests/features/save_test.py

@@ -1,6 +1,7 @@
 import pytest
 
 from testing.runner import and_exit
+from testing.runner import trigger_command_mode
 
 
 def test_mixed_newlines(run, tmpdir):
@@ -25,6 +26,18 @@ def test_modify_file_with_windows_newlines(run, tmpdir):
     assert f.read_binary() == b'\r\nfoo\r\nbar\r\n'
 
 
+def test_saving_file_with_multiple_lines_at_end_maintains_those(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('foo\n\n')
+    with run(str(f)) as h, and_exit(h):
+        h.press('a')
+        h.await_text('*')
+        h.press('^S')
+        h.await_text('saved!')
+
+    assert f.read() == 'afoo\n\n'
+
+
 def test_new_file(run):
     with run('this_is_a_new_file') as h, and_exit(h):
         h.await_text('this_is_a_new_file')
@@ -202,3 +215,38 @@ def test_save_on_exit_resize(run, tmpdir):
         h.await_text('file is modified - save [yes, no]?')
         h.press('^C')
         h.await_text('cancelled')
+
+
+def test_vim_save_on_exit_cancel_yn(run):
+    with run() as h, and_exit(h):
+        h.press('hello')
+        h.await_text('hello')
+        trigger_command_mode(h)
+        h.press_and_enter(':q')
+        h.await_text('file is modified - save [yes, no]?')
+        h.press('^C')
+        h.await_text('cancelled')
+
+
+def test_vim_save_on_exit(run, tmpdir):
+    f = tmpdir.join('f')
+    with run(str(f)) as h:
+        h.press('hello')
+        h.await_text('hello')
+        trigger_command_mode(h)
+        h.press_and_enter(':q')
+        h.await_text('file is modified - save [yes, no]?')
+        h.press('y')
+        h.await_text(f'enter filename: ')
+        h.press('Enter')
+        h.await_exit()
+
+
+def test_vim_force_exit(run, tmpdir):
+    f = tmpdir.join('f')
+    with run(str(f)) as h:
+        h.press('hello')
+        h.await_text('hello')
+        trigger_command_mode(h)
+        h.press_and_enter(':q!')
+        h.await_exit()

tests/features/syntax_highlight_test.py

@@ -113,3 +113,43 @@ def test_syntax_highlighting_one_off_left_of_screen(run, tmpdir):
         h.press('End')
         h.await_text_missing('?')
         h.assert_screen_attr_equals(1, [(236, 40, 0)] * 20)
+
+
+def test_syntax_highlighting_to_edge_of_screen(run, tmpdir):
+    f = tmpdir.join('f.demo')
+    f.write(f'# {"x" * 18}')
+
+    with run(str(f), term='screen-256color', width=20) as h, and_exit(h):
+        h.await_text('# xxx')
+        h.assert_screen_attr_equals(1, [(243, 40, 0)] * 20)
+
+
+def test_syntax_highlighting_with_tabs(run, tmpdir):
+    f = tmpdir.join('f.demo')
+    f.write('\t# 12345678901234567890\n')
+
+    with run(str(f), term='screen-256color', width=20) as h, and_exit(h):
+        h.await_text('1234567890')
+        expected = 4 * [(236, 40, 0)] + 15 * [(243, 40, 0)] + [(236, 40, 0)]
+        h.assert_screen_attr_equals(1, expected)
+
+
+def test_syntax_highlighting_tabs_after_line_creation(run, tmpdir):
+    f = tmpdir.join('f')
+    # trailing whitespace is used to trigger highlighting
+    f.write('foo\n\txx \ny    \n')
+
+    with run(str(f), term='screen-256color') as h, and_exit(h):
+        # this looks weird, but it populates the width cache
+        h.press('Down')
+        h.press('Down')
+        h.press('Down')
+
+        # press enter after the tab
+        h.press('Up')
+        h.press('Up')
+        h.press('Right')
+        h.press('Right')
+        h.press('Enter')
+
+        h.await_text('foo\n    x\nx\ny\n')

tests/features/text_editing_test.py

@@ -97,6 +97,24 @@ def test_delete_at_end_of_line(run, tmpdir):
         h.await_text('f *')
 
 
+def test_delete_at_end_of_last_line(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('hello\n')
+
+    with run(str(f)) as h, and_exit(h):
+        h.await_text('hello')
+        h.press('End')
+        h.press('DC')
+        # should not make the file modified
+        h.await_text_missing('*')
+
+        # delete should still be functional
+        h.press('Left')
+        h.press('Left')
+        h.press('DC')
+        h.await_text('helo')
+
+
 def test_press_enter_beginning_of_file(run, tmpdir):
     f = tmpdir.join('f')
     f.write('hello world')

tests/highlight_test.py

@@ -441,6 +441,38 @@ def test_include_repository_rule(compiler_state):
     )
 
 
+def test_include_with_nested_repositories(compiler_state):
+    compiler, state = compiler_state({
+        'scopeName': 'test',
+        'patterns': [{
+            'begin': '<', 'end': '>', 'name': 'b',
+            'patterns': [
+                {'include': '#rule1'},
+                {'include': '#rule2'},
+                {'include': '#rule3'},
+            ],
+            'repository': {
+                'rule2': {'match': '2', 'name': 'inner2'},
+                'rule3': {'match': '3', 'name': 'inner3'},
+            },
+        }],
+        'repository': {
+            'rule1': {'match': '1', 'name': 'root1'},
+            'rule2': {'match': '2', 'name': 'root2'},
+        },
+    })
+
+    state, regions = highlight_line(compiler, state, '<123>', first_line=True)
+
+    assert regions == (
+        Region(0, 1, ('test', 'b')),
+        Region(1, 2, ('test', 'b', 'root1')),
+        Region(2, 3, ('test', 'b', 'inner2')),
+        Region(3, 4, ('test', 'b', 'inner3')),
+        Region(4, 5, ('test', 'b')),
+    )
+
+
 def test_include_other_grammar(compiler_state):
     compiler, state = compiler_state(
         {

tests/hl/syntax_test.py

@@ -4,6 +4,7 @@ from unittest import mock
 
 import pytest
 
+from babi.buf import Buf
 from babi.color_manager import ColorManager
 from babi.hl.interface import HL
 from babi.hl.syntax import Syntax
@@ -161,7 +162,7 @@ def test_syntax_highlight_cache_first_line(stdscr, make_grammars):
         syntax = Syntax(grammars, THEME, ColorManager.make())
         syntax._init_screen(stdscr)
         file_hl = syntax.file_highlighter('foo.demo', '')
-        file_hl.highlight_until(['int', 'int'], 2)
+        file_hl.highlight_until(Buf(['int', 'int']), 2)
         assert file_hl.regions == [
             (HL(0, 3, curses.A_BOLD | 2 << 8),),
             (),

tests/list_spy_test.py

@@ -1,144 +0,0 @@
-import pytest
-
-from babi.list_spy import ListSpy
-
-
-def test_list_spy_repr():
-    assert repr(ListSpy(['a', 'b', 'c'])) == "ListSpy(['a', 'b', 'c'])"
-
-
-def test_list_spy_item_retrieval():
-    spy = ListSpy(['a', 'b', 'c'])
-    assert spy[1] == 'b'
-    assert spy[-1] == 'c'
-    with pytest.raises(IndexError):
-        spy[3]
-
-
-def test_list_spy_del():
-    lst = ['a', 'b', 'c']
-
-    spy = ListSpy(lst)
-    del spy[1]
-
-    assert lst == ['a', 'c']
-
-    spy.undo(lst)
-
-    assert lst == ['a', 'b', 'c']
-
-
-def test_list_spy_del_with_negative():
-    lst = ['a', 'b', 'c']
-
-    spy = ListSpy(lst)
-    del spy[-1]
-
-    assert lst == ['a', 'b']
-
-    spy.undo(lst)
-
-    assert lst == ['a', 'b', 'c']
-
-
-def test_list_spy_insert():
-    lst = ['a', 'b', 'c']
-
-    spy = ListSpy(lst)
-    spy.insert(1, 'q')
-
-    assert lst == ['a', 'q', 'b', 'c']
-
-    spy.undo(lst)
-
-    assert lst == ['a', 'b', 'c']
-
-
-def test_list_spy_insert_with_negative():
-    lst = ['a', 'b', 'c']
-
-    spy = ListSpy(lst)
-    spy.insert(-1, 'q')
-
-    assert lst == ['a', 'b', 'q', 'c']
-
-    spy.undo(lst)
-
-    assert lst == ['a', 'b', 'c']
-
-
-def test_list_spy_set_value():
-    lst = ['a', 'b', 'c']
-
-    spy = ListSpy(lst)
-    spy[1] = 'hello'
-
-    assert lst == ['a', 'hello', 'c']
-
-    spy.undo(lst)
-
-    assert lst == ['a', 'b', 'c']
-
-
-def test_list_spy_multiple_modifications():
-    lst = ['a', 'b', 'c']
-
-    spy = ListSpy(lst)
-    spy[1] = 'hello'
-    spy.insert(1, 'ohai')
-    del spy[0]
-
-    assert lst == ['ohai', 'hello', 'c']
-
-    spy.undo(lst)
-
-    assert lst == ['a', 'b', 'c']
-
-
-def test_list_spy_iter():
-    spy = ListSpy(['a', 'b', 'c'])
-    spy_iter = iter(spy)
-    assert next(spy_iter) == 'a'
-    assert next(spy_iter) == 'b'
-    assert next(spy_iter) == 'c'
-    with pytest.raises(StopIteration):
-        next(spy_iter)
-
-
-def test_list_spy_append():
-    lst = ['a', 'b', 'c']
-
-    spy = ListSpy(lst)
-    spy.append('q')
-
-    assert lst == ['a', 'b', 'c', 'q']
-
-    spy.undo(lst)
-
-    assert lst == ['a', 'b', 'c']
-
-
-def test_list_spy_pop_default():
-    lst = ['a', 'b', 'c']
-
-    spy = ListSpy(lst)
-    spy.pop()
-
-    assert lst == ['a', 'b']
-
-    spy.undo(lst)
-
-    assert lst == ['a', 'b', 'c']
-
-
-def test_list_spy_pop_idx():
-    lst = ['a', 'b', 'c']
-
-    spy = ListSpy(lst)
-    spy.pop(1)
-
-    assert lst == ['a', 'c']
-
-    spy.undo(lst)
-
-    assert lst == ['a', 'b', 'c']