v0.0.22

drop python 3.6

.github/FUNDING.yml

@@ -0,0 +1 @@
+github: asottile

.gitignore

@@ -5,4 +5,6 @@
 /.mypy_cache
 /.pytest_cache
 /.tox
+/build
+/dist
 /venv*

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.5.0
+    rev: v4.0.1
     hooks:
     -   id: trailing-whitespace
     -   id: end-of-file-fixer
@@ -10,31 +10,35 @@ repos:
     -   id: double-quote-string-fixer
     -   id: name-tests-test
     -   id: requirements-txt-fixer
--   repo: https://gitlab.com/pycqa/flake8
-    rev: 3.7.9
+-   repo: https://github.com/PyCQA/flake8
+    rev: 3.9.2
     hooks:
     -   id: flake8
         additional_dependencies: [flake8-typing-imports==1.7.0]
 -   repo: https://github.com/pre-commit/mirrors-autopep8
-    rev: v1.5
+    rev: v1.5.7
     hooks:
     -   id: autopep8
 -   repo: https://github.com/asottile/reorder_python_imports
-    rev: v1.9.0
+    rev: v2.6.0
     hooks:
     -   id: reorder-python-imports
-        args: [--py3-plus]
+        args: [--py3-plus, --add-import, 'from __future__ import annotations']
 -   repo: https://github.com/asottile/add-trailing-comma
-    rev: v1.5.0
+    rev: v2.1.0
     hooks:
     -   id: add-trailing-comma
         args: [--py36-plus]
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.1.0
+    rev: v2.25.0
     hooks:
     -   id: pyupgrade
-        args: [--py36-plus]
+        args: [--py37-plus]
+-   repo: https://github.com/asottile/setup-cfg-fmt
+    rev: v1.17.0
+    hooks:
+    -   id: setup-cfg-fmt
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.770
+    rev: v0.910
     hooks:
     -   id: mypy

README.md

@@ -1,16 +1,30 @@
 [![Build Status](https://dev.azure.com/asottile/asottile/_apis/build/status/asottile.babi?branchName=master)](https://dev.azure.com/asottile/asottile/_build/latest?definitionId=29&branchName=master)
 [![Azure DevOps coverage](https://img.shields.io/azure-devops/coverage/asottile/asottile/29/master.svg)](https://dev.azure.com/asottile/asottile/_build/latest?definitionId=29&branchName=master)
+[![pre-commit.ci status](https://results.pre-commit.ci/badge/github/asottile/babi/master.svg)](https://results.pre-commit.ci/latest/github/asottile/babi/master)
+
+![babi logo](https://user-images.githubusercontent.com/1810591/89981369-9ed84e80-dc28-11ea-9708-5f4c49c09632.png)
 
 babi
 ====
 
 a text editor, eventually...
 
+### installation
+
+`pip install babi`
+
 ### why is it called babi?
 
-I usually use the text editor `nano`, frequently I typo this.  on a qwerty
+I used to use the text editor `nano`, frequently I typo this.  on a qwerty
 keyboard, when the right hand is shifted left by one, `nano` becomes `babi`.
 
+### babi vs. nano
+
+here is a youtube video where I discuss the motivation for creating and using
+`babi` instead of `nano`:
+
+[![youtube video about babi](https://img.youtube.com/vi/WyR1hAGmR3g/mqdefault.jpg)](https://youtu.be/WyR1hAGmR3g)
+
 ### quitting babi
 
 currently you can quit `babi` by using <kbd>^X</kbd> (or via <kbd>esc</kbd> +
@@ -23,6 +37,7 @@ these are all of the current key bindings in babi
 - <kbd>^S</kbd>: save
 - <kbd>^O</kbd>: save as
 - <kbd>^X</kbd>: quit
+- <kbd>^P</kbd>: open file
 - arrow keys: movement
 - <kbd>^A</kbd> / <kbd>home</kbd>: move to beginning of line
 - <kbd>^E</kbd> / <kbd>end</kbd>: move to end of line
@@ -40,7 +55,7 @@ these are all of the current key bindings in babi
 - <kbd>tab</kbd> / <kbd>shift-tab</kbd>: indent or dedent current line (or
   selection)
 - <kbd>^K</kbd> / <kbd>^U</kbd>: cut and uncut the current line (or selection)
-- <kbd>M-u</kbd> / <kbd>M-U</kbd>: undo / redo
+- <kbd>M-u</kbd> / <kbd>M-U</kbd> or <kbd>M-e</kbd>: undo / redo
 - <kbd>^W</kbd>: search
 - <kbd>^\\</kbd>: search and replace
 - <kbd>^C</kbd>: show the current position in the file
@@ -62,10 +77,34 @@ in prompts (search, search replace, command):
 
 the syntax highlighting setup is a bit manual right now
 
-1. from a clone of babi, run `./bin/download-syntax` -- you will likely need
-   to install some additional packages to download them (`pip install cson`)
-2. find a visual studio code theme, convert it to json (if it is not already
-   json) and put it at `~/.config/babi/theme.json`
+1. find a visual studio code theme, convert it to json (if it is not already
+   json) and put it at `~/.config/babi/theme.json`.  a helper script is
+   provided to make this easier: `./bin/download-theme NAME URL`
+
+here's a modified vs dark plus theme that works:
+
+```bash
+./bin/download-theme vs-dark-asottile https://gist.github.com/asottile/b465856c82b1aaa4ba8c7c6314a72e13/raw/22d602fb355fb12b04f176a733941ba5713bc36c/vs_dark_asottile.json
+```
+
+### keyboard shortcuts on macos
+
+to get the most out of babi's built in keyboard shortcuts, a few settings must
+be changed on macos with Terminal.app:
+
+- in **System Preferences**: **Keyboard** > **Shortcuts** >
+  **Mission Control**: disable or rebind "Move left a space" and
+  "Move right a space" (the defaults `⌃ →` and `⌃ ←` conflict)
+- in **Terminal.app**: **Terminal** > **Preferences** > **Profiles** >
+  **Keyboard**:
+    - check **Use Option as Meta key**
+    - ensure the following keys are enabled:
+        - `⌃ →`: `\033[1;5C`
+        - `⌃ ←`: `\033[1;5D`
+        - `⇧ ↑`: `\033[1;2A`
+        - `⇧ ↓`: `\033[1;2B`
+        - `⇧ →`: `\033[1;2C`
+        - `⇧ ←`: `\033[1;2D`
 
 ## demos
 
@@ -77,7 +116,7 @@ this opens the file, displays it, and can be edited and can save! unknown keys
 are displayed as errors in the status bar.  babi will scroll if the cursor
 goes off screen either from resize events or from movement.  babi can edit
 multiple files.  babi has a command mode (so you can quit it like vim
-<kbd>:q</kbd>!).  babi also support syntax highlighting
+<kbd>:q</kbd>!).  babi also supports syntax highlighting
 
 ![](https://i.fluffy.cc/5WFZBJ4mWs7wtThD9strQnGlJqw4Z9KS.png)
 

azure-pipelines.yml

@@ -10,11 +10,10 @@ resources:
       type: github
       endpoint: github
       name: asottile/azure-pipeline-templates
-      ref: refs/tags/v1.0.0
+      ref: refs/tags/v2.1.0
 
 jobs:
-- template: job--pre-commit.yml@asottile
 - template: job--python-tox.yml@asottile
   parameters:
-    toxenvs: [py36, py37, py38]
+    toxenvs: [py37, py38, py39]
     os: linux

babi/__main__.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from babi.main import main
 
 if __name__ == '__main__':

babi/_types.py

@@ -1,8 +1,8 @@
+from __future__ import annotations
+
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from typing import Protocol  # python3.8+
-    from typing_extensions import TypedDict  # python3.8+
 else:
     Protocol = object
-    TypedDict = dict

babi/buf.py

@@ -0,0 +1,314 @@
+from __future__ import annotations
+
+import bisect
+import contextlib
+from typing import Callable
+from typing import Generator
+from typing import Iterator
+from typing import NamedTuple
+
+from babi._types import Protocol
+from babi.horizontal_scrolling import line_x
+from babi.horizontal_scrolling import scrolled_line
+from babi.horizontal_scrolling import wcwidth
+from babi.margin import Margin
+
+SetCallback = Callable[['Buf', int, str], None]
+DelCallback = Callable[['Buf', int, str], None]
+InsCallback = Callable[['Buf', int], None]
+
+
+def _offsets(s: str, tab_size: int) -> tuple[int, ...]:
+    ret = [0]
+    for c in s:
+        if c == '\t':
+            ret.append(ret[-1] + (tab_size - ret[-1] % tab_size))
+        else:
+            ret.append(ret[-1] + wcwidth(c))
+    return tuple(ret)
+
+
+class Modification(Protocol):
+    def __call__(self, buf: Buf) -> None: ...
+
+
+class SetModification(NamedTuple):
+    idx: int
+    s: str
+
+    def __call__(self, buf: Buf) -> None:
+        buf[self.idx] = self.s
+
+
+class InsModification(NamedTuple):
+    idx: int
+    s: str
+
+    def __call__(self, buf: Buf) -> None:
+        buf.insert(self.idx, self.s)
+
+
+class DelModification(NamedTuple):
+    idx: int
+
+    def __call__(self, buf: Buf) -> None:
+        del buf[self.idx]
+
+
+class Buf:
+    def __init__(self, lines: list[str], tab_size: int = 4) -> None:
+        self._lines = lines
+        self.expandtabs = True
+        self.tab_size = tab_size
+        self.file_y = self.y = self._x = self._x_hint = 0
+
+        self._set_callbacks: list[SetCallback] = [self._set_cb]
+        self._del_callbacks: list[DelCallback] = [self._del_cb]
+        self._ins_callbacks: list[InsCallback] = [self._ins_cb]
+
+        self._positions: list[tuple[int, ...] | None] = []
+
+    # read only interface
+
+    def __repr__(self) -> str:
+        return (
+            f'{type(self).__name__}('
+            f'{self._lines!r}, x={self.x}, y={self.y}, file_y={self.file_y}'
+            f')'
+        )
+
+    def __bool__(self) -> bool:
+        return bool(self._lines)
+
+    def __getitem__(self, idx: int) -> str:
+        return self._lines[idx]
+
+    def __iter__(self) -> Iterator[str]:
+        yield from self._lines
+
+    def __len__(self) -> int:
+        return len(self._lines)
+
+    # mutators
+
+    def __setitem__(self, idx: int, val: str) -> None:
+        if idx < 0:
+            idx %= len(self)
+        victim = self._lines[idx]
+
+        self._lines[idx] = val
+
+        for set_callback in self._set_callbacks:
+            set_callback(self, idx, victim)
+
+    def __delitem__(self, idx: int) -> None:
+        if idx < 0:
+            idx %= len(self)
+        victim = self._lines[idx]
+
+        del self._lines[idx]
+
+        for del_callback in self._del_callbacks:
+            del_callback(self, idx, victim)
+
+    def insert(self, idx: int, val: str) -> None:
+        if idx < 0:
+            idx %= len(self)
+
+        self._lines.insert(idx, val)
+
+        for ins_callback in self._ins_callbacks:
+            ins_callback(self, idx)
+
+    # also mutators, but implemented using above functions
+
+    def append(self, val: str) -> None:
+        self.insert(len(self), val)
+
+    def pop(self, idx: int = -1) -> str:
+        victim = self[idx]
+        del self[idx]
+        return victim
+
+    def restore_eof_invariant(self) -> None:
+        """the file lines will always contain a blank empty string at the end'
+        to simplify rendering.  call this whenever the last line may change
+        """
+        if self[-1] != '':
+            self.append('')
+
+    def set_tab_size(self, tab_size: int) -> None:
+        self.tab_size = tab_size
+        self._positions = [None]
+
+    # event handling
+
+    def add_set_callback(self, cb: SetCallback) -> None:
+        self._set_callbacks.append(cb)
+
+    def remove_set_callback(self, cb: SetCallback) -> None:
+        self._set_callbacks.remove(cb)
+
+    def add_del_callback(self, cb: DelCallback) -> None:
+        self._del_callbacks.append(cb)
+
+    def remove_del_callback(self, cb: DelCallback) -> None:
+        self._del_callbacks.remove(cb)
+
+    def add_ins_callback(self, cb: InsCallback) -> None:
+        self._ins_callbacks.append(cb)
+
+    def remove_ins_callback(self, cb: InsCallback) -> None:
+        self._ins_callbacks.remove(cb)
+
+    @contextlib.contextmanager
+    def record(self) -> Generator[list[Modification], None, None]:
+        modifications: list[Modification] = []
+
+        def set_cb(buf: Buf, idx: int, victim: str) -> None:
+            modifications.append(SetModification(idx, victim))
+
+        def del_cb(buf: Buf, idx: int, victim: str) -> None:
+            modifications.append(InsModification(idx, victim))
+
+        def ins_cb(buf: Buf, idx: int) -> None:
+            modifications.append(DelModification(idx))
+
+        self.add_set_callback(set_cb)
+        self.add_del_callback(del_cb)
+        self.add_ins_callback(ins_cb)
+        try:
+            yield modifications
+        finally:
+            self.remove_ins_callback(ins_cb)
+            self.remove_del_callback(del_cb)
+            self.remove_set_callback(set_cb)
+
+    def apply(self, modifications: list[Modification]) -> list[Modification]:
+        with self.record() as ret_modifications:
+            for modification in reversed(modifications):
+                modification(self)
+        return ret_modifications
+
+    # position properties
+
+    @property
+    def displayable_count(self) -> int:
+        return len(self._lines) - self.file_y
+
+    @property
+    def x(self) -> int:
+        return self._x
+
+    @x.setter
+    def x(self, x: int) -> None:
+        self._x = x
+        self._x_hint = self._cursor_x
+
+    def _extend_positions(self, idx: int) -> None:
+        self._positions.extend([None] * (1 + idx - len(self._positions)))
+
+    def _set_cb(self, buf: Buf, idx: int, victim: str) -> None:
+        self._extend_positions(idx)
+        self._positions[idx] = None
+
+    def _del_cb(self, buf: Buf, idx: int, victim: str) -> None:
+        self._extend_positions(idx)
+        del self._positions[idx]
+
+    def _ins_cb(self, buf: Buf, idx: int) -> None:
+        self._extend_positions(idx)
+        self._positions.insert(idx, None)
+
+    def line_positions(self, idx: int) -> tuple[int, ...]:
+        self._extend_positions(idx)
+        value = self._positions[idx]
+        if value is None:
+            value = _offsets(self._lines[idx], self.tab_size)
+            self._positions[idx] = value
+        return value
+
+    def line_x(self, margin: Margin) -> int:
+        return line_x(self._cursor_x, margin.cols)
+
+    @property
+    def _cursor_x(self) -> int:
+        return self.line_positions(self.y)[self.x]
+
+    def cursor_position(self, margin: Margin) -> tuple[int, int]:
+        y = self.y - self.file_y + margin.header
+        x = self._cursor_x - self.line_x(margin)
+        return y, x
+
+    # rendered lines
+
+    @property
+    def tab_string(self) -> str:
+        if self.expandtabs:
+            return ' ' * self.tab_size
+        else:
+            return '\t'
+
+    def rendered_line(self, idx: int, margin: Margin) -> str:
+        x = self._cursor_x if idx == self.y else 0
+        expanded = self._lines[idx].expandtabs(self.tab_size)
+        return scrolled_line(expanded, x, margin.cols)
+
+    # movement
+
+    def scroll_screen_if_needed(self, margin: Margin) -> None:
+        # if the `y` is not on screen, make it so
+        if not (self.file_y <= self.y < self.file_y + margin.body_lines):
+            self.file_y = max(self.y - margin.body_lines // 2, 0)
+
+    def _set_x_after_vertical_movement(self) -> None:
+        positions = self.line_positions(self.y)
+        x = bisect.bisect_left(positions, self._x_hint)
+        x = min(len(self._lines[self.y]), x)
+        if positions[x] > self._x_hint:
+            x -= 1
+        self._x = x
+
+    def up(self, margin: Margin) -> None:
+        if self.y > 0:
+            self.y -= 1
+            if self.y < self.file_y:
+                self.file_y = max(self.file_y - margin.scroll_amount, 0)
+            self._set_x_after_vertical_movement()
+
+    def down(self, margin: Margin) -> None:
+        if self.y < len(self._lines) - 1:
+            self.y += 1
+            if self.y >= self.file_y + margin.body_lines:
+                self.file_y += margin.scroll_amount
+            self._set_x_after_vertical_movement()
+
+    def right(self, margin: Margin) -> None:
+        if self.x >= len(self._lines[self.y]):
+            if self.y < len(self._lines) - 1:
+                self.down(margin)
+                self.x = 0
+        else:
+            self.x += 1
+
+    def left(self, margin: Margin) -> None:
+        if self.x == 0:
+            if self.y > 0:
+                self.up(margin)
+                self.x = len(self._lines[self.y])
+        else:
+            self.x -= 1
+
+    # screen movement
+
+    def file_up(self, margin: Margin) -> None:
+        if self.file_y > 0:
+            self.file_y -= 1
+            if self.y > self.file_y + margin.body_lines - 1:
+                self.up(margin)
+
+    def file_down(self, margin: Margin) -> None:
+        if self.file_y < len(self._lines) - 1:
+            self.file_y += 1
+            if self.y < self.file_y:
+                self.down(margin)

babi/cached_property.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import sys
 
 if sys.version_info >= (3, 8):  # pragma: no cover (>=py38)
@@ -5,8 +7,6 @@ if sys.version_info >= (3, 8):  # pragma: no cover (>=py38)
 else:  # pragma: no cover (<py38)
     from typing import Callable
     from typing import Generic
-    from typing import Optional
-    from typing import Type
     from typing import TypeVar
 
     TSelf = TypeVar('TSelf')
@@ -18,8 +18,8 @@ else:  # pragma: no cover (<py38)
 
         def __get__(
                 self,
-                instance: Optional[TSelf],
-                owner: Optional[Type[TSelf]] = None,
+                instance: TSelf | None,
+                owner: type[TSelf] | None = None,
         ) -> TRet:
             assert instance is not None
             ret = instance.__dict__[self._func.__name__] = self._func(instance)

babi/color.py

@@ -1,5 +1,11 @@
+from __future__ import annotations
+
 from typing import NamedTuple
 
+# TODO: find a standard which defines these
+# limited number of "named" colors
+NAMED_COLORS = {'white': '#ffffff', 'black': '#000000'}
+
 
 class Color(NamedTuple):
     r: int
@@ -7,5 +13,10 @@ class Color(NamedTuple):
     b: int
 
     @classmethod
-    def parse(cls, s: str) -> 'Color':
-        return cls(r=int(s[1:3], 16), g=int(s[3:5], 16), b=int(s[5:7], 16))
+    def parse(cls, s: str) -> Color:
+        if s.startswith('#') and len(s) >= 7:
+            return cls(r=int(s[1:3], 16), g=int(s[3:5], 16), b=int(s[5:7], 16))
+        elif s.startswith('#'):
+            return cls.parse(f'#{s[1] * 2}{s[2] * 2}{s[3] * 2}')
+        else:
+            return cls.parse(NAMED_COLORS[s])

babi/color_kd.py

@@ -1,9 +1,8 @@
+from __future__ import annotations
+
 import functools
 import itertools
-from typing import List
 from typing import NamedTuple
-from typing import Optional
-from typing import Tuple
 
 from babi._types import Protocol
 from babi.color import Color
@@ -19,19 +18,19 @@ class KD(Protocol):
     @property
     def n(self) -> int: ...
     @property
-    def left(self) -> Optional['KD']: ...
+    def left(self) -> KD | None: ...
     @property
-    def right(self) -> Optional['KD']: ...
+    def right(self) -> KD | None: ...
 
 
 class _KD(NamedTuple):
     color: Color
     n: int
-    left: Optional[KD]
-    right: Optional[KD]
+    left: KD | None
+    right: KD | None
 
 
-def _build(colors: List[Tuple[Color, int]], depth: int = 0) -> Optional[KD]:
+def _build(colors: list[tuple[Color, int]], depth: int = 0) -> KD | None:
     if not colors:
         return None
 
@@ -46,11 +45,11 @@ def _build(colors: List[Tuple[Color, int]], depth: int = 0) -> Optional[KD]:
     )
 
 
-def nearest(color: Color, colors: Optional[KD]) -> int:
+def nearest(color: Color, colors: KD | None) -> int:
     best = 0
     dist = 2 ** 32
 
-    def _search(kd: Optional[KD], *, depth: int) -> None:
+    def _search(kd: KD | None, *, depth: int) -> None:
         nonlocal best
         nonlocal dist
 
@@ -77,7 +76,7 @@ def nearest(color: Color, colors: Optional[KD]) -> int:
 
 
 @functools.lru_cache(maxsize=1)
-def make_256() -> Optional[KD]:
+def make_256() -> KD | None:
     vals = (0, 95, 135, 175, 215, 255)
     colors = [
         (Color(r, g, b), i)

babi/color_manager.py

@@ -1,49 +1,49 @@
-import contextlib
+from __future__ import annotations
+
 import curses
-from typing import Dict
 from typing import NamedTuple
-from typing import Optional
-from typing import Tuple
 
 from babi import color_kd
 from babi.color import Color
 
 
-def _color_to_curses(color: Color) -> Tuple[int, int, int]:
+def _color_to_curses(color: Color) -> tuple[int, int, int]:
     factor = 1000 / 255
     return int(color.r * factor), int(color.g * factor), int(color.b * factor)
 
 
 class ColorManager(NamedTuple):
-    colors: Dict[Color, int]
-    raw_pairs: Dict[Tuple[int, int], int]
+    colors: dict[Color, int]
+    raw_pairs: dict[tuple[int, int], int]
 
     def init_color(self, color: Color) -> None:
-        if curses.COLORS < 256:
-            return
-        elif curses.can_change_color():
+        if curses.can_change_color():
             n = min(self.colors.values(), default=256) - 1
             self.colors[color] = n
             curses.init_color(n, *_color_to_curses(color))
-        else:
+        elif curses.COLORS >= 256:
             self.colors[color] = color_kd.nearest(color, color_kd.make_256())
+        else:
+            self.colors[color] = -1
 
-    def color_pair(self, fg: Optional[Color], bg: Optional[Color]) -> int:
-        if curses.COLORS < 256:
-            return 0
-
+    def color_pair(self, fg: Color | None, bg: Color | None) -> int:
         fg_i = self.colors[fg] if fg is not None else -1
         bg_i = self.colors[bg] if bg is not None else -1
         return self.raw_color_pair(fg_i, bg_i)
 
     def raw_color_pair(self, fg: int, bg: int) -> int:
-        with contextlib.suppress(KeyError):
-            return self.raw_pairs[(fg, bg)]
+        if curses.COLORS > 0:
+            try:
+                return self.raw_pairs[(fg, bg)]
+            except KeyError:
+                pass
 
-        n = self.raw_pairs[(fg, bg)] = len(self.raw_pairs) + 1
-        curses.init_pair(n, fg, bg)
-        return n
+            n = self.raw_pairs[(fg, bg)] = len(self.raw_pairs) + 1
+            curses.init_pair(n, fg, bg)
+            return n
+        else:
+            return 0
 
     @classmethod
-    def make(cls) -> 'ColorManager':
+    def make(cls) -> ColorManager:
         return cls({}, {})

babi/fdict.py

@@ -1,10 +1,14 @@
+from __future__ import annotations
+
 from typing import Generic
 from typing import Iterable
 from typing import Mapping
 from typing import TypeVar
 
-TKey = TypeVar('TKey')
-TValue = TypeVar('TValue')
+from babi._types import Protocol
+
+TKey = TypeVar('TKey', contravariant=True)
+TValue = TypeVar('TValue', covariant=True)
 
 
 class FDict(Generic[TKey, TValue]):
@@ -22,3 +26,21 @@ class FDict(Generic[TKey, TValue]):
 
     def values(self) -> Iterable[TValue]:
         return self._dct.values()
+
+
+class Indexable(Generic[TKey, TValue], Protocol):
+    def __getitem__(self, key: TKey) -> TValue: ...
+
+
+class FChainMap(Generic[TKey, TValue]):
+    def __init__(self, *mappings: Indexable[TKey, TValue]) -> None:
+        self._mappings = mappings
+
+    def __getitem__(self, key: TKey) -> TValue:
+        for mapping in reversed(self._mappings):
+            try:
+                return mapping[key]
+            except KeyError:
+                pass
+        else:
+            raise KeyError(key)

babi/highlight.py

@@ -1,30 +1,38 @@
-import contextlib
+from __future__ import annotations
+
 import functools
 import json
 import os.path
 from typing import Any
-from typing import Dict
-from typing import FrozenSet
-from typing import List
 from typing import Match
 from typing import NamedTuple
-from typing import Optional
 from typing import Tuple
+from typing import TypeVar
+
+from identify.identify import tags_from_filename
 
 from babi._types import Protocol
-from babi.fdict import FDict
+from babi.fdict import FChainMap
 from babi.reg import _Reg
 from babi.reg import _RegSet
 from babi.reg import ERR_REG
+from babi.reg import expand_escaped
 from babi.reg import make_reg
 from babi.reg import make_regset
 
+T = TypeVar('T')
 Scope = Tuple[str, ...]
 Regions = Tuple['Region', ...]
 Captures = Tuple[Tuple[int, '_Rule'], ...]
 
 
-def _split_name(s: Optional[str]) -> Tuple[str, ...]:
+def uniquely_constructed(t: T) -> T:
+    """avoid tuple.__hash__ for "singleton" constructed objects"""
+    t.__hash__ = object.__hash__  # type: ignore
+    return t
+
+
+def _split_name(s: str | None) -> tuple[str, ...]:
     if s is None:
         return ()
     else:
@@ -34,17 +42,17 @@ def _split_name(s: Optional[str]) -> Tuple[str, ...]:
 class _Rule(Protocol):
     """hax for recursive types python/mypy#731"""
     @property
-    def name(self) -> Tuple[str, ...]: ...
+    def name(self) -> tuple[str, ...]: ...
     @property
-    def match(self) -> Optional[str]: ...
+    def match(self) -> str | None: ...
     @property
-    def begin(self) -> Optional[str]: ...
+    def begin(self) -> str | None: ...
     @property
-    def end(self) -> Optional[str]: ...
+    def end(self) -> str | None: ...
     @property
-    def while_(self) -> Optional[str]: ...
+    def while_(self) -> str | None: ...
     @property
-    def content_name(self) -> Tuple[str, ...]: ...
+    def content_name(self) -> tuple[str, ...]: ...
     @property
     def captures(self) -> Captures: ...
     @property
@@ -54,27 +62,45 @@ class _Rule(Protocol):
     @property
     def while_captures(self) -> Captures: ...
     @property
-    def include(self) -> Optional[str]: ...
+    def include(self) -> str | None: ...
     @property
-    def patterns(self) -> 'Tuple[_Rule, ...]': ...
+    def patterns(self) -> tuple[_Rule, ...]: ...
+    @property
+    def repository(self) -> FChainMap[str, _Rule]: ...
 
 
+@uniquely_constructed
 class Rule(NamedTuple):
-    name: Tuple[str, ...]
-    match: Optional[str]
-    begin: Optional[str]
-    end: Optional[str]
-    while_: Optional[str]
-    content_name: Tuple[str, ...]
+    name: tuple[str, ...]
+    match: str | None
+    begin: str | None
+    end: str | None
+    while_: str | None
+    content_name: tuple[str, ...]
     captures: Captures
     begin_captures: Captures
     end_captures: Captures
     while_captures: Captures
-    include: Optional[str]
-    patterns: Tuple[_Rule, ...]
+    include: str | None
+    patterns: tuple[_Rule, ...]
+    repository: FChainMap[str, _Rule]
 
     @classmethod
-    def from_dct(cls, dct: Dict[str, Any]) -> _Rule:
+    def make(
+            cls,
+            dct: dict[str, Any],
+            parent_repository: FChainMap[str, _Rule],
+    ) -> _Rule:
+        if 'repository' in dct:
+            # this looks odd, but it's so we can have a self-referential
+            # immutable-after-construction chain map
+            repository_dct: dict[str, _Rule] = {}
+            repository = FChainMap(parent_repository, repository_dct)
+            for k, sub_dct in dct['repository'].items():
+                repository_dct[k] = Rule.make(sub_dct, repository)
+        else:
+            repository = parent_repository
+
         name = _split_name(dct.get('name'))
         match = dct.get('match')
         begin = dct.get('begin')
@@ -84,7 +110,7 @@ class Rule(NamedTuple):
 
         if 'captures' in dct:
             captures = tuple(
-                (int(k), Rule.from_dct(v))
+                (int(k), Rule.make(v, repository))
                 for k, v in dct['captures'].items()
             )
         else:
@@ -92,7 +118,7 @@ class Rule(NamedTuple):
 
         if 'beginCaptures' in dct:
             begin_captures = tuple(
-                (int(k), Rule.from_dct(v))
+                (int(k), Rule.make(v, repository))
                 for k, v in dct['beginCaptures'].items()
             )
         else:
@@ -100,7 +126,7 @@ class Rule(NamedTuple):
 
         if 'endCaptures' in dct:
             end_captures = tuple(
-                (int(k), Rule.from_dct(v))
+                (int(k), Rule.make(v, repository))
                 for k, v in dct['endCaptures'].items()
             )
         else:
@@ -108,12 +134,16 @@ class Rule(NamedTuple):
 
         if 'whileCaptures' in dct:
             while_captures = tuple(
-                (int(k), Rule.from_dct(v))
+                (int(k), Rule.make(v, repository))
                 for k, v in dct['whileCaptures'].items()
             )
         else:
             while_captures = ()
 
+        # some grammars (at least xml) have begin rules with no end
+        if begin is not None and end is None and while_ is None:
+            end = '$impossible^'
+
         # Using the captures key for a begin/end/while rule is short-hand for
         # giving both beginCaptures and endCaptures with same values
         if begin and end and captures:
@@ -126,7 +156,7 @@ class Rule(NamedTuple):
         include = dct.get('include')
 
         if 'patterns' in dct:
-            patterns = tuple(Rule.from_dct(d) for d in dct['patterns'])
+            patterns = tuple(Rule.make(d, repository) for d in dct['patterns'])
         else:
             patterns = ()
 
@@ -143,69 +173,34 @@ class Rule(NamedTuple):
             while_captures=while_captures,
             include=include,
             patterns=patterns,
-        )
-
-
-class Grammar(NamedTuple):
-    scope_name: str
-    first_line_match: Optional[_Reg]
-    file_types: FrozenSet[str]
-    patterns: Tuple[_Rule, ...]
-    repository: FDict[str, _Rule]
-
-    @classmethod
-    def from_data(cls, data: Dict[str, Any]) -> 'Grammar':
-        scope_name = data['scopeName']
-        if 'firstLineMatch' in data:
-            first_line_match: Optional[_Reg] = make_reg(data['firstLineMatch'])
-        else:
-            first_line_match = None
-        if 'fileTypes' in data:
-            file_types = frozenset(data['fileTypes'])
-        else:
-            file_types = frozenset()
-        patterns = tuple(Rule.from_dct(dct) for dct in data['patterns'])
-        if 'repository' in data:
-            repository = FDict({
-                k: Rule.from_dct(dct) for k, dct in data['repository'].items()
-            })
-        else:
-            repository = FDict({})
-        return cls(
-            scope_name=scope_name,
-            first_line_match=first_line_match,
-            file_types=file_types,
-            patterns=patterns,
             repository=repository,
         )
 
-    @classmethod
-    def parse(cls, filename: str) -> 'Grammar':
-        with open(filename) as f:
-            return cls.from_data(json.load(f))
+
+@uniquely_constructed
+class Grammar(NamedTuple):
+    scope_name: str
+    repository: FChainMap[str, _Rule]
+    patterns: tuple[_Rule, ...]
 
     @classmethod
-    def blank(cls) -> 'Grammar':
-        return cls(
-            scope_name='source.unknown',
-            first_line_match=None,
-            file_types=frozenset(),
-            patterns=(),
-            repository=FDict({}),
-        )
-
-    def matches_file(self, filename: str, first_line: str) -> bool:
-        _, ext = os.path.splitext(filename)
-        if ext.lstrip('.') in self.file_types:
-            return True
-        elif self.first_line_match is not None:
-            return bool(
-                self.first_line_match.match(
-                    first_line, 0, first_line=True, boundary=True,
-                ),
-            )
+    def make(cls, data: dict[str, Any]) -> Grammar:
+        scope_name = data['scopeName']
+        if 'repository' in data:
+            # this looks odd, but it's so we can have a self-referential
+            # immutable-after-construction chain map
+            repository_dct: dict[str, _Rule] = {}
+            repository = FChainMap(repository_dct)
+            for k, dct in data['repository'].items():
+                repository_dct[k] = Rule.make(dct, repository)
         else:
-            return False
+            repository = FChainMap()
+        patterns = tuple(Rule.make(d, repository) for d in data['patterns'])
+        return cls(
+            scope_name=scope_name,
+            repository=repository,
+            patterns=patterns,
+        )
 
 
 class Region(NamedTuple):
@@ -215,54 +210,54 @@ class Region(NamedTuple):
 
 
 class State(NamedTuple):
-    entries: Tuple['Entry', ...]
-    while_stack: Tuple[Tuple['WhileRule', int], ...]
+    entries: tuple[Entry, ...]
+    while_stack: tuple[tuple[WhileRule, int], ...]
 
     @classmethod
-    def root(cls, entry: 'Entry') -> 'State':
+    def root(cls, entry: Entry) -> State:
         return cls((entry,), ())
 
     @property
-    def cur(self) -> 'Entry':
+    def cur(self) -> Entry:
         return self.entries[-1]
 
-    def push(self, entry: 'Entry') -> 'State':
+    def push(self, entry: Entry) -> State:
         return self._replace(entries=(*self.entries, entry))
 
-    def pop(self) -> 'State':
+    def pop(self) -> State:
         return self._replace(entries=self.entries[:-1])
 
-    def push_while(self, rule: 'WhileRule', entry: 'Entry') -> 'State':
+    def push_while(self, rule: WhileRule, entry: Entry) -> State:
         entries = (*self.entries, entry)
         while_stack = (*self.while_stack, (rule, len(entries)))
         return self._replace(entries=entries, while_stack=while_stack)
 
-    def pop_while(self) -> 'State':
+    def pop_while(self) -> State:
         entries, while_stack = self.entries[:-1], self.while_stack[:-1]
         return self._replace(entries=entries, while_stack=while_stack)
 
 
 class CompiledRule(Protocol):
     @property
-    def name(self) -> Tuple[str, ...]: ...
+    def name(self) -> tuple[str, ...]: ...
 
     def start(
             self,
-            compiler: 'Compiler',
+            compiler: Compiler,
             match: Match[str],
             state: State,
-    ) -> Tuple[State, bool, Regions]:
+    ) -> tuple[State, bool, Regions]:
         ...
 
     def search(
             self,
-            compiler: 'Compiler',
+            compiler: Compiler,
             state: State,
             line: str,
             pos: int,
             first_line: bool,
             boundary: bool,
-    ) -> Optional[Tuple[State, int, bool, Regions]]:
+    ) -> tuple[State, int, bool, Regions] | None:
         ...
 
 
@@ -270,24 +265,25 @@ class CompiledRegsetRule(CompiledRule, Protocol):
     @property
     def regset(self) -> _RegSet: ...
     @property
-    def u_rules(self) -> Tuple[_Rule, ...]: ...
+    def u_rules(self) -> tuple[_Rule, ...]: ...
 
 
 class Entry(NamedTuple):
-    scope: Tuple[str, ...]
+    scope: tuple[str, ...]
     rule: CompiledRule
+    start: tuple[str, int]
     reg: _Reg = ERR_REG
     boundary: bool = False
 
 
 def _inner_capture_parse(
-        compiler: 'Compiler',
+        compiler: Compiler,
         start: int,
         s: str,
         scope: Scope,
         rule: CompiledRule,
 ) -> Regions:
-    state = State.root(Entry(scope + rule.name, rule))
+    state = State.root(Entry(scope + rule.name, rule, (s, 0)))
     _, regions = highlight_line(compiler, state, s, first_line=False)
     return tuple(
         r._replace(start=r.start + start, end=r.end + start) for r in regions
@@ -295,12 +291,12 @@ def _inner_capture_parse(
 
 
 def _captures(
-        compiler: 'Compiler',
+        compiler: Compiler,
         scope: Scope,
         match: Match[str],
         captures: Captures,
 ) -> Regions:
-    ret: List[Region] = []
+    ret: list[Region] = []
     pos, pos_end = match.span()
     for i, u_rule in captures:
         try:
@@ -349,12 +345,12 @@ def _captures(
 
 def _do_regset(
         idx: int,
-        match: Optional[Match[str]],
+        match: Match[str] | None,
         rule: CompiledRegsetRule,
-        compiler: 'Compiler',
+        compiler: Compiler,
         state: State,
         pos: int,
-) -> Optional[Tuple[State, int, bool, Regions]]:
+) -> tuple[State, int, bool, Regions] | None:
     if match is None:
         return None
 
@@ -369,103 +365,116 @@ def _do_regset(
     return state, match.end(), boundary, tuple(ret)
 
 
+@uniquely_constructed
 class PatternRule(NamedTuple):
-    name: Tuple[str, ...]
+    name: tuple[str, ...]
     regset: _RegSet
-    u_rules: Tuple[_Rule, ...]
+    u_rules: tuple[_Rule, ...]
 
     def start(
             self,
-            compiler: 'Compiler',
+            compiler: Compiler,
             match: Match[str],
             state: State,
-    ) -> Tuple[State, bool, Regions]:
+    ) -> tuple[State, bool, Regions]:
         raise AssertionError(f'unreachable {self}')
 
     def search(
             self,
-            compiler: 'Compiler',
+            compiler: Compiler,
             state: State,
             line: str,
             pos: int,
             first_line: bool,
             boundary: bool,
-    ) -> Optional[Tuple[State, int, bool, Regions]]:
+    ) -> tuple[State, int, bool, Regions] | None:
         idx, match = self.regset.search(line, pos, first_line, boundary)
         return _do_regset(idx, match, self, compiler, state, pos)
 
 
+@uniquely_constructed
 class MatchRule(NamedTuple):
-    name: Tuple[str, ...]
+    name: tuple[str, ...]
     captures: Captures
 
     def start(
             self,
-            compiler: 'Compiler',
+            compiler: Compiler,
             match: Match[str],
             state: State,
-    ) -> Tuple[State, bool, Regions]:
+    ) -> tuple[State, bool, Regions]:
         scope = state.cur.scope + self.name
         return state, False, _captures(compiler, scope, match, self.captures)
 
     def search(
             self,
-            compiler: 'Compiler',
+            compiler: Compiler,
             state: State,
             line: str,
             pos: int,
             first_line: bool,
             boundary: bool,
-    ) -> Optional[Tuple[State, int, bool, Regions]]:
+    ) -> tuple[State, int, bool, Regions] | None:
         raise AssertionError(f'unreachable {self}')
 
 
+@uniquely_constructed
 class EndRule(NamedTuple):
-    name: Tuple[str, ...]
-    content_name: Tuple[str, ...]
+    name: tuple[str, ...]
+    content_name: tuple[str, ...]
     begin_captures: Captures
     end_captures: Captures
     end: str
     regset: _RegSet
-    u_rules: Tuple[_Rule, ...]
+    u_rules: tuple[_Rule, ...]
 
     def start(
             self,
-            compiler: 'Compiler',
+            compiler: Compiler,
             match: Match[str],
             state: State,
-    ) -> Tuple[State, bool, Regions]:
+    ) -> tuple[State, bool, Regions]:
         scope = state.cur.scope + self.name
         next_scope = scope + self.content_name
 
         boundary = match.end() == len(match.string)
-        reg = make_reg(match.expand(self.end))
-        state = state.push(Entry(next_scope, self, reg, boundary))
+        reg = make_reg(expand_escaped(match, self.end))
+        start = (match.string, match.start())
+        state = state.push(Entry(next_scope, self, start, reg, boundary))
         regions = _captures(compiler, scope, match, self.begin_captures)
         return state, True, regions
 
     def _end_ret(
             self,
-            compiler: 'Compiler',
+            compiler: Compiler,
             state: State,
             pos: int,
             m: Match[str],
-    ) -> Tuple[State, int, bool, Regions]:
+    ) -> tuple[State, int, bool, Regions]:
         ret = []
         if m.start() > pos:
             ret.append(Region(pos, m.start(), state.cur.scope))
         ret.extend(_captures(compiler, state.cur.scope, m, self.end_captures))
-        return state.pop(), m.end(), False, tuple(ret)
+        # this is probably a bug in the grammar, but it pushed and popped at
+        # the same position.
+        # we'll advance the highlighter by one position to get past the loop
+        # this appears to be what vs code does as well
+        if state.entries[-1].start == (m.string, m.end()):
+            ret.append(Region(m.end(), m.end() + 1, state.cur.scope))
+            end = m.end() + 1
+        else:
+            end = m.end()
+        return state.pop(), end, False, tuple(ret)
 
     def search(
             self,
-            compiler: 'Compiler',
+            compiler: Compiler,
             state: State,
             line: str,
             pos: int,
             first_line: bool,
             boundary: bool,
-    ) -> Optional[Tuple[State, int, bool, Regions]]:
+    ) -> tuple[State, int, bool, Regions] | None:
         end_match = state.cur.reg.search(line, pos, first_line, boundary)
         if end_match is not None and end_match.start() == pos:
             return self._end_ret(compiler, state, pos, end_match)
@@ -480,39 +489,42 @@ class EndRule(NamedTuple):
                 return _do_regset(idx, match, self, compiler, state, pos)
 
 
+@uniquely_constructed
 class WhileRule(NamedTuple):
-    name: Tuple[str, ...]
-    content_name: Tuple[str, ...]
+    name: tuple[str, ...]
+    content_name: tuple[str, ...]
     begin_captures: Captures
     while_captures: Captures
     while_: str
     regset: _RegSet
-    u_rules: Tuple[_Rule, ...]
+    u_rules: tuple[_Rule, ...]
 
     def start(
             self,
-            compiler: 'Compiler',
+            compiler: Compiler,
             match: Match[str],
             state: State,
-    ) -> Tuple[State, bool, Regions]:
+    ) -> tuple[State, bool, Regions]:
         scope = state.cur.scope + self.name
         next_scope = scope + self.content_name
 
         boundary = match.end() == len(match.string)
-        reg = make_reg(match.expand(self.while_))
-        state = state.push_while(self, Entry(next_scope, self, reg, boundary))
+        reg = make_reg(expand_escaped(match, self.while_))
+        start = (match.string, match.start())
+        entry = Entry(next_scope, self, start, reg, boundary)
+        state = state.push_while(self, entry)
         regions = _captures(compiler, scope, match, self.begin_captures)
         return state, True, regions
 
     def continues(
             self,
-            compiler: 'Compiler',
+            compiler: Compiler,
             state: State,
             line: str,
             pos: int,
             first_line: bool,
             boundary: bool,
-    ) -> Optional[Tuple[int, bool, Regions]]:
+    ) -> tuple[int, bool, Regions] | None:
         match = state.cur.reg.match(line, pos, first_line, boundary)
         if match is None:
             return None
@@ -522,25 +534,25 @@ class WhileRule(NamedTuple):
 
     def search(
             self,
-            compiler: 'Compiler',
+            compiler: Compiler,
             state: State,
             line: str,
             pos: int,
             first_line: bool,
             boundary: bool,
-    ) -> Optional[Tuple[State, int, bool, Regions]]:
+    ) -> tuple[State, int, bool, Regions] | None:
         idx, match = self.regset.search(line, pos, first_line, boundary)
         return _do_regset(idx, match, self, compiler, state, pos)
 
 
 class Compiler:
-    def __init__(self, grammar: Grammar, grammars: Dict[str, Grammar]) -> None:
+    def __init__(self, grammar: Grammar, grammars: Grammars) -> None:
         self._root_scope = grammar.scope_name
         self._grammars = grammars
-        self._rule_to_grammar: Dict[_Rule, Grammar] = {}
-        self._c_rules: Dict[_Rule, CompiledRule] = {}
+        self._rule_to_grammar: dict[_Rule, Grammar] = {}
+        self._c_rules: dict[_Rule, CompiledRule] = {}
         root = self._compile_root(grammar)
-        self.root_state = State.root(Entry(root.name, root))
+        self.root_state = State.root(Entry(root.name, root, ('', 0)))
 
     def _visit_rule(self, grammar: Grammar, rule: _Rule) -> _Rule:
         self._rule_to_grammar[rule] = grammar
@@ -550,31 +562,37 @@ class Compiler:
     def _include(
             self,
             grammar: Grammar,
+            repository: FChainMap[str, _Rule],
             s: str,
-    ) -> Tuple[List[str], Tuple[_Rule, ...]]:
+    ) -> tuple[list[str], tuple[_Rule, ...]]:
         if s == '$self':
             return self._patterns(grammar, grammar.patterns)
         elif s == '$base':
-            return self._include(self._grammars[self._root_scope], '$self')
+            grammar = self._grammars.grammar_for_scope(self._root_scope)
+            return self._include(grammar, grammar.repository, '$self')
         elif s.startswith('#'):
-            return self._patterns(grammar, (grammar.repository[s[1:]],))
+            return self._patterns(grammar, (repository[s[1:]],))
         elif '#' not in s:
-            return self._include(self._grammars[s], '$self')
+            grammar = self._grammars.grammar_for_scope(s)
+            return self._include(grammar, grammar.repository, '$self')
         else:
             scope, _, s = s.partition('#')
-            return self._include(self._grammars[scope], f'#{s}')
+            grammar = self._grammars.grammar_for_scope(scope)
+            return self._include(grammar, grammar.repository, f'#{s}')
 
     @functools.lru_cache(maxsize=None)
     def _patterns(
             self,
             grammar: Grammar,
-            rules: Tuple[_Rule, ...],
-    ) -> Tuple[List[str], Tuple[_Rule, ...]]:
+            rules: tuple[_Rule, ...],
+    ) -> tuple[list[str], tuple[_Rule, ...]]:
         ret_regs = []
-        ret_rules: List[_Rule] = []
+        ret_rules: list[_Rule] = []
         for rule in rules:
             if rule.include is not None:
-                tmp_regs, tmp_rules = self._include(grammar, rule.include)
+                tmp_regs, tmp_rules = self._include(
+                    grammar, rule.repository, rule.include,
+                )
                 ret_regs.extend(tmp_regs)
                 ret_rules.extend(tmp_rules)
             elif rule.match is None and rule.begin is None and rule.patterns:
@@ -634,8 +652,10 @@ class Compiler:
             return PatternRule(rule.name, make_regset(*regs), rules)
 
     def compile_rule(self, rule: _Rule) -> CompiledRule:
-        with contextlib.suppress(KeyError):
+        try:
             return self._c_rules[rule]
+        except KeyError:
+            pass
 
         grammar = self._rule_to_grammar[rule]
         ret = self._c_rules[rule] = self._compile_rule(grammar, rule)
@@ -643,56 +663,94 @@ class Compiler:
 
 
 class Grammars:
-    def __init__(self, grammars: List[Grammar]) -> None:
-        self.grammars = {grammar.scope_name: grammar for grammar in grammars}
-        self._compilers: Dict[Grammar, Compiler] = {}
+    def __init__(self, *directories: str) -> None:
+        self._scope_to_files = {
+            os.path.splitext(filename)[0]: os.path.join(directory, filename)
+            for directory in directories
+            if os.path.exists(directory)
+            for filename in sorted(os.listdir(directory))
+            if filename.endswith('.json')
+        }
 
-    @classmethod
-    def from_syntax_dir(cls, syntax_dir: str) -> 'Grammars':
-        grammars = [Grammar.blank()]
-        if os.path.exists(syntax_dir):
-            grammars.extend(
-                Grammar.parse(os.path.join(syntax_dir, filename))
-                for filename in os.listdir(syntax_dir)
-            )
-        return cls(grammars)
+        unknown_grammar = {'scopeName': 'source.unknown', 'patterns': []}
+        self._raw = {'source.unknown': unknown_grammar}
+        self._file_types: list[tuple[frozenset[str], str]] = []
+        self._first_line: list[tuple[_Reg, str]] = []
+        self._parsed: dict[str, Grammar] = {}
+        self._compiled: dict[str, Compiler] = {}
 
-    def _compiler_for_grammar(self, grammar: Grammar) -> Compiler:
-        with contextlib.suppress(KeyError):
-            return self._compilers[grammar]
+    def _raw_for_scope(self, scope: str) -> dict[str, Any]:
+        try:
+            return self._raw[scope]
+        except KeyError:
+            pass
 
-        ret = self._compilers[grammar] = Compiler(grammar, self.grammars)
+        grammar_path = self._scope_to_files.pop(scope)
+        with open(grammar_path, encoding='UTF-8') as f:
+            ret = self._raw[scope] = json.load(f)
+
+        file_types = frozenset(ret.get('fileTypes', ()))
+        first_line = make_reg(ret.get('firstLineMatch', '$impossible^'))
+
+        self._file_types.append((file_types, scope))
+        self._first_line.append((first_line, scope))
+
+        return ret
+
+    def grammar_for_scope(self, scope: str) -> Grammar:
+        try:
+            return self._parsed[scope]
+        except KeyError:
+            pass
+
+        raw = self._raw_for_scope(scope)
+        ret = self._parsed[scope] = Grammar.make(raw)
         return ret
 
     def compiler_for_scope(self, scope: str) -> Compiler:
-        return self._compiler_for_grammar(self.grammars[scope])
+        try:
+            return self._compiled[scope]
+        except KeyError:
+            pass
+
+        grammar = self.grammar_for_scope(scope)
+        ret = self._compiled[scope] = Compiler(grammar, self)
+        return ret
 
     def blank_compiler(self) -> Compiler:
         return self.compiler_for_scope('source.unknown')
 
-    def compiler_for_file(self, filename: str) -> Compiler:
-        if os.path.exists(filename):
-            with open(filename) as f:
-                first_line = next(f, '')
-        else:
-            first_line = ''
-        for grammar in self.grammars.values():
-            if grammar.matches_file(filename, first_line):
-                break
-        else:
-            grammar = self.grammars['source.unknown']
+    def compiler_for_file(self, filename: str, first_line: str) -> Compiler:
+        for tag in tags_from_filename(filename) - {'text'}:
+            try:
+                # TODO: this doesn't always match even if we detect it
+                return self.compiler_for_scope(f'source.{tag}')
+            except KeyError:
+                pass
 
-        return self._compiler_for_grammar(grammar)
+        # didn't find it in the fast path, need to read all the json
+        for k in tuple(self._scope_to_files):
+            self._raw_for_scope(k)
+
+        _, _, ext = os.path.basename(filename).rpartition('.')
+        for extensions, scope in self._file_types:
+            if ext in extensions:
+                return self.compiler_for_scope(scope)
+
+        for reg, scope in self._first_line:
+            if reg.match(first_line, 0, first_line=True, boundary=True):
+                return self.compiler_for_scope(scope)
+
+        return self.compiler_for_scope('source.unknown')
 
 
-@functools.lru_cache(maxsize=None)
 def highlight_line(
-        compiler: 'Compiler',
+        compiler: Compiler,
         state: State,
         line: str,
         first_line: bool,
-) -> Tuple[State, Regions]:
-    ret: List[Region] = []
+) -> tuple[State, Regions]:
+    ret: list[Region] = []
     pos = 0
     boundary = state.cur.boundary
 

babi/history.py

@@ -1,25 +1,26 @@
+from __future__ import annotations
+
 import collections
 import contextlib
 import os.path
-from typing import Dict
 from typing import Generator
-from typing import List
 
 from babi.user_data import xdg_data
 
 
 class History:
     def __init__(self) -> None:
-        self._orig_len: Dict[str, int] = collections.defaultdict(int)
-        self.data: Dict[str, List[str]] = collections.defaultdict(list)
-        self.prev: Dict[str, str] = {}
+        self._orig_len: dict[str, int] = collections.defaultdict(int)
+        self.data: dict[str, list[str]] = collections.defaultdict(list)
+        self.prev: dict[str, str] = {}
 
     @contextlib.contextmanager
     def save(self) -> Generator[None, None, None]:
         history_dir = xdg_data('history')
         os.makedirs(history_dir, exist_ok=True)
         for filename in os.listdir(history_dir):
-            with open(os.path.join(history_dir, filename)) as f:
+            history_filename = os.path.join(history_dir, filename)
+            with open(history_filename, encoding='UTF-8') as f:
                 self.data[filename] = f.read().splitlines()
                 self._orig_len[filename] = len(self.data[filename])
         try:
@@ -28,5 +29,6 @@ class History:
             for k, v in self.data.items():
                 new_history = v[self._orig_len[k]:]
                 if new_history:
-                    with open(os.path.join(history_dir, k), 'a+') as f:
+                    history_filename = os.path.join(history_dir, k)
+                    with open(history_filename, 'a+', encoding='UTF-8') as f:
                         f.write('\n'.join(new_history) + '\n')

babi/hl/interface.py

@@ -1,27 +1,34 @@
-from typing import Sequence
+from __future__ import annotations
+
+from typing import NamedTuple
 from typing import Tuple
 
 from babi._types import Protocol
-from babi._types import TypedDict
-from babi.list_spy import SequenceNoSlice
+from babi.buf import Buf
 
 
-class CursesRegion(TypedDict):
+class HL(NamedTuple):
     x: int
-    n: int
-    color: int
+    end: int
+    attr: int
 
 
-CursesRegions = Tuple[CursesRegion, ...]
+HLs = Tuple[HL, ...]
+
+
+class RegionsMapping(Protocol):
+    def __getitem__(self, idx: int) -> HLs: ...
 
 
 class FileHL(Protocol):
     @property
-    def regions(self) -> Sequence[CursesRegions]: ...
-    def highlight_until(self, lines: SequenceNoSlice, idx: int) -> None: ...
-    def touch(self, lineno: int) -> None: ...
+    def include_edge(self) -> bool: ...
+    @property
+    def regions(self) -> RegionsMapping: ...
+    def highlight_until(self, lines: Buf, idx: int) -> None: ...
+    def register_callbacks(self, buf: Buf) -> None: ...
 
 
 class HLFactory(Protocol):
-    def get_file_highlighter(self, filename: str) -> FileHL: ...
-    def get_blank_file_highlighter(self) -> FileHL: ...
+    def file_highlighter(self, filename: str, first_line: str) -> FileHL: ...
+    def blank_file_highlighter(self) -> FileHL: ...

babi/hl/replace.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+import collections
+import contextlib
+import curses
+from typing import Generator
+
+from babi.buf import Buf
+from babi.hl.interface import HL
+from babi.hl.interface import HLs
+
+
+class Replace:
+    include_edge = True
+
+    def __init__(self) -> None:
+        self.regions: dict[int, HLs] = collections.defaultdict(tuple)
+
+    def highlight_until(self, lines: Buf, idx: int) -> None:
+        """our highlight regions are populated in other ways"""
+
+    def register_callbacks(self, buf: Buf) -> None:
+        """our highlight regions are populated in other ways"""
+
+    @contextlib.contextmanager
+    def region(self, y: int, x: int, end: int) -> Generator[None, None, None]:
+        # XXX: this assumes pair 1 is the background
+        attr = curses.A_REVERSE | curses.A_DIM | curses.color_pair(1)
+        self.regions[y] = (HL(x=x, end=end, attr=attr),)
+        try:
+            yield
+        finally:
+            del self.regions[y]

babi/hl/selection.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+import collections
+import curses
+
+from babi.buf import Buf
+from babi.hl.interface import HL
+from babi.hl.interface import HLs
+
+
+class Selection:
+    include_edge = True
+
+    def __init__(self) -> None:
+        self.regions: dict[int, HLs] = collections.defaultdict(tuple)
+        self.start: tuple[int, int] | None = None
+        self.end: tuple[int, int] | None = None
+
+    def register_callbacks(self, buf: Buf) -> None:
+        """our highlight regions are populated in other ways"""
+
+    def highlight_until(self, lines: Buf, idx: int) -> None:
+        if self.start is None or self.end is None:
+            return
+
+        # XXX: this assumes pair 1 is the background
+        attr = curses.A_REVERSE | curses.A_DIM | curses.color_pair(1)
+        (s_y, s_x), (e_y, e_x) = self.get()
+        if s_y == e_y:
+            self.regions[s_y] = (HL(x=s_x, end=e_x, attr=attr),)
+        else:
+            self.regions[s_y] = (
+                HL(x=s_x, end=len(lines[s_y]) + 1, attr=attr),
+            )
+            for l_y in range(s_y + 1, e_y):
+                self.regions[l_y] = (
+                    HL(x=0, end=len(lines[l_y]) + 1, attr=attr),
+                )
+            self.regions[e_y] = (HL(x=0, end=e_x, attr=attr),)
+
+    def get(self) -> tuple[tuple[int, int], tuple[int, int]]:
+        assert self.start is not None and self.end is not None
+        if self.start < self.end:
+            return self.start, self.end
+        else:
+            return self.end, self.start
+
+    def clear(self) -> None:
+        if self.start is not None and self.end is not None:
+            (s_y, _), (e_y, _) = self.get()
+            for l_y in range(s_y, e_y + 1):
+                del self.regions[l_y]
+        self.start = self.end = None
+
+    def set(self, s_y: int, s_x: int, e_y: int, e_x: int) -> None:
+        self.clear()
+        self.start, self.end = (s_y, s_x), (e_y, e_x)

babi/hl/syntax.py

@@ -1,26 +1,29 @@
-import curses
-from typing import Dict
-from typing import List
-from typing import NamedTuple
-from typing import Tuple
+from __future__ import annotations
 
+import curses
+import functools
+import math
+from typing import Callable
+from typing import NamedTuple
+
+from babi.buf import Buf
 from babi.color_manager import ColorManager
 from babi.highlight import Compiler
 from babi.highlight import Grammars
 from babi.highlight import highlight_line
 from babi.highlight import State
-from babi.hl.interface import CursesRegion
-from babi.hl.interface import CursesRegions
-from babi.list_spy import SequenceNoSlice
+from babi.hl.interface import HL
+from babi.hl.interface import HLs
 from babi.theme import Style
 from babi.theme import Theme
+from babi.user_data import prefix_data
 from babi.user_data import xdg_config
 from babi.user_data import xdg_data
 
-A_ITALIC = getattr(curses, 'A_ITALIC', 0x80000000)  # new in py37
-
 
 class FileSyntax:
+    include_edge = False
+
     def __init__(
             self,
             compiler: Compiler,
@@ -31,91 +34,103 @@ class FileSyntax:
         self._theme = theme
         self._color_manager = color_manager
 
-        self.regions: List[CursesRegions] = []
-        self._states: List[State] = []
+        self.regions: list[HLs] = []
+        self._states: list[State] = []
 
-        self._hl_cache: Dict[str, Dict[State, Tuple[State, CursesRegions]]]
-        self._hl_cache = {}
+        # this will be assigned a functools.lru_cache per instance for
+        # better hit rate and memory usage
+        self._hl: Callable[[State, str, bool], tuple[State, HLs]] | None
+        self._hl = None
 
     def attr(self, style: Style) -> int:
         pair = self._color_manager.color_pair(style.fg, style.bg)
         return (
             curses.color_pair(pair) |
             curses.A_BOLD * style.b |
-            A_ITALIC * style.i |
+            curses.A_ITALIC * style.i |
             curses.A_UNDERLINE * style.u
         )
 
-    def _hl(
+    def _hl_uncached(
             self,
             state: State,
             line: str,
-            i: int,
-    ) -> Tuple[State, CursesRegions]:
-        try:
-            return self._hl_cache[line][state]
-        except KeyError:
-            pass
-
+            first_line: bool,
+    ) -> tuple[State, HLs]:
         new_state, regions = highlight_line(
-            self._compiler, state, f'{line}\n', first_line=i == 0,
+            self._compiler, state, f'{line}\n', first_line=first_line,
         )
 
         # remove the trailing newline
         new_end = regions[-1]._replace(end=regions[-1].end - 1)
         regions = regions[:-1] + (new_end,)
 
-        regs: List[CursesRegion] = []
+        regs: list[HL] = []
         for r in regions:
             style = self._theme.select(r.scope)
             if style == self._theme.default:
                 continue
 
-            n = r.end - r.start
             attr = self.attr(style)
             if (
                     regs and
-                    regs[-1]['color'] == attr and
-                    regs[-1]['x'] + regs[-1]['n'] == r.start
+                    regs[-1].attr == attr and
+                    regs[-1].end == r.start
             ):
-                regs[-1]['n'] += n
+                regs[-1] = regs[-1]._replace(end=r.end)
             else:
-                regs.append(CursesRegion(x=r.start, n=n, color=attr))
+                regs.append(HL(x=r.start, end=r.end, attr=attr))
 
-        dct = self._hl_cache.setdefault(line, {})
-        ret = dct[state] = (new_state, tuple(regs))
-        return ret
+        return new_state, tuple(regs)
+
+    def _set_cb(self, lines: Buf, idx: int, victim: str) -> None:
+        del self.regions[idx:]
+        del self._states[idx:]
+
+    def _del_cb(self, lines: Buf, idx: int, victim: str) -> None:
+        del self.regions[idx:]
+        del self._states[idx:]
+
+    def _ins_cb(self, lines: Buf, idx: int) -> None:
+        del self.regions[idx:]
+        del self._states[idx:]
+
+    def register_callbacks(self, buf: Buf) -> None:
+        buf.add_set_callback(self._set_cb)
+        buf.add_del_callback(self._del_cb)
+        buf.add_ins_callback(self._ins_cb)
+
+    def highlight_until(self, lines: Buf, idx: int) -> None:
+        if self._hl is None:
+            # the docs claim better performance with power of two sizing
+            size = max(4096, 2 ** (int(math.log(len(lines), 2)) + 2))
+            self._hl = functools.lru_cache(maxsize=size)(self._hl_uncached)
 
-    def highlight_until(self, lines: SequenceNoSlice, idx: int) -> None:
         if not self._states:
             state = self._compiler.root_state
         else:
             state = self._states[-1]
 
         for i in range(len(self._states), idx):
-            state, regions = self._hl(state, lines[i], i)
+            state, regions = self._hl(state, lines[i], i == 0)
             self._states.append(state)
             self.regions.append(regions)
 
-    def touch(self, lineno: int) -> None:
-        del self._states[lineno:]
-        del self.regions[lineno:]
-
 
 class Syntax(NamedTuple):
     grammars: Grammars
     theme: Theme
     color_manager: ColorManager
 
-    def get_file_highlighter(self, filename: str) -> FileSyntax:
-        compiler = self.grammars.compiler_for_file(filename)
+    def file_highlighter(self, filename: str, first_line: str) -> FileSyntax:
+        compiler = self.grammars.compiler_for_file(filename, first_line)
         return FileSyntax(compiler, self.theme, self.color_manager)
 
-    def get_blank_file_highlighter(self) -> FileSyntax:
+    def blank_file_highlighter(self) -> FileSyntax:
         compiler = self.grammars.blank_compiler()
         return FileSyntax(compiler, self.theme, self.color_manager)
 
-    def _init_screen(self, stdscr: 'curses._CursesWindow') -> None:
+    def _init_screen(self, stdscr: curses._CursesWindow) -> None:
         default_fg, default_bg = self.theme.default.fg, self.theme.default.bg
         all_colors = {c for c in (default_fg, default_bg) if c is not None}
         todo = list(self.theme.rules.children.values())
@@ -136,10 +151,10 @@ class Syntax(NamedTuple):
     @classmethod
     def from_screen(
             cls,
-            stdscr: 'curses._CursesWindow',
+            stdscr: curses._CursesWindow,
             color_manager: ColorManager,
-    ) -> 'Syntax':
-        grammars = Grammars.from_syntax_dir(xdg_data('textmate_syntax'))
+    ) -> Syntax:
+        grammars = Grammars(prefix_data('grammar_v1'), xdg_data('grammar_v1'))
         theme = Theme.from_filename(xdg_config('theme.json'))
         ret = cls(grammars, theme, color_manager)
         ret._init_screen(stdscr)

babi/hl/trailing_whitespace.py

@@ -1,20 +1,22 @@
+from __future__ import annotations
+
 import curses
-from typing import List
-from typing import NamedTuple
 
+from babi.buf import Buf
 from babi.color_manager import ColorManager
-from babi.hl.interface import CursesRegion
-from babi.hl.interface import CursesRegions
-from babi.list_spy import SequenceNoSlice
+from babi.hl.interface import HL
+from babi.hl.interface import HLs
 
 
-class FileTrailingWhitespace:
+class TrailingWhitespace:
+    include_edge = False
+
     def __init__(self, color_manager: ColorManager) -> None:
         self._color_manager = color_manager
 
-        self.regions: List[CursesRegions] = []
+        self.regions: list[HLs] = []
 
-    def _trailing_ws(self, line: str) -> CursesRegions:
+    def _trailing_ws(self, line: str) -> HLs:
         if not line:
             return ()
 
@@ -27,22 +29,25 @@ class FileTrailingWhitespace:
         else:
             pair = self._color_manager.raw_color_pair(-1, curses.COLOR_RED)
             attr = curses.color_pair(pair)
-            return (CursesRegion(x=i, n=len(line) - i, color=attr),)
+            return (HL(x=i, end=len(line), attr=attr),)
 
-    def highlight_until(self, lines: SequenceNoSlice, idx: int) -> None:
+    def _set_cb(self, lines: Buf, idx: int, victim: str) -> None:
+        if idx < len(self.regions):
+            self.regions[idx] = self._trailing_ws(lines[idx])
+
+    def _del_cb(self, lines: Buf, idx: int, victim: str) -> None:
+        if idx < len(self.regions):
+            del self.regions[idx]
+
+    def _ins_cb(self, lines: Buf, idx: int) -> None:
+        if idx < len(self.regions):
+            self.regions.insert(idx, self._trailing_ws(lines[idx]))
+
+    def register_callbacks(self, buf: Buf) -> None:
+        buf.add_set_callback(self._set_cb)
+        buf.add_del_callback(self._del_cb)
+        buf.add_ins_callback(self._ins_cb)
+
+    def highlight_until(self, lines: Buf, idx: int) -> None:
         for i in range(len(self.regions), idx):
             self.regions.append(self._trailing_ws(lines[i]))
-
-    def touch(self, lineno: int) -> None:
-        del self.regions[lineno:]
-
-
-class TrailingWhitespace(NamedTuple):
-    color_manager: ColorManager
-
-    def get_file_highlighter(self, filename: str) -> FileTrailingWhitespace:
-        # no file-specific behaviour
-        return self.get_blank_file_highlighter()
-
-    def get_blank_file_highlighter(self) -> FileTrailingWhitespace:
-        return FileTrailingWhitespace(self.color_manager)

babi/horizontal_scrolling.py

@@ -1,10 +1,17 @@
+from __future__ import annotations
+
+import curses
+
+from babi.cached_property import cached_property
+
+
 def line_x(x: int, width: int) -> int:
-    margin = min(width - 3, 6)
     if x + 1 < width:
         return 0
     elif width == 1:
         return x
     else:
+        margin = min(width - 3, 6)
         return (
             width - margin - 2 +
             (x + 1 - width) //
@@ -17,7 +24,7 @@ def scrolled_line(s: str, x: int, width: int) -> str:
     l_x = line_x(x, width)
     if l_x:
         s = f'«{s[l_x + 1:]}'
-        if l_x and len(s) > width:
+        if len(s) > width:
             return f'{s[:width - 1]}»'
         else:
             return s.ljust(width)
@@ -25,3 +32,17 @@ def scrolled_line(s: str, x: int, width: int) -> str:
         return f'{s[:width - 1]}»'
     else:
         return s.ljust(width)
+
+
+class _CalcWidth:
+    @cached_property
+    def _window(self) -> curses._CursesWindow:
+        return curses.newwin(1, 10)
+
+    def wcwidth(self, c: str) -> int:
+        self._window.addstr(0, 0, c)
+        return self._window.getyx()[1]
+
+
+wcwidth = _CalcWidth().wcwidth
+del _CalcWidth

babi/list_spy.py

@@ -1,85 +0,0 @@
-import functools
-import sys
-from typing import Callable
-from typing import Iterator
-from typing import List
-
-from babi._types import Protocol
-
-
-class SequenceNoSlice(Protocol):
-    def __len__(self) -> int: ...
-    def __getitem__(self, idx: int) -> str: ...
-
-    def __iter__(self) -> Iterator[str]:
-        for i in range(len(self)):
-            yield self[i]
-
-
-class MutableSequenceNoSlice(SequenceNoSlice, Protocol):
-    def __setitem__(self, idx: int, val: str) -> None: ...
-    def __delitem__(self, idx: int) -> None: ...
-    def insert(self, idx: int, val: str) -> None: ...
-
-    def append(self, val: str) -> None:
-        self.insert(len(self), val)
-
-    def pop(self, idx: int = -1) -> str:
-        victim = self[idx]
-        del self[idx]
-        return victim
-
-
-def _del(lst: MutableSequenceNoSlice, *, idx: int) -> None:
-    del lst[idx]
-
-
-def _set(lst: MutableSequenceNoSlice, *, idx: int, val: str) -> None:
-    lst[idx] = val
-
-
-def _ins(lst: MutableSequenceNoSlice, *, idx: int, val: str) -> None:
-    lst.insert(idx, val)
-
-
-class ListSpy(MutableSequenceNoSlice):
-    def __init__(self, lst: MutableSequenceNoSlice) -> None:
-        self._lst = lst
-        self._undo: List[Callable[[MutableSequenceNoSlice], None]] = []
-        self.min_line_touched = sys.maxsize
-
-    def __repr__(self) -> str:
-        return f'{type(self).__name__}({self._lst})'
-
-    def __len__(self) -> int:
-        return len(self._lst)
-
-    def __getitem__(self, idx: int) -> str:
-        return self._lst[idx]
-
-    def __setitem__(self, idx: int, val: str) -> None:
-        self._undo.append(functools.partial(_set, idx=idx, val=self._lst[idx]))
-        self.min_line_touched = min(idx, self.min_line_touched)
-        self._lst[idx] = val
-
-    def __delitem__(self, idx: int) -> None:
-        if idx < 0:
-            idx %= len(self)
-        self._undo.append(functools.partial(_ins, idx=idx, val=self._lst[idx]))
-        self.min_line_touched = min(idx, self.min_line_touched)
-        del self._lst[idx]
-
-    def insert(self, idx: int, val: str) -> None:
-        if idx < 0:
-            idx %= len(self)
-        self._undo.append(functools.partial(_del, idx=idx))
-        self.min_line_touched = min(idx, self.min_line_touched)
-        self._lst.insert(idx, val)
-
-    def undo(self, lst: MutableSequenceNoSlice) -> None:
-        for fn in reversed(self._undo):
-            fn(lst)
-
-    @property
-    def has_modifications(self) -> bool:
-        return bool(self._undo)

babi/main.py

@@ -1,17 +1,27 @@
+from __future__ import annotations
+
 import argparse
 import curses
-from typing import Optional
+import os
+import re
+import signal
+import sys
 from typing import Sequence
 
+from babi.buf import Buf
 from babi.file import File
+from babi.perf import Perf
 from babi.perf import perf_log
 from babi.screen import EditResult
 from babi.screen import make_stdscr
 from babi.screen import Screen
 
+CONSOLE = 'CONIN$' if sys.platform == 'win32' else '/dev/tty'
+POSITION_RE = re.compile(r'^\+-?\d+$')
 
-def _edit(screen: Screen) -> EditResult:
-    screen.file.ensure_loaded(screen.status)
+
+def _edit(screen: Screen, stdin: str) -> EditResult:
+    screen.file.ensure_loaded(screen.status, screen.margin, stdin)
 
     while True:
         screen.status.tick(screen.margin)
@@ -32,35 +42,113 @@ def _edit(screen: Screen) -> EditResult:
             screen.status.update(f'unknown key: {key}')
 
 
-def c_main(stdscr: 'curses._CursesWindow', args: argparse.Namespace) -> int:
-    with perf_log(args.perf_log) as perf:
-        screen = Screen(stdscr, args.filenames or [None], perf)
-        with screen.history.save():
-            while screen.files:
-                screen.i = screen.i % len(screen.files)
-                res = _edit(screen)
-                if res == EditResult.EXIT:
-                    del screen.files[screen.i]
-                    screen.status.clear()
-                elif res == EditResult.NEXT:
-                    screen.i += 1
-                    screen.status.clear()
-                elif res == EditResult.PREV:
-                    screen.i -= 1
-                    screen.status.clear()
-                else:
-                    raise AssertionError(f'unreachable {res}')
+def c_main(
+        stdscr: curses._CursesWindow,
+        filenames: list[str | None],
+        positions: list[int],
+        stdin: str,
+        perf: Perf,
+) -> int:
+    screen = Screen(stdscr, filenames, positions, perf)
+    with screen.history.save():
+        while screen.files:
+            screen.i = screen.i % len(screen.files)
+            res = _edit(screen, stdin)
+            if res == EditResult.EXIT:
+                del screen.files[screen.i]
+                # always go to the next file except at the end
+                screen.i = min(screen.i, len(screen.files) - 1)
+                screen.status.clear()
+            elif res == EditResult.NEXT:
+                screen.i += 1
+                screen.status.clear()
+            elif res == EditResult.PREV:
+                screen.i -= 1
+                screen.status.clear()
+            elif res == EditResult.OPEN:
+                screen.i = len(screen.files) - 1
+            else:
+                raise AssertionError(f'unreachable {res}')
     return 0
 
 
-def main(argv: Optional[Sequence[str]] = None) -> int:
+def _key_debug(stdscr: curses._CursesWindow, perf: Perf) -> int:
+    screen = Screen(stdscr, ['<<key debug>>'], [0], perf)
+    screen.file.buf = Buf([''])
+
+    while True:
+        screen.status.update('press q to quit')
+        screen.draw()
+        screen.file.move_cursor(screen.stdscr, screen.margin)
+
+        key = screen.get_char()
+        screen.file.buf.insert(-1, f'{key.wch!r} {key.keyname.decode()!r}')
+        screen.file.down(screen.margin)
+        if key.wch == curses.KEY_RESIZE:
+            screen.resize()
+        if key.wch == 'q':
+            return 0
+
+
+def _filenames(filenames: list[str]) -> tuple[list[str | None], list[int]]:
+    if not filenames:
+        return [None], [0]
+
+    ret_filenames: list[str | None] = []
+    ret_positions = []
+
+    filenames_iter = iter(filenames)
+    for filename in filenames_iter:
+        if POSITION_RE.match(filename):
+            # in the success case we get:
+            #
+            # position_s = +...
+            # filename = (the next thing)
+            #
+            # in the error case we only need to reset `position_s` as
+            # `filename` is already correct
+            position_s = filename
+            try:
+                filename = next(filenames_iter)
+            except StopIteration:
+                position_s = '+0'
+            ret_positions.append(int(position_s[1:]))
+            ret_filenames.append(filename)
+        else:
+            ret_positions.append(0)
+            ret_filenames.append(filename)
+
+    return ret_filenames, ret_positions
+
+
+def main(argv: Sequence[str] | None = None) -> int:
     parser = argparse.ArgumentParser()
     parser.add_argument('filenames', metavar='filename', nargs='*')
     parser.add_argument('--perf-log')
+    parser.add_argument(
+        '--key-debug', action='store_true', help=argparse.SUPPRESS,
+    )
     args = parser.parse_args(argv)
 
-    with make_stdscr() as stdscr:
-        return c_main(stdscr, args)
+    if '-' in args.filenames:
+        print('reading stdin...', file=sys.stderr)
+        stdin = sys.stdin.buffer.read().decode()
+        tty = os.open(CONSOLE, os.O_RDONLY)
+        os.dup2(tty, sys.stdin.fileno())
+    else:
+        stdin = ''
+
+    # ignore backgrounding signals, we'll handle those in curses
+    # fixes a problem with ^Z on termination which would break the terminal
+    if sys.platform != 'win32':  # pragma: win32 no cover  # pragma: no branch
+        signal.signal(signal.SIGTSTP, signal.SIG_IGN)
+
+    with perf_log(args.perf_log) as perf, make_stdscr() as stdscr:
+        if args.key_debug:
+            return _key_debug(stdscr, perf)
+        else:
+            filenames, positions = _filenames(args.filenames)
+            return c_main(stdscr, filenames, positions, stdin, perf)
 
 
 if __name__ == '__main__':

babi/margin.py

@@ -1,14 +1,24 @@
+from __future__ import annotations
+
 import curses
 from typing import NamedTuple
 
 
 class Margin(NamedTuple):
-    header: bool
-    footer: bool
+    lines: int
+    cols: int
+
+    @property
+    def header(self) -> bool:
+        return self.lines > 2
+
+    @property
+    def footer(self) -> bool:
+        return self.lines > 1
 
     @property
     def body_lines(self) -> int:
-        return curses.LINES - self.header - self.footer
+        return self.lines - self.header - self.footer
 
     @property
     def page_size(self) -> int:
@@ -17,11 +27,11 @@ class Margin(NamedTuple):
         else:
             return self.body_lines - 2
 
+    @property
+    def scroll_amount(self) -> int:
+        # integer round up without banker's rounding (so 1/2 => 1 instead of 0)
+        return int(self.lines / 2 + .5)
+
     @classmethod
-    def from_current_screen(cls) -> 'Margin':
-        if curses.LINES == 1:
-            return cls(header=False, footer=False)
-        elif curses.LINES == 2:
-            return cls(header=False, footer=True)
-        else:
-            return cls(header=True, footer=True)
+    def from_current_screen(cls) -> Margin:
+        return cls(curses.LINES, curses.COLS)

babi/perf.py

@@ -1,18 +1,17 @@
+from __future__ import annotations
+
 import contextlib
 import cProfile
 import time
 from typing import Generator
-from typing import List
-from typing import Optional
-from typing import Tuple
 
 
 class Perf:
     def __init__(self) -> None:
-        self._prof: Optional[cProfile.Profile] = None
-        self._records: List[Tuple[str, float]] = []
-        self._name: Optional[str] = None
-        self._time: Optional[float] = None
+        self._prof: cProfile.Profile | None = None
+        self._records: list[tuple[str, float]] = []
+        self._name: str | None = None
+        self._time: float | None = None
 
     def start(self, name: str) -> None:
         if self._prof:
@@ -36,14 +35,14 @@ class Perf:
     def save_profiles(self, filename: str) -> None:
         assert self._prof is not None
         self._prof.dump_stats(f'{filename}.pstats')
-        with open(filename, 'w') as f:
+        with open(filename, 'w', encoding='UTF-8') as f:
             f.write('μs\tevent\n')
             for name, duration in self._records:
                 f.write(f'{int(duration * 1000 * 1000)}\t{name}\n')
 
 
 @contextlib.contextmanager
-def perf_log(filename: Optional[str]) -> Generator[Perf, None, None]:
+def perf_log(filename: str | None) -> Generator[Perf, None, None]:
     perf = Perf()
     if filename is None:
         yield perf

babi/prompt.py

@@ -1,10 +1,8 @@
+from __future__ import annotations
+
 import curses
 import enum
-from typing import List
-from typing import Optional
-from typing import Tuple
 from typing import TYPE_CHECKING
-from typing import Union
 
 from babi.horizontal_scrolling import line_x
 from babi.horizontal_scrolling import scrolled_line
@@ -16,7 +14,7 @@ PromptResult = enum.Enum('PromptResult', 'CANCELLED')
 
 
 class Prompt:
-    def __init__(self, screen: 'Screen', prompt: str, lst: List[str]) -> None:
+    def __init__(self, screen: Screen, prompt: str, lst: list[str]) -> None:
         self._screen = screen
         self._prompt = prompt
         self._lst = lst
@@ -31,20 +29,21 @@ class Prompt:
     def _s(self, s: str) -> None:
         self._lst[self._y] = s
 
-    def _render_prompt(self, *, base: Optional[str] = None) -> None:
+    def _render_prompt(self, *, base: str | None = None) -> None:
         base = base or self._prompt
-        if not base or curses.COLS < 7:
+        if not base or self._screen.margin.cols < 7:
             prompt_s = ''
-        elif len(base) > curses.COLS - 6:
-            prompt_s = f'{base[:curses.COLS - 7]}…: '
+        elif len(base) > self._screen.margin.cols - 6:
+            prompt_s = f'{base[:self._screen.margin.cols - 7]}…: '
         else:
             prompt_s = f'{base}: '
-        width = curses.COLS - len(prompt_s)
+        width = self._screen.margin.cols - len(prompt_s)
         line = scrolled_line(self._s, self._x, width)
         cmd = f'{prompt_s}{line}'
-        self._screen.stdscr.insstr(curses.LINES - 1, 0, cmd, curses.A_REVERSE)
+        prompt_line = self._screen.margin.lines - 1
+        self._screen.stdscr.insstr(prompt_line, 0, cmd, curses.A_REVERSE)
         x = len(prompt_s) + self._x - line_x(self._x, width)
-        self._screen.stdscr.move(curses.LINES - 1, x)
+        self._screen.stdscr.move(prompt_line, x)
 
     def _up(self) -> None:
         self._y = max(0, self._y - 1)
@@ -99,7 +98,7 @@ class Prompt:
     def _resize(self) -> None:
         self._screen.resize()
 
-    def _check_failed(self, idx: int, s: str) -> Tuple[bool, int]:
+    def _check_failed(self, idx: int, s: str) -> tuple[bool, int]:
         failed = False
         for search_idx in range(idx, -1, -1):
             if s in self._lst[search_idx]:
@@ -110,7 +109,7 @@ class Prompt:
             failed = True
         return failed, idx
 
-    def _reverse_search(self) -> Union[None, str, PromptResult]:
+    def _reverse_search(self) -> None | str | PromptResult:
         reverse_s = ''
         idx = self._y
         while True:
@@ -126,7 +125,7 @@ class Prompt:
             key = self._screen.get_char()
             if key.keyname == b'KEY_RESIZE':
                 self._screen.resize()
-            elif key.keyname == b'KEY_BACKSPACE' or key.keyname == b'^H':
+            elif key.keyname == b'KEY_BACKSPACE':
                 reverse_s = reverse_s[:-1]
             elif key.keyname == b'^R':
                 idx = max(0, idx - 1)
@@ -163,7 +162,6 @@ class Prompt:
         b'kLFT5': _ctrl_left,
         # editing
         b'KEY_BACKSPACE': _backspace,
-        b'^H': _backspace,  # ^Backspace
         b'KEY_DC': _delete,
         b'^K': _cut_to_end,
         # misc
@@ -177,7 +175,7 @@ class Prompt:
         self._s = self._s[:self._x] + c + self._s[self._x:]
         self._x += len(c)
 
-    def run(self) -> Union[PromptResult, str]:
+    def run(self) -> PromptResult | str:
         while True:
             self._render_prompt()
 

babi/reg.py

@@ -1,90 +1,49 @@
+from __future__ import annotations
+
 import functools
+import re
 from typing import Match
-from typing import Optional
-from typing import Tuple
 
 import onigurumacffi
 
-from babi.cached_property import cached_property
+_BACKREF_RE = re.compile(r'((?<!\\)(?:\\\\)*)\\([0-9]+)')
 
 
-def _replace_esc(s: str, chars: str) -> str:
-    """replace the given escape sequences of `chars` with \\uffff"""
-    for c in chars:
-        if f'\\{c}' in s:
-            break
-    else:
-        return s
-
-    b = []
-    i = 0
-    length = len(s)
-    while i < length:
-        try:
-            sbi = s.index('\\', i)
-        except ValueError:
-            b.append(s[i:])
-            break
-        if sbi > i:
-            b.append(s[i:sbi])
-        b.append('\\')
-        i = sbi + 1
-        if i < length:
-            if s[i] in chars:
-                b.append('\uffff')
-            else:
-                b.append(s[i])
-        i += 1
-    return ''.join(b)
+_FLAGS = {
+    # (first_line, boundary)
+    (False, False): (
+        onigurumacffi.OnigSearchOption.NOT_END_STRING |
+        onigurumacffi.OnigSearchOption.NOT_BEGIN_STRING |
+        onigurumacffi.OnigSearchOption.NOT_BEGIN_POSITION
+    ),
+    (False, True): (
+        onigurumacffi.OnigSearchOption.NOT_END_STRING |
+        onigurumacffi.OnigSearchOption.NOT_BEGIN_STRING
+    ),
+    (True, False): (
+        onigurumacffi.OnigSearchOption.NOT_END_STRING |
+        onigurumacffi.OnigSearchOption.NOT_BEGIN_POSITION
+    ),
+    (True, True): onigurumacffi.OnigSearchOption.NOT_END_STRING,
+}
 
 
 class _Reg:
     def __init__(self, s: str) -> None:
         self._pattern = s
+        self._reg = onigurumacffi.compile(self._pattern)
 
     def __repr__(self) -> str:
         return f'{type(self).__name__}({self._pattern!r})'
 
-    @cached_property
-    def _reg(self) -> onigurumacffi._Pattern:
-        return onigurumacffi.compile(self._pattern)
-
-    @cached_property
-    def _reg_no_A(self) -> onigurumacffi._Pattern:
-        return onigurumacffi.compile(_replace_esc(self._pattern, 'A'))
-
-    @cached_property
-    def _reg_no_G(self) -> onigurumacffi._Pattern:
-        return onigurumacffi.compile(_replace_esc(self._pattern, 'G'))
-
-    @cached_property
-    def _reg_no_A_no_G(self) -> onigurumacffi._Pattern:
-        return onigurumacffi.compile(_replace_esc(self._pattern, 'AG'))
-
-    def _get_reg(
-            self,
-            first_line: bool,
-            boundary: bool,
-    ) -> onigurumacffi._Pattern:
-        if boundary:
-            if first_line:
-                return self._reg
-            else:
-                return self._reg_no_A
-        else:
-            if first_line:
-                return self._reg_no_G
-            else:
-                return self._reg_no_A_no_G
-
     def search(
             self,
             line: str,
             pos: int,
             first_line: bool,
             boundary: bool,
-    ) -> Optional[Match[str]]:
-        return self._get_reg(first_line, boundary).search(line, pos)
+    ) -> Match[str] | None:
+        return self._reg.search(line, pos, flags=_FLAGS[first_line, boundary])
 
     def match(
             self,
@@ -92,56 +51,33 @@ class _Reg:
             pos: int,
             first_line: bool,
             boundary: bool,
-    ) -> Optional[Match[str]]:
-        return self._get_reg(first_line, boundary).match(line, pos)
+    ) -> Match[str] | None:
+        return self._reg.match(line, pos, flags=_FLAGS[first_line, boundary])
 
 
 class _RegSet:
     def __init__(self, *s: str) -> None:
         self._patterns = s
+        self._set = onigurumacffi.compile_regset(*self._patterns)
 
     def __repr__(self) -> str:
         args = ', '.join(repr(s) for s in self._patterns)
         return f'{type(self).__name__}({args})'
 
-    @cached_property
-    def _set(self) -> onigurumacffi._RegSet:
-        return onigurumacffi.compile_regset(*self._patterns)
-
-    @cached_property
-    def _set_no_A(self) -> onigurumacffi._RegSet:
-        patterns = (_replace_esc(p, 'A') for p in self._patterns)
-        return onigurumacffi.compile_regset(*patterns)
-
-    @cached_property
-    def _set_no_G(self) -> onigurumacffi._RegSet:
-        patterns = (_replace_esc(p, 'G') for p in self._patterns)
-        return onigurumacffi.compile_regset(*patterns)
-
-    @cached_property
-    def _set_no_A_no_G(self) -> onigurumacffi._RegSet:
-        patterns = (_replace_esc(p, 'AG') for p in self._patterns)
-        return onigurumacffi.compile_regset(*patterns)
-
     def search(
             self,
             line: str,
             pos: int,
             first_line: bool,
             boundary: bool,
-    ) -> Tuple[int, Optional[Match[str]]]:
-        if boundary:
-            if first_line:
-                return self._set.search(line, pos)
-            else:
-                return self._set_no_A.search(line, pos)
-        else:
-            if first_line:
-                return self._set_no_G.search(line, pos)
-            else:
-                return self._set_no_A_no_G.search(line, pos)
+    ) -> tuple[int, Match[str] | None]:
+        return self._set.search(line, pos, flags=_FLAGS[first_line, boundary])
+
+
+def expand_escaped(match: Match[str], s: str) -> str:
+    return _BACKREF_RE.sub(lambda m: f'{m[1]}{re.escape(match[int(m[2])])}', s)
 
 
 make_reg = functools.lru_cache(maxsize=None)(_Reg)
 make_regset = functools.lru_cache(maxsize=None)(_RegSet)
-ERR_REG = make_reg(')this pattern always triggers an error when used(')
+ERR_REG = make_reg('$ ^')

babi/screen.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import contextlib
 import curses
 import enum
@@ -6,14 +8,9 @@ import os
 import re
 import signal
 import sys
-from typing import Callable
 from typing import Generator
-from typing import List
 from typing import NamedTuple
-from typing import Optional
 from typing import Pattern
-from typing import Tuple
-from typing import Union
 
 from babi.color_manager import ColorManager
 from babi.file import Action
@@ -21,20 +18,26 @@ from babi.file import File
 from babi.file import get_lines
 from babi.history import History
 from babi.hl.syntax import Syntax
-from babi.hl.trailing_whitespace import TrailingWhitespace
 from babi.margin import Margin
 from babi.perf import Perf
 from babi.prompt import Prompt
 from babi.prompt import PromptResult
 from babi.status import Status
 
-VERSION_STR = 'babi v0'
-EditResult = enum.Enum('EditResult', 'EXIT NEXT PREV')
+if sys.version_info >= (3, 8):  # pragma: no cover (py38+)
+    import importlib.metadata as importlib_metadata
+else:  # pragma: no cover (<py38)
+    import importlib_metadata
+
+VERSION_STR = f'babi v{importlib_metadata.version("babi")}'
+EditResult = enum.Enum('EditResult', 'EXIT NEXT PREV OPEN')
 
 # TODO: find a place to populate these, surely there's a database somewhere
 SEQUENCE_KEYNAME = {
     '\x1bOH': b'KEY_HOME',
     '\x1bOF': b'KEY_END',
+    '\x1b[1~': b'KEY_HOME',
+    '\x1b[4~': b'KEY_END',
     '\x1b[1;2A': b'KEY_SR',
     '\x1b[1;2B': b'KEY_SF',
     '\x1b[1;2C': b'KEY_SRIGHT',
@@ -57,37 +60,73 @@ SEQUENCE_KEYNAME = {
     '\x1b[1;6D': b'kLFT6',  # Shift + ^Left
     '\x1b[1;6H': b'kHOM6',  # Shift + ^Home
     '\x1b[1;6F': b'kEND6',  # Shift + ^End
+    '\x1b[~': b'KEY_BTAB',  # Shift + Tab
+}
+KEYNAME_REWRITE = {
+    # windows-curses: numeric pad arrow keys
+    # - some overlay keyboards pick these as well
+    # - in xterm it seems these are mapped automatically
+    b'KEY_A2': b'KEY_UP',
+    b'KEY_C2': b'KEY_DOWN',
+    b'KEY_B3': b'KEY_RIGHT',
+    b'KEY_B1': b'KEY_LEFT',
+    b'PADSTOP': b'KEY_DC',
+    b'KEY_A3': b'KEY_PPAGE',
+    b'KEY_C3': b'KEY_NPAGE',
+    b'KEY_A1': b'KEY_HOME',
+    b'KEY_C1': b'KEY_END',
+    # windows-curses: map to our M- names
+    b'ALT_U': b'M-u',
+    # windows-curses: arguably these names are better than the xterm names
+    b'CTL_UP': b'kUP5',
+    b'CTL_DOWN': b'kDN5',
+    b'CTL_RIGHT': b'kRIT5',
+    b'CTL_LEFT': b'kLFT5',
+    b'CTL_HOME': b'kHOM5',
+    b'CTL_END': b'kEND5',
+    b'ALT_RIGHT': b'kRIT3',
+    b'ALT_LEFT': b'kLFT3',
+    b'ALT_E': b'M-e',
+    # windows-curses: idk why these are different
+    b'KEY_SUP': b'KEY_SR',
+    b'KEY_SDOWN': b'KEY_SF',
+    # macos: (sends this for backspace key, others interpret this as well)
+    b'^?': b'KEY_BACKSPACE',
+    # linux, perhaps others
+    b'^H': b'KEY_BACKSPACE',  # ^Backspace on my keyboard
+    b'^D': b'KEY_DC',
+    b'PADENTER': b'^M',  # Enter on numpad
 }
 
 
 class Key(NamedTuple):
-    wch: Union[int, str]
+    wch: int | str
     keyname: bytes
 
 
 class Screen:
     def __init__(
             self,
-            stdscr: 'curses._CursesWindow',
-            filenames: List[Optional[str]],
+            stdscr: curses._CursesWindow,
+            filenames: list[str | None],
+            initial_lines: list[int],
             perf: Perf,
     ) -> None:
         self.stdscr = stdscr
-        color_manager = ColorManager.make()
-        hl_factories = (
-            Syntax.from_screen(stdscr, color_manager),
-            TrailingWhitespace(color_manager),
-        )
-        self.files = [File(f, hl_factories) for f in filenames]
+        self.color_manager = ColorManager.make()
+        self.hl_factories = (Syntax.from_screen(stdscr, self.color_manager),)
+        self.files = [
+            File(filename, line, self.color_manager, self.hl_factories)
+            for filename, line in zip(filenames, initial_lines)
+        ]
         self.i = 0
         self.history = History()
         self.perf = perf
         self.status = Status()
         self.margin = Margin.from_current_screen()
-        self.cut_buffer: Tuple[str, ...] = ()
+        self.cut_buffer: tuple[str, ...] = ()
         self.cut_selection = False
-        self._resize_cb: Optional[Callable[[], None]] = None
-        self._buffered_input: Union[int, str, None] = None
+        self._buffered_input: int | str | None = None
 
     @property
     def file(self) -> File:
@@ -103,7 +142,7 @@ class Screen:
         else:
             files = ''
             version_width = len(VERSION_STR) + 2
-        centered = filename.center(curses.COLS)[version_width:]
+        centered = filename.center(self.margin.cols)[version_width:]
         s = f' {VERSION_STR} {files}{centered}{files}'
         self.stdscr.insstr(0, 0, s, curses.A_REVERSE)
 
@@ -189,7 +228,10 @@ class Screen:
         if self._buffered_input is not None:
             wch, self._buffered_input = self._buffered_input, None
         else:
-            wch = self.stdscr.get_wch()
+            try:
+                wch = self.stdscr.get_wch()
+            except curses.error:  # pragma: no cover (macos bug?)
+                wch = self.stdscr.get_wch()
         if isinstance(wch, str) and wch == '\x1b':
             wch = self._get_sequence(wch)
             if len(wch) == 2:
@@ -200,12 +242,10 @@ class Screen:
         elif isinstance(wch, str) and wch.isprintable():
             wch = self._get_string(wch)
             return Key(wch, b'STRING')
-        elif wch == '\x7f':  # pragma: no cover (macos)
-            keyname = curses.keyname(curses.KEY_BACKSPACE)
-            return Key(wch, keyname)
 
         key = wch if isinstance(wch, int) else ord(wch)
         keyname = curses.keyname(key)
+        keyname = KEYNAME_REWRITE.get(keyname, keyname)
         return Key(wch, keyname)
 
     def get_char(self) -> Key:
@@ -220,49 +260,66 @@ class Screen:
         self.file.draw(self.stdscr, self.margin)
         self.status.draw(self.stdscr, self.margin)
 
-    @contextlib.contextmanager
-    def resize_cb(self, f: Callable[[], None]) -> Generator[None, None, None]:
-        assert self._resize_cb is None, self._resize_cb
-        self._resize_cb = f
-        try:
-            yield
-        finally:
-            self._resize_cb = None
-
     def resize(self) -> None:
         curses.update_lines_cols()
         self.margin = Margin.from_current_screen()
-        self.file.scroll_screen_if_needed(self.margin)
+        self.file.buf.scroll_screen_if_needed(self.margin)
         self.draw()
-        if self._resize_cb is not None:
-            self._resize_cb()
 
-    def quick_prompt(self, prompt: str, opts: str) -> Union[str, PromptResult]:
+    def quick_prompt(
+            self,
+            prompt: str,
+            opt_strs: tuple[str, ...],
+    ) -> str | PromptResult:
+        opts = {opt[0] for opt in opt_strs}
         while True:
-            s = prompt.ljust(curses.COLS)
-            if len(s) > curses.COLS:
-                s = f'{s[:curses.COLS - 1]}…'
-            self.stdscr.insstr(curses.LINES - 1, 0, s, curses.A_REVERSE)
-            x = min(curses.COLS - 1, len(prompt) + 1)
-            self.stdscr.move(curses.LINES - 1, x)
+            x = 0
+            prompt_line = self.margin.lines - 1
+
+            def _write(s: str, *, attr: int = curses.A_REVERSE) -> None:
+                nonlocal x
+
+                if x >= self.margin.cols:
+                    return
+                self.stdscr.insstr(prompt_line, x, s, attr)
+                x += len(s)
+
+            _write(prompt)
+            _write(' [')
+            for i, opt_str in enumerate(opt_strs):
+                _write(opt_str[0], attr=curses.A_REVERSE | curses.A_BOLD)
+                _write(opt_str[1:])
+                if i != len(opt_strs) - 1:
+                    _write(', ')
+            _write(']?')
+
+            if x < self.margin.cols - 1:
+                s = ' ' * (self.margin.cols - x)
+                self.stdscr.insstr(prompt_line, x, s, curses.A_REVERSE)
+                x += 1
+            else:
+                x = self.margin.cols - 1
+                self.stdscr.insstr(prompt_line, x, '…', curses.A_REVERSE)
+
+            self.stdscr.move(prompt_line, x)
 
             key = self.get_char()
             if key.keyname == b'KEY_RESIZE':
                 self.resize()
             elif key.keyname == b'^C':
                 return self.status.cancelled()
-            elif isinstance(key.wch, str) and key.wch in opts:
-                return key.wch
+            elif isinstance(key.wch, str) and key.wch.lower() in opts:
+                return key.wch.lower()
 
     def prompt(
             self,
             prompt: str,
             *,
             allow_empty: bool = False,
-            history: Optional[str] = None,
+            history: str | None = None,
             default_prev: bool = False,
-            default: Optional[str] = None,
-    ) -> Union[str, PromptResult]:
+            default: str | None = None,
+    ) -> str | PromptResult:
         default = default or ''
         self.status.clear()
         if history is not None:
@@ -299,14 +356,14 @@ class Screen:
                 self.file.go_to_line(lineno, self.margin)
 
     def current_position(self) -> None:
-        line = f'line {self.file.y + 1}'
-        col = f'col {self.file.x + 1}'
-        line_count = max(len(self.file.lines) - 1, 1)
+        line = f'line {self.file.buf.y + 1}'
+        col = f'col {self.file.buf.x + 1}'
+        line_count = max(len(self.file.buf) - 1, 1)
         lines_word = 'line' if line_count == 1 else 'lines'
         self.status.update(f'{line}, {col} (of {line_count} {lines_word})')
 
     def cut(self) -> None:
-        if self.file.select_start:
+        if self.file.selection.start:
             self.cut_buffer = self.file.cut_selection(self.margin)
             self.cut_selection = True
         else:
@@ -319,7 +376,7 @@ class Screen:
         else:
             self.file.uncut(self.cut_buffer, self.margin)
 
-    def _get_search_re(self, prompt: str) -> Union[Pattern[str], PromptResult]:
+    def _get_search_re(self, prompt: str) -> Pattern[str] | PromptResult:
         response = self.prompt(prompt, history='search', default_prev=True)
         if response is PromptResult.CANCELLED:
             return response
@@ -332,16 +389,17 @@ class Screen:
     def _undo_redo(
             self,
             op: str,
-            from_stack: List[Action],
-            to_stack: List[Action],
+            from_stack: list[Action],
+            to_stack: list[Action],
     ) -> None:
         if not from_stack:
             self.status.update(f'nothing to {op}!')
         else:
             action = from_stack.pop()
             to_stack.append(action.apply(self.file))
-            self.file.scroll_screen_if_needed(self.margin)
+            self.file.buf.scroll_screen_if_needed(self.margin)
             self.status.update(f'{op}: {action.name}')
+            self.file.selection.clear()
 
     def undo(self) -> None:
         self._undo_redo('undo', self.file.undo_stack, self.file.redo_stack)
@@ -363,9 +421,13 @@ class Screen:
             if response is not PromptResult.CANCELLED:
                 self.file.replace(self, search_response, response)
 
-    def command(self) -> Optional[EditResult]:
+    def command(self) -> EditResult | None:
         response = self.prompt('', history='command')
-        if response == ':q':
+        if response is PromptResult.CANCELLED:
+            pass
+        elif response == ':q':
+            return self.quit_save_modified()
+        elif response == ':q!':
             return EditResult.EXIT
         elif response == ':w':
             self.save()
@@ -373,16 +435,50 @@ class Screen:
             self.save()
             return EditResult.EXIT
         elif response == ':sort':
-            if self.file.select_start:
+            if self.file.selection.start:
                 self.file.sort_selection(self.margin)
             else:
                 self.file.sort(self.margin)
             self.status.update('sorted!')
-        elif response is not PromptResult.CANCELLED:
+        elif response == ':sort!':
+            if self.file.selection.start:
+                self.file.sort_selection(self.margin, reverse=True)
+            else:
+                self.file.sort(self.margin, reverse=True)
+            self.status.update('sorted!')
+        elif response.startswith((':tabstop ', ':tabsize ')):
+            _, _, tab_size = response.partition(' ')
+            try:
+                parsed_tab_size = int(tab_size)
+            except ValueError:
+                self.status.update(f'invalid size: {tab_size}')
+            else:
+                if parsed_tab_size <= 0:
+                    self.status.update(f'invalid size: {parsed_tab_size}')
+                else:
+                    for file in self.files:
+                        file.buf.set_tab_size(parsed_tab_size)
+                    self.status.update('updated!')
+        elif response.startswith(':expandtabs'):
+            for file in self.files:
+                file.buf.expandtabs = True
+            self.status.update('updated!')
+        elif response.startswith(':noexpandtabs'):
+            for file in self.files:
+                file.buf.expandtabs = False
+            self.status.update('updated!')
+        elif response == ':comment' or response.startswith(':comment '):
+            _, _, comment = response.partition(' ')
+            comment = (comment or '#').strip()
+            if self.file.selection.start:
+                self.file.toggle_comment_selection(comment)
+            else:
+                self.file.toggle_comment(comment)
+        else:
             self.status.update(f'invalid command: {response}')
         return None
 
-    def save(self) -> Optional[PromptResult]:
+    def save(self) -> PromptResult | None:
         self.file.finalize_previous_action()
 
         # TODO: make directories if they don't exist
@@ -396,26 +492,32 @@ class Screen:
             else:
                 self.file.filename = filename
 
-        if os.path.isfile(self.file.filename):
-            with open(self.file.filename) as f:
-                *_, sha256 = get_lines(f)
+        if not os.path.isfile(self.file.filename):
+            sha256: str | None = None
         else:
-            sha256 = hashlib.sha256(b'').hexdigest()
+            with open(self.file.filename, encoding='UTF-8', newline='') as f:
+                *_, sha256 = get_lines(f)
 
-        contents = self.file.nl.join(self.file.lines)
+        contents = self.file.nl.join(self.file.buf)
         sha256_to_save = hashlib.sha256(contents.encode()).hexdigest()
 
         # the file on disk is the same as when we opened it
-        if sha256 not in (self.file.sha256, sha256_to_save):
+        if sha256 not in (None, self.file.sha256, sha256_to_save):
             self.status.update('(file changed on disk, not implemented)')
             return PromptResult.CANCELLED
 
-        with open(self.file.filename, 'w') as f:
-            f.write(contents)
+        try:
+            with open(
+                self.file.filename, 'w', encoding='UTF-8', newline='',
+            ) as f:
+                f.write(contents)
+        except OSError as e:
+            self.status.update(f'cannot save file: {e}')
+            return PromptResult.CANCELLED
 
         self.file.modified = False
         self.file.sha256 = sha256_to_save
-        num_lines = len(self.file.lines) - 1
+        num_lines = len(self.file.buf) - 1
         lines = 'lines' if num_lines != 1 else 'line'
         self.status.update(f'saved! ({num_lines} {lines} written)')
 
@@ -428,7 +530,7 @@ class Screen:
                 first = False
         return None
 
-    def save_filename(self) -> Optional[PromptResult]:
+    def save_filename(self) -> PromptResult | None:
         response = self.prompt('enter filename', default=self.file.filename)
         if response is PromptResult.CANCELLED:
             return PromptResult.CANCELLED
@@ -436,10 +538,19 @@ class Screen:
             self.file.filename = response
             return self.save()
 
-    def quit_save_modified(self) -> Optional[EditResult]:
+    def open_file(self) -> EditResult | None:
+        response = self.prompt('enter filename', history='open')
+        if response is not PromptResult.CANCELLED:
+            opened = File(response, 0, self.color_manager, self.hl_factories)
+            self.files.append(opened)
+            return EditResult.OPEN
+        else:
+            return None
+
+    def quit_save_modified(self) -> EditResult | None:
         if self.file.modified:
             response = self.quick_prompt(
-                'file is modified - save [y(es), n(o)]?', 'yn',
+                'file is modified - save', ('yes', 'no'),
             )
             if response == 'y':
                 if self.save_filename() is not PromptResult.CANCELLED:
@@ -454,10 +565,13 @@ class Screen:
         return EditResult.EXIT
 
     def background(self) -> None:
-        curses.endwin()
-        os.kill(os.getpid(), signal.SIGSTOP)
-        self.stdscr = _init_screen()
-        self.resize()
+        if sys.platform == 'win32':  # pragma: win32 cover
+            self.status.update('cannot run babi in background on Windows')
+        else:  # pragma: win32 no cover
+            curses.endwin()
+            os.kill(os.getpid(), signal.SIGSTOP)
+            self.stdscr = _init_screen()
+            self.resize()
 
     DISPATCH = {
         b'KEY_RESIZE': resize,
@@ -467,21 +581,26 @@ class Screen:
         b'^U': uncut,
         b'M-u': undo,
         b'M-U': redo,
+        b'M-e': redo,
         b'^W': search,
         b'^\\': replace,
         b'^[': command,
         b'^S': save,
         b'^O': save_filename,
         b'^X': quit_save_modified,
+        b'^P': open_file,
         b'kLFT3': lambda screen: EditResult.PREV,
         b'kRIT3': lambda screen: EditResult.NEXT,
         b'^Z': background,
     }
 
 
-def _init_screen() -> 'curses._CursesWindow':
+def _init_screen() -> curses._CursesWindow:
     # set the escape delay so curses does not pause waiting for sequences
-    if sys.version_info >= (3, 9):  # pragma: no cover
+    if (
+            sys.version_info >= (3, 9) and
+            hasattr(curses, 'set_escdelay')
+    ):  # pragma: no cover
         curses.set_escdelay(25)
     else:  # pragma: no cover
         os.environ.setdefault('ESCDELAY', '25')
@@ -502,7 +621,7 @@ def _init_screen() -> 'curses._CursesWindow':
 
 
 @contextlib.contextmanager
-def make_stdscr() -> Generator['curses._CursesWindow', None, None]:
+def make_stdscr() -> Generator[curses._CursesWindow, None, None]:
     """essentially `curses.wrapper` but split out to implement ^Z"""
     try:
         yield _init_screen()

babi/status.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import curses
 
 from babi.margin import Margin
@@ -16,16 +18,16 @@ class Status:
     def clear(self) -> None:
         self._status = ''
 
-    def draw(self, stdscr: 'curses._CursesWindow', margin: Margin) -> None:
+    def draw(self, stdscr: curses._CursesWindow, margin: Margin) -> None:
         if margin.footer or self._status:
-            stdscr.insstr(curses.LINES - 1, 0, ' ' * curses.COLS)
+            stdscr.insstr(margin.lines - 1, 0, ' ' * margin.cols)
             if self._status:
                 status = f' {self._status} '
-                x = (curses.COLS - len(status)) // 2
+                x = (margin.cols - len(status)) // 2
                 if x < 0:
                     x = 0
                     status = status.strip()
-                stdscr.insstr(curses.LINES - 1, x, status, curses.A_REVERSE)
+                stdscr.insstr(margin.lines - 1, x, status, curses.A_REVERSE)
 
     def tick(self, margin: Margin) -> None:
         # when the window is only 1-tall, hide the status quicker

babi/textmate_demo.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+import argparse
+from typing import Sequence
+
+from babi.highlight import Compiler
+from babi.highlight import Grammars
+from babi.highlight import highlight_line
+from babi.theme import Style
+from babi.theme import Theme
+from babi.user_data import prefix_data
+from babi.user_data import xdg_config
+
+
+def print_styled(s: str, style: Style) -> None:
+    color_s = ''
+    undo_s = ''
+    if style.fg is not None:
+        color_s += '\x1b[38;2;{r};{g};{b}m'.format(**style.fg._asdict())
+        undo_s += '\x1b[39m'
+    if style.bg is not None:
+        color_s += '\x1b[48;2;{r};{g};{b}m'.format(**style.bg._asdict())
+        undo_s += '\x1b[49m'
+    if style.b:
+        color_s += '\x1b[1m'
+        undo_s += '\x1b[22m'
+    if style.i:
+        color_s += '\x1b[3m'
+        undo_s += '\x1b[23m'
+    if style.u:
+        color_s += '\x1b[4m'
+        undo_s += '\x1b[24m'
+    print(f'{color_s}{s}{undo_s}', end='', flush=True)
+
+
+def _highlight_output(theme: Theme, compiler: Compiler, filename: str) -> int:
+    state = compiler.root_state
+
+    if theme.default.bg is not None:
+        print('\x1b[48;2;{r};{g};{b}m'.format(**theme.default.bg._asdict()))
+    with open(filename, encoding='UTF-8') as f:
+        for line_idx, line in enumerate(f):
+            first_line = line_idx == 0
+            state, regions = highlight_line(compiler, state, line, first_line)
+            for start, end, scope in regions:
+                print_styled(line[start:end], theme.select(scope))
+    print('\x1b[m', end='')
+    return 0
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--theme', default=xdg_config('theme.json'))
+    parser.add_argument('--grammar-dir', default=prefix_data('grammar_v1'))
+    parser.add_argument('filename')
+    args = parser.parse_args(argv)
+
+    with open(args.filename, encoding='UTF-8') as f:
+        first_line = next(f, '')
+
+    theme = Theme.from_filename(args.theme)
+
+    grammars = Grammars(args.grammar_dir)
+    compiler = grammars.compiler_for_file(args.filename, first_line)
+
+    return _highlight_output(theme, compiler, args.filename)
+
+
+if __name__ == '__main__':
+    exit(main())

babi/theme.py

@@ -1,48 +1,43 @@
+from __future__ import annotations
+
 import functools
 import json
 import os.path
-import re
 from typing import Any
-from typing import Dict
 from typing import NamedTuple
-from typing import Optional
-from typing import Tuple
 
 from babi._types import Protocol
 from babi.color import Color
 from babi.fdict import FDict
 
-# yes I know this is wrong, but it's good enough for now
-UN_COMMENT = re.compile(r'^\s*//.*$', re.MULTILINE)
-
 
 class Style(NamedTuple):
-    fg: Optional[Color]
-    bg: Optional[Color]
+    fg: Color | None
+    bg: Color | None
     b: bool
     i: bool
     u: bool
 
     @classmethod
-    def blank(cls) -> 'Style':
+    def blank(cls) -> Style:
         return cls(fg=None, bg=None, b=False, i=False, u=False)
 
 
 class PartialStyle(NamedTuple):
-    fg: Optional[Color] = None
-    bg: Optional[Color] = None
-    b: Optional[bool] = None
-    i: Optional[bool] = None
-    u: Optional[bool] = None
+    fg: Color | None = None
+    bg: Color | None = None
+    b: bool | None = None
+    i: bool | None = None
+    u: bool | None = None
 
-    def overlay_on(self, dct: Dict[str, Any]) -> None:
+    def overlay_on(self, dct: dict[str, Any]) -> None:
         for attr in self._fields:
             value = getattr(self, attr)
             if value is not None:
                 dct[attr] = value
 
     @classmethod
-    def from_dct(cls, dct: Dict[str, Any]) -> 'PartialStyle':
+    def from_dct(cls, dct: dict[str, Any]) -> PartialStyle:
         kv = cls()._asdict()
         if 'foreground' in dct:
             kv['fg'] = Color.parse(dct['foreground'])
@@ -61,7 +56,7 @@ class _TrieNode(Protocol):
     @property
     def style(self) -> PartialStyle: ...
     @property
-    def children(self) -> FDict[str, '_TrieNode']: ...
+    def children(self) -> FDict[str, _TrieNode]: ...
 
 
 class TrieNode(NamedTuple):
@@ -69,7 +64,7 @@ class TrieNode(NamedTuple):
     children: FDict[str, _TrieNode]
 
     @classmethod
-    def from_dct(cls, dct: Dict[str, Any]) -> _TrieNode:
+    def from_dct(cls, dct: dict[str, Any]) -> _TrieNode:
         children = FDict({
             k: TrieNode.from_dct(v) for k, v in dct['children'].items()
         })
@@ -81,7 +76,7 @@ class Theme(NamedTuple):
     rules: _TrieNode
 
     @functools.lru_cache(maxsize=None)
-    def select(self, scope: Tuple[str, ...]) -> Style:
+    def select(self, scope: tuple[str, ...]) -> Style:
         if not scope:
             return self.default
         else:
@@ -96,7 +91,7 @@ class Theme(NamedTuple):
             return Style(**style)
 
     @classmethod
-    def from_dct(cls, data: Dict[str, Any]) -> 'Theme':
+    def from_dct(cls, data: dict[str, Any]) -> Theme:
         default = Style.blank()._asdict()
 
         for k in ('foreground', 'editor.foreground'):
@@ -109,15 +104,19 @@ class Theme(NamedTuple):
                 default['bg'] = Color.parse(data['colors'][k])
                 break
 
-        root: Dict[str, Any] = {'children': {}}
+        root: dict[str, Any] = {'children': {}}
         rules = data.get('tokenColors', []) + data.get('settings', [])
         for rule in rules:
             if 'scope' not in rule:
                 scopes = ['']
+            elif rule['scope'] == '':
+                scopes = ['']
             elif isinstance(rule['scope'], str):
                 scopes = [
-                    # some themes have a buggy trailing comma
-                    s.strip() for s in rule['scope'].strip(',').split(',')
+                    s.strip()
+                    # some themes have a buggy trailing/leading comma
+                    for s in rule['scope'].strip().strip(',').split(',')
+                    if s.strip()
                 ]
             else:
                 scopes = rule['scope']
@@ -139,14 +138,13 @@ class Theme(NamedTuple):
         return cls(Style(**default), TrieNode.from_dct(root))
 
     @classmethod
-    def blank(cls) -> 'Theme':
+    def blank(cls) -> Theme:
         return cls(Style.blank(), TrieNode.from_dct({'children': {}}))
 
     @classmethod
-    def from_filename(cls, filename: str) -> 'Theme':
+    def from_filename(cls, filename: str) -> Theme:
         if not os.path.exists(filename):
             return cls.blank()
         else:
-            with open(filename) as f:
-                contents = UN_COMMENT.sub('', f.read())
-                return cls.from_dct(json.loads(contents))
+            with open(filename, encoding='UTF-8') as f:
+                return cls.from_dct(json.load(f))

babi/user_data.py

@@ -1,4 +1,7 @@
+from __future__ import annotations
+
 import os.path
+import sys
 
 
 def _xdg(*path: str, env: str, default: str) -> str:
@@ -14,3 +17,7 @@ def xdg_data(*path: str) -> str:
 
 def xdg_config(*path: str) -> str:
     return _xdg(*path, env='XDG_CONFIG_HOME', default='~/.config')
+
+
+def prefix_data(*path: str) -> str:
+    return os.path.join(sys.prefix, 'share/babi', *path)

bin/download-syntax

@@ -1,81 +0,0 @@
-#!/usr/bin/env python3
-import argparse
-import enum
-import json
-import os.path
-import plistlib
-import urllib.request
-from typing import NamedTuple
-
-import cson  # pip install cson
-
-
-DEFAULT_DIR = os.path.join(
-    os.environ.get('XDG_DATA_HOME') or
-    os.path.expanduser('~/.local/share'),
-    'babi/textmate_syntax',
-)
-Ext = enum.Enum('Ext', 'CSON PLIST JSON')
-
-
-def _convert_cson(src: bytes) -> str:
-    return json.dumps(cson.loads(src))
-
-
-def _convert_json(src: bytes) -> str:
-    return json.dumps(json.loads(src))
-
-
-def _convert_plist(src: bytes) -> str:
-    return json.dumps(plistlib.loads(src))
-
-
-EXT_CONVERT = {
-    Ext.CSON: _convert_cson,
-    Ext.JSON: _convert_json,
-    Ext.PLIST: _convert_plist,
-}
-
-
-class Syntax(NamedTuple):
-    name: str
-    ext: Ext
-    url: str
-
-
-SYNTAXES = (
-    Syntax('c', Ext.JSON, 'https://raw.githubusercontent.com/jeff-hykin/cpp-textmate-grammar/53e39b1c/syntaxes/c.tmLanguage.json'),  # noqa: E501
-    Syntax('css', Ext.CSON, 'https://raw.githubusercontent.com/atom/language-css/9feb69c081308b63f78bb0d6a2af2ff5eb7d869b/grammars/css.cson'),  # noqa: E501
-    Syntax('diff', Ext.PLIST, 'https://raw.githubusercontent.com/textmate/diff.tmbundle/0593bb77/Syntaxes/Diff.plist'),  # noqa: E501
-    Syntax('html', Ext.PLIST, 'https://raw.githubusercontent.com/textmate/html.tmbundle/0c3d5ee5/Syntaxes/HTML.plist'),  # noqa: E501
-    Syntax('html-derivative', Ext.PLIST, 'https://raw.githubusercontent.com/textmate/html.tmbundle/0c3d5ee54de3a993f747f54186b73a4d2d3c44a2/Syntaxes/HTML%20(Derivative).tmLanguage'),  # noqa: E501
-    Syntax('ini', Ext.PLIST, 'https://raw.githubusercontent.com/textmate/ini.tmbundle/7d8c7b55/Syntaxes/Ini.plist'),  # noqa: E501
-    Syntax('json', Ext.PLIST, 'https://raw.githubusercontent.com/microsoft/vscode-JSON.tmLanguage/d113e90937ed3ecc31ac54750aac2e8efa08d784/JSON.tmLanguage'),  # noqa: E501
-    Syntax('markdown', Ext.PLIST, 'https://raw.githubusercontent.com/microsoft/vscode-markdown-tm-grammar/59a5962/syntaxes/markdown.tmLanguage'),  # noqa: E501
-    Syntax('powershell', Ext.PLIST, 'https://raw.githubusercontent.com/PowerShell/EditorSyntax/4a0a0766/PowerShellSyntax.tmLanguage'),  # noqa: E501
-    Syntax('python', Ext.PLIST, 'https://raw.githubusercontent.com/MagicStack/MagicPython/c9b3409d/grammars/MagicPython.tmLanguage'),  # noqa: E501
-    # TODO: https://github.com/zargony/atom-language-rust/pull/149
-    Syntax('rust', Ext.CSON, 'https://raw.githubusercontent.com/asottile/atom-language-rust/e113ca67/grammars/rust.cson'),  # noqa: E501
-    Syntax('shell', Ext.CSON, 'https://raw.githubusercontent.com/atom/language-shellscript/7008ea926867d8a231003e78094091471c4fccf8/grammars/shell-unix-bash.cson'),  # noqa: E501
-    Syntax('yaml', Ext.PLIST, 'https://raw.githubusercontent.com/textmate/yaml.tmbundle/e54ceae3/Syntaxes/YAML.tmLanguage'),  # noqa: E501
-)
-
-
-def main() -> int:
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--dest', default=DEFAULT_DIR)
-    args = parser.parse_args()
-
-    os.makedirs(args.dest, exist_ok=True)
-    for syntax in SYNTAXES:
-        print(f'downloading {syntax.name}...')
-        resp = urllib.request.urlopen(syntax.url).read()
-        converted = EXT_CONVERT[syntax.ext](resp)
-        with open(os.path.join(args.dest, f'{syntax.name}.json'), 'w') as f:
-            f.write(converted)
-
-    return 0
-
-
-if __name__ == '__main__':
-    exit(main())

bin/download-theme

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import io
+import json
+import os.path
+import plistlib
+import re
+import urllib.request
+from typing import Any
+
+import cson  # pip install cson
+
+TOKEN = re.compile(br'(\\\\|\\"|"|//|\n)')
+
+
+def json_with_comments(s: bytes) -> Any:
+    bio = io.BytesIO()
+
+    idx = 0
+    in_string = False
+    in_comment = False
+
+    match = TOKEN.search(s, idx)
+    while match:
+        if not in_comment:
+            bio.write(s[idx:match.start()])
+
+        tok = match[0]
+        if not in_comment and tok == b'"':
+            in_string = not in_string
+        elif in_comment and tok == b'\n':
+            in_comment = False
+        elif not in_string and tok == b'//':
+            in_comment = True
+
+        if not in_comment:
+            bio.write(tok)
+
+        idx = match.end()
+        match = TOKEN.search(s, idx)
+
+    bio.seek(0)
+    return json.load(bio)
+
+
+STRATEGIES = (json.loads, plistlib.loads, cson.loads, json_with_comments)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument('name')
+    parser.add_argument('url')
+    args = parser.parse_args()
+
+    if '/blob/' in args.url:
+        url = args.url.replace('/blob/', '/raw/')
+    else:
+        url = args.url
+
+    contents = urllib.request.urlopen(url).read()
+
+    errors = []
+    for strategy in STRATEGIES:
+        try:
+            loaded = strategy(contents)
+        except Exception as e:
+            errors.append((f'{strategy.__module__}.{strategy.__name__}', e))
+        else:
+            break
+    else:
+        errors_s = '\n'.join(f'\t{name}: {error}' for name, error in errors)
+        raise AssertionError(f'could not load as json/plist/cson:\n{errors_s}')
+
+    config_dir = os.path.expanduser('~/.config/babi')
+    os.makedirs(config_dir, exist_ok=True)
+    dest = os.path.join(config_dir, f'{args.name}.json')
+    with open(dest, 'w') as f:
+        json.dump(loaded, f)
+
+    theme_json = os.path.join(config_dir, 'theme.json')
+    if os.path.lexists(theme_json):
+        os.remove(theme_json)
+    os.symlink(dest, theme_json)
+    return 0
+
+
+if __name__ == '__main__':
+    exit(main())

requirements-dev.txt

@@ -1,5 +1,5 @@
 covdefaults
 coverage
-git+https://github.com/asottile/hecate@ebe6dfb
+git+https://github.com/asottile/hecate@875567f
 pytest
 remote-pdb

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = babi
-version = 0.0.1
+version = 0.0.22
 description = a text editor
 long_description = file: README.md
 long_description_content_type = text/markdown
@@ -13,27 +13,32 @@ classifiers =
     License :: OSI Approved :: MIT License
     Programming Language :: Python :: 3
     Programming Language :: Python :: 3 :: Only
-    Programming Language :: Python :: 3.6
     Programming Language :: Python :: 3.7
     Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
     Programming Language :: Python :: Implementation :: CPython
     Programming Language :: Python :: Implementation :: PyPy
 
 [options]
 packages = find:
 install_requires =
-    onigurumacffi>=0.0.10
-python_requires = >=3.6.1
-
-[options.entry_points]
-console_scripts =
-    babi = babi.main:main
+    babi-grammars
+    identify
+    onigurumacffi>=0.0.18
+    importlib_metadata>=1;python_version<"3.8"
+    windows-curses;sys_platform=="win32"
+python_requires = >=3.7
 
 [options.packages.find]
 exclude =
     tests*
     testing*
 
+[options.entry_points]
+console_scripts =
+    babi = babi.main:main
+    babi-textmate-demo = babi.textmate_demo:main
+
 [bdist_wheel]
 universal = True
 
@@ -47,6 +52,8 @@ disallow_any_generics = true
 disallow_incomplete_defs = true
 disallow_untyped_defs = true
 no_implicit_optional = true
+warn_redundant_casts = true
+warn_unused_ignores = true
 
 [mypy-testing.*]
 disallow_untyped_defs = false

setup.py

@@ -1,2 +1,4 @@
+from __future__ import annotations
+
 from setuptools import setup
 setup()

testing/runner.py

@@ -1,9 +1,9 @@
+from __future__ import annotations
+
 import contextlib
 import curses
 import enum
 import re
-from typing import List
-from typing import Tuple
 
 from hecate import Runner
 
@@ -34,7 +34,7 @@ def to_attrs(screen, width):
     fg = bg = -1
     attr = 0
     idx = 0
-    ret: List[List[Tuple[int, int, int]]]
+    ret: list[list[tuple[int, int, int]]]
     ret = [[] for _ in range(len(screen.splitlines()))]
 
     for tp, match in tokenize_colors(screen):

testing/vsc_test/.gitignore

@@ -0,0 +1,2 @@
+/node_modules
+/package-lock.json

testing/vsc_test/package.json

@@ -0,0 +1,5 @@
+{
+  "dependencies": [
+    "vscode-textmate"
+  ]
+}

testing/vsc_test/vsc.js

@@ -0,0 +1,51 @@
+const fs = require('fs');
+const vsctm = require('vscode-textmate');
+
+if (process.argv.length < 4) {
+    console.log('usage: t.js GRAMMAR FILE');
+    process.exit(1);
+}
+
+const grammar = process.argv[2];
+const file = process.argv[3];
+
+const scope = JSON.parse(fs.readFileSync(grammar, {encoding: 'UTF-8'})).scopeName;
+
+/**
+ * Utility to read a file as a promise
+ */
+function readFile(path) {
+    return new Promise((resolve, reject) => {
+        fs.readFile(path, (error, data) => error ? reject(error) : resolve(data));
+    })
+}
+
+// Create a registry that can create a grammar from a scope name.
+const registry = new vsctm.Registry({
+    loadGrammar: (scopeName) => {
+        if (scopeName === scope) {
+            return readFile(grammar).then(data => vsctm.parseRawGrammar(data.toString(), grammar))
+        }
+        console.log(`Unknown scope name: ${scopeName}`);
+        return null;
+    }
+});
+
+// Load the JavaScript grammar and any other grammars included by it async.
+registry.loadGrammar(scope).then(grammar => {
+    const text = fs.readFileSync(file, {encoding: 'UTF-8'}).trimEnd('\n').split(/\n/);
+    let ruleStack = vsctm.INITIAL;
+    for (let i = 0; i < text.length; i++) {
+        const line = text[i];
+        const lineTokens = grammar.tokenizeLine(line, ruleStack);
+        console.log(`\nTokenizing line: ${line}`);
+        for (let j = 0; j < lineTokens.tokens.length; j++) {
+            const token = lineTokens.tokens[j];
+            console.log(` - token from ${token.startIndex} to ${token.endIndex} ` +
+              `(${line.substring(token.startIndex, token.endIndex)}) ` +
+              `with scopes ${token.scopes.join(', ')}`
+            );
+        }
+        ruleStack = lineTokens.ruleStack;
+    }
+});

tests/buf_test.py

@@ -0,0 +1,180 @@
+from __future__ import annotations
+
+import pytest
+
+from babi.buf import Buf
+
+
+def test_buf_repr():
+    ret = repr(Buf(['a', 'b', 'c']))
+    assert ret == "Buf(['a', 'b', 'c'], x=0, y=0, file_y=0)"
+
+
+def test_buf_item_retrieval():
+    buf = Buf(['a', 'b', 'c'])
+    assert buf[1] == 'b'
+    assert buf[-1] == 'c'
+    with pytest.raises(IndexError):
+        buf[3]
+
+
+def test_buf_del():
+    lst = ['a', 'b', 'c']
+
+    buf = Buf(lst)
+
+    with buf.record() as modifications:
+        del buf[1]
+
+    assert lst == ['a', 'c']
+
+    buf.apply(modifications)
+
+    assert lst == ['a', 'b', 'c']
+
+
+def test_buf_del_with_negative():
+    lst = ['a', 'b', 'c']
+
+    buf = Buf(lst)
+
+    with buf.record() as modifications:
+        del buf[-1]
+
+    assert lst == ['a', 'b']
+
+    buf.apply(modifications)
+
+    assert lst == ['a', 'b', 'c']
+
+
+def test_buf_insert():
+    lst = ['a', 'b', 'c']
+
+    buf = Buf(lst)
+
+    with buf.record() as modifications:
+        buf.insert(1, 'q')
+
+    assert lst == ['a', 'q', 'b', 'c']
+
+    buf.apply(modifications)
+
+    assert lst == ['a', 'b', 'c']
+
+
+def test_buf_insert_with_negative():
+    lst = ['a', 'b', 'c']
+
+    buf = Buf(lst)
+
+    with buf.record() as modifications:
+        buf.insert(-1, 'q')
+
+    assert lst == ['a', 'b', 'q', 'c']
+
+    buf.apply(modifications)
+
+    assert lst == ['a', 'b', 'c']
+
+
+def test_buf_set_value():
+    lst = ['a', 'b', 'c']
+
+    buf = Buf(lst)
+
+    with buf.record() as modifications:
+        buf[1] = 'hello'
+
+    assert lst == ['a', 'hello', 'c']
+
+    buf.apply(modifications)
+
+    assert lst == ['a', 'b', 'c']
+
+
+def test_buf_set_value_idx_negative():
+    lst = ['a', 'b', 'c']
+
+    buf = Buf(lst)
+
+    with buf.record() as modifications:
+        buf[-1] = 'hello'
+
+    assert lst == ['a', 'b', 'hello']
+
+    buf.apply(modifications)
+
+    assert lst == ['a', 'b', 'c']
+
+
+def test_buf_multiple_modifications():
+    lst = ['a', 'b', 'c']
+
+    buf = Buf(lst)
+
+    with buf.record() as modifications:
+        buf[1] = 'hello'
+        buf.insert(1, 'ohai')
+        del buf[0]
+
+    assert lst == ['ohai', 'hello', 'c']
+
+    buf.apply(modifications)
+
+    assert lst == ['a', 'b', 'c']
+
+
+def test_buf_iter():
+    buf = Buf(['a', 'b', 'c'])
+    buf_iter = iter(buf)
+    assert next(buf_iter) == 'a'
+    assert next(buf_iter) == 'b'
+    assert next(buf_iter) == 'c'
+    with pytest.raises(StopIteration):
+        next(buf_iter)
+
+
+def test_buf_append():
+    lst = ['a', 'b', 'c']
+
+    buf = Buf(lst)
+
+    with buf.record() as modifications:
+        buf.append('q')
+
+    assert lst == ['a', 'b', 'c', 'q']
+
+    buf.apply(modifications)
+
+    assert lst == ['a', 'b', 'c']
+
+
+def test_buf_pop_default():
+    lst = ['a', 'b', 'c']
+
+    buf = Buf(lst)
+
+    with buf.record() as modifications:
+        buf.pop()
+
+    assert lst == ['a', 'b']
+
+    buf.apply(modifications)
+
+    assert lst == ['a', 'b', 'c']
+
+
+def test_buf_pop_idx():
+    lst = ['a', 'b', 'c']
+
+    buf = Buf(lst)
+
+    with buf.record() as modifications:
+        buf.pop(1)
+
+    assert lst == ['a', 'c']
+
+    buf.apply(modifications)
+
+    assert lst == ['a', 'b', 'c']

tests/color_kd_test.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from babi import color_kd
 from babi.color import Color
 

tests/color_manager_test.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import pytest
 
 from babi.color import Color

tests/color_test.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+import pytest
+
+from babi.color import Color
+
+
+@pytest.mark.parametrize(
+    ('s', 'expected'),
+    (
+        ('#1e77d3', Color(0x1e, 0x77, 0xd3)),
+        ('white', Color(0xff, 0xff, 0xff)),
+        ('black', Color(0x00, 0x00, 0x00)),
+        ('#ccc', Color(0xcc, 0xcc, 0xcc)),
+    ),
+)
+def test_color_parse(s, expected):
+    assert Color.parse(s) == expected

tests/conftest.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from babi.highlight import Grammars
+
+
+@pytest.fixture
+def make_grammars(tmpdir):
+    grammar_dir = tmpdir.join('grammars').ensure_dir()
+
+    def make_grammars(*grammar_dcts):
+        for grammar in grammar_dcts:
+            filename = f'{grammar["scopeName"]}.json'
+            grammar_dir.join(filename).write(json.dumps(grammar))
+        return Grammars(grammar_dir)
+    return make_grammars

tests/fdict_test.py

@@ -1,3 +1,8 @@
+from __future__ import annotations
+
+import pytest
+
+from babi.fdict import FChainMap
 from babi.fdict import FDict
 
 
@@ -5,3 +10,21 @@ def test_fdict_repr():
     # mostly because this shouldn't get hit elsewhere but is uesful for
     # debugging purposes
     assert repr(FDict({1: 2, 3: 4})) == 'FDict({1: 2, 3: 4})'
+
+
+def test_f_chain_map():
+    chain_map = FChainMap({1: 2}, {3: 4}, FDict({1: 5}))
+    assert chain_map[1] == 5
+    assert chain_map[3] == 4
+
+    with pytest.raises(KeyError) as excinfo:
+        chain_map[2]
+    k, = excinfo.value.args
+    assert k == 2
+
+
+def test_f_chain_map_extend():
+    chain_map = FChainMap({1: 2})
+    assert chain_map[1] == 2
+    chain_map = FChainMap(chain_map, {1: 5})
+    assert chain_map[1] == 5

tests/features/command_mode_test.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import pytest
 
 from testing.runner import and_exit

tests/features/comment_test.py

@@ -0,0 +1,207 @@
+from __future__ import annotations
+
+import pytest
+
+from testing.runner import and_exit
+from testing.runner import trigger_command_mode
+
+
+@pytest.fixture
+def three_lines_with_indentation(tmpdir):
+    f = tmpdir.join('f')
+    f.write('line_0\n    line_1\n    line_2')
+    return f
+
+
+def test_comment_some_code(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('S-Down')
+
+        trigger_command_mode(h)
+        h.press_and_enter(':comment')
+
+        h.await_text('# line_0\n# line_1\nline_2\n')
+
+
+def test_comment_empty_line_trailing_whitespace(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('1\n\n2\n')
+
+    with run(str(f)) as h, and_exit(h):
+        h.press('S-Down')
+        h.press('S-Down')
+
+        trigger_command_mode(h)
+        h.press_and_enter(':comment')
+
+        h.await_text('# 1\n#\n# 2')
+
+
+def test_comment_some_code_with_alternate_comment_character(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('S-Down')
+
+        trigger_command_mode(h)
+        h.press_and_enter(':comment //')
+
+        h.await_text('// line_0\n// line_1\nline_2\n')
+
+
+def test_comment_partially_commented(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('#')
+        h.press('S-Down')
+        h.await_text('#line_0\nline_1\nline_2')
+
+        trigger_command_mode(h)
+        h.press_and_enter(':comment')
+
+        h.await_text('line_0\nline_1\nline_2\n')
+
+
+def test_comment_partially_uncommented(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('Down')
+        h.press('#')
+        h.press('Up')
+        h.press('S-Down')
+        h.await_text('line_0\n#line_1\nline_2')
+
+        trigger_command_mode(h)
+        h.press_and_enter(':comment')
+
+        h.await_text('# line_0\n# #line_1\nline_2\n')
+
+
+def test_comment_single_line(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        trigger_command_mode(h)
+        h.press_and_enter(':comment')
+
+        h.await_text('# line_0\nline_1\n')
+
+
+def test_uncomment_single_line(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('#')
+        h.await_text('#line_0\nline_1\n')
+
+        trigger_command_mode(h)
+        h.press_and_enter(':comment')
+
+        h.await_text('line_0\nline_1\n')
+
+
+def test_comment_with_trailing_whitespace(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        trigger_command_mode(h)
+        h.press_and_enter(':comment //   ')
+
+        h.await_text('// line_0\nline_1\n')
+
+
+def test_comment_some_code_with_indentation(run, three_lines_with_indentation):
+    with run(str(three_lines_with_indentation)) as h, and_exit(h):
+        h.press('S-Down')
+
+        trigger_command_mode(h)
+        h.press_and_enter(':comment')
+
+        h.await_text('# line_0\n#     line_1\n    line_2\n')
+
+        h.press('S-Up')
+        trigger_command_mode(h)
+        h.press_and_enter(':comment')
+
+        h.await_text('line_0\n    line_1\n    line_2\n')
+
+
+def test_comment_some_code_on_indent_part(run, three_lines_with_indentation):
+    with run(str(three_lines_with_indentation)) as h, and_exit(h):
+        h.press('Down')
+        h.press('S-Down')
+
+        trigger_command_mode(h)
+        h.press_and_enter(':comment')
+
+        h.await_text('line_0\n    # line_1\n    # line_2\n')
+
+        h.press('S-Up')
+
+        trigger_command_mode(h)
+        h.press_and_enter(':comment')
+
+        h.await_text('line_0\n    line_1\n    line_2\n')
+
+
+def test_comment_some_code_on_tabs_part(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('line_0\n\tline_1\n\t\tline_2')
+
+    with run(str(f)) as h, and_exit(h):
+        h.await_text('line_0\n    line_1\n        line_2')
+        h.press('Down')
+        h.press('S-Down')
+
+        trigger_command_mode(h)
+        h.press_and_enter(':comment')
+
+        h.await_text('line_0\n    # line_1\n    #   line_2')
+
+        h.press('S-Up')
+
+        trigger_command_mode(h)
+        h.press_and_enter(':comment')
+
+        h.await_text('line_0\n    line_1\n        line_2')
+
+
+def test_comment_cursor_at_end_of_line(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('# ')
+        h.press('End')
+        h.await_cursor_position(x=8, y=1)
+
+        trigger_command_mode(h)
+        h.press_and_enter(':comment')
+
+        h.await_cursor_position(x=6, y=1)
+
+
+def test_add_comment_moves_cursor(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('End')
+
+        h.await_cursor_position(x=6, y=1)
+
+        trigger_command_mode(h)
+        h.press_and_enter(':comment')
+
+        h.await_cursor_position(x=8, y=1)
+
+
+def test_do_not_move_if_cursor_before_comment(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('\t\tfoo')
+
+    with run(str(f)) as h, and_exit(h):
+        h.press('Right')
+
+        h.await_cursor_position(x=4, y=1)
+
+        trigger_command_mode(h)
+        h.press_and_enter(':comment')
+
+        h.await_cursor_position(x=4, y=1)
+
+
+@pytest.mark.parametrize('comment', ('# ', '#'))
+def test_remove_comment_with_comment_elsewhere_in_line(run, tmpdir, comment):
+    f = tmpdir.join('f')
+    f.write(f'{comment}print("not a # comment here!")\n')
+
+    with run(str(f)) as h, and_exit(h):
+        trigger_command_mode(h)
+        h.press_and_enter(':comment')
+
+        h.await_text('\nprint("not a # comment here!")\n')

tests/features/conftest.py

@@ -1,11 +1,10 @@
+from __future__ import annotations
+
 import contextlib
 import curses
 import os
 import sys
-from typing import List
 from typing import NamedTuple
-from typing import Tuple
-from typing import Union
 from unittest import mock
 
 import pytest
@@ -16,6 +15,13 @@ from babi.screen import VERSION_STR
 from testing.runner import PrintsErrorRunner
 
 
+@pytest.fixture(autouse=True)
+def prefix_home(tmpdir):
+    prefix_home = tmpdir.join('prefix_home')
+    with mock.patch.object(sys, 'prefix', str(prefix_home)):
+        yield prefix_home
+
+
 @pytest.fixture(autouse=True)
 def xdg_data_home(tmpdir):
     data_home = tmpdir.join('data_home')
@@ -39,7 +45,6 @@ def ten_lines(tmpdir):
 
 class Screen:
     def __init__(self, width, height):
-        self.disabled = True
         self.nodelay = False
         self.width = width
         self.height = height
@@ -57,6 +62,16 @@ class Screen:
             self._prev_screenshot = ret
         return ret
 
+    def addstr(self, y, x, s, attr):
+        self.lines[y] = self.lines[y][:x] + s + self.lines[y][x + len(s):]
+
+        line_attr = self.attrs[y]
+        new = [attr] * len(s)
+        self.attrs[y] = line_attr[:x] + new + line_attr[x + len(s):]
+
+        self.y = y
+        self.x = x + len(s)
+
     def insstr(self, y, x, s, attr):
         line = self.lines[y]
         self.lines[y] = (line[:x] + s + line[x:])[:self.width]
@@ -66,6 +81,7 @@ class Screen:
         self.attrs[y] = (line_attr[:x] + new + line_attr[x:])[:self.width]
 
     def chgat(self, y, x, n, attr):
+        assert n >= 0  # TODO: switch to > 0, we should never do 0-length
         self.attrs[y][x:x + n] = [attr] * n
 
     def move(self, y, x):
@@ -131,7 +147,7 @@ class AssertScreenLineEquals(NamedTuple):
 
 class AssertScreenAttrEquals(NamedTuple):
     n: int
-    attr: List[Tuple[int, int, int]]
+    attr: list[tuple[int, int, int]]
 
     def __call__(self, screen: Screen) -> None:
         assert screen.attrs[self.n] == self.attr
@@ -153,7 +169,7 @@ class Resize(NamedTuple):
 
 
 class KeyPress(NamedTuple):
-    wch: Union[int, str]
+    wch: int | str
 
     def __call__(self, screen: Screen) -> None:
         raise AssertionError('unreachable')
@@ -166,7 +182,8 @@ class CursesError(NamedTuple):
 
 
 class CursesScreen:
-    def __init__(self, runner):
+    def __init__(self, screen, runner):
+        self._screen = screen
         self._runner = runner
         self._bkgd_attr = (-1, -1, 0)
 
@@ -190,20 +207,26 @@ class CursesScreen:
         pass
 
     def nodelay(self, val):
-        self._runner.screen.nodelay = val
+        self._screen.nodelay = val
+
+    def addstr(self, y, x, s, attr=0):
+        self._screen.addstr(y, x, s, self._to_attr(attr))
 
     def insstr(self, y, x, s, attr=0):
-        self._runner.screen.insstr(y, x, s, self._to_attr(attr))
+        self._screen.insstr(y, x, s, self._to_attr(attr))
 
     def clrtoeol(self):
-        s = self._runner.screen.width * ' '
-        self.insstr(self._runner.screen.y, self._runner.screen.x, s)
+        s = self._screen.width * ' '
+        self.insstr(self._screen.y, self._screen.x, s)
 
     def chgat(self, y, x, n, attr):
-        self._runner.screen.chgat(y, x, n, self._to_attr(attr))
+        self._screen.chgat(y, x, n, self._to_attr(attr))
 
     def move(self, y, x):
-        self._runner.screen.move(y, x)
+        self._screen.move(y, x)
+
+    def getyx(self):
+        return self._screen.y, self._screen.x
 
     def get_wch(self):
         return self._runner._get_wch()
@@ -212,7 +235,7 @@ class CursesScreen:
 class Key(NamedTuple):
     tmux: str
     curses: bytes
-    wch: Union[int, str]
+    wch: int | str
 
     @property
     def value(self) -> int:
@@ -254,6 +277,7 @@ KEYS = [
     Key('^E', b'^E', '\x05'),
     Key('^J', b'^J', '\n'),
     Key('^O', b'^O', '\x0f'),
+    Key('^P', b'^P', '\x10'),
     Key('^R', b'^R', '\x12'),
     Key('^S', b'^S', '\x13'),
     Key('^U', b'^U', '\x15'),
@@ -265,8 +289,9 @@ KEYS = [
     Key('^_', b'^_', '\x1f'),
     Key('^\\', b'^\\', '\x1c'),
     Key('!resize', b'KEY_RESIZE', curses.KEY_RESIZE),
+    Key('^D', b'^D', '\x04'),
 ]
-KEYS_TMUX = {k.tmux: k.value for k in KEYS}
+KEYS_TMUX = {k.tmux: k.wch for k in KEYS}
 KEYS_CURSES = {k.value: k.curses for k in KEYS}
 
 
@@ -274,10 +299,11 @@ class DeferredRunner:
     def __init__(self, command, width=80, height=24, term='screen'):
         self.command = command
         self._i = 0
-        self._ops: List[Op] = []
+        self._ops: list[Op] = []
         self.color_pairs = {0: (7, 0)}
         self.screen = Screen(width, height)
         self._n_colors, self._can_change_color = {
+            'xterm-mono': (0, False),
             'screen': (8, False),
             'screen-256color': (256, False),
             'xterm-256color': (256, True),
@@ -297,7 +323,7 @@ class DeferredRunner:
         print(f'KEY: {keypress_event.wch!r}')
         return keypress_event.wch
 
-    def await_text(self, text):
+    def await_text(self, text, timeout=1):
         self._ops.append(AwaitText(text))
 
     def await_text_missing(self, text):
@@ -364,8 +390,9 @@ class DeferredRunner:
     def _curses__noop(self, *_, **__):
         pass
 
-    _curses_cbreak = _curses_noecho = _curses_nonl = _curses__noop
-    _curses_raw = _curses_use_default_colors = _curses__noop
+    _curses_cbreak = _curses_endwin = _curses_noecho = _curses__noop
+    _curses_nonl = _curses_raw = _curses_use_default_colors = _curses__noop
+    _curses_set_escdelay = _curses__noop
 
     _curses_error = curses.error  # so we don't mock the exception
 
@@ -391,11 +418,10 @@ class DeferredRunner:
 
     def _curses_initscr(self):
         self._curses_update_lines_cols()
-        self.screen.disabled = False
-        return CursesScreen(self)
+        return CursesScreen(self.screen, self)
 
-    def _curses_endwin(self):
-        self.screen.disabled = True
+    def _curses_newwin(self, height, width):
+        return CursesScreen(Screen(width, height), self)
 
     def _curses_not_implemented(self, fn):
         def fn_inner(*args, **kwargs):

tests/features/current_position_test.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from testing.runner import and_exit
 
 

tests/features/cut_uncut_test.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from testing.runner import and_exit
 
 
@@ -132,3 +134,21 @@ def test_selection_cut_uncut_selection_offscreen_x(run):
         h.await_text_missing('hello')
         h.press('^K')
         h.await_text('hello\n')
+
+
+def test_selection_cut_uncut_at_end_of_file(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('S-Down')
+        h.press('S-Right')
+        h.press('^K')
+        h.await_text_missing('line_0')
+        h.await_text_missing('line_1')
+        h.await_text('ine_1')
+
+        h.press('^End')
+        h.press('^U')
+        h.await_text('line_0\nl\n')
+        h.await_cursor_position(x=1, y=11)
+
+        h.press('Down')
+        h.await_cursor_position(x=0, y=12)

tests/features/expandtabs_test.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from testing.runner import and_exit
+from testing.runner import trigger_command_mode
+
+
+def test_set_expandtabs(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('a')
+
+    with run(str(f)) as h, and_exit(h):
+        h.press('Left')
+        trigger_command_mode(h)
+        h.press_and_enter(':expandtabs')
+        h.await_text('updated!')
+        h.press('Tab')
+        h.press('^S')
+    assert f.read() == '    a\n'
+
+
+def test_set_noexpandtabs(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('a')
+
+    with run(str(f)) as h, and_exit(h):
+        h.press('Left')
+        trigger_command_mode(h)
+        h.press_and_enter(':noexpandtabs')
+        h.await_text('updated!')
+        h.press('Tab')
+        h.press('^S')
+    assert f.read() == '\ta\n'
+
+
+def test_indent_with_expandtabs(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('a\nb\nc')
+
+    with run(str(f)) as h, and_exit(h):
+        trigger_command_mode(h)
+        h.press_and_enter(':noexpandtabs')
+        h.await_text('updated!')
+        for _ in range(3):
+            h.press('S-Down')
+        h.press('Tab')
+        h.press('^S')
+    assert f.read() == '\ta\n\tb\n\tc\n'

tests/features/go_to_line_test.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import pytest
 
 from testing.runner import and_exit

tests/features/indent_test.py

@@ -1,4 +1,7 @@
+from __future__ import annotations
+
 from testing.runner import and_exit
+from testing.runner import trigger_command_mode
 
 
 def test_indent_at_beginning_of_line(run):
@@ -12,11 +15,12 @@ def test_indent_at_beginning_of_line(run):
 
 def test_indent_not_full_tab(run):
     with run() as h, and_exit(h):
-        h.press('h')
+        h.press('hello')
+        h.press('Home')
+        h.press('Right')
         h.press('Tab')
-        h.press('ello')
         h.await_text('h   ello')
-        h.await_cursor_position(x=8, y=1)
+        h.await_cursor_position(x=4, y=1)
 
 
 def test_indent_fixes_eof(run):
@@ -86,6 +90,20 @@ def test_dedent_selection(run, tmpdir):
         h.await_text('\n1\n2\n    3\n')
 
 
+def test_dedent_selection_with_noexpandtabs(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('1\n\t2\n\t\t3\n')
+    with run(str(f)) as h, and_exit(h):
+        trigger_command_mode(h)
+        h.press_and_enter(':noexpandtabs')
+        h.await_text('updated!')
+        for _ in range(3):
+            h.press('S-Down')
+        h.press('BTab')
+        h.press('^S')
+    assert f.read() == '1\n2\n\t3\n'
+
+
 def test_dedent_beginning_of_line(run, tmpdir):
     f = tmpdir.join('f')
     f.write('    hi\n')

tests/features/initial_position_test.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from testing.runner import and_exit
+
+
+def test_open_file_named_plus_something(run):
+    with run('+3') as h, and_exit(h):
+        h.await_text(' +3')
+
+
+def test_initial_position_one_file(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('hello\nworld\n')
+
+    with run('+2', str(f)) as h, and_exit(h):
+        h.await_cursor_position(x=0, y=2)
+
+
+def test_initial_position_multiple_files(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('1\n2\n3\n4\n')
+    g = tmpdir.join('g')
+    g.write('5\n6\n7\n8\n')
+
+    with run('+2', str(f), '+3', str(g)) as h, and_exit(h):
+        h.await_cursor_position(x=0, y=2)
+
+        h.press('^X')
+
+        h.await_cursor_position(x=0, y=3)

tests/features/key_debug_test.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+import curses
+
+from babi.screen import VERSION_STR
+
+
+def test_key_debug(run):
+    with run('--key-debug') as h:
+        h.await_text(VERSION_STR, timeout=2)
+
+        h.await_text('press q to quit')
+
+        h.press('a')
+        h.await_text("'a' 'STRING'")
+
+        h.press('^X')
+        h.await_text(r"'\x18' '^X'")
+
+        with h.resize(width=20, height=20):
+            h.await_text(f"{curses.KEY_RESIZE} 'KEY_RESIZE'")
+
+        h.press('q')
+        h.await_exit()

tests/features/movement_test.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import pytest
 
 from testing.runner import and_exit
@@ -411,3 +413,30 @@ def test_sequence_handling(run_only_fake):
         h.press(' test7')
         h.await_text('test1 test2 test3 test4 test5 test6 test7')
         h.await_text(r'\x1b[1;')
+
+
+def test_indentation_using_tabs(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write(f'123456789\n\t12\t{"x" * 20}\n')
+
+    with run(str(f), width=20) as h, and_exit(h):
+        h.await_text('123456789\n    12  xxxxxxxxxxx»\n')
+
+        h.press('Down')
+        h.await_cursor_position(x=0, y=2)
+        h.press('Up')
+        h.await_cursor_position(x=0, y=1)
+
+        h.press('Right')
+        h.await_cursor_position(x=1, y=1)
+        h.press('Down')
+        h.await_cursor_position(x=0, y=2)
+        h.press('Up')
+        h.await_cursor_position(x=1, y=1)
+
+        h.press('Down')
+        h.await_cursor_position(x=0, y=2)
+        h.press('Right')
+        h.await_cursor_position(x=4, y=2)
+        h.press('Up')
+        h.await_cursor_position(x=4, y=1)

tests/features/multiple_files_test.py

@@ -1,10 +1,21 @@
-def test_multiple_files(run, tmpdir):
+from __future__ import annotations
+
+import pytest
+
+
+@pytest.fixture
+def abc(tmpdir):
     a = tmpdir.join('file_a')
     a.write('a text')
     b = tmpdir.join('file_b')
     b.write('b text')
     c = tmpdir.join('file_c')
     c.write('c text')
+    yield a, b, c
+
+
+def test_multiple_files(run, abc):
+    a, b, c = abc
 
     with run(str(a), str(b), str(c)) as h:
         h.await_text('file_a')
@@ -44,9 +55,36 @@ def test_multiple_files(run, tmpdir):
         h.press('^J')
         h.await_text('unknown key')
         h.press('^X')
-        h.await_text('file_a')
+        h.await_text('file_b')
         h.await_text_missing('unknown key')
+        h.press('^X')
+        h.await_text('file_a')
+        h.press('^X')
+        h.await_exit()
+
+
+def test_multiple_files_close_from_beginning(run, abc):
+    a, b, c = abc
+
+    with run(str(a), str(b), str(c)) as h:
         h.press('^X')
         h.await_text('file_b')
         h.press('^X')
+        h.await_text('file_c')
+        h.press('^X')
+        h.await_exit()
+
+
+def test_multiple_files_close_from_end(run, abc):
+    a, b, c = abc
+
+    with run(str(a), str(b), str(c)) as h:
+        h.press('M-Right')
+        h.await_text('file_b')
+
+        h.press('^X')
+        h.await_text('file_c')
+        h.press('^X')
+        h.await_text('file_a')
+        h.press('^X')
         h.await_exit()

tests/features/open_test.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from testing.runner import and_exit
+
+
+def test_open_cancelled(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('hello world')
+
+    with run(str(f)) as h, and_exit(h):
+        h.await_text('hello world')
+
+        h.press('^P')
+        h.await_text('enter filename:')
+        h.press('^C')
+
+        h.await_text('cancelled')
+        h.await_text('hello world')
+
+
+def test_open(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('hello world')
+    g = tmpdir.join('g')
+    g.write('goodbye world')
+
+    with run(str(f)) as h:
+        h.await_text('hello world')
+
+        h.press('^P')
+        h.press_and_enter(str(g))
+
+        h.await_text('[2/2]')
+        h.await_text('goodbye world')
+
+        h.press('^X')
+        h.await_text('hello world')
+
+        h.press('^X')
+        h.await_exit()

tests/features/perf_log_test.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from testing.runner import and_exit
 
 

tests/features/replace_test.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import pytest
 
 from testing.runner import and_exit
@@ -30,15 +32,16 @@ def test_replace_cancel_at_replace_string(run):
         h.await_text('cancelled')
 
 
-def test_replace_actual_contents(run, ten_lines):
+@pytest.mark.parametrize('key', ('y', 'Y'))
+def test_replace_actual_contents(run, ten_lines, key):
     with run(str(ten_lines)) as h, and_exit(h):
         h.press('^\\')
         h.await_text('search (to replace):')
         h.press_and_enter('line_0')
         h.await_text('replace with:')
         h.press_and_enter('ohai')
-        h.await_text('replace [y(es), n(o), a(ll)]?')
-        h.press('y')
+        h.await_text('replace [yes, no, all]?')
+        h.press(key)
         h.await_text_missing('line_0')
         h.await_text('ohai')
         h.await_text(' *')
@@ -59,7 +62,7 @@ match me!
         h.press_and_enter('me!')
         h.await_text('replace with:')
         h.press_and_enter('youuuu')
-        h.await_text('replace [y(es), n(o), a(ll)]?')
+        h.await_text('replace [yes, no, all]?')
         h.press('y')
         h.await_cursor_position(x=6, y=3)
         h.press('Up')
@@ -74,7 +77,7 @@ def test_replace_cancel_at_individual_replace(run, ten_lines):
         h.press_and_enter(r'line_\d')
         h.await_text('replace with:')
         h.press_and_enter('ohai')
-        h.await_text('replace [y(es), n(o), a(ll)]?')
+        h.await_text('replace [yes, no, all]?')
         h.press('^C')
         h.await_text('cancelled')
 
@@ -86,7 +89,7 @@ def test_replace_unknown_characters_at_individual_replace(run, ten_lines):
         h.press_and_enter(r'line_\d')
         h.await_text('replace with:')
         h.press_and_enter('ohai')
-        h.await_text('replace [y(es), n(o), a(ll)]?')
+        h.await_text('replace [yes, no, all]?')
         h.press('?')
         h.press('^C')
         h.await_text('cancelled')
@@ -99,7 +102,7 @@ def test_replace_say_no_to_individual_replace(run, ten_lines):
         h.press_and_enter('line_[135]')
         h.await_text('replace with:')
         h.press_and_enter('ohai')
-        h.await_text('replace [y(es), n(o), a(ll)]?')
+        h.await_text('replace [yes, no, all]?')
         h.press('y')
         h.await_text_missing('line_1')
         h.press('n')
@@ -116,7 +119,7 @@ def test_replace_all(run, ten_lines):
         h.press_and_enter(r'line_(\d)')
         h.await_text('replace with:')
         h.press_and_enter(r'ohai+\1')
-        h.await_text('replace [y(es), n(o), a(ll)]?')
+        h.await_text('replace [yes, no, all]?')
         h.press('a')
         h.await_text_missing('line')
         h.await_text('ohai+1')
@@ -130,7 +133,7 @@ def test_replace_with_empty_string(run, ten_lines):
         h.press_and_enter('line_1')
         h.await_text('replace with:')
         h.press('Enter')
-        h.await_text('replace [y(es), n(o), a(ll)]?')
+        h.await_text('replace [yes, no, all]?')
         h.press('y')
         h.await_text_missing('line_1')
 
@@ -153,7 +156,7 @@ def test_replace_small_window_size(run, ten_lines):
         h.press_and_enter('line')
         h.await_text('replace with:')
         h.press_and_enter('wat')
-        h.await_text('replace [y(es), n(o), a(ll)]?')
+        h.await_text('replace [yes, no, all]?')
 
         with h.resize(width=8, height=24):
             h.await_text('replace…')
@@ -170,7 +173,7 @@ def test_replace_height_1_highlight(run, tmpdir):
         h.press_and_enter('^x+$')
         h.await_text('replace with:')
         h.press('Enter')
-        h.await_text('replace [y(es), n(o), a(ll)]?')
+        h.await_text('replace [yes, no, all]?')
 
         with h.resize(width=80, height=1):
             h.await_text_missing('xxxxx')
@@ -189,7 +192,7 @@ def test_replace_line_goes_off_screen(run):
         h.press_and_enter('b+')
         h.await_text('replace with:')
         h.press_and_enter('wat')
-        h.await_text('replace [y(es), n(o), a(ll)]?')
+        h.await_text('replace [yes, no, all]?')
         h.await_text(f'{"a" * 20}{"b" * 59}»')
         h.press('y')
         h.await_text(f'{"a" * 20}wat')
@@ -221,7 +224,7 @@ def test_replace_multiple_occurrences_in_line(run):
         h.press_and_enter('a+')
         h.await_text('replace with:')
         h.press_and_enter('q')
-        h.await_text('replace [y(es), n(o), a(ll)]?')
+        h.await_text('replace [yes, no, all]?')
         h.press('a')
         h.await_text('bqbq')
 
@@ -234,7 +237,7 @@ def test_replace_after_wrapping(run, ten_lines):
         h.press_and_enter('line_[02]')
         h.await_text('replace with:')
         h.press_and_enter('ohai')
-        h.await_text('replace [y(es), n(o), a(ll)]?')
+        h.await_text('replace [yes, no, all]?')
         h.press('y')
         h.await_text_missing('line_2')
         h.press('y')
@@ -251,7 +254,7 @@ def test_replace_after_cursor_after_wrapping(run):
         h.press_and_enter('b')
         h.await_text('replace with:')
         h.press_and_enter('q')
-        h.await_text('replace [y(es), n(o), a(ll)]?')
+        h.await_text('replace [yes, no, all]?')
         h.press('n')
         h.press('y')
         h.await_text('replaced 1 occurrence')
@@ -267,8 +270,36 @@ def test_replace_separate_line_after_wrapping(run, ten_lines):
         h.press_and_enter('line_[01]')
         h.await_text('replace with:')
         h.press_and_enter('_')
-        h.await_text('replace [y(es), n(o), a(ll)]?')
+        h.await_text('replace [yes, no, all]?')
         h.press('y')
         h.await_text_missing('line_0')
         h.press('y')
         h.await_text_missing('line_1')
+
+
+def test_replace_with_newline_characters(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('^\\')
+        h.await_text('search (to replace):')
+        h.press_and_enter('(line)_([01])')
+        h.await_text('replace with:')
+        h.press_and_enter(r'\1\n\2')
+        h.await_text('replace [yes, no, all]?')
+        h.press('a')
+        h.await_text_missing('line_0')
+        h.await_text_missing('line_1')
+        h.await_text('line\n0\nline\n1\n')
+
+
+def test_replace_with_multiple_newline_characters(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('^\\')
+        h.await_text('search (to replace):')
+        h.press_and_enter('(li)(ne)_(1)')
+        h.await_text('replace with:')
+        h.press_and_enter(r'\1\n\2\n\3\n')
+        h.await_text('replace [yes, no, all]?')
+        h.press('a')
+
+        h.await_text_missing('line_1')
+        h.await_text('li\nne\n1\n\nline_2')

tests/features/resize_test.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from babi.screen import VERSION_STR
 from testing.runner import and_exit
 

tests/features/save_test.py

@@ -1,6 +1,9 @@
+from __future__ import annotations
+
 import pytest
 
 from testing.runner import and_exit
+from testing.runner import trigger_command_mode
 
 
 def test_mixed_newlines(run, tmpdir):
@@ -12,6 +15,31 @@ def test_mixed_newlines(run, tmpdir):
         h.await_text(r"mixed newlines will be converted to '\n'")
 
 
+def test_modify_file_with_windows_newlines(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write_binary(b'foo\r\nbar\r\n')
+    with run(str(f)) as h, and_exit(h):
+        # should not start modified
+        h.await_text_missing('*')
+        h.press('Enter')
+        h.await_text('*')
+        h.press('^S')
+        h.await_text('saved!')
+    assert f.read_binary() == b'\r\nfoo\r\nbar\r\n'
+
+
+def test_saving_file_with_multiple_lines_at_end_maintains_those(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('foo\n\n')
+    with run(str(f)) as h, and_exit(h):
+        h.press('a')
+        h.await_text('*')
+        h.press('^S')
+        h.await_text('saved!')
+
+    assert f.read() == 'afoo\n\n'
+
+
 def test_new_file(run):
     with run('this_is_a_new_file') as h, and_exit(h):
         h.await_text('this_is_a_new_file')
@@ -102,12 +130,24 @@ def test_save_file_when_it_did_not_exist(run, tmpdir):
     assert f.read() == 'hello world\n'
 
 
+def test_saving_file_permission_denied(run, tmpdir):
+    f = tmpdir.join('f').ensure()
+    f.chmod(0o400)
+
+    with run(str(f)) as h, and_exit(h):
+        h.press('hello world')
+        h.press('^S')
+        # the filename message is missing as it is too long to be captured
+        h.await_text('cannot save file: [Errno 13] Permission denied:')
+        h.await_text(' *')
+
+
 def test_save_via_ctrl_o(run, tmpdir):
     f = tmpdir.join('f')
     with run(str(f)) as h, and_exit(h):
         h.press('hello world')
         h.press('^O')
-        h.await_text(f'enter filename: ')
+        h.await_text('enter filename: ')
         h.press('Enter')
         h.await_text('saved! (1 line written)')
     assert f.read() == 'hello world\n'
@@ -124,6 +164,18 @@ def test_save_via_ctrl_o_set_filename(run, tmpdir):
     assert f.read() == 'hello world\n'
 
 
+def test_save_via_ctrl_o_new_filename(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('wat\n')
+    with run(str(f)) as h, and_exit(h):
+        h.press('^O')
+        h.await_text('enter filename: ')
+        h.press_and_enter('new')
+        h.await_text('saved! (1 line written)')
+    assert f.read() == 'wat\n'
+    assert tmpdir.join('fnew').read() == 'wat\n'
+
+
 @pytest.mark.parametrize('key', ('^C', 'Enter'))
 def test_save_via_ctrl_o_cancelled(run, key):
     with run() as h, and_exit(h):
@@ -148,7 +200,7 @@ def test_save_on_exit_cancel_yn(run):
         h.press('hello')
         h.await_text('hello')
         h.press('^X')
-        h.await_text('file is modified - save [y(es), n(o)]?')
+        h.await_text('file is modified - save [yes, no]?')
         h.press('^C')
         h.await_text('cancelled')
 
@@ -158,7 +210,7 @@ def test_save_on_exit_cancel_filename(run):
         h.press('hello')
         h.await_text('hello')
         h.press('^X')
-        h.await_text('file is modified - save [y(es), n(o)]?')
+        h.await_text('file is modified - save [yes, no]?')
         h.press('y')
         h.await_text('enter filename:')
         h.press('^C')
@@ -171,7 +223,7 @@ def test_save_on_exit(run, tmpdir):
         h.press('hello')
         h.await_text('hello')
         h.press('^X')
-        h.await_text('file is modified - save [y(es), n(o)]?')
+        h.await_text('file is modified - save [yes, no]?')
         h.press('y')
         h.await_text(f'enter filename: {f}')
         h.press('Enter')
@@ -183,9 +235,44 @@ def test_save_on_exit_resize(run, tmpdir):
         h.press('hello')
         h.await_text('hello')
         h.press('^X')
-        h.await_text('file is modified - save [y(es), n(o)]?')
+        h.await_text('file is modified - save [yes, no]?')
         with h.resize(width=10, height=24):
             h.await_text('file is m…')
-        h.await_text('file is modified - save [y(es), n(o)]?')
+        h.await_text('file is modified - save [yes, no]?')
         h.press('^C')
         h.await_text('cancelled')
+
+
+def test_vim_save_on_exit_cancel_yn(run):
+    with run() as h, and_exit(h):
+        h.press('hello')
+        h.await_text('hello')
+        trigger_command_mode(h)
+        h.press_and_enter(':q')
+        h.await_text('file is modified - save [yes, no]?')
+        h.press('^C')
+        h.await_text('cancelled')
+
+
+def test_vim_save_on_exit(run, tmpdir):
+    f = tmpdir.join('f')
+    with run(str(f)) as h:
+        h.press('hello')
+        h.await_text('hello')
+        trigger_command_mode(h)
+        h.press_and_enter(':q')
+        h.await_text('file is modified - save [yes, no]?')
+        h.press('y')
+        h.await_text('enter filename: ')
+        h.press('Enter')
+        h.await_exit()
+
+
+def test_vim_force_exit(run, tmpdir):
+    f = tmpdir.join('f')
+    with run(str(f)) as h:
+        h.press('hello')
+        h.await_text('hello')
+        trigger_command_mode(h)
+        h.press_and_enter(':q!')
+        h.await_exit()

tests/features/search_test.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import pytest
 
 from testing.runner import and_exit

tests/features/sort_test.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import pytest
 
 from testing.runner import and_exit
@@ -21,6 +23,16 @@ def test_sort_entire_file(run, unsorted):
     assert unsorted.read() == 'a\nb\nc\nd\n'
 
 
+def test_reverse_sort_entire_file(run, unsorted):
+    with run(str(unsorted)) as h, and_exit(h):
+        trigger_command_mode(h)
+        h.press_and_enter(':sort!')
+        h.await_text('sorted!')
+        h.await_cursor_position(x=0, y=1)
+        h.press('^S')
+    assert unsorted.read() == 'd\nc\nb\na\n'
+
+
 def test_sort_selection(run, unsorted):
     with run(str(unsorted)) as h, and_exit(h):
         h.press('S-Down')
@@ -32,6 +44,18 @@ def test_sort_selection(run, unsorted):
     assert unsorted.read() == 'b\nd\nc\na\n'
 
 
+def test_reverse_sort_selection(run, unsorted):
+    with run(str(unsorted)) as h, and_exit(h):
+        h.press('Down')
+        h.press('S-Down')
+        trigger_command_mode(h)
+        h.press_and_enter(':sort!')
+        h.await_text('sorted!')
+        h.await_cursor_position(x=0, y=2)
+        h.press('^S')
+    assert unsorted.read() == 'd\nc\nb\na\n'
+
+
 def test_sort_selection_does_not_include_eof(run, unsorted):
     with run(str(unsorted)) as h, and_exit(h):
         for _ in range(5):

tests/features/status_test.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from testing.runner import and_exit
 
 

tests/features/stdin_test.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+import shlex
+import sys
+
+from babi.screen import VERSION_STR
+from testing.runner import PrintsErrorRunner
+
+
+def test_open_from_stdin():
+    with PrintsErrorRunner('env', 'TERM=screen', 'bash', '--norc') as h:
+        cmd = (sys.executable, '-mcoverage', 'run', '-m', 'babi', '-')
+        babi_cmd = ' '.join(shlex.quote(part) for part in cmd)
+        h.press_and_enter(fr"echo $'hello\nworld' | {babi_cmd}")
+
+        h.await_text(VERSION_STR, timeout=2)
+        h.await_text('<<new file>> *')
+        h.await_text('hello\nworld')
+
+        h.press('^X')
+        h.press('n')
+        h.await_text_missing('<<new file>>')
+        h.press_and_enter('exit')
+        h.await_exit()

tests/features/suspend_test.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import shlex
 import sys
 

tests/features/syntax_highlight_test.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import curses
 import json
 
@@ -15,6 +17,7 @@ THEME = json.dumps({
             'settings': {'foreground': '#5f0000', 'background': '#ff5f5f'},
         },
         {'scope': 'tqs', 'settings': {'foreground': '#00005f'}},
+        {'scope': 'qmark', 'settings': {'foreground': '#5f0000'}},
         {'scope': 'b', 'settings': {'fontStyle': 'bold'}},
         {'scope': 'i', 'settings': {'fontStyle': 'italic'}},
         {'scope': 'u', 'settings': {'fontStyle': 'underline'}},
@@ -28,6 +31,7 @@ SYNTAX = json.dumps({
         {'match': r'#.*$\n?', 'name': 'comment'},
         {'match': r'^-.*$\n?', 'name': 'diffremove'},
         {'begin': '"""', 'end': '"""', 'name': 'tqs'},
+        {'match': r'\?', 'name': 'qmark'},
     ],
 })
 DEMO_S = '''\
@@ -43,7 +47,7 @@ still more
 @pytest.fixture(autouse=True)
 def theme_and_grammar(xdg_data_home, xdg_config_home):
     xdg_config_home.join('babi/theme.json').ensure().write(THEME)
-    xdg_data_home.join('babi/textmate_syntax/demo.json').ensure().write(SYNTAX)
+    xdg_data_home.join('babi/grammar_v1/demo.json').ensure().write(SYNTAX)
 
 
 @pytest.fixture
@@ -78,3 +82,81 @@ def test_syntax_highlighting_does_not_highlight_arrows(run, tmpdir):
     with run(str(f), term='screen-256color', width=20) as h, and_exit(h):
         h.await_text('loooo')
         h.assert_screen_attr_equals(2, [(243, 40, 0)] * 19 + [(236, 40, 0)])
+
+        h.press('Down')
+        h.press('^E')
+        h.await_text_missing('loooo')
+        expected = [(236, 40, 0)] + [(243, 40, 0)] * 15 + [(236, 40, 0)] * 4
+        h.assert_screen_attr_equals(2, expected)
+
+
+def test_syntax_highlighting_off_screen_does_not_crash(run, tmpdir):
+    f = tmpdir.join('f.demo')
+    f.write(f'"""a"""{"x" * 40}"""b"""')
+
+    with run(str(f), term='screen-256color', width=20) as h, and_exit(h):
+        h.await_text('"""a"""')
+        h.assert_screen_attr_equals(1, [(17, 40, 0)] * 7 + [(236, 40, 0)] * 13)
+        h.press('^E')
+        h.await_text('"""b"""')
+        expected = [(236, 40, 0)] * 11 + [(17, 40, 0)] * 7 + [(236, 40, 0)] * 2
+        h.assert_screen_attr_equals(1, expected)
+
+
+def test_syntax_highlighting_one_off_left_of_screen(run, tmpdir):
+    f = tmpdir.join('f.demo')
+    f.write(f'{"x" * 11}?123456789')
+
+    with run(str(f), term='screen-256color', width=20) as h, and_exit(h):
+        h.await_text('xxx?123')
+        expected = [(236, 40, 0)] * 11 + [(52, 40, 0)] + [(236, 40, 0)] * 8
+        h.assert_screen_attr_equals(1, expected)
+
+        h.press('End')
+        h.await_text_missing('?')
+        h.assert_screen_attr_equals(1, [(236, 40, 0)] * 20)
+
+
+def test_syntax_highlighting_to_edge_of_screen(run, tmpdir):
+    f = tmpdir.join('f.demo')
+    f.write(f'# {"x" * 18}')
+
+    with run(str(f), term='screen-256color', width=20) as h, and_exit(h):
+        h.await_text('# xxx')
+        h.assert_screen_attr_equals(1, [(243, 40, 0)] * 20)
+
+
+def test_syntax_highlighting_with_tabs(run, tmpdir):
+    f = tmpdir.join('f.demo')
+    f.write('\t# 12345678901234567890\n')
+
+    with run(str(f), term='screen-256color', width=20) as h, and_exit(h):
+        h.await_text('1234567890')
+        expected = 4 * [(236, 40, 0)] + 15 * [(243, 40, 0)] + [(236, 40, 0)]
+        h.assert_screen_attr_equals(1, expected)
+
+
+def test_syntax_highlighting_tabs_after_line_creation(run, tmpdir):
+    f = tmpdir.join('f')
+    # trailing whitespace is used to trigger highlighting
+    f.write('foo\n\txx \ny    \n')
+
+    with run(str(f), term='screen-256color') as h, and_exit(h):
+        # this looks weird, but it populates the width cache
+        h.press('Down')
+        h.press('Down')
+        h.press('Down')
+
+        # press enter after the tab
+        h.press('Up')
+        h.press('Up')
+        h.press('Right')
+        h.press('Right')
+        h.press('Enter')
+
+        h.await_text('foo\n    x\nx\ny\n')
+
+
+def test_does_not_crash_with_no_color_support(run):
+    with run(term='xterm-mono') as h, and_exit(h):
+        pass

tests/features/tabsize_test.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+import pytest
+
+from testing.runner import and_exit
+from testing.runner import trigger_command_mode
+
+
+@pytest.mark.parametrize('setting', ('tabsize', 'tabstop'))
+def test_set_tabstop(run, setting):
+    with run() as h, and_exit(h):
+        h.press('a')
+        h.press('Left')
+        trigger_command_mode(h)
+        h.press_and_enter(f':{setting} 2')
+        h.await_text('updated!')
+        h.press('Tab')
+        h.await_text('\n  a')
+        h.await_cursor_position(x=2, y=1)
+
+
+@pytest.mark.parametrize('tabsize', ('-1', '0', 'wat'))
+def test_set_invalid_tabstop(run, tabsize):
+    with run() as h, and_exit(h):
+        h.press('a')
+        h.press('Left')
+        trigger_command_mode(h)
+        h.press_and_enter(f':tabstop {tabsize}')
+        h.await_text(f'invalid size: {tabsize}')
+        h.press('Tab')
+        h.await_text('    a')
+        h.await_cursor_position(x=4, y=1)

tests/features/text_editing_test.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import pytest
 
 from testing.runner import and_exit
@@ -50,6 +52,18 @@ def test_backspace_at_end_of_file_still_allows_scrolling_down(run, tmpdir):
         h.await_text_missing('*')
 
 
+def test_backspace_deletes_newline_at_end_of_file(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('foo\n\n')
+
+    with run(str(f)) as h, and_exit(h):
+        h.press('^End')
+        h.press('BSpace')
+        h.press('^S')
+
+    assert f.read() == 'foo\n'
+
+
 @pytest.mark.parametrize('key', ('BSpace', '^H'))
 def test_backspace_deletes_text(run, tmpdir, key):
     f = tmpdir.join('f')
@@ -72,14 +86,15 @@ def test_delete_at_end_of_file(run, tmpdir):
         h.await_text_missing('*')
 
 
-def test_delete_removes_character_afterwards(run, tmpdir):
+@pytest.mark.parametrize('key', ('DC', '^D'))
+def test_delete_removes_character_afterwards(run, tmpdir, key):
     f = tmpdir.join('f')
     f.write('hello world')
 
     with run(str(f)) as h, and_exit(h):
         h.await_text('hello world')
         h.press('Right')
-        h.press('DC')
+        h.press(key)
         h.await_text('hllo world')
         h.await_text('f *')
 
@@ -97,6 +112,24 @@ def test_delete_at_end_of_line(run, tmpdir):
         h.await_text('f *')
 
 
+def test_delete_at_end_of_last_line(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('hello\n')
+
+    with run(str(f)) as h, and_exit(h):
+        h.await_text('hello')
+        h.press('End')
+        h.press('DC')
+        # should not make the file modified
+        h.await_text_missing('*')
+
+        # delete should still be functional
+        h.press('Left')
+        h.press('Left')
+        h.press('DC')
+        h.await_text('helo')
+
+
 def test_press_enter_beginning_of_file(run, tmpdir):
     f = tmpdir.join('f')
     f.write('hello world')

tests/features/trailing_whitespace_test.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import curses
 
 from testing.runner import and_exit

tests/features/undo_redo_test.py

@@ -1,3 +1,7 @@
+from __future__ import annotations
+
+import pytest
+
 from testing.runner import and_exit
 
 
@@ -9,7 +13,8 @@ def test_nothing_to_undo_redo(run):
         h.await_text('nothing to redo!')
 
 
-def test_undo_redo(run):
+@pytest.mark.parametrize('r', ('M-U', 'M-e'))
+def test_undo_redo(run, r):
     with run() as h, and_exit(h):
         h.press('hello')
         h.await_text('hello')
@@ -17,7 +22,7 @@ def test_undo_redo(run):
         h.await_text('undo: text')
         h.await_text_missing('hello')
         h.await_text_missing(' *')
-        h.press('M-U')
+        h.press(r)
         h.await_text('redo: text')
         h.await_text('hello')
         h.await_text(' *')
@@ -127,3 +132,16 @@ def test_undo_redo_causes_scroll(run):
         h.await_cursor_position(x=0, y=1)
         h.press('M-U')
         h.await_cursor_position(x=0, y=4)
+
+
+def test_undo_redo_clears_selection(run, ten_lines):
+    # maintaining the selection across undo/redo is both difficult and not all
+    # that useful.  prior to this it was buggy anyway (a negative selection
+    # indented and then undone would highlight out of bounds)
+    with run(str(ten_lines), width=20) as h, and_exit(h):
+        h.press('S-Down')
+        h.press('Tab')
+        h.await_cursor_position(x=4, y=2)
+        h.press('M-u')
+        h.await_cursor_position(x=0, y=2)
+        h.assert_screen_attr_equals(1, [(-1, -1, 0)] * 20)

tests/file_test.py

@@ -1,13 +1,16 @@
+from __future__ import annotations
+
 import io
 
 import pytest
 
+from babi.color_manager import ColorManager
 from babi.file import File
 from babi.file import get_lines
 
 
 def test_position_repr():
-    ret = repr(File('f.txt', ()))
+    ret = repr(File('f.txt', 0, ColorManager.make(), ()))
     assert ret == "<File 'f.txt'>"
 
 

tests/highlight_test.py

@@ -1,22 +1,39 @@
-from babi.highlight import Grammar
-from babi.highlight import Grammars
+from __future__ import annotations
+
+import pytest
+
 from babi.highlight import highlight_line
 from babi.highlight import Region
 
 
-def _compiler_state(grammar_dct, *others):
-    grammar = Grammar.from_data(grammar_dct)
-    grammars = [grammar, *(Grammar.from_data(dct) for dct in others)]
-    compiler = Grammars(grammars).compiler_for_scope(grammar.scope_name)
-    return compiler, compiler.root_state
+def test_grammar_matches_extension_only_name(make_grammars):
+    data = {'scopeName': 'shell', 'patterns': [], 'fileTypes': ['bashrc']}
+    grammars = make_grammars(data)
+    compiler = grammars.compiler_for_file('.bashrc', 'alias nano=babi')
+    assert compiler.root_state.entries[0].scope[0] == 'shell'
 
 
-def test_backslash_a():
+def test_grammar_matches_via_identify_tag(make_grammars):
+    grammars = make_grammars({'scopeName': 'source.ini', 'patterns': []})
+    compiler = grammars.compiler_for_file('setup.cfg', '')
+    assert compiler.root_state.entries[0].scope[0] == 'source.ini'
+
+
+@pytest.fixture
+def compiler_state(make_grammars):
+    def _compiler_state(*grammar_dcts):
+        grammars = make_grammars(*grammar_dcts)
+        compiler = grammars.compiler_for_scope(grammar_dcts[0]['scopeName'])
+        return compiler, compiler.root_state
+    return _compiler_state
+
+
+def test_backslash_a(compiler_state):
     grammar = {
         'scopeName': 'test',
         'patterns': [{'name': 'aaa', 'match': r'\Aa+'}],
     }
-    compiler, state = _compiler_state(grammar)
+    compiler, state = compiler_state(grammar)
 
     state, (region_0,) = highlight_line(compiler, state, 'aaa', True)
     state, (region_1,) = highlight_line(compiler, state, 'aaa', False)
@@ -39,8 +56,8 @@ BEGIN_END_NO_NL = {
 }
 
 
-def test_backslash_g_inline():
-    compiler, state = _compiler_state(BEGIN_END_NO_NL)
+def test_backslash_g_inline(compiler_state):
+    compiler, state = compiler_state(BEGIN_END_NO_NL)
 
     _, regions = highlight_line(compiler, state, 'xaax', True)
     assert regions == (
@@ -51,8 +68,8 @@ def test_backslash_g_inline():
     )
 
 
-def test_backslash_g_next_line():
-    compiler, state = _compiler_state(BEGIN_END_NO_NL)
+def test_backslash_g_next_line(compiler_state):
+    compiler, state = compiler_state(BEGIN_END_NO_NL)
 
     state, regions1 = highlight_line(compiler, state, 'x\n', True)
     state, regions2 = highlight_line(compiler, state, 'aax\n', False)
@@ -69,8 +86,8 @@ def test_backslash_g_next_line():
     )
 
 
-def test_end_before_other_match():
-    compiler, state = _compiler_state(BEGIN_END_NO_NL)
+def test_end_before_other_match(compiler_state):
+    compiler, state = compiler_state(BEGIN_END_NO_NL)
 
     state, regions = highlight_line(compiler, state, 'xazzx', True)
 
@@ -95,8 +112,8 @@ BEGIN_END_NL = {
 }
 
 
-def test_backslash_g_captures_nl():
-    compiler, state = _compiler_state(BEGIN_END_NL)
+def test_backslash_g_captures_nl(compiler_state):
+    compiler, state = compiler_state(BEGIN_END_NL)
 
     state, regions1 = highlight_line(compiler, state, 'x\n', True)
     state, regions2 = highlight_line(compiler, state, 'aax\n', False)
@@ -112,8 +129,8 @@ def test_backslash_g_captures_nl():
     )
 
 
-def test_backslash_g_captures_nl_next_line():
-    compiler, state = _compiler_state(BEGIN_END_NL)
+def test_backslash_g_captures_nl_next_line(compiler_state):
+    compiler, state = compiler_state(BEGIN_END_NL)
 
     state, regions1 = highlight_line(compiler, state, 'x\n', True)
     state, regions2 = highlight_line(compiler, state, 'aa\n', False)
@@ -135,8 +152,8 @@ def test_backslash_g_captures_nl_next_line():
     )
 
 
-def test_while_no_nl():
-    compiler, state = _compiler_state({
+def test_while_no_nl(compiler_state):
+    compiler, state = compiler_state({
         'scopeName': 'test',
         'patterns': [{
             'begin': '> ',
@@ -170,8 +187,8 @@ def test_while_no_nl():
     )
 
 
-def test_complex_captures():
-    compiler, state = _compiler_state({
+def test_complex_captures(compiler_state):
+    compiler, state = compiler_state({
         'scopeName': 'test',
         'patterns': [
             {
@@ -201,8 +218,8 @@ def test_complex_captures():
     )
 
 
-def test_captures_multiple_applied_to_same_capture():
-    compiler, state = _compiler_state({
+def test_captures_multiple_applied_to_same_capture(compiler_state):
+    compiler, state = compiler_state({
         'scopeName': 'test',
         'patterns': [
             {
@@ -244,8 +261,8 @@ def test_captures_multiple_applied_to_same_capture():
     )
 
 
-def test_captures_ignores_empty():
-    compiler, state = _compiler_state({
+def test_captures_ignores_empty(compiler_state):
+    compiler, state = compiler_state({
         'scopeName': 'test',
         'patterns': [{
             'match': '(.*) hi',
@@ -267,8 +284,8 @@ def test_captures_ignores_empty():
     )
 
 
-def test_captures_ignores_invalid_out_of_bounds():
-    compiler, state = _compiler_state({
+def test_captures_ignores_invalid_out_of_bounds(compiler_state):
+    compiler, state = compiler_state({
         'scopeName': 'test',
         'patterns': [{'match': '.', 'captures': {'1': {'name': 'oob'}}}],
     })
@@ -280,8 +297,8 @@ def test_captures_ignores_invalid_out_of_bounds():
     )
 
 
-def test_captures_begin_end():
-    compiler, state = _compiler_state({
+def test_captures_begin_end(compiler_state):
+    compiler, state = compiler_state({
         'scopeName': 'test',
         'patterns': [
             {
@@ -302,8 +319,8 @@ def test_captures_begin_end():
     )
 
 
-def test_captures_while_captures():
-    compiler, state = _compiler_state({
+def test_captures_while_captures(compiler_state):
+    compiler, state = compiler_state({
         'scopeName': 'test',
         'patterns': [
             {
@@ -331,8 +348,8 @@ def test_captures_while_captures():
     )
 
 
-def test_captures_implies_begin_end_captures():
-    compiler, state = _compiler_state({
+def test_captures_implies_begin_end_captures(compiler_state):
+    compiler, state = compiler_state({
         'scopeName': 'test',
         'patterns': [
             {
@@ -352,8 +369,8 @@ def test_captures_implies_begin_end_captures():
     )
 
 
-def test_captures_implies_begin_while_captures():
-    compiler, state = _compiler_state({
+def test_captures_implies_begin_while_captures(compiler_state):
+    compiler, state = compiler_state({
         'scopeName': 'test',
         'patterns': [
             {
@@ -380,8 +397,8 @@ def test_captures_implies_begin_while_captures():
     )
 
 
-def test_include_self():
-    compiler, state = _compiler_state({
+def test_include_self(compiler_state):
+    compiler, state = compiler_state({
         'scopeName': 'test',
         'patterns': [
             {
@@ -404,8 +421,8 @@ def test_include_self():
     )
 
 
-def test_include_repository_rule():
-    compiler, state = _compiler_state({
+def test_include_repository_rule(compiler_state):
+    compiler, state = compiler_state({
         'scopeName': 'test',
         'patterns': [{'include': '#impl'}],
         'repository': {
@@ -426,8 +443,40 @@ def test_include_repository_rule():
     )
 
 
-def test_include_other_grammar():
-    compiler, state = _compiler_state(
+def test_include_with_nested_repositories(compiler_state):
+    compiler, state = compiler_state({
+        'scopeName': 'test',
+        'patterns': [{
+            'begin': '<', 'end': '>', 'name': 'b',
+            'patterns': [
+                {'include': '#rule1'},
+                {'include': '#rule2'},
+                {'include': '#rule3'},
+            ],
+            'repository': {
+                'rule2': {'match': '2', 'name': 'inner2'},
+                'rule3': {'match': '3', 'name': 'inner3'},
+            },
+        }],
+        'repository': {
+            'rule1': {'match': '1', 'name': 'root1'},
+            'rule2': {'match': '2', 'name': 'root2'},
+        },
+    })
+
+    state, regions = highlight_line(compiler, state, '<123>', first_line=True)
+
+    assert regions == (
+        Region(0, 1, ('test', 'b')),
+        Region(1, 2, ('test', 'b', 'root1')),
+        Region(2, 3, ('test', 'b', 'inner2')),
+        Region(3, 4, ('test', 'b', 'inner3')),
+        Region(4, 5, ('test', 'b')),
+    )
+
+
+def test_include_other_grammar(compiler_state):
+    compiler, state = compiler_state(
         {
             'scopeName': 'test',
             'patterns': [
@@ -482,8 +531,8 @@ def test_include_other_grammar():
     )
 
 
-def test_include_base():
-    compiler, state = _compiler_state(
+def test_include_base(compiler_state):
+    compiler, state = compiler_state(
         {
             'scopeName': 'test',
             'patterns': [
@@ -528,3 +577,87 @@ def test_include_base():
         Region(2, 3, ('test', 'tick')),
         Region(3, 4, ('test',)),
     )
+
+
+def test_rule_with_begin_and_no_end(compiler_state):
+    compiler, state = compiler_state({
+        'scopeName': 'test',
+        'patterns': [
+            {
+                'begin': '!', 'end': '!', 'name': 'bang',
+                'patterns': [{'begin': '--', 'name': 'invalid'}],
+            },
+        ],
+    })
+
+    state, regions = highlight_line(compiler, state, '!x! !--!', True)
+
+    assert regions == (
+        Region(0, 1, ('test', 'bang')),
+        Region(1, 2, ('test', 'bang')),
+        Region(2, 3, ('test', 'bang')),
+        Region(3, 4, ('test',)),
+        Region(4, 5, ('test', 'bang')),
+        Region(5, 7, ('test', 'bang', 'invalid')),
+        Region(7, 8, ('test', 'bang', 'invalid')),
+    )
+
+
+def test_begin_end_substitute_special_chars(compiler_state):
+    compiler, state = compiler_state({
+        'scopeName': 'test',
+        'patterns': [{'begin': r'(\*)', 'end': r'\1', 'name': 'italic'}],
+    })
+
+    state, regions = highlight_line(compiler, state, '*italic*', True)
+
+    assert regions == (
+        Region(0, 1, ('test', 'italic')),
+        Region(1, 7, ('test', 'italic')),
+        Region(7, 8, ('test', 'italic')),
+    )
+
+
+def test_backslash_z(compiler_state):
+    # similar to text.git-commit grammar, \z matches nothing!
+    compiler, state = compiler_state({
+        'scopeName': 'test',
+        'patterns': [
+            {'begin': '#', 'end': r'\z', 'name': 'comment'},
+            {'name': 'other', 'match': '.'},
+        ],
+    })
+
+    state, regions1 = highlight_line(compiler, state, '# comment', True)
+    state, regions2 = highlight_line(compiler, state, 'other?', False)
+
+    assert regions1 == (
+        Region(0, 1, ('test', 'comment')),
+        Region(1, 9, ('test', 'comment')),
+    )
+
+    assert regions2 == (
+        Region(0, 6, ('test', 'comment')),
+    )
+
+
+def test_buggy_begin_end_grammar(compiler_state):
+    # before this would result in an infinite loop of start / end
+    compiler, state = compiler_state({
+        'scopeName': 'test',
+        'patterns': [
+            {
+                'begin': '(?=</style)',
+                'end': '(?=</style)',
+                'name': 'css',
+            },
+        ],
+    })
+
+    state, regions = highlight_line(compiler, state, 'test </style', True)
+
+    assert regions == (
+        Region(0, 5, ('test',)),
+        Region(5, 6, ('test', 'css')),
+        Region(6, 12, ('test',)),
+    )

tests/hl/syntax_test.py

@@ -1,12 +1,14 @@
+from __future__ import annotations
+
 import contextlib
 import curses
 from unittest import mock
 
 import pytest
 
+from babi.buf import Buf
 from babi.color_manager import ColorManager
-from babi.highlight import Grammar
-from babi.highlight import Grammars
+from babi.hl.interface import HL
 from babi.hl.syntax import Syntax
 from babi.theme import Color
 from babi.theme import Theme
@@ -72,18 +74,23 @@ THEME = Theme.from_dct({
 
 
 @pytest.fixture
-def syntax():
-    return Syntax(Grammars([Grammar.blank()]), THEME, ColorManager.make())
+def syntax(make_grammars):
+    return Syntax(make_grammars(), THEME, ColorManager.make())
 
 
 def test_init_screen_low_color(stdscr, syntax):
     with FakeCurses.patch(n_colors=16, can_change_color=False) as fake_curses:
         syntax._init_screen(stdscr)
-    assert syntax.color_manager.colors == {}
-    assert syntax.color_manager.raw_pairs == {}
+    assert syntax.color_manager.colors == {
+        Color.parse('#cccccc'): -1,
+        Color.parse('#333333'): -1,
+        Color.parse('#000000'): -1,
+        Color.parse('#009900'): -1,
+    }
+    assert syntax.color_manager.raw_pairs == {(-1, -1): 1}
     assert fake_curses.colors == {}
-    assert fake_curses.pairs == {}
-    assert stdscr.attr == 0
+    assert fake_curses.pairs == {1: (-1, -1)}
+    assert stdscr.attr == 1 << 8
 
 
 def test_init_screen_256_color(stdscr, syntax):
@@ -131,7 +138,7 @@ def test_lazily_instantiated_pairs(stdscr, syntax):
         assert len(fake_curses.pairs) == 1
 
         style = THEME.select(('string.python',))
-        attr = syntax.get_blank_file_highlighter().attr(style)
+        attr = syntax.blank_file_highlighter().attr(style)
         assert attr == 2 << 8
 
         assert len(syntax.color_manager.raw_pairs) == 2
@@ -143,5 +150,22 @@ def test_style_attributes_applied(stdscr, syntax):
         syntax._init_screen(stdscr)
 
         style = THEME.select(('keyword.python',))
-        attr = syntax.get_blank_file_highlighter().attr(style)
+        attr = syntax.blank_file_highlighter().attr(style)
         assert attr == 2 << 8 | curses.A_BOLD
+
+
+def test_syntax_highlight_cache_first_line(stdscr, make_grammars):
+    with FakeCurses.patch(n_colors=256, can_change_color=False):
+        grammars = make_grammars({
+            'scopeName': 'source.demo',
+            'fileTypes': ['demo'],
+            'patterns': [{'match': r'\Aint', 'name': 'keyword'}],
+        })
+        syntax = Syntax(grammars, THEME, ColorManager.make())
+        syntax._init_screen(stdscr)
+        file_hl = syntax.file_highlighter('foo.demo', '')
+        file_hl.highlight_until(Buf(['int', 'int']), 2)
+        assert file_hl.regions == [
+            (HL(0, 3, curses.A_BOLD | 2 << 8),),
+            (),
+        ]

tests/list_spy_test.py

@@ -1,144 +0,0 @@
-import pytest
-
-from babi.list_spy import ListSpy
-
-
-def test_list_spy_repr():
-    assert repr(ListSpy(['a', 'b', 'c'])) == "ListSpy(['a', 'b', 'c'])"
-
-
-def test_list_spy_item_retrieval():
-    spy = ListSpy(['a', 'b', 'c'])
-    assert spy[1] == 'b'
-    assert spy[-1] == 'c'
-    with pytest.raises(IndexError):
-        spy[3]
-
-
-def test_list_spy_del():
-    lst = ['a', 'b', 'c']
-
-    spy = ListSpy(lst)
-    del spy[1]
-
-    assert lst == ['a', 'c']
-
-    spy.undo(lst)
-
-    assert lst == ['a', 'b', 'c']
-
-
-def test_list_spy_del_with_negative():
-    lst = ['a', 'b', 'c']
-
-    spy = ListSpy(lst)
-    del spy[-1]
-
-    assert lst == ['a', 'b']
-
-    spy.undo(lst)
-
-    assert lst == ['a', 'b', 'c']
-
-
-def test_list_spy_insert():
-    lst = ['a', 'b', 'c']
-
-    spy = ListSpy(lst)
-    spy.insert(1, 'q')
-
-    assert lst == ['a', 'q', 'b', 'c']
-
-    spy.undo(lst)
-
-    assert lst == ['a', 'b', 'c']
-
-
-def test_list_spy_insert_with_negative():
-    lst = ['a', 'b', 'c']
-
-    spy = ListSpy(lst)
-    spy.insert(-1, 'q')
-
-    assert lst == ['a', 'b', 'q', 'c']
-
-    spy.undo(lst)
-
-    assert lst == ['a', 'b', 'c']
-
-
-def test_list_spy_set_value():
-    lst = ['a', 'b', 'c']
-
-    spy = ListSpy(lst)
-    spy[1] = 'hello'
-
-    assert lst == ['a', 'hello', 'c']
-
-    spy.undo(lst)
-
-    assert lst == ['a', 'b', 'c']
-
-
-def test_list_spy_multiple_modifications():
-    lst = ['a', 'b', 'c']
-
-    spy = ListSpy(lst)
-    spy[1] = 'hello'
-    spy.insert(1, 'ohai')
-    del spy[0]
-
-    assert lst == ['ohai', 'hello', 'c']
-
-    spy.undo(lst)
-
-    assert lst == ['a', 'b', 'c']
-
-
-def test_list_spy_iter():
-    spy = ListSpy(['a', 'b', 'c'])
-    spy_iter = iter(spy)
-    assert next(spy_iter) == 'a'
-    assert next(spy_iter) == 'b'
-    assert next(spy_iter) == 'c'
-    with pytest.raises(StopIteration):
-        next(spy_iter)
-
-
-def test_list_spy_append():
-    lst = ['a', 'b', 'c']
-
-    spy = ListSpy(lst)
-    spy.append('q')
-
-    assert lst == ['a', 'b', 'c', 'q']
-
-    spy.undo(lst)
-
-    assert lst == ['a', 'b', 'c']
-
-
-def test_list_spy_pop_default():
-    lst = ['a', 'b', 'c']
-
-    spy = ListSpy(lst)
-    spy.pop()
-
-    assert lst == ['a', 'b']
-
-    spy.undo(lst)
-
-    assert lst == ['a', 'b', 'c']
-
-
-def test_list_spy_pop_idx():
-    lst = ['a', 'b', 'c']
-
-    spy = ListSpy(lst)
-    spy.pop(1)
-
-    assert lst == ['a', 'c']
-
-    spy.undo(lst)
-
-    assert lst == ['a', 'b', 'c']

tests/main_test.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+import pytest
+
+from babi import main
+
+
+@pytest.mark.parametrize(
+    ('in_filenames', 'expected_filenames', 'expected_positions'),
+    (
+        ([], [None], [0]),
+        (['+3'], ['+3'], [0]),
+        (['f'], ['f'], [0]),
+        (['+3', 'f'], ['f'], [3]),
+        (['+-3', 'f'], ['f'], [-3]),
+        (['+3', '+3'], ['+3'], [3]),
+        (['+2', 'f', '+5', 'g'], ['f', 'g'], [2, 5]),
+    ),
+)
+def test_filenames(in_filenames, expected_filenames, expected_positions):
+    filenames, positions = main._filenames(in_filenames)

tests/reg_test.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import onigurumacffi
 import pytest
 
@@ -35,9 +37,8 @@ def test_reg_other_escapes_left_untouched():
 def test_reg_not_out_of_bounds_at_end():
     # the only way this is triggerable is with an illegal regex, we'd rather
     # produce an error about the regex being wrong than an IndexError
-    reg = _Reg('\\A\\')
     with pytest.raises(onigurumacffi.OnigError) as excinfo:
-        reg.search('\\', 0, first_line=False, boundary=False)
+        _Reg('\\A\\')
     msg, = excinfo.value.args
     assert msg == 'end pattern at escape'
 

tests/textmate_demo_test.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from babi.textmate_demo import main
+
+THEME = {
+    'colors': {'foreground': '#ffffff', 'background': '#000000'},
+    'tokenColors': [
+        {'scope': 'bold', 'settings': {'fontStyle': 'bold'}},
+        {'scope': 'italic', 'settings': {'fontStyle': 'italic'}},
+        {'scope': 'underline', 'settings': {'fontStyle': 'underline'}},
+        {'scope': 'comment', 'settings': {'foreground': '#1e77d3'}},
+    ],
+}
+
+GRAMMAR = {
+    'scopeName': 'source.demo',
+    'fileTypes': ['demo'],
+    'patterns': [
+        {'match': r'\*[^*]*\*', 'name': 'bold'},
+        {'match': '/[^/]*/', 'name': 'italic'},
+        {'match': '_[^_]*_', 'name': 'underline'},
+        {'match': '#.*', 'name': 'comment'},
+    ],
+}
+
+
+@pytest.fixture
+def theme_grammars(tmpdir):
+    theme = tmpdir.join('config/theme.json').ensure()
+    theme.write(json.dumps(THEME))
+    grammars = tmpdir.join('grammar_v1').ensure_dir()
+    grammars.join('source.demo.json').write(json.dumps(GRAMMAR))
+    return theme, grammars
+
+
+def test_basic(theme_grammars, tmpdir, capsys):
+    theme, grammars = theme_grammars
+
+    f = tmpdir.join('f.demo')
+    f.write('*bold*/italic/_underline_# comment\n')
+
+    assert not main((
+        '--theme', str(theme), '--grammar-dir', str(grammars),
+        str(f),
+    ))
+
+    out, _ = capsys.readouterr()
+
+    assert out == (
+        '\x1b[48;2;0;0;0m\n'
+        '\x1b[38;2;255;255;255m\x1b[48;2;0;0;0m\x1b[1m'
+        '*bold*'
+        '\x1b[39m\x1b[49m\x1b[22m'
+        '\x1b[38;2;255;255;255m\x1b[48;2;0;0;0m\x1b[3m'
+        '/italic/'
+        '\x1b[39m\x1b[49m\x1b[23m'
+        '\x1b[38;2;255;255;255m\x1b[48;2;0;0;0m\x1b[4m'
+        '_underline_'
+        '\x1b[39m\x1b[49m\x1b[24m'
+        '\x1b[38;2;30;119;211m\x1b[48;2;0;0;0m'
+        '# comment'
+        '\x1b[39m\x1b[49m\x1b'
+        '[38;2;255;255;255m\x1b[48;2;0;0;0m\n\x1b[39m\x1b[49m'
+        '\x1b[m'
+    )
+
+
+def test_basic_with_blank_theme(theme_grammars, tmpdir, capsys):
+    theme, grammars = theme_grammars
+    theme.write('{}')
+
+    f = tmpdir.join('f.demo')
+    f.write('*bold*/italic/_underline_# comment\n')
+
+    assert not main((
+        '--theme', str(theme), '--grammar-dir', str(grammars),
+        str(f),
+    ))
+
+    out, _ = capsys.readouterr()
+
+    assert out == '*bold*/italic/_underline_# comment\n\x1b[m'

tests/theme_test.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import pytest
 
 from babi.color import Color
@@ -72,6 +74,31 @@ def test_theme_scope_split_by_commas():
     assert theme.select(('c',)).i is True
 
 
+def test_theme_scope_comma_at_beginning_and_end():
+    theme = Theme.from_dct({
+        'colors': {'foreground': '#cccccc', 'background': '#333333'},
+        'tokenColors': [
+            {'scope': '\n,a,b,\n', 'settings': {'fontStyle': 'italic'}},
+        ],
+    })
+    assert theme.select(('d',)).i is False
+    assert theme.select(('a',)).i is True
+    assert theme.select(('b',)).i is True
+
+
+def test_theme_scope_internal_newline_commas():
+    # this is arguably malformed, but `cobalt2` in the wild has this issue
+    theme = Theme.from_dct({
+        'colors': {'foreground': '#cccccc', 'background': '#333333'},
+        'tokenColors': [
+            {'scope': '\n,a,\n,b,\n', 'settings': {'fontStyle': 'italic'}},
+        ],
+    })
+    assert theme.select(('d',)).i is False
+    assert theme.select(('a',)).i is True
+    assert theme.select(('b',)).i is True
+
+
 def test_theme_scope_as_A_list():
     theme = Theme.from_dct({
         'colors': {'foreground': '#cccccc', 'background': '#333333'},

tests/user_data_test.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import os
 from unittest import mock
 

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py36,py37,pre-commit
+envlist = py37,pre-commit
 
 [testenv]
 deps = -rrequirements-dev.txt