v0.0.4

Fix replacing with embedded newline characters
Resolves #39
2020-03-28 08:01:02 -07:00 · 2020-03-27 20:32:43 -07:00 · 2020-03-27 19:29:52 -07:00 · 2020-03-27 19:10:11 -07:00 · 2020-03-27 19:06:50 -07:00 · 2020-03-27 18:42:37 -07:00
43 changed files with 1279 additions and 486 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -5,4 +5,6 @@
 /.mypy_cache
 /.pytest_cache
 /.tox
+/build
+/dist
 /venv*
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -20,12 +20,12 @@ repos:
    hooks:
    -   id: autopep8
 -   repo: https://github.com/asottile/reorder_python_imports
-    rev: v1.9.0
+    rev: v2.1.0
    hooks:
    -   id: reorder-python-imports
        args: [--py3-plus]
 -   repo: https://github.com/asottile/add-trailing-comma
-    rev: v1.5.0
+    rev: v2.0.1
    hooks:
    -   id: add-trailing-comma
        args: [--py36-plus]
@@ -34,6 +34,10 @@ repos:
    hooks:
    -   id: pyupgrade
        args: [--py36-plus]
+-   repo: https://github.com/asottile/setup-cfg-fmt
+    rev: v1.7.0
+    hooks:
+    -   id: setup-cfg-fmt
 -   repo: https://github.com/pre-commit/mirrors-mypy
    rev: v0.770
    hooks:
--- a/README.md
+++ b/README.md
@@ -23,6 +23,7 @@ these are all of the current key bindings in babi
 - <kbd>^S</kbd>: save
 - <kbd>^O</kbd>: save as
 - <kbd>^X</kbd>: quit
+- <kbd>^P</kbd>: open file
 - arrow keys: movement
 - <kbd>^A</kbd> / <kbd>home</kbd>: move to beginning of line
 - <kbd>^E</kbd> / <kbd>end</kbd>: move to end of line
@@ -62,10 +63,15 @@ in prompts (search, search replace, command):

 the syntax highlighting setup is a bit manual right now

-1. from a clone of babi, run `./bin/download-syntax` -- you will likely need
-   to install some additional packages to download them (`pip install cson`)
-2. find a visual studio code theme, convert it to json (if it is not already
-   json) and put it at `~/.config/babi/theme.json`
+1. find a visual studio code theme, convert it to json (if it is not already
+   json) and put it at `~/.config/babi/theme.json`.  a helper script is
+   provided to make this easier: `./bin/download-theme NAME URL`
+
+here's a modified vs dark plus theme that works:
+
+```bash
+./bin/download-theme vs-dark-asottile https://gist.github.com/asottile/b465856c82b1aaa4ba8c7c6314a72e13/raw/22d602fb355fb12b04f176a733941ba5713bc36c/vs_dark_asottile.json
+```

 ## demos

--- a/babi/_types.py
+++ b/babi/_types.py
@@ -2,7 +2,5 @@ from typing import TYPE_CHECKING

 if TYPE_CHECKING:
    from typing import Protocol  # python3.8+
-    from typing_extensions import TypedDict  # python3.8+
 else:
    Protocol = object
-    TypedDict = dict
--- a/babi/color.py
+++ b/babi/color.py
@@ -1,5 +1,9 @@
 from typing import NamedTuple

+# TODO: find a standard which defines these
+# limited number of "named" colors
+NAMED_COLORS = {'white': '#ffffff', 'black': '#000000'}
+

 class Color(NamedTuple):
    r: int
@@ -8,4 +12,9 @@ class Color(NamedTuple):

    @classmethod
    def parse(cls, s: str) -> 'Color':
-        return cls(r=int(s[1:3], 16), g=int(s[3:5], 16), b=int(s[5:7], 16))
+        if s.startswith('#') and len(s) >= 7:
+            return cls(r=int(s[1:3], 16), g=int(s[3:5], 16), b=int(s[5:7], 16))
+        elif s.startswith('#'):
+            return cls.parse(f'#{s[1] * 2}{s[2] * 2}{s[3] * 2}')
+        else:
+            return cls.parse(NAMED_COLORS[s])
--- a/babi/color_manager.py
+++ b/babi/color_manager.py
@@ -1,4 +1,3 @@
-import contextlib
 import curses
 from typing import Dict
 from typing import NamedTuple
@@ -19,26 +18,25 @@ class ColorManager(NamedTuple):
    raw_pairs: Dict[Tuple[int, int], int]

    def init_color(self, color: Color) -> None:
-        if curses.COLORS < 256:
-            return
-        elif curses.can_change_color():
+        if curses.can_change_color():
            n = min(self.colors.values(), default=256) - 1
            self.colors[color] = n
            curses.init_color(n, *_color_to_curses(color))
-        else:
+        elif curses.COLORS >= 256:
            self.colors[color] = color_kd.nearest(color, color_kd.make_256())
+        else:
+            self.colors[color] = -1

    def color_pair(self, fg: Optional[Color], bg: Optional[Color]) -> int:
-        if curses.COLORS < 256:
-            return 0
-
        fg_i = self.colors[fg] if fg is not None else -1
        bg_i = self.colors[bg] if bg is not None else -1
        return self.raw_color_pair(fg_i, bg_i)

    def raw_color_pair(self, fg: int, bg: int) -> int:
-        with contextlib.suppress(KeyError):
+        try:
            return self.raw_pairs[(fg, bg)]
+        except KeyError:
+            pass

        n = self.raw_pairs[(fg, bg)] = len(self.raw_pairs) + 1
        curses.init_pair(n, fg, bg)
--- a/babi/file.py
+++ b/babi/file.py
@@ -21,8 +21,12 @@ from typing import TYPE_CHECKING
 from typing import TypeVar
 from typing import Union

+from babi.color_manager import ColorManager
 from babi.hl.interface import FileHL
 from babi.hl.interface import HLFactory
+from babi.hl.replace import Replace
+from babi.hl.selection import Selection
+from babi.hl.trailing_whitespace import TrailingWhitespace
 from babi.horizontal_scrolling import line_x
 from babi.horizontal_scrolling import scrolled_line
 from babi.list_spy import ListSpy
@@ -36,8 +40,6 @@ if TYPE_CHECKING:

 TCallable = TypeVar('TCallable', bound=Callable[..., Any])

-HIGHLIGHT = curses.A_REVERSE | curses.A_DIM
-

 def _restore_lines_eof_invariant(lines: MutableSequenceNoSlice) -> None:
    """The file lines will always contain a blank empty string at the end to
@@ -138,7 +140,7 @@ def clear_selection(func: TCallable) -> TCallable:
    @functools.wraps(func)
    def clear_selection_inner(self: 'File', *args: Any, **kwargs: Any) -> Any:
        ret = func(self, *args, **kwargs)
-        self.select_start = None
+        self.selection.clear()
        return ret
    return cast(TCallable, clear_selection_inner)

@@ -209,6 +211,7 @@ class File:
    def __init__(
            self,
            filename: Optional[str],
+            color_manager: ColorManager,
            hl_factories: Tuple[HLFactory, ...],
    ) -> None:
        self.filename = filename
@@ -219,15 +222,23 @@ class File:
        self.sha256: Optional[str] = None
        self.undo_stack: List[Action] = []
        self.redo_stack: List[Action] = []
-        self.select_start: Optional[Tuple[int, int]] = None
        self._hl_factories = hl_factories
+        self._trailing_whitespace = TrailingWhitespace(color_manager)
+        self._replace_hl = Replace()
+        self.selection = Selection()
        self._file_hls: Tuple[FileHL, ...] = ()

-    def ensure_loaded(self, status: Status) -> None:
+    def ensure_loaded(self, status: Status, stdin: str) -> None:
        if self.lines:
            return

-        if self.filename is not None and os.path.isfile(self.filename):
+        if self.filename == '-':
+            status.update('(from stdin)')
+            self.filename = None
+            self.modified = True
+            sio = io.StringIO(stdin)
+            self.lines, self.nl, mixed, self.sha256 = get_lines(sio)
+        elif self.filename is not None and os.path.isfile(self.filename):
            with open(self.filename, newline='') as f:
                self.lines, self.nl, mixed, self.sha256 = get_lines(f)
        else:
@@ -247,11 +258,14 @@ class File:
        file_hls = []
        for factory in self._hl_factories:
            if self.filename is not None:
-                # TODO: this does an extra read
-                file_hls.append(factory.get_file_highlighter(self.filename))
+                hl = factory.file_highlighter(self.filename, self.lines[0])
+                file_hls.append(hl)
            else:
-                file_hls.append(factory.get_blank_file_highlighter())
-        self._file_hls = tuple(file_hls)
+                file_hls.append(factory.blank_file_highlighter())
+        self._file_hls = (
+            *file_hls,
+            self._trailing_whitespace, self._replace_hl, self.selection,
+        )

    def __repr__(self) -> str:
        return f'<{type(self).__name__} {self.filename!r}>'
@@ -434,13 +448,6 @@ class File:
    ) -> None:
        self.finalize_previous_action()

-        def highlight() -> None:
-            self.highlight(
-                screen.stdscr, screen.margin,
-                y=self.y, x=self.x, n=len(match[0]),
-                color=HIGHLIGHT, include_edge=True,
-            )
-
        count = 0
        res: Union[str, PromptResult] = ''
        search = _SearchIter(self, reg, offset=0)
@@ -449,20 +456,36 @@ class File:
            self.x = self.x_hint = match.start()
            self.scroll_screen_if_needed(screen.margin)
            if res != 'a':  # make `a` replace the rest of them
-                screen.draw()
-                highlight()
-                with screen.resize_cb(highlight):
-                    res = screen.quick_prompt(
-                        'replace [y(es), n(o), a(ll)]?', 'yna',
-                    )
+                with self._replace_hl.region(self.y, self.x, match.end()):
+                    screen.draw()
+                    res = screen.quick_prompt('replace', ('yes', 'no', 'all'))
            if res in {'y', 'a'}:
                count += 1
                with self.edit_action_context('replace', final=True):
                    replaced = match.expand(replace)
                    line = screen.file.lines[line_y]
-                    line = line[:match.start()] + replaced + line[match.end():]
-                    screen.file.lines[line_y] = line
-                search.offset = len(replaced)
+                    if '\n' in replaced:
+                        replaced_lines = replaced.split('\n')
+                        self.lines[line_y] = (
+                            f'{line[:match.start()]}{replaced_lines[0]}'
+                        )
+                        for i, ins_line in enumerate(replaced_lines[1:-1], 1):
+                            self.lines.insert(line_y + i, ins_line)
+                        last_insert = line_y + len(replaced_lines) - 1
+                        self.lines.insert(
+                            last_insert,
+                            f'{replaced_lines[-1]}{line[match.end():]}',
+                        )
+                        self.y = last_insert
+                        self.x = self.x_hint = 0
+                        search.offset = len(replaced_lines[-1])
+                    else:
+                        self.lines[line_y] = (
+                            f'{line[:match.start()]}'
+                            f'{replaced}'
+                            f'{line[match.end():]}'
+                        )
+                        search.offset = len(replaced)
            elif res == 'n':
                search.offset = 1
            else:
@@ -543,16 +566,17 @@ class File:

    @edit_action('indent selection', final=True)
    def _indent_selection(self, margin: Margin) -> None:
-        assert self.select_start is not None
-        sel_y, sel_x = self.select_start
-        (s_y, _), (e_y, _) = self._get_selection()
+        assert self.selection.start is not None
+        sel_y, sel_x = self.selection.start
+        (s_y, _), (e_y, _) = self.selection.get()
        for l_y in range(s_y, e_y + 1):
            if self.lines[l_y]:
                self.lines[l_y] = ' ' * 4 + self.lines[l_y]
-                if l_y == sel_y and sel_x != 0:
-                    self.select_start = (sel_y, sel_x + 4)
                if l_y == self.y:
                    self.x = self.x_hint = self.x + 4
+                if l_y == sel_y and sel_x != 0:
+                    sel_x += 4
+        self.selection.set(sel_y, sel_x, self.y, self.x)

    @edit_action('insert tab', final=False)
    def _tab(self, margin: Margin) -> None:
@@ -563,7 +587,7 @@ class File:
        _restore_lines_eof_invariant(self.lines)

    def tab(self, margin: Margin) -> None:
-        if self.select_start:
+        if self.selection.start is not None:
            self._indent_selection(margin)
        else:
            self._tab(margin)
@@ -578,17 +602,18 @@ class File:

    @edit_action('dedent selection', final=True)
    def _dedent_selection(self, margin: Margin) -> None:
-        assert self.select_start is not None
-        sel_y, sel_x = self.select_start
-        (s_y, _), (e_y, _) = self._get_selection()
+        assert self.selection.start is not None
+        sel_y, sel_x = self.selection.start
+        (s_y, _), (e_y, _) = self.selection.get()
        for l_y in range(s_y, e_y + 1):
            n = self._dedent_line(self.lines[l_y])
            if n:
                self.lines[l_y] = self.lines[l_y][n:]
-                if l_y == sel_y:
-                    self.select_start = (sel_y, max(sel_x - n, 0))
                if l_y == self.y:
                    self.x = self.x_hint = max(self.x - n, 0)
+                if l_y == sel_y:
+                    sel_x = max(sel_x - n, 0)
+        self.selection.set(sel_y, sel_x, self.y, self.x)

    @edit_action('dedent', final=True)
    def _dedent(self, margin: Margin) -> None:
@@ -598,7 +623,7 @@ class File:
            self.x = self.x_hint = max(self.x - n, 0)

    def shift_tab(self, margin: Margin) -> None:
-        if self.select_start:
+        if self.selection.start is not None:
            self._dedent_selection(margin)
        else:
            self._dedent(margin)
@@ -607,7 +632,7 @@ class File:
    @clear_selection
    def cut_selection(self, margin: Margin) -> Tuple[str, ...]:
        ret = []
-        (s_y, s_x), (e_y, e_x) = self._get_selection()
+        (s_y, s_x), (e_y, e_x) = self.selection.get()
        if s_y == e_y:
            ret.append(self.lines[s_y][s_x:e_x])
            self.lines[s_y] = self.lines[s_y][:s_x] + self.lines[s_y][e_x:]
@@ -680,7 +705,7 @@ class File:
    @edit_action('sort selection', final=True)
    @clear_selection
    def sort_selection(self, margin: Margin) -> None:
-        (s_y, _), (e_y, _) = self._get_selection()
+        (s_y, _), (e_y, _) = self.selection.get()
        e_y = min(e_y + 1, len(self.lines) - 1)
        if self.lines[e_y - 1] == '':
            e_y -= 1
@@ -738,7 +763,7 @@ class File:

    def finalize_previous_action(self) -> None:
        assert not isinstance(self.lines, ListSpy), 'nested edit/movement'
-        self.select_start = None
+        self.selection.clear()
        if self.undo_stack:
            self.undo_stack[-1].final = True

@@ -791,14 +816,14 @@ class File:

    @contextlib.contextmanager
    def select(self) -> Generator[None, None, None]:
-        if self.select_start is None:
-            select_start = (self.y, self.x)
+        if self.selection.start is None:
+            start = (self.y, self.x)
        else:
-            select_start = self.select_start
+            start = self.selection.start
        try:
            yield
        finally:
-            self.select_start = select_start
+            self.selection.set(*start, self.y, self.x)

    # positioning

@@ -815,95 +840,50 @@ class File:
    ) -> None:
        stdscr.move(self.rendered_y(margin), self.rendered_x())

-    def _get_selection(self) -> Tuple[Tuple[int, int], Tuple[int, int]]:
-        assert self.select_start is not None
-        select_end = (self.y, self.x)
-        if select_end < self.select_start:
-            return select_end, self.select_start
-        else:
-            return self.select_start, select_end
-
    def touch(self, lineno: int) -> None:
        for file_hl in self._file_hls:
            file_hl.touch(lineno)

    def draw(self, stdscr: 'curses._CursesWindow', margin: Margin) -> None:
        to_display = min(len(self.lines) - self.file_y, margin.body_lines)
-        for i in range(to_display):
-            line_idx = self.file_y + i
-            line = self.lines[line_idx]
-            x = self.x if line_idx == self.y else 0
-            line = scrolled_line(line, x, curses.COLS)
-            stdscr.insstr(i + margin.header, 0, line)
-        for i in range(to_display, margin.body_lines):
-            stdscr.move(i + margin.header, 0)
-            stdscr.clrtoeol()

        for file_hl in self._file_hls:
            file_hl.highlight_until(self.lines, self.file_y + to_display)

-        for i in range(self.file_y, self.file_y + to_display):
+        for i in range(to_display):
+            draw_y = i + margin.header
+            l_y = self.file_y + i
+            x = self.x if l_y == self.y else 0
+            line = scrolled_line(self.lines[l_y], x, curses.COLS)
+            stdscr.insstr(draw_y, 0, line)
+
+            l_x = line_x(x, curses.COLS)
+            l_x_max = l_x + curses.COLS
            for file_hl in self._file_hls:
-                for region in file_hl.regions[i]:
-                    self.highlight(
-                        stdscr, margin,
-                        y=i, include_edge=False, **region,
-                    )
+                for region in file_hl.regions[l_y]:
+                    if region.x >= l_x_max:
+                        break
+                    elif region.end <= l_x:
+                        continue

-        if self.select_start is not None:
-            (s_y, s_x), (e_y, e_x) = self._get_selection()
+                    if l_x and region.x <= l_x:
+                        if file_hl.include_edge:
+                            h_s_x = 0
+                        else:
+                            h_s_x = 1
+                    else:
+                        h_s_x = region.x - l_x

-            if s_y == e_y:
-                self.highlight(
-                    stdscr, margin,
-                    y=s_y, x=s_x, n=e_x - s_x,
-                    color=HIGHLIGHT, include_edge=True,
-                )
-            else:
-                self.highlight(
-                    stdscr, margin,
-                    y=s_y, x=s_x, n=len(self.lines[s_y]) - s_x + 1,
-                    color=HIGHLIGHT, include_edge=True,
-                )
-                for l_y in range(s_y + 1, e_y):
-                    self.highlight(
-                        stdscr, margin,
-                        y=l_y, x=0, n=len(self.lines[l_y]) + 1,
-                        color=HIGHLIGHT, include_edge=True,
-                    )
-                self.highlight(
-                    stdscr, margin,
-                    y=e_y, x=0, n=e_x,
-                    color=HIGHLIGHT, include_edge=True,
-                )
+                    if region.end >= l_x_max:
+                        if file_hl.include_edge:
+                            h_e_x = curses.COLS
+                        else:
+                            h_e_x = curses.COLS - 1
+                    else:
+                        h_e_x = region.end - l_x

-    def highlight(
-            self,
-            stdscr: 'curses._CursesWindow', margin: Margin,
-            *,
-            y: int, x: int, n: int, color: int,
-            include_edge: bool,
-    ) -> None:
-        h_y = y - self.file_y + margin.header
-        if y == self.y:
-            l_x = line_x(self.x, curses.COLS)
-            # TODO: include edge left detection
-            if x < l_x:
-                h_x = 0
-                n -= l_x - x
-            else:
-                h_x = x - l_x
-        else:
-            l_x = 0
-            h_x = x
-        if not include_edge and len(self.lines[y]) > l_x + curses.COLS:
-            h_n = min(curses.COLS - h_x - 1, n)
-        else:
-            h_n = n
-        if (
-                h_y < margin.header or
-                h_y > margin.header + margin.body_lines or
-                h_x >= curses.COLS
-        ):
-            return
-        stdscr.chgat(h_y, h_x, h_n, color)
+                    stdscr.chgat(draw_y, h_s_x, h_e_x - h_s_x, region.attr)
+
+        for i in range(to_display, margin.body_lines):
+            stdscr.move(i + margin.header, 0)
+            stdscr.clrtoeol()
--- a/babi/highlight.py
+++ b/babi/highlight.py
@@ -1,4 +1,3 @@
-import contextlib
 import functools
 import json
 import os.path
@@ -10,20 +9,31 @@ from typing import Match
 from typing import NamedTuple
 from typing import Optional
 from typing import Tuple
+from typing import TypeVar
+
+from identify.identify import tags_from_filename

 from babi._types import Protocol
 from babi.fdict import FDict
 from babi.reg import _Reg
 from babi.reg import _RegSet
 from babi.reg import ERR_REG
+from babi.reg import expand_escaped
 from babi.reg import make_reg
 from babi.reg import make_regset

+T = TypeVar('T')
 Scope = Tuple[str, ...]
 Regions = Tuple['Region', ...]
 Captures = Tuple[Tuple[int, '_Rule'], ...]


+def uniquely_constructed(t: T) -> T:
+    """avoid tuple.__hash__ for "singleton" constructed objects"""
+    t.__hash__ = object.__hash__  # type: ignore
+    return t
+
+
 def _split_name(s: Optional[str]) -> Tuple[str, ...]:
    if s is None:
        return ()
@@ -59,6 +69,7 @@ class _Rule(Protocol):
    def patterns(self) -> 'Tuple[_Rule, ...]': ...


+@uniquely_constructed
 class Rule(NamedTuple):
    name: Tuple[str, ...]
    match: Optional[str]
@@ -114,6 +125,10 @@ class Rule(NamedTuple):
        else:
            while_captures = ()

+        # some grammars (at least xml) have begin rules with no end
+        if begin is not None and end is None and while_ is None:
+            end = '$impossible^'
+
        # Using the captures key for a begin/end/while rule is short-hand for
        # giving both beginCaptures and endCaptures with same values
        if begin and end and captures:
@@ -146,24 +161,15 @@ class Rule(NamedTuple):
        )


+@uniquely_constructed
 class Grammar(NamedTuple):
    scope_name: str
-    first_line_match: Optional[_Reg]
-    file_types: FrozenSet[str]
    patterns: Tuple[_Rule, ...]
    repository: FDict[str, _Rule]

    @classmethod
    def from_data(cls, data: Dict[str, Any]) -> 'Grammar':
        scope_name = data['scopeName']
-        if 'firstLineMatch' in data:
-            first_line_match: Optional[_Reg] = make_reg(data['firstLineMatch'])
-        else:
-            first_line_match = None
-        if 'fileTypes' in data:
-            file_types = frozenset(data['fileTypes'])
-        else:
-            file_types = frozenset()
        patterns = tuple(Rule.from_dct(dct) for dct in data['patterns'])
        if 'repository' in data:
            repository = FDict({
@@ -173,40 +179,10 @@ class Grammar(NamedTuple):
            repository = FDict({})
        return cls(
            scope_name=scope_name,
-            first_line_match=first_line_match,
-            file_types=file_types,
            patterns=patterns,
            repository=repository,
        )

-    @classmethod
-    def parse(cls, filename: str) -> 'Grammar':
-        with open(filename) as f:
-            return cls.from_data(json.load(f))
-
-    @classmethod
-    def blank(cls) -> 'Grammar':
-        return cls(
-            scope_name='source.unknown',
-            first_line_match=None,
-            file_types=frozenset(),
-            patterns=(),
-            repository=FDict({}),
-        )
-
-    def matches_file(self, filename: str, first_line: str) -> bool:
-        _, ext = os.path.splitext(filename)
-        if ext.lstrip('.') in self.file_types:
-            return True
-        elif self.first_line_match is not None:
-            return bool(
-                self.first_line_match.match(
-                    first_line, 0, first_line=True, boundary=True,
-                ),
-            )
-        else:
-            return False
-

 class Region(NamedTuple):
    start: int
@@ -369,6 +345,7 @@ def _do_regset(
    return state, match.end(), boundary, tuple(ret)


+@uniquely_constructed
 class PatternRule(NamedTuple):
    name: Tuple[str, ...]
    regset: _RegSet
@@ -395,6 +372,7 @@ class PatternRule(NamedTuple):
        return _do_regset(idx, match, self, compiler, state, pos)


+@uniquely_constructed
 class MatchRule(NamedTuple):
    name: Tuple[str, ...]
    captures: Captures
@@ -420,6 +398,7 @@ class MatchRule(NamedTuple):
        raise AssertionError(f'unreachable {self}')


+@uniquely_constructed
 class EndRule(NamedTuple):
    name: Tuple[str, ...]
    content_name: Tuple[str, ...]
@@ -439,7 +418,7 @@ class EndRule(NamedTuple):
        next_scope = scope + self.content_name

        boundary = match.end() == len(match.string)
-        reg = make_reg(match.expand(self.end))
+        reg = make_reg(expand_escaped(match, self.end))
        state = state.push(Entry(next_scope, self, reg, boundary))
        regions = _captures(compiler, scope, match, self.begin_captures)
        return state, True, regions
@@ -480,6 +459,7 @@ class EndRule(NamedTuple):
                return _do_regset(idx, match, self, compiler, state, pos)


+@uniquely_constructed
 class WhileRule(NamedTuple):
    name: Tuple[str, ...]
    content_name: Tuple[str, ...]
@@ -499,7 +479,7 @@ class WhileRule(NamedTuple):
        next_scope = scope + self.content_name

        boundary = match.end() == len(match.string)
-        reg = make_reg(match.expand(self.while_))
+        reg = make_reg(expand_escaped(match, self.while_))
        state = state.push_while(self, Entry(next_scope, self, reg, boundary))
        regions = _captures(compiler, scope, match, self.begin_captures)
        return state, True, regions
@@ -534,7 +514,7 @@ class WhileRule(NamedTuple):


 class Compiler:
-    def __init__(self, grammar: Grammar, grammars: Dict[str, Grammar]) -> None:
+    def __init__(self, grammar: Grammar, grammars: 'Grammars') -> None:
        self._root_scope = grammar.scope_name
        self._grammars = grammars
        self._rule_to_grammar: Dict[_Rule, Grammar] = {}
@@ -555,14 +535,17 @@ class Compiler:
        if s == '$self':
            return self._patterns(grammar, grammar.patterns)
        elif s == '$base':
-            return self._include(self._grammars[self._root_scope], '$self')
+            grammar = self._grammars.grammar_for_scope(self._root_scope)
+            return self._include(grammar, '$self')
        elif s.startswith('#'):
            return self._patterns(grammar, (grammar.repository[s[1:]],))
        elif '#' not in s:
-            return self._include(self._grammars[s], '$self')
+            grammar = self._grammars.grammar_for_scope(s)
+            return self._include(grammar, '$self')
        else:
            scope, _, s = s.partition('#')
-            return self._include(self._grammars[scope], f'#{s}')
+            grammar = self._grammars.grammar_for_scope(scope)
+            return self._include(grammar, f'#{s}')

    @functools.lru_cache(maxsize=None)
    def _patterns(
@@ -634,8 +617,10 @@ class Compiler:
            return PatternRule(rule.name, make_regset(*regs), rules)

    def compile_rule(self, rule: _Rule) -> CompiledRule:
-        with contextlib.suppress(KeyError):
+        try:
            return self._c_rules[rule]
+        except KeyError:
+            pass

        grammar = self._rule_to_grammar[rule]
        ret = self._c_rules[rule] = self._compile_rule(grammar, rule)
@@ -643,49 +628,87 @@ class Compiler:


 class Grammars:
-    def __init__(self, grammars: List[Grammar]) -> None:
-        self.grammars = {grammar.scope_name: grammar for grammar in grammars}
-        self._compilers: Dict[Grammar, Compiler] = {}
+    def __init__(self, *directories: str) -> None:
+        self._scope_to_files = {
+            os.path.splitext(filename)[0]: os.path.join(directory, filename)
+            for directory in directories
+            if os.path.exists(directory)
+            for filename in os.listdir(directory)
+            if filename.endswith('.json')
+        }

-    @classmethod
-    def from_syntax_dir(cls, syntax_dir: str) -> 'Grammars':
-        grammars = [Grammar.blank()]
-        if os.path.exists(syntax_dir):
-            grammars.extend(
-                Grammar.parse(os.path.join(syntax_dir, filename))
-                for filename in os.listdir(syntax_dir)
-            )
-        return cls(grammars)
+        unknown_grammar = {'scopeName': 'source.unknown', 'patterns': []}
+        self._raw = {'source.unknown': unknown_grammar}
+        self._file_types: List[Tuple[FrozenSet[str], str]] = []
+        self._first_line: List[Tuple[_Reg, str]] = []
+        self._parsed: Dict[str, Grammar] = {}
+        self._compiled: Dict[str, Compiler] = {}

-    def _compiler_for_grammar(self, grammar: Grammar) -> Compiler:
-        with contextlib.suppress(KeyError):
-            return self._compilers[grammar]
+    def _raw_for_scope(self, scope: str) -> Dict[str, Any]:
+        try:
+            return self._raw[scope]
+        except KeyError:
+            pass

-        ret = self._compilers[grammar] = Compiler(grammar, self.grammars)
+        grammar_path = self._scope_to_files.pop(scope)
+        with open(grammar_path) as f:
+            ret = self._raw[scope] = json.load(f)
+
+        file_types = frozenset(ret.get('fileTypes', ()))
+        first_line = make_reg(ret.get('firstLineMatch', '$impossible^'))
+
+        self._file_types.append((file_types, scope))
+        self._first_line.append((first_line, scope))
+
+        return ret
+
+    def grammar_for_scope(self, scope: str) -> Grammar:
+        try:
+            return self._parsed[scope]
+        except KeyError:
+            pass
+
+        raw = self._raw_for_scope(scope)
+        ret = self._parsed[scope] = Grammar.from_data(raw)
        return ret

    def compiler_for_scope(self, scope: str) -> Compiler:
-        return self._compiler_for_grammar(self.grammars[scope])
+        try:
+            return self._compiled[scope]
+        except KeyError:
+            pass
+
+        grammar = self.grammar_for_scope(scope)
+        ret = self._compiled[scope] = Compiler(grammar, self)
+        return ret

    def blank_compiler(self) -> Compiler:
        return self.compiler_for_scope('source.unknown')

-    def compiler_for_file(self, filename: str) -> Compiler:
-        if os.path.exists(filename):
-            with open(filename) as f:
-                first_line = next(f, '')
-        else:
-            first_line = ''
-        for grammar in self.grammars.values():
-            if grammar.matches_file(filename, first_line):
-                break
-        else:
-            grammar = self.grammars['source.unknown']
+    def compiler_for_file(self, filename: str, first_line: str) -> Compiler:
+        for tag in tags_from_filename(filename) - {'text'}:
+            try:
+                # TODO: this doesn't always match even if we detect it
+                return self.compiler_for_scope(f'source.{tag}')
+            except KeyError:
+                pass

-        return self._compiler_for_grammar(grammar)
+        # didn't find it in the fast path, need to read all the json
+        for k in tuple(self._scope_to_files):
+            self._raw_for_scope(k)
+
+        _, _, ext = os.path.basename(filename).rpartition('.')
+        for extensions, scope in self._file_types:
+            if ext in extensions:
+                return self.compiler_for_scope(scope)
+
+        for reg, scope in self._first_line:
+            if reg.match(first_line, 0, first_line=True, boundary=True):
+                return self.compiler_for_scope(scope)
+
+        return self.compiler_for_scope('source.unknown')


-@functools.lru_cache(maxsize=None)
 def highlight_line(
        compiler: 'Compiler',
        state: State,
--- a/babi/hl/interface.py
+++ b/babi/hl/interface.py
@@ -1,27 +1,32 @@
-from typing import Sequence
+from typing import NamedTuple
 from typing import Tuple

 from babi._types import Protocol
-from babi._types import TypedDict
 from babi.list_spy import SequenceNoSlice


-class CursesRegion(TypedDict):
+class HL(NamedTuple):
    x: int
-    n: int
-    color: int
+    end: int
+    attr: int


-CursesRegions = Tuple[CursesRegion, ...]
+HLs = Tuple[HL, ...]
+
+
+class RegionsMapping(Protocol):
+    def __getitem__(self, idx: int) -> HLs: ...


 class FileHL(Protocol):
    @property
-    def regions(self) -> Sequence[CursesRegions]: ...
+    def include_edge(self) -> bool: ...
+    @property
+    def regions(self) -> RegionsMapping: ...
    def highlight_until(self, lines: SequenceNoSlice, idx: int) -> None: ...
    def touch(self, lineno: int) -> None: ...


 class HLFactory(Protocol):
-    def get_file_highlighter(self, filename: str) -> FileHL: ...
-    def get_blank_file_highlighter(self) -> FileHL: ...
+    def file_highlighter(self, filename: str, first_line: str) -> FileHL: ...
+    def blank_file_highlighter(self) -> FileHL: ...
--- a/babi/hl/replace.py
+++ b/babi/hl/replace.py
@@ -0,0 +1,32 @@
+import collections
+import contextlib
+import curses
+from typing import Dict
+from typing import Generator
+
+from babi.hl.interface import HL
+from babi.hl.interface import HLs
+from babi.list_spy import SequenceNoSlice
+
+
+class Replace:
+    include_edge = True
+
+    def __init__(self) -> None:
+        self.regions: Dict[int, HLs] = collections.defaultdict(tuple)
+
+    def highlight_until(self, lines: SequenceNoSlice, idx: int) -> None:
+        """our highlight regions are populated in other ways"""
+
+    def touch(self, lineno: int) -> None:
+        """our highlight regions are populated in other ways"""
+
+    @contextlib.contextmanager
+    def region(self, y: int, x: int, end: int) -> Generator[None, None, None]:
+        # XXX: this assumes pair 1 is the background
+        attr = curses.A_REVERSE | curses.A_DIM | curses.color_pair(1)
+        self.regions[y] = (HL(x=x, end=end, attr=attr),)
+        try:
+            yield
+        finally:
+            del self.regions[y]
--- a/babi/hl/selection.py
+++ b/babi/hl/selection.py
@@ -0,0 +1,58 @@
+import collections
+import curses
+from typing import Dict
+from typing import Optional
+from typing import Tuple
+
+from babi.hl.interface import HL
+from babi.hl.interface import HLs
+from babi.list_spy import SequenceNoSlice
+
+
+class Selection:
+    include_edge = True
+
+    def __init__(self) -> None:
+        self.regions: Dict[int, HLs] = collections.defaultdict(tuple)
+        self.start: Optional[Tuple[int, int]] = None
+        self.end: Optional[Tuple[int, int]] = None
+
+    def highlight_until(self, lines: SequenceNoSlice, idx: int) -> None:
+        if self.start is None or self.end is None:
+            return
+
+        # XXX: this assumes pair 1 is the background
+        attr = curses.A_REVERSE | curses.A_DIM | curses.color_pair(1)
+        (s_y, s_x), (e_y, e_x) = self.get()
+        if s_y == e_y:
+            self.regions[s_y] = (HL(x=s_x, end=e_x, attr=attr),)
+        else:
+            self.regions[s_y] = (
+                HL(x=s_x, end=len(lines[s_y]) + 1, attr=attr),
+            )
+            for l_y in range(s_y + 1, e_y):
+                self.regions[l_y] = (
+                    HL(x=0, end=len(lines[l_y]) + 1, attr=attr),
+                )
+            self.regions[e_y] = (HL(x=0, end=e_x, attr=attr),)
+
+    def touch(self, lineno: int) -> None:
+        """our highlight regions are populated in other ways"""
+
+    def get(self) -> Tuple[Tuple[int, int], Tuple[int, int]]:
+        assert self.start is not None and self.end is not None
+        if self.start < self.end:
+            return self.start, self.end
+        else:
+            return self.end, self.start
+
+    def clear(self) -> None:
+        if self.start is not None and self.end is not None:
+            (s_y, _), (e_y, _) = self.get()
+            for l_y in range(s_y, e_y + 1):
+                del self.regions[l_y]
+        self.start = self.end = None
+
+    def set(self, s_y: int, s_x: int, e_y: int, e_x: int) -> None:
+        self.clear()
+        self.start, self.end = (s_y, s_x), (e_y, e_x)
--- a/babi/hl/syntax.py
+++ b/babi/hl/syntax.py
@@ -1,7 +1,10 @@
 import curses
-from typing import Dict
+import functools
+import math
+from typing import Callable
 from typing import List
 from typing import NamedTuple
+from typing import Optional
 from typing import Tuple

 from babi.color_manager import ColorManager
@@ -9,11 +12,12 @@ from babi.highlight import Compiler
 from babi.highlight import Grammars
 from babi.highlight import highlight_line
 from babi.highlight import State
-from babi.hl.interface import CursesRegion
-from babi.hl.interface import CursesRegions
+from babi.hl.interface import HL
+from babi.hl.interface import HLs
 from babi.list_spy import SequenceNoSlice
 from babi.theme import Style
 from babi.theme import Theme
+from babi.user_data import prefix_data
 from babi.user_data import xdg_config
 from babi.user_data import xdg_data

@@ -21,6 +25,8 @@ A_ITALIC = getattr(curses, 'A_ITALIC', 0x80000000)  # new in py37


 class FileSyntax:
+    include_edge = False
+
    def __init__(
            self,
            compiler: Compiler,
@@ -31,11 +37,13 @@ class FileSyntax:
        self._theme = theme
        self._color_manager = color_manager

-        self.regions: List[CursesRegions] = []
+        self.regions: List[HLs] = []
        self._states: List[State] = []

-        self._hl_cache: Dict[str, Dict[State, Tuple[State, CursesRegions]]]
-        self._hl_cache = {}
+        # this will be assigned a functools.lru_cache per instance for
+        # better hit rate and memory usage
+        self._hl: Optional[Callable[[State, str, bool], Tuple[State, HLs]]]
+        self._hl = None

    def attr(self, style: Style) -> int:
        pair = self._color_manager.color_pair(style.fg, style.bg)
@@ -46,54 +54,52 @@ class FileSyntax:
            curses.A_UNDERLINE * style.u
        )

-    def _hl(
+    def _hl_uncached(
            self,
            state: State,
            line: str,
-            i: int,
-    ) -> Tuple[State, CursesRegions]:
-        try:
-            return self._hl_cache[line][state]
-        except KeyError:
-            pass
-
+            first_line: bool,
+    ) -> Tuple[State, HLs]:
        new_state, regions = highlight_line(
-            self._compiler, state, f'{line}\n', first_line=i == 0,
+            self._compiler, state, f'{line}\n', first_line=first_line,
        )

        # remove the trailing newline
        new_end = regions[-1]._replace(end=regions[-1].end - 1)
        regions = regions[:-1] + (new_end,)

-        regs: List[CursesRegion] = []
+        regs: List[HL] = []
        for r in regions:
            style = self._theme.select(r.scope)
            if style == self._theme.default:
                continue

-            n = r.end - r.start
            attr = self.attr(style)
            if (
                    regs and
-                    regs[-1]['color'] == attr and
-                    regs[-1]['x'] + regs[-1]['n'] == r.start
+                    regs[-1].attr == attr and
+                    regs[-1].end == r.start
            ):
-                regs[-1]['n'] += n
+                regs[-1] = regs[-1]._replace(end=r.end)
            else:
-                regs.append(CursesRegion(x=r.start, n=n, color=attr))
+                regs.append(HL(x=r.start, end=r.end, attr=attr))

-        dct = self._hl_cache.setdefault(line, {})
-        ret = dct[state] = (new_state, tuple(regs))
-        return ret
+        return new_state, tuple(regs)

    def highlight_until(self, lines: SequenceNoSlice, idx: int) -> None:
+        if self._hl is None:
+            # the docs claim better performance with power of two sizing
+            size = max(4096, 2 ** (int(math.log(len(lines), 2)) + 2))
+            self._hl = functools.lru_cache(maxsize=size)(self._hl_uncached)
+
        if not self._states:
            state = self._compiler.root_state
        else:
            state = self._states[-1]

        for i in range(len(self._states), idx):
-            state, regions = self._hl(state, lines[i], i)
+            # https://github.com/python/mypy/issues/8579
+            state, regions = self._hl(state, lines[i], i == 0)  # type: ignore
            self._states.append(state)
            self.regions.append(regions)

@@ -107,11 +113,11 @@ class Syntax(NamedTuple):
    theme: Theme
    color_manager: ColorManager

-    def get_file_highlighter(self, filename: str) -> FileSyntax:
-        compiler = self.grammars.compiler_for_file(filename)
+    def file_highlighter(self, filename: str, first_line: str) -> FileSyntax:
+        compiler = self.grammars.compiler_for_file(filename, first_line)
        return FileSyntax(compiler, self.theme, self.color_manager)

-    def get_blank_file_highlighter(self) -> FileSyntax:
+    def blank_file_highlighter(self) -> FileSyntax:
        compiler = self.grammars.blank_compiler()
        return FileSyntax(compiler, self.theme, self.color_manager)

@@ -139,7 +145,7 @@ class Syntax(NamedTuple):
            stdscr: 'curses._CursesWindow',
            color_manager: ColorManager,
    ) -> 'Syntax':
-        grammars = Grammars.from_syntax_dir(xdg_data('textmate_syntax'))
+        grammars = Grammars(prefix_data('grammar_v1'), xdg_data('grammar_v1'))
        theme = Theme.from_filename(xdg_config('theme.json'))
        ret = cls(grammars, theme, color_manager)
        ret._init_screen(stdscr)
--- a/babi/hl/trailing_whitespace.py
+++ b/babi/hl/trailing_whitespace.py
@@ -1,20 +1,21 @@
 import curses
 from typing import List
-from typing import NamedTuple

 from babi.color_manager import ColorManager
-from babi.hl.interface import CursesRegion
-from babi.hl.interface import CursesRegions
+from babi.hl.interface import HL
+from babi.hl.interface import HLs
 from babi.list_spy import SequenceNoSlice


-class FileTrailingWhitespace:
+class TrailingWhitespace:
+    include_edge = False
+
    def __init__(self, color_manager: ColorManager) -> None:
        self._color_manager = color_manager

-        self.regions: List[CursesRegions] = []
+        self.regions: List[HLs] = []

-    def _trailing_ws(self, line: str) -> CursesRegions:
+    def _trailing_ws(self, line: str) -> HLs:
        if not line:
            return ()

@@ -27,7 +28,7 @@ class FileTrailingWhitespace:
        else:
            pair = self._color_manager.raw_color_pair(-1, curses.COLOR_RED)
            attr = curses.color_pair(pair)
-            return (CursesRegion(x=i, n=len(line) - i, color=attr),)
+            return (HL(x=i, end=len(line), attr=attr),)

    def highlight_until(self, lines: SequenceNoSlice, idx: int) -> None:
        for i in range(len(self.regions), idx):
@@ -35,14 +36,3 @@ class FileTrailingWhitespace:

    def touch(self, lineno: int) -> None:
        del self.regions[lineno:]
-
-
-class TrailingWhitespace(NamedTuple):
-    color_manager: ColorManager
-
-    def get_file_highlighter(self, filename: str) -> FileTrailingWhitespace:
-        # no file-specific behaviour
-        return self.get_blank_file_highlighter()
-
-    def get_blank_file_highlighter(self) -> FileTrailingWhitespace:
-        return FileTrailingWhitespace(self.color_manager)
--- a/babi/horizontal_scrolling.py
+++ b/babi/horizontal_scrolling.py
@@ -1,10 +1,10 @@
 def line_x(x: int, width: int) -> int:
-    margin = min(width - 3, 6)
    if x + 1 < width:
        return 0
    elif width == 1:
        return x
    else:
+        margin = min(width - 3, 6)
        return (
            width - margin - 2 +
            (x + 1 - width) //
@@ -17,7 +17,7 @@ def scrolled_line(s: str, x: int, width: int) -> str:
    l_x = line_x(x, width)
    if l_x:
        s = f'«{s[l_x + 1:]}'
-        if l_x and len(s) > width:
+        if len(s) > width:
            return f'{s[:width - 1]}»'
        else:
            return s.ljust(width)
--- a/babi/main.py
+++ b/babi/main.py
@@ -1,17 +1,22 @@
 import argparse
 import curses
+import os
+import sys
 from typing import Optional
 from typing import Sequence

 from babi.file import File
+from babi.perf import Perf
 from babi.perf import perf_log
 from babi.screen import EditResult
 from babi.screen import make_stdscr
 from babi.screen import Screen

+CONSOLE = 'CONIN$' if sys.platform == 'win32' else '/dev/tty'

-def _edit(screen: Screen) -> EditResult:
-    screen.file.ensure_loaded(screen.status)
+
+def _edit(screen: Screen, stdin: str) -> EditResult:
+    screen.file.ensure_loaded(screen.status, stdin)

    while True:
        screen.status.tick(screen.margin)
@@ -32,15 +37,21 @@ def _edit(screen: Screen) -> EditResult:
            screen.status.update(f'unknown key: {key}')


-def c_main(stdscr: 'curses._CursesWindow', args: argparse.Namespace) -> int:
+def c_main(
+        stdscr: 'curses._CursesWindow',
+        args: argparse.Namespace,
+        stdin: str,
+) -> int:
    with perf_log(args.perf_log) as perf:
        screen = Screen(stdscr, args.filenames or [None], perf)
        with screen.history.save():
            while screen.files:
                screen.i = screen.i % len(screen.files)
-                res = _edit(screen)
+                res = _edit(screen, stdin)
                if res == EditResult.EXIT:
                    del screen.files[screen.i]
+                    # always go to the next file except at the end
+                    screen.i = min(screen.i, len(screen.files) - 1)
                    screen.status.clear()
                elif res == EditResult.NEXT:
                    screen.i += 1
@@ -48,19 +59,53 @@ def c_main(stdscr: 'curses._CursesWindow', args: argparse.Namespace) -> int:
                elif res == EditResult.PREV:
                    screen.i -= 1
                    screen.status.clear()
+                elif res == EditResult.OPEN:
+                    screen.i = len(screen.files) - 1
                else:
                    raise AssertionError(f'unreachable {res}')
    return 0


+def _key_debug(stdscr: 'curses._CursesWindow') -> int:
+    screen = Screen(stdscr, ['<<key debug>>'], Perf())
+    screen.file.lines = ['']
+
+    while True:
+        screen.status.update('press q to quit')
+        screen.draw()
+        screen.file.move_cursor(screen.stdscr, screen.margin)
+
+        key = screen.get_char()
+        screen.file.lines.insert(-1, f'{key.wch!r} {key.keyname.decode()!r}')
+        screen.file.down(screen.margin)
+        if key.wch == curses.KEY_RESIZE:
+            screen.resize()
+        if key.wch == 'q':
+            return 0
+
+
 def main(argv: Optional[Sequence[str]] = None) -> int:
    parser = argparse.ArgumentParser()
    parser.add_argument('filenames', metavar='filename', nargs='*')
    parser.add_argument('--perf-log')
+    parser.add_argument(
+        '--key-debug', action='store_true', help=argparse.SUPPRESS,
+    )
    args = parser.parse_args(argv)

+    if '-' in args.filenames:
+        print('reading stdin...', file=sys.stderr)
+        stdin = sys.stdin.read()
+        tty = os.open(CONSOLE, os.O_RDONLY)
+        os.dup2(tty, sys.stdin.fileno())
+    else:
+        stdin = ''
+
    with make_stdscr() as stdscr:
-        return c_main(stdscr, args)
+        if args.key_debug:
+            return _key_debug(stdscr)
+        else:
+            return c_main(stdscr, args, stdin)


 if __name__ == '__main__':
--- a/babi/reg.py
+++ b/babi/reg.py
@@ -1,4 +1,5 @@
 import functools
+import re
 from typing import Match
 from typing import Optional
 from typing import Tuple
@@ -7,6 +8,8 @@ import onigurumacffi

 from babi.cached_property import cached_property

+_BACKREF_RE = re.compile(r'((?<!\\)(?:\\\\)*)\\([0-9]+)')
+

 def _replace_esc(s: str, chars: str) -> str:
    """replace the given escape sequences of `chars` with \\uffff"""
@@ -40,7 +43,7 @@ def _replace_esc(s: str, chars: str) -> str:

 class _Reg:
    def __init__(self, s: str) -> None:
-        self._pattern = s
+        self._pattern = _replace_esc(s, 'z')

    def __repr__(self) -> str:
        return f'{type(self).__name__}({self._pattern!r})'
@@ -142,6 +145,10 @@ class _RegSet:
                return self._set_no_A_no_G.search(line, pos)


+def expand_escaped(match: Match[str], s: str) -> str:
+    return _BACKREF_RE.sub(lambda m: f'{m[1]}{re.escape(match[int(m[2])])}', s)
+
+
 make_reg = functools.lru_cache(maxsize=None)(_Reg)
 make_regset = functools.lru_cache(maxsize=None)(_RegSet)
 ERR_REG = make_reg(')this pattern always triggers an error when used(')
--- a/babi/screen.py
+++ b/babi/screen.py
@@ -6,7 +6,6 @@ import os
 import re
 import signal
 import sys
-from typing import Callable
 from typing import Generator
 from typing import List
 from typing import NamedTuple
@@ -21,15 +20,19 @@ from babi.file import File
 from babi.file import get_lines
 from babi.history import History
 from babi.hl.syntax import Syntax
-from babi.hl.trailing_whitespace import TrailingWhitespace
 from babi.margin import Margin
 from babi.perf import Perf
 from babi.prompt import Prompt
 from babi.prompt import PromptResult
 from babi.status import Status

-VERSION_STR = 'babi v0'
-EditResult = enum.Enum('EditResult', 'EXIT NEXT PREV')
+if sys.version_info >= (3, 8):  # pragma: no cover (py38+)
+    import importlib.metadata as importlib_metadata
+else:  # pragma: no cover (<py38)
+    import importlib_metadata
+
+VERSION_STR = f'babi v{importlib_metadata.version("babi")}'
+EditResult = enum.Enum('EditResult', 'EXIT NEXT PREV OPEN')

 # TODO: find a place to populate these, surely there's a database somewhere
 SEQUENCE_KEYNAME = {
@@ -73,12 +76,12 @@ class Screen:
            perf: Perf,
    ) -> None:
        self.stdscr = stdscr
-        color_manager = ColorManager.make()
-        hl_factories = (
-            Syntax.from_screen(stdscr, color_manager),
-            TrailingWhitespace(color_manager),
-        )
-        self.files = [File(f, hl_factories) for f in filenames]
+        self.color_manager = ColorManager.make()
+        self.hl_factories = (Syntax.from_screen(stdscr, self.color_manager),)
+        self.files = [
+            File(filename, self.color_manager, self.hl_factories)
+            for filename in filenames
+        ]
        self.i = 0
        self.history = History()
        self.perf = perf
@@ -86,7 +89,6 @@ class Screen:
        self.margin = Margin.from_current_screen()
        self.cut_buffer: Tuple[str, ...] = ()
        self.cut_selection = False
-        self._resize_cb: Optional[Callable[[], None]] = None
        self._buffered_input: Union[int, str, None] = None

    @property
@@ -220,30 +222,46 @@ class Screen:
        self.file.draw(self.stdscr, self.margin)
        self.status.draw(self.stdscr, self.margin)

-    @contextlib.contextmanager
-    def resize_cb(self, f: Callable[[], None]) -> Generator[None, None, None]:
-        assert self._resize_cb is None, self._resize_cb
-        self._resize_cb = f
-        try:
-            yield
-        finally:
-            self._resize_cb = None
-
    def resize(self) -> None:
        curses.update_lines_cols()
        self.margin = Margin.from_current_screen()
        self.file.scroll_screen_if_needed(self.margin)
        self.draw()
-        if self._resize_cb is not None:
-            self._resize_cb()

-    def quick_prompt(self, prompt: str, opts: str) -> Union[str, PromptResult]:
+    def quick_prompt(
+            self,
+            prompt: str,
+            opt_strs: Tuple[str, ...],
+    ) -> Union[str, PromptResult]:
+        opts = [opt[0] for opt in opt_strs]
        while True:
-            s = prompt.ljust(curses.COLS)
-            if len(s) > curses.COLS:
-                s = f'{s[:curses.COLS - 1]}…'
-            self.stdscr.insstr(curses.LINES - 1, 0, s, curses.A_REVERSE)
-            x = min(curses.COLS - 1, len(prompt) + 1)
+            x = 0
+
+            def _write(s: str, *, attr: int = curses.A_REVERSE) -> None:
+                nonlocal x
+
+                if x >= curses.COLS:
+                    return
+                self.stdscr.insstr(curses.LINES - 1, x, s, attr)
+                x += len(s)
+
+            _write(prompt)
+            _write(' [')
+            for i, opt_str in enumerate(opt_strs):
+                _write(opt_str[0], attr=curses.A_REVERSE | curses.A_BOLD)
+                _write(opt_str[1:])
+                if i != len(opt_strs) - 1:
+                    _write(', ')
+            _write(']?')
+
+            if x < curses.COLS - 1:
+                s = ' ' * (curses.COLS - x)
+                self.stdscr.insstr(curses.LINES - 1, x, s, curses.A_REVERSE)
+                x += 1
+            else:
+                x = curses.COLS - 1
+                self.stdscr.insstr(curses.LINES - 1, x, '…', curses.A_REVERSE)
+
            self.stdscr.move(curses.LINES - 1, x)

            key = self.get_char()
@@ -306,7 +324,7 @@ class Screen:
        self.status.update(f'{line}, {col} (of {line_count} {lines_word})')

    def cut(self) -> None:
-        if self.file.select_start:
+        if self.file.selection.start:
            self.cut_buffer = self.file.cut_selection(self.margin)
            self.cut_selection = True
        else:
@@ -342,6 +360,7 @@ class Screen:
            to_stack.append(action.apply(self.file))
            self.file.scroll_screen_if_needed(self.margin)
            self.status.update(f'{op}: {action.name}')
+            self.file.selection.clear()

    def undo(self) -> None:
        self._undo_redo('undo', self.file.undo_stack, self.file.redo_stack)
@@ -373,7 +392,7 @@ class Screen:
            self.save()
            return EditResult.EXIT
        elif response == ':sort':
-            if self.file.select_start:
+            if self.file.selection.start:
                self.file.sort_selection(self.margin)
            else:
                self.file.sort(self.margin)
@@ -397,7 +416,7 @@ class Screen:
                self.file.filename = filename

        if os.path.isfile(self.file.filename):
-            with open(self.file.filename) as f:
+            with open(self.file.filename, newline='') as f:
                *_, sha256 = get_lines(f)
        else:
            sha256 = hashlib.sha256(b'').hexdigest()
@@ -436,10 +455,19 @@ class Screen:
            self.file.filename = response
            return self.save()

+    def open_file(self) -> Optional[EditResult]:
+        response = self.prompt('enter filename', history='open')
+        if response is not PromptResult.CANCELLED:
+            opened = File(response, self.color_manager, self.hl_factories)
+            self.files.append(opened)
+            return EditResult.OPEN
+        else:
+            return None
+
    def quit_save_modified(self) -> Optional[EditResult]:
        if self.file.modified:
            response = self.quick_prompt(
-                'file is modified - save [y(es), n(o)]?', 'yn',
+                'file is modified - save', ('yes', 'no'),
            )
            if response == 'y':
                if self.save_filename() is not PromptResult.CANCELLED:
@@ -473,6 +501,7 @@ class Screen:
        b'^S': save,
        b'^O': save_filename,
        b'^X': quit_save_modified,
+        b'^P': open_file,
        b'kLFT3': lambda screen: EditResult.PREV,
        b'kRIT3': lambda screen: EditResult.NEXT,
        b'^Z': background,
--- a/babi/textmate_demo.py
+++ b/babi/textmate_demo.py
@@ -0,0 +1,69 @@
+import argparse
+from typing import Optional
+from typing import Sequence
+
+from babi.highlight import Compiler
+from babi.highlight import Grammars
+from babi.highlight import highlight_line
+from babi.theme import Style
+from babi.theme import Theme
+from babi.user_data import prefix_data
+from babi.user_data import xdg_config
+
+
+def print_styled(s: str, style: Style) -> None:
+    color_s = ''
+    undo_s = ''
+    if style.fg is not None:
+        color_s += '\x1b[38;2;{r};{g};{b}m'.format(**style.fg._asdict())
+        undo_s += '\x1b[39m'
+    if style.bg is not None:
+        color_s += '\x1b[48;2;{r};{g};{b}m'.format(**style.bg._asdict())
+        undo_s += '\x1b[49m'
+    if style.b:
+        color_s += '\x1b[1m'
+        undo_s += '\x1b[22m'
+    if style.i:
+        color_s += '\x1b[3m'
+        undo_s += '\x1b[23m'
+    if style.u:
+        color_s += '\x1b[4m'
+        undo_s += '\x1b[24m'
+    print(f'{color_s}{s}{undo_s}', end='', flush=True)
+
+
+def _highlight_output(theme: Theme, compiler: Compiler, filename: str) -> int:
+    state = compiler.root_state
+
+    if theme.default.bg is not None:
+        print('\x1b[48;2;{r};{g};{b}m'.format(**theme.default.bg._asdict()))
+    with open(filename) as f:
+        for line_idx, line in enumerate(f):
+            first_line = line_idx == 0
+            state, regions = highlight_line(compiler, state, line, first_line)
+            for start, end, scope in regions:
+                print_styled(line[start:end], theme.select(scope))
+    print('\x1b[m', end='')
+    return 0
+
+
+def main(argv: Optional[Sequence[str]] = None) -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--theme', default=xdg_config('theme.json'))
+    parser.add_argument('--grammar-dir', default=prefix_data('grammar_v1'))
+    parser.add_argument('filename')
+    args = parser.parse_args(argv)
+
+    with open(args.filename) as f:
+        first_line = next(f, '')
+
+    theme = Theme.from_filename(args.theme)
+
+    grammars = Grammars(args.grammar_dir)
+    compiler = grammars.compiler_for_file(args.filename, first_line)
+
+    return _highlight_output(theme, compiler, args.filename)
+
+
+if __name__ == '__main__':
+    exit(main())
--- a/babi/theme.py
+++ b/babi/theme.py
@@ -1,7 +1,6 @@
 import functools
 import json
 import os.path
-import re
 from typing import Any
 from typing import Dict
 from typing import NamedTuple
@@ -12,9 +11,6 @@ from babi._types import Protocol
 from babi.color import Color
 from babi.fdict import FDict

-# yes I know this is wrong, but it's good enough for now
-UN_COMMENT = re.compile(r'^\s*//.*$', re.MULTILINE)
-

 class Style(NamedTuple):
    fg: Optional[Color]
@@ -114,10 +110,14 @@ class Theme(NamedTuple):
        for rule in rules:
            if 'scope' not in rule:
                scopes = ['']
+            elif rule['scope'] == '':
+                scopes = ['']
            elif isinstance(rule['scope'], str):
                scopes = [
-                    # some themes have a buggy trailing comma
-                    s.strip() for s in rule['scope'].strip(',').split(',')
+                    s.strip()
+                    # some themes have a buggy trailing/leading comma
+                    for s in rule['scope'].strip().strip(',').split(',')
+                    if s.strip()
                ]
            else:
                scopes = rule['scope']
@@ -148,5 +148,4 @@ class Theme(NamedTuple):
            return cls.blank()
        else:
            with open(filename) as f:
-                contents = UN_COMMENT.sub('', f.read())
-                return cls.from_dct(json.loads(contents))
+                return cls.from_dct(json.load(f))
--- a/babi/user_data.py
+++ b/babi/user_data.py
@@ -1,4 +1,5 @@
 import os.path
+import sys


 def _xdg(*path: str, env: str, default: str) -> str:
@@ -14,3 +15,7 @@ def xdg_data(*path: str) -> str:

 def xdg_config(*path: str) -> str:
    return _xdg(*path, env='XDG_CONFIG_HOME', default='~/.config')
+
+
+def prefix_data(*path: str) -> str:
+    return os.path.join(sys.prefix, 'share/babi', *path)
--- a/bin/download-syntax
+++ b/bin/download-syntax
@@ -1,81 +0,0 @@
-#!/usr/bin/env python3
-import argparse
-import enum
-import json
-import os.path
-import plistlib
-import urllib.request
-from typing import NamedTuple
-
-import cson  # pip install cson
-
-
-DEFAULT_DIR = os.path.join(
-    os.environ.get('XDG_DATA_HOME') or
-    os.path.expanduser('~/.local/share'),
-    'babi/textmate_syntax',
-)
-Ext = enum.Enum('Ext', 'CSON PLIST JSON')
-
-
-def _convert_cson(src: bytes) -> str:
-    return json.dumps(cson.loads(src))
-
-
-def _convert_json(src: bytes) -> str:
-    return json.dumps(json.loads(src))
-
-
-def _convert_plist(src: bytes) -> str:
-    return json.dumps(plistlib.loads(src))
-
-
-EXT_CONVERT = {
-    Ext.CSON: _convert_cson,
-    Ext.JSON: _convert_json,
-    Ext.PLIST: _convert_plist,
-}
-
-
-class Syntax(NamedTuple):
-    name: str
-    ext: Ext
-    url: str
-
-
-SYNTAXES = (
-    Syntax('c', Ext.JSON, 'https://raw.githubusercontent.com/jeff-hykin/cpp-textmate-grammar/53e39b1c/syntaxes/c.tmLanguage.json'),  # noqa: E501
-    Syntax('css', Ext.CSON, 'https://raw.githubusercontent.com/atom/language-css/9feb69c081308b63f78bb0d6a2af2ff5eb7d869b/grammars/css.cson'),  # noqa: E501
-    Syntax('diff', Ext.PLIST, 'https://raw.githubusercontent.com/textmate/diff.tmbundle/0593bb77/Syntaxes/Diff.plist'),  # noqa: E501
-    Syntax('html', Ext.PLIST, 'https://raw.githubusercontent.com/textmate/html.tmbundle/0c3d5ee5/Syntaxes/HTML.plist'),  # noqa: E501
-    Syntax('html-derivative', Ext.PLIST, 'https://raw.githubusercontent.com/textmate/html.tmbundle/0c3d5ee54de3a993f747f54186b73a4d2d3c44a2/Syntaxes/HTML%20(Derivative).tmLanguage'),  # noqa: E501
-    Syntax('ini', Ext.PLIST, 'https://raw.githubusercontent.com/textmate/ini.tmbundle/7d8c7b55/Syntaxes/Ini.plist'),  # noqa: E501
-    Syntax('json', Ext.PLIST, 'https://raw.githubusercontent.com/microsoft/vscode-JSON.tmLanguage/d113e90937ed3ecc31ac54750aac2e8efa08d784/JSON.tmLanguage'),  # noqa: E501
-    Syntax('markdown', Ext.PLIST, 'https://raw.githubusercontent.com/microsoft/vscode-markdown-tm-grammar/59a5962/syntaxes/markdown.tmLanguage'),  # noqa: E501
-    Syntax('powershell', Ext.PLIST, 'https://raw.githubusercontent.com/PowerShell/EditorSyntax/4a0a0766/PowerShellSyntax.tmLanguage'),  # noqa: E501
-    Syntax('python', Ext.PLIST, 'https://raw.githubusercontent.com/MagicStack/MagicPython/c9b3409d/grammars/MagicPython.tmLanguage'),  # noqa: E501
-    # TODO: https://github.com/zargony/atom-language-rust/pull/149
-    Syntax('rust', Ext.CSON, 'https://raw.githubusercontent.com/asottile/atom-language-rust/e113ca67/grammars/rust.cson'),  # noqa: E501
-    Syntax('shell', Ext.CSON, 'https://raw.githubusercontent.com/atom/language-shellscript/7008ea926867d8a231003e78094091471c4fccf8/grammars/shell-unix-bash.cson'),  # noqa: E501
-    Syntax('yaml', Ext.PLIST, 'https://raw.githubusercontent.com/textmate/yaml.tmbundle/e54ceae3/Syntaxes/YAML.tmLanguage'),  # noqa: E501
-)
-
-
-def main() -> int:
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--dest', default=DEFAULT_DIR)
-    args = parser.parse_args()
-
-    os.makedirs(args.dest, exist_ok=True)
-    for syntax in SYNTAXES:
-        print(f'downloading {syntax.name}...')
-        resp = urllib.request.urlopen(syntax.url).read()
-        converted = EXT_CONVERT[syntax.ext](resp)
-        with open(os.path.join(args.dest, f'{syntax.name}.json'), 'w') as f:
-            f.write(converted)
-
-    return 0
-
-
-if __name__ == '__main__':
-    exit(main())
--- a/bin/download-theme
+++ b/bin/download-theme
@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+import argparse
+import io
+import json
+import os.path
+import plistlib
+import re
+import urllib.request
+from typing import Any
+
+import cson  # pip install cson
+
+TOKEN = re.compile(br'(\\\\|\\"|"|//|\n)')
+
+
+def json_with_comments(s: bytes) -> Any:
+    bio = io.BytesIO()
+
+    idx = 0
+    in_string = False
+    in_comment = False
+
+    match = TOKEN.search(s, idx)
+    while match:
+        if not in_comment:
+            bio.write(s[idx:match.start()])
+
+        tok = match[0]
+        if not in_comment and tok == b'"':
+            in_string = not in_string
+        elif in_comment and tok == b'\n':
+            in_comment = False
+        elif not in_string and tok == b'//':
+            in_comment = True
+
+        if not in_comment:
+            bio.write(tok)
+
+        idx = match.end()
+        match = TOKEN.search(s, idx)
+
+    print(bio.getvalue())
+    bio.seek(0)
+    return json.load(bio)
+
+
+STRATEGIES = (json.loads, plistlib.loads, cson.loads, json_with_comments)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument('name')
+    parser.add_argument('url')
+    args = parser.parse_args()
+
+    if '/blob/' in args.url:
+        url = args.url.replace('/blob/', '/raw/')
+    else:
+        url = args.url
+
+    contents = urllib.request.urlopen(url).read()
+
+    errors = []
+    for strategy in STRATEGIES:
+        try:
+            loaded = strategy(contents)
+        except Exception as e:
+            errors.append((f'{strategy.__module__}.{strategy.__name__}', e))
+        else:
+            break
+    else:
+        errors_s = '\n'.join(f'\t{name}: {error}' for name, error in errors)
+        raise AssertionError(f'could not load as json/plist/cson:\n{errors_s}')
+
+    config_dir = os.path.expanduser('~/.config/babi')
+    os.makedirs(config_dir, exist_ok=True)
+    dest = os.path.join(config_dir, f'{args.name}.json')
+    with open(dest, 'w') as f:
+        json.dump(loaded, f)
+
+    theme_json = os.path.join(config_dir, 'theme.json')
+    if os.path.lexists(theme_json):
+        os.remove(theme_json)
+    os.symlink(dest, theme_json)
+    return 0
+
+
+if __name__ == '__main__':
+    exit(main())
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -1,5 +1,5 @@
 covdefaults
 coverage
-git+https://github.com/asottile/hecate@ebe6dfb
+git+https://github.com/asottile/hecate@875567f
 pytest
 remote-pdb
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,6 +1,6 @@
 [metadata]
 name = babi
-version = 0.0.1
+version = 0.0.4
 description = a text editor
 long_description = file: README.md
 long_description_content_type = text/markdown
@@ -22,12 +22,17 @@ classifiers =
 [options]
 packages = find:
 install_requires =
+    babi-grammars
+    identify
    onigurumacffi>=0.0.10
+    importlib_metadata>=1;python_version<"3.8"
+    windows-curses;sys_platform=="win32"
 python_requires = >=3.6.1

 [options.entry_points]
 console_scripts =
    babi = babi.main:main
+    babi-textmate-demo = babi.textmate_demo:main

 [options.packages.find]
 exclude =
--- a/testing/vsc_test/.gitignore
+++ b/testing/vsc_test/.gitignore
@@ -0,0 +1,2 @@
+/node_modules
+/package-lock.json
--- a/testing/vsc_test/package.json
+++ b/testing/vsc_test/package.json
@@ -0,0 +1,5 @@
+{
+  "dependencies": [
+    "vscode-textmate"
+  ]
+}
--- a/testing/vsc_test/vsc.js
+++ b/testing/vsc_test/vsc.js
@@ -0,0 +1,51 @@
+const fs = require('fs');
+const vsctm = require('vscode-textmate');
+
+if (process.argv.length < 4) {
+    console.log('usage: t.js GRAMMAR FILE');
+    process.exit(1);
+}
+
+const grammar = process.argv[2];
+const file = process.argv[3];
+
+const scope = JSON.parse(fs.readFileSync(grammar, {encoding: 'UTF-8'})).scopeName;
+
+/**
+ * Utility to read a file as a promise
+ */
+function readFile(path) {
+    return new Promise((resolve, reject) => {
+        fs.readFile(path, (error, data) => error ? reject(error) : resolve(data));
+    })
+}
+
+// Create a registry that can create a grammar from a scope name.
+const registry = new vsctm.Registry({
+    loadGrammar: (scopeName) => {
+        if (scopeName === scope) {
+            return readFile(grammar).then(data => vsctm.parseRawGrammar(data.toString(), grammar))
+        }
+        console.log(`Unknown scope name: ${scopeName}`);
+        return null;
+    }
+});
+
+// Load the JavaScript grammar and any other grammars included by it async.
+registry.loadGrammar(scope).then(grammar => {
+    const text = fs.readFileSync(file, {encoding: 'UTF-8'}).trimEnd('\n').split(/\n/);
+    let ruleStack = vsctm.INITIAL;
+    for (let i = 0; i < text.length; i++) {
+        const line = text[i];
+        const lineTokens = grammar.tokenizeLine(line, ruleStack);
+        console.log(`\nTokenizing line: ${line}`);
+        for (let j = 0; j < lineTokens.tokens.length; j++) {
+            const token = lineTokens.tokens[j];
+            console.log(` - token from ${token.startIndex} to ${token.endIndex} ` +
+              `(${line.substring(token.startIndex, token.endIndex)}) ` +
+              `with scopes ${token.scopes.join(', ')}`
+            );
+        }
+        ruleStack = lineTokens.ruleStack;
+    }
+});
--- a/tests/color_test.py
+++ b/tests/color_test.py
@@ -0,0 +1,16 @@
+import pytest
+
+from babi.color import Color
+
+
+@pytest.mark.parametrize(
+    ('s', 'expected'),
+    (
+        ('#1e77d3', Color(0x1e, 0x77, 0xd3)),
+        ('white', Color(0xff, 0xff, 0xff)),
+        ('black', Color(0x00, 0x00, 0x00)),
+        ('#ccc', Color(0xcc, 0xcc, 0xcc)),
+    ),
+)
+def test_color_parse(s, expected):
+    assert Color.parse(s) == expected
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -0,0 +1,17 @@
+import json
+
+import pytest
+
+from babi.highlight import Grammars
+
+
+@pytest.fixture
+def make_grammars(tmpdir):
+    grammar_dir = tmpdir.join('grammars').ensure_dir()
+
+    def make_grammars(*grammar_dcts):
+        for grammar in grammar_dcts:
+            filename = f'{grammar["scopeName"]}.json'
+            grammar_dir.join(filename).write(json.dumps(grammar))
+        return Grammars(grammar_dir)
+    return make_grammars
--- a/tests/features/conftest.py
+++ b/tests/features/conftest.py
@@ -16,6 +16,13 @@ from babi.screen import VERSION_STR
 from testing.runner import PrintsErrorRunner


+@pytest.fixture(autouse=True)
+def prefix_home(tmpdir):
+    prefix_home = tmpdir.join('prefix_home')
+    with mock.patch.object(sys, 'prefix', str(prefix_home)):
+        yield prefix_home
+
+
@pytest.fixture(autouse=True)
 def xdg_data_home(tmpdir):
    data_home = tmpdir.join('data_home')
@@ -66,6 +73,7 @@ class Screen:
        self.attrs[y] = (line_attr[:x] + new + line_attr[x:])[:self.width]

    def chgat(self, y, x, n, attr):
+        assert n >= 0  # TODO: switch to > 0, we should never do 0-length
        self.attrs[y][x:x + n] = [attr] * n

    def move(self, y, x):
@@ -254,6 +262,7 @@ KEYS = [
    Key('^E', b'^E', '\x05'),
    Key('^J', b'^J', '\n'),
    Key('^O', b'^O', '\x0f'),
+    Key('^P', b'^P', '\x10'),
    Key('^R', b'^R', '\x12'),
    Key('^S', b'^S', '\x13'),
    Key('^U', b'^U', '\x15'),
@@ -266,7 +275,7 @@ KEYS = [
    Key('^\\', b'^\\', '\x1c'),
    Key('!resize', b'KEY_RESIZE', curses.KEY_RESIZE),
 ]
-KEYS_TMUX = {k.tmux: k.value for k in KEYS}
+KEYS_TMUX = {k.tmux: k.wch for k in KEYS}
 KEYS_CURSES = {k.value: k.curses for k in KEYS}


@@ -297,7 +306,7 @@ class DeferredRunner:
        print(f'KEY: {keypress_event.wch!r}')
        return keypress_event.wch

-    def await_text(self, text):
+    def await_text(self, text, timeout=1):
        self._ops.append(AwaitText(text))

    def await_text_missing(self, text):
--- a/tests/features/key_debug_test.py
+++ b/tests/features/key_debug_test.py
@@ -0,0 +1,22 @@
+import curses
+
+from babi.screen import VERSION_STR
+
+
+def test_key_debug(run):
+    with run('--key-debug') as h:
+        h.await_text(VERSION_STR, timeout=2)
+
+        h.await_text('press q to quit')
+
+        h.press('a')
+        h.await_text("'a' 'STRING'")
+
+        h.press('^X')
+        h.await_text(r"'\x18' '^X'")
+
+        with h.resize(width=20, height=20):
+            h.await_text(f"{curses.KEY_RESIZE} 'KEY_RESIZE'")
+
+        h.press('q')
+        h.await_exit()
--- a/tests/features/multiple_files_test.py
+++ b/tests/features/multiple_files_test.py
@@ -1,10 +1,19 @@
-def test_multiple_files(run, tmpdir):
+import pytest
+
+
+@pytest.fixture
+def abc(tmpdir):
    a = tmpdir.join('file_a')
    a.write('a text')
    b = tmpdir.join('file_b')
    b.write('b text')
    c = tmpdir.join('file_c')
    c.write('c text')
+    yield a, b, c
+
+
+def test_multiple_files(run, abc):
+    a, b, c = abc

    with run(str(a), str(b), str(c)) as h:
        h.await_text('file_a')
@@ -44,9 +53,36 @@ def test_multiple_files(run, tmpdir):
        h.press('^J')
        h.await_text('unknown key')
        h.press('^X')
-        h.await_text('file_a')
+        h.await_text('file_b')
        h.await_text_missing('unknown key')
+        h.press('^X')
+        h.await_text('file_a')
+        h.press('^X')
+        h.await_exit()
+
+
+def test_multiple_files_close_from_beginning(run, abc):
+    a, b, c = abc
+
+    with run(str(a), str(b), str(c)) as h:
        h.press('^X')
        h.await_text('file_b')
        h.press('^X')
+        h.await_text('file_c')
+        h.press('^X')
+        h.await_exit()
+
+
+def test_multiple_files_close_from_end(run, abc):
+    a, b, c = abc
+
+    with run(str(a), str(b), str(c)) as h:
+        h.press('M-Right')
+        h.await_text('file_b')
+
+        h.press('^X')
+        h.await_text('file_c')
+        h.press('^X')
+        h.await_text('file_a')
+        h.press('^X')
        h.await_exit()
--- a/tests/features/open_test.py
+++ b/tests/features/open_test.py
@@ -0,0 +1,38 @@
+from testing.runner import and_exit
+
+
+def test_open_cancelled(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('hello world')
+
+    with run(str(f)) as h, and_exit(h):
+        h.await_text('hello world')
+
+        h.press('^P')
+        h.await_text('enter filename:')
+        h.press('^C')
+
+        h.await_text('cancelled')
+        h.await_text('hello world')
+
+
+def test_open(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('hello world')
+    g = tmpdir.join('g')
+    g.write('goodbye world')
+
+    with run(str(f)) as h:
+        h.await_text('hello world')
+
+        h.press('^P')
+        h.press_and_enter(str(g))
+
+        h.await_text('[2/2]')
+        h.await_text('goodbye world')
+
+        h.press('^X')
+        h.await_text('hello world')
+
+        h.press('^X')
+        h.await_exit()
--- a/tests/features/replace_test.py
+++ b/tests/features/replace_test.py
@@ -37,7 +37,7 @@ def test_replace_actual_contents(run, ten_lines):
        h.press_and_enter('line_0')
        h.await_text('replace with:')
        h.press_and_enter('ohai')
-        h.await_text('replace [y(es), n(o), a(ll)]?')
+        h.await_text('replace [yes, no, all]?')
        h.press('y')
        h.await_text_missing('line_0')
        h.await_text('ohai')
@@ -59,7 +59,7 @@ match me!
        h.press_and_enter('me!')
        h.await_text('replace with:')
        h.press_and_enter('youuuu')
-        h.await_text('replace [y(es), n(o), a(ll)]?')
+        h.await_text('replace [yes, no, all]?')
        h.press('y')
        h.await_cursor_position(x=6, y=3)
        h.press('Up')
@@ -74,7 +74,7 @@ def test_replace_cancel_at_individual_replace(run, ten_lines):
        h.press_and_enter(r'line_\d')
        h.await_text('replace with:')
        h.press_and_enter('ohai')
-        h.await_text('replace [y(es), n(o), a(ll)]?')
+        h.await_text('replace [yes, no, all]?')
        h.press('^C')
        h.await_text('cancelled')

@@ -86,7 +86,7 @@ def test_replace_unknown_characters_at_individual_replace(run, ten_lines):
        h.press_and_enter(r'line_\d')
        h.await_text('replace with:')
        h.press_and_enter('ohai')
-        h.await_text('replace [y(es), n(o), a(ll)]?')
+        h.await_text('replace [yes, no, all]?')
        h.press('?')
        h.press('^C')
        h.await_text('cancelled')
@@ -99,7 +99,7 @@ def test_replace_say_no_to_individual_replace(run, ten_lines):
        h.press_and_enter('line_[135]')
        h.await_text('replace with:')
        h.press_and_enter('ohai')
-        h.await_text('replace [y(es), n(o), a(ll)]?')
+        h.await_text('replace [yes, no, all]?')
        h.press('y')
        h.await_text_missing('line_1')
        h.press('n')
@@ -116,7 +116,7 @@ def test_replace_all(run, ten_lines):
        h.press_and_enter(r'line_(\d)')
        h.await_text('replace with:')
        h.press_and_enter(r'ohai+\1')
-        h.await_text('replace [y(es), n(o), a(ll)]?')
+        h.await_text('replace [yes, no, all]?')
        h.press('a')
        h.await_text_missing('line')
        h.await_text('ohai+1')
@@ -130,7 +130,7 @@ def test_replace_with_empty_string(run, ten_lines):
        h.press_and_enter('line_1')
        h.await_text('replace with:')
        h.press('Enter')
-        h.await_text('replace [y(es), n(o), a(ll)]?')
+        h.await_text('replace [yes, no, all]?')
        h.press('y')
        h.await_text_missing('line_1')

@@ -153,7 +153,7 @@ def test_replace_small_window_size(run, ten_lines):
        h.press_and_enter('line')
        h.await_text('replace with:')
        h.press_and_enter('wat')
-        h.await_text('replace [y(es), n(o), a(ll)]?')
+        h.await_text('replace [yes, no, all]?')

        with h.resize(width=8, height=24):
            h.await_text('replace…')
@@ -170,7 +170,7 @@ def test_replace_height_1_highlight(run, tmpdir):
        h.press_and_enter('^x+$')
        h.await_text('replace with:')
        h.press('Enter')
-        h.await_text('replace [y(es), n(o), a(ll)]?')
+        h.await_text('replace [yes, no, all]?')

        with h.resize(width=80, height=1):
            h.await_text_missing('xxxxx')
@@ -189,7 +189,7 @@ def test_replace_line_goes_off_screen(run):
        h.press_and_enter('b+')
        h.await_text('replace with:')
        h.press_and_enter('wat')
-        h.await_text('replace [y(es), n(o), a(ll)]?')
+        h.await_text('replace [yes, no, all]?')
        h.await_text(f'{"a" * 20}{"b" * 59}»')
        h.press('y')
        h.await_text(f'{"a" * 20}wat')
@@ -221,7 +221,7 @@ def test_replace_multiple_occurrences_in_line(run):
        h.press_and_enter('a+')
        h.await_text('replace with:')
        h.press_and_enter('q')
-        h.await_text('replace [y(es), n(o), a(ll)]?')
+        h.await_text('replace [yes, no, all]?')
        h.press('a')
        h.await_text('bqbq')

@@ -234,7 +234,7 @@ def test_replace_after_wrapping(run, ten_lines):
        h.press_and_enter('line_[02]')
        h.await_text('replace with:')
        h.press_and_enter('ohai')
-        h.await_text('replace [y(es), n(o), a(ll)]?')
+        h.await_text('replace [yes, no, all]?')
        h.press('y')
        h.await_text_missing('line_2')
        h.press('y')
@@ -251,7 +251,7 @@ def test_replace_after_cursor_after_wrapping(run):
        h.press_and_enter('b')
        h.await_text('replace with:')
        h.press_and_enter('q')
-        h.await_text('replace [y(es), n(o), a(ll)]?')
+        h.await_text('replace [yes, no, all]?')
        h.press('n')
        h.press('y')
        h.await_text('replaced 1 occurrence')
@@ -267,8 +267,36 @@ def test_replace_separate_line_after_wrapping(run, ten_lines):
        h.press_and_enter('line_[01]')
        h.await_text('replace with:')
        h.press_and_enter('_')
-        h.await_text('replace [y(es), n(o), a(ll)]?')
+        h.await_text('replace [yes, no, all]?')
        h.press('y')
        h.await_text_missing('line_0')
        h.press('y')
        h.await_text_missing('line_1')
+
+
+def test_replace_with_newline_characters(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('^\\')
+        h.await_text('search (to replace):')
+        h.press_and_enter('(line)_([01])')
+        h.await_text('replace with:')
+        h.press_and_enter(r'\1\n\2')
+        h.await_text('replace [yes, no, all]?')
+        h.press('a')
+        h.await_text_missing('line_0')
+        h.await_text_missing('line_1')
+        h.await_text('line\n0\nline\n1\n')
+
+
+def test_replace_with_multiple_newline_characters(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('^\\')
+        h.await_text('search (to replace):')
+        h.press_and_enter('(li)(ne)_(1)')
+        h.await_text('replace with:')
+        h.press_and_enter(r'\1\n\2\n\3\n')
+        h.await_text('replace [yes, no, all]?')
+        h.press('a')
+
+        h.await_text_missing('line_1')
+        h.await_text('li\nne\n1\n\nline_2')
--- a/tests/features/save_test.py
+++ b/tests/features/save_test.py
@@ -12,6 +12,19 @@ def test_mixed_newlines(run, tmpdir):
        h.await_text(r"mixed newlines will be converted to '\n'")


+def test_modify_file_with_windows_newlines(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write_binary(b'foo\r\nbar\r\n')
+    with run(str(f)) as h, and_exit(h):
+        # should not start modified
+        h.await_text_missing('*')
+        h.press('Enter')
+        h.await_text('*')
+        h.press('^S')
+        h.await_text('saved!')
+    assert f.read_binary() == b'\r\nfoo\r\nbar\r\n'
+
+
 def test_new_file(run):
    with run('this_is_a_new_file') as h, and_exit(h):
        h.await_text('this_is_a_new_file')
@@ -148,7 +161,7 @@ def test_save_on_exit_cancel_yn(run):
        h.press('hello')
        h.await_text('hello')
        h.press('^X')
-        h.await_text('file is modified - save [y(es), n(o)]?')
+        h.await_text('file is modified - save [yes, no]?')
        h.press('^C')
        h.await_text('cancelled')

@@ -158,7 +171,7 @@ def test_save_on_exit_cancel_filename(run):
        h.press('hello')
        h.await_text('hello')
        h.press('^X')
-        h.await_text('file is modified - save [y(es), n(o)]?')
+        h.await_text('file is modified - save [yes, no]?')
        h.press('y')
        h.await_text('enter filename:')
        h.press('^C')
@@ -171,7 +184,7 @@ def test_save_on_exit(run, tmpdir):
        h.press('hello')
        h.await_text('hello')
        h.press('^X')
-        h.await_text('file is modified - save [y(es), n(o)]?')
+        h.await_text('file is modified - save [yes, no]?')
        h.press('y')
        h.await_text(f'enter filename: {f}')
        h.press('Enter')
@@ -183,9 +196,9 @@ def test_save_on_exit_resize(run, tmpdir):
        h.press('hello')
        h.await_text('hello')
        h.press('^X')
-        h.await_text('file is modified - save [y(es), n(o)]?')
+        h.await_text('file is modified - save [yes, no]?')
        with h.resize(width=10, height=24):
            h.await_text('file is m…')
-        h.await_text('file is modified - save [y(es), n(o)]?')
+        h.await_text('file is modified - save [yes, no]?')
        h.press('^C')
        h.await_text('cancelled')
--- a/tests/features/stdin_test.py
+++ b/tests/features/stdin_test.py
@@ -0,0 +1,22 @@
+import shlex
+import sys
+
+from babi.screen import VERSION_STR
+from testing.runner import PrintsErrorRunner
+
+
+def test_open_from_stdin():
+    with PrintsErrorRunner('env', 'TERM=screen', 'bash', '--norc') as h:
+        cmd = (sys.executable, '-mcoverage', 'run', '-m', 'babi', '-')
+        babi_cmd = ' '.join(shlex.quote(part) for part in cmd)
+        h.press_and_enter(fr"echo $'hello\nworld' | {babi_cmd}")
+
+        h.await_text(VERSION_STR, timeout=2)
+        h.await_text('<<new file>> *')
+        h.await_text('hello\nworld')
+
+        h.press('^X')
+        h.press('n')
+        h.await_text_missing('<<new file>>')
+        h.press_and_enter('exit')
+        h.await_exit()
--- a/tests/features/syntax_highlight_test.py
+++ b/tests/features/syntax_highlight_test.py
@@ -15,6 +15,7 @@ THEME = json.dumps({
            'settings': {'foreground': '#5f0000', 'background': '#ff5f5f'},
        },
        {'scope': 'tqs', 'settings': {'foreground': '#00005f'}},
+        {'scope': 'qmark', 'settings': {'foreground': '#5f0000'}},
        {'scope': 'b', 'settings': {'fontStyle': 'bold'}},
        {'scope': 'i', 'settings': {'fontStyle': 'italic'}},
        {'scope': 'u', 'settings': {'fontStyle': 'underline'}},
@@ -28,6 +29,7 @@ SYNTAX = json.dumps({
        {'match': r'#.*$\n?', 'name': 'comment'},
        {'match': r'^-.*$\n?', 'name': 'diffremove'},
        {'begin': '"""', 'end': '"""', 'name': 'tqs'},
+        {'match': r'\?', 'name': 'qmark'},
    ],
 })
 DEMO_S = '''\
@@ -43,7 +45,7 @@ still more
@pytest.fixture(autouse=True)
 def theme_and_grammar(xdg_data_home, xdg_config_home):
    xdg_config_home.join('babi/theme.json').ensure().write(THEME)
-    xdg_data_home.join('babi/textmate_syntax/demo.json').ensure().write(SYNTAX)
+    xdg_data_home.join('babi/grammar_v1/demo.json').ensure().write(SYNTAX)


@pytest.fixture
@@ -78,3 +80,36 @@ def test_syntax_highlighting_does_not_highlight_arrows(run, tmpdir):
    with run(str(f), term='screen-256color', width=20) as h, and_exit(h):
        h.await_text('loooo')
        h.assert_screen_attr_equals(2, [(243, 40, 0)] * 19 + [(236, 40, 0)])
+
+        h.press('Down')
+        h.press('^E')
+        h.await_text_missing('loooo')
+        expected = [(236, 40, 0)] + [(243, 40, 0)] * 15 + [(236, 40, 0)] * 4
+        h.assert_screen_attr_equals(2, expected)
+
+
+def test_syntax_highlighting_off_screen_does_not_crash(run, tmpdir):
+    f = tmpdir.join('f.demo')
+    f.write(f'"""a"""{"x" * 40}"""b"""')
+
+    with run(str(f), term='screen-256color', width=20) as h, and_exit(h):
+        h.await_text('"""a"""')
+        h.assert_screen_attr_equals(1, [(17, 40, 0)] * 7 + [(236, 40, 0)] * 13)
+        h.press('^E')
+        h.await_text('"""b"""')
+        expected = [(236, 40, 0)] * 11 + [(17, 40, 0)] * 7 + [(236, 40, 0)] * 2
+        h.assert_screen_attr_equals(1, expected)
+
+
+def test_syntax_highlighting_one_off_left_of_screen(run, tmpdir):
+    f = tmpdir.join('f.demo')
+    f.write(f'{"x" * 11}?123456789')
+
+    with run(str(f), term='screen-256color', width=20) as h, and_exit(h):
+        h.await_text('xxx?123')
+        expected = [(236, 40, 0)] * 11 + [(52, 40, 0)] + [(236, 40, 0)] * 8
+        h.assert_screen_attr_equals(1, expected)
+
+        h.press('End')
+        h.await_text_missing('?')
+        h.assert_screen_attr_equals(1, [(236, 40, 0)] * 20)
--- a/tests/features/undo_redo_test.py
+++ b/tests/features/undo_redo_test.py
@@ -127,3 +127,16 @@ def test_undo_redo_causes_scroll(run):
        h.await_cursor_position(x=0, y=1)
        h.press('M-U')
        h.await_cursor_position(x=0, y=4)
+
+
+def test_undo_redo_clears_selection(run, ten_lines):
+    # maintaining the selection across undo/redo is both difficult and not all
+    # that useful.  prior to this it was buggy anyway (a negative selection
+    # indented and then undone would highlight out of bounds)
+    with run(str(ten_lines), width=20) as h, and_exit(h):
+        h.press('S-Down')
+        h.press('Tab')
+        h.await_cursor_position(x=4, y=2)
+        h.press('M-u')
+        h.await_cursor_position(x=0, y=2)
+        h.assert_screen_attr_equals(1, [(-1, -1, 0)] * 20)
--- a/tests/file_test.py
+++ b/tests/file_test.py
@@ -2,12 +2,13 @@ import io

 import pytest

+from babi.color_manager import ColorManager
 from babi.file import File
 from babi.file import get_lines


 def test_position_repr():
-    ret = repr(File('f.txt', ()))
+    ret = repr(File('f.txt', ColorManager.make(), ()))
    assert ret == "<File 'f.txt'>"


--- a/tests/highlight_test.py
+++ b/tests/highlight_test.py
@@ -1,22 +1,37 @@
-from babi.highlight import Grammar
-from babi.highlight import Grammars
+import pytest
+
 from babi.highlight import highlight_line
 from babi.highlight import Region


-def _compiler_state(grammar_dct, *others):
-    grammar = Grammar.from_data(grammar_dct)
-    grammars = [grammar, *(Grammar.from_data(dct) for dct in others)]
-    compiler = Grammars(grammars).compiler_for_scope(grammar.scope_name)
-    return compiler, compiler.root_state
+def test_grammar_matches_extension_only_name(make_grammars):
+    data = {'scopeName': 'shell', 'patterns': [], 'fileTypes': ['bashrc']}
+    grammars = make_grammars(data)
+    compiler = grammars.compiler_for_file('.bashrc', 'alias nano=babi')
+    assert compiler.root_state.entries[0].scope[0] == 'shell'


-def test_backslash_a():
+def test_grammar_matches_via_identify_tag(make_grammars):
+    grammars = make_grammars({'scopeName': 'source.ini', 'patterns': []})
+    compiler = grammars.compiler_for_file('setup.cfg', '')
+    assert compiler.root_state.entries[0].scope[0] == 'source.ini'
+
+
+@pytest.fixture
+def compiler_state(make_grammars):
+    def _compiler_state(*grammar_dcts):
+        grammars = make_grammars(*grammar_dcts)
+        compiler = grammars.compiler_for_scope(grammar_dcts[0]['scopeName'])
+        return compiler, compiler.root_state
+    return _compiler_state
+
+
+def test_backslash_a(compiler_state):
    grammar = {
        'scopeName': 'test',
        'patterns': [{'name': 'aaa', 'match': r'\Aa+'}],
    }
-    compiler, state = _compiler_state(grammar)
+    compiler, state = compiler_state(grammar)

    state, (region_0,) = highlight_line(compiler, state, 'aaa', True)
    state, (region_1,) = highlight_line(compiler, state, 'aaa', False)
@@ -39,8 +54,8 @@ BEGIN_END_NO_NL = {
 }


-def test_backslash_g_inline():
-    compiler, state = _compiler_state(BEGIN_END_NO_NL)
+def test_backslash_g_inline(compiler_state):
+    compiler, state = compiler_state(BEGIN_END_NO_NL)

    _, regions = highlight_line(compiler, state, 'xaax', True)
    assert regions == (
@@ -51,8 +66,8 @@ def test_backslash_g_inline():
    )


-def test_backslash_g_next_line():
-    compiler, state = _compiler_state(BEGIN_END_NO_NL)
+def test_backslash_g_next_line(compiler_state):
+    compiler, state = compiler_state(BEGIN_END_NO_NL)

    state, regions1 = highlight_line(compiler, state, 'x\n', True)
    state, regions2 = highlight_line(compiler, state, 'aax\n', False)
@@ -69,8 +84,8 @@ def test_backslash_g_next_line():
    )


-def test_end_before_other_match():
-    compiler, state = _compiler_state(BEGIN_END_NO_NL)
+def test_end_before_other_match(compiler_state):
+    compiler, state = compiler_state(BEGIN_END_NO_NL)

    state, regions = highlight_line(compiler, state, 'xazzx', True)

@@ -95,8 +110,8 @@ BEGIN_END_NL = {
 }


-def test_backslash_g_captures_nl():
-    compiler, state = _compiler_state(BEGIN_END_NL)
+def test_backslash_g_captures_nl(compiler_state):
+    compiler, state = compiler_state(BEGIN_END_NL)

    state, regions1 = highlight_line(compiler, state, 'x\n', True)
    state, regions2 = highlight_line(compiler, state, 'aax\n', False)
@@ -112,8 +127,8 @@ def test_backslash_g_captures_nl():
    )


-def test_backslash_g_captures_nl_next_line():
-    compiler, state = _compiler_state(BEGIN_END_NL)
+def test_backslash_g_captures_nl_next_line(compiler_state):
+    compiler, state = compiler_state(BEGIN_END_NL)

    state, regions1 = highlight_line(compiler, state, 'x\n', True)
    state, regions2 = highlight_line(compiler, state, 'aa\n', False)
@@ -135,8 +150,8 @@ def test_backslash_g_captures_nl_next_line():
    )


-def test_while_no_nl():
-    compiler, state = _compiler_state({
+def test_while_no_nl(compiler_state):
+    compiler, state = compiler_state({
        'scopeName': 'test',
        'patterns': [{
            'begin': '> ',
@@ -170,8 +185,8 @@ def test_while_no_nl():
    )


-def test_complex_captures():
-    compiler, state = _compiler_state({
+def test_complex_captures(compiler_state):
+    compiler, state = compiler_state({
        'scopeName': 'test',
        'patterns': [
            {
@@ -201,8 +216,8 @@ def test_complex_captures():
    )


-def test_captures_multiple_applied_to_same_capture():
-    compiler, state = _compiler_state({
+def test_captures_multiple_applied_to_same_capture(compiler_state):
+    compiler, state = compiler_state({
        'scopeName': 'test',
        'patterns': [
            {
@@ -244,8 +259,8 @@ def test_captures_multiple_applied_to_same_capture():
    )


-def test_captures_ignores_empty():
-    compiler, state = _compiler_state({
+def test_captures_ignores_empty(compiler_state):
+    compiler, state = compiler_state({
        'scopeName': 'test',
        'patterns': [{
            'match': '(.*) hi',
@@ -267,8 +282,8 @@ def test_captures_ignores_empty():
    )


-def test_captures_ignores_invalid_out_of_bounds():
-    compiler, state = _compiler_state({
+def test_captures_ignores_invalid_out_of_bounds(compiler_state):
+    compiler, state = compiler_state({
        'scopeName': 'test',
        'patterns': [{'match': '.', 'captures': {'1': {'name': 'oob'}}}],
    })
@@ -280,8 +295,8 @@ def test_captures_ignores_invalid_out_of_bounds():
    )


-def test_captures_begin_end():
-    compiler, state = _compiler_state({
+def test_captures_begin_end(compiler_state):
+    compiler, state = compiler_state({
        'scopeName': 'test',
        'patterns': [
            {
@@ -302,8 +317,8 @@ def test_captures_begin_end():
    )


-def test_captures_while_captures():
-    compiler, state = _compiler_state({
+def test_captures_while_captures(compiler_state):
+    compiler, state = compiler_state({
        'scopeName': 'test',
        'patterns': [
            {
@@ -331,8 +346,8 @@ def test_captures_while_captures():
    )


-def test_captures_implies_begin_end_captures():
-    compiler, state = _compiler_state({
+def test_captures_implies_begin_end_captures(compiler_state):
+    compiler, state = compiler_state({
        'scopeName': 'test',
        'patterns': [
            {
@@ -352,8 +367,8 @@ def test_captures_implies_begin_end_captures():
    )


-def test_captures_implies_begin_while_captures():
-    compiler, state = _compiler_state({
+def test_captures_implies_begin_while_captures(compiler_state):
+    compiler, state = compiler_state({
        'scopeName': 'test',
        'patterns': [
            {
@@ -380,8 +395,8 @@ def test_captures_implies_begin_while_captures():
    )


-def test_include_self():
-    compiler, state = _compiler_state({
+def test_include_self(compiler_state):
+    compiler, state = compiler_state({
        'scopeName': 'test',
        'patterns': [
            {
@@ -404,8 +419,8 @@ def test_include_self():
    )


-def test_include_repository_rule():
-    compiler, state = _compiler_state({
+def test_include_repository_rule(compiler_state):
+    compiler, state = compiler_state({
        'scopeName': 'test',
        'patterns': [{'include': '#impl'}],
        'repository': {
@@ -426,8 +441,8 @@ def test_include_repository_rule():
    )


-def test_include_other_grammar():
-    compiler, state = _compiler_state(
+def test_include_other_grammar(compiler_state):
+    compiler, state = compiler_state(
        {
            'scopeName': 'test',
            'patterns': [
@@ -482,8 +497,8 @@ def test_include_other_grammar():
    )


-def test_include_base():
-    compiler, state = _compiler_state(
+def test_include_base(compiler_state):
+    compiler, state = compiler_state(
        {
            'scopeName': 'test',
            'patterns': [
@@ -528,3 +543,65 @@ def test_include_base():
        Region(2, 3, ('test', 'tick')),
        Region(3, 4, ('test',)),
    )
+
+
+def test_rule_with_begin_and_no_end(compiler_state):
+    compiler, state = compiler_state({
+        'scopeName': 'test',
+        'patterns': [
+            {
+                'begin': '!', 'end': '!', 'name': 'bang',
+                'patterns': [{'begin': '--', 'name': 'invalid'}],
+            },
+        ],
+    })
+
+    state, regions = highlight_line(compiler, state, '!x! !--!', True)
+
+    assert regions == (
+        Region(0, 1, ('test', 'bang')),
+        Region(1, 2, ('test', 'bang')),
+        Region(2, 3, ('test', 'bang')),
+        Region(3, 4, ('test',)),
+        Region(4, 5, ('test', 'bang')),
+        Region(5, 7, ('test', 'bang', 'invalid')),
+        Region(7, 8, ('test', 'bang', 'invalid')),
+    )
+
+
+def test_begin_end_substitute_special_chars(compiler_state):
+    compiler, state = compiler_state({
+        'scopeName': 'test',
+        'patterns': [{'begin': r'(\*)', 'end': r'\1', 'name': 'italic'}],
+    })
+
+    state, regions = highlight_line(compiler, state, '*italic*', True)
+
+    assert regions == (
+        Region(0, 1, ('test', 'italic')),
+        Region(1, 7, ('test', 'italic')),
+        Region(7, 8, ('test', 'italic')),
+    )
+
+
+def test_backslash_z(compiler_state):
+    # similar to text.git-commit grammar, \z matches nothing!
+    compiler, state = compiler_state({
+        'scopeName': 'test',
+        'patterns': [
+            {'begin': '#', 'end': r'\z', 'name': 'comment'},
+            {'name': 'other', 'match': '.'},
+        ],
+    })
+
+    state, regions1 = highlight_line(compiler, state, '# comment', True)
+    state, regions2 = highlight_line(compiler, state, 'other?', False)
+
+    assert regions1 == (
+        Region(0, 1, ('test', 'comment')),
+        Region(1, 9, ('test', 'comment')),
+    )
+
+    assert regions2 == (
+        Region(0, 6, ('test', 'comment')),
+    )
--- a/tests/hl/syntax_test.py
+++ b/tests/hl/syntax_test.py
@@ -5,8 +5,7 @@ from unittest import mock
 import pytest

 from babi.color_manager import ColorManager
-from babi.highlight import Grammar
-from babi.highlight import Grammars
+from babi.hl.interface import HL
 from babi.hl.syntax import Syntax
 from babi.theme import Color
 from babi.theme import Theme
@@ -72,18 +71,23 @@ THEME = Theme.from_dct({


@pytest.fixture
-def syntax():
-    return Syntax(Grammars([Grammar.blank()]), THEME, ColorManager.make())
+def syntax(make_grammars):
+    return Syntax(make_grammars(), THEME, ColorManager.make())


 def test_init_screen_low_color(stdscr, syntax):
    with FakeCurses.patch(n_colors=16, can_change_color=False) as fake_curses:
        syntax._init_screen(stdscr)
-    assert syntax.color_manager.colors == {}
-    assert syntax.color_manager.raw_pairs == {}
+    assert syntax.color_manager.colors == {
+        Color.parse('#cccccc'): -1,
+        Color.parse('#333333'): -1,
+        Color.parse('#000000'): -1,
+        Color.parse('#009900'): -1,
+    }
+    assert syntax.color_manager.raw_pairs == {(-1, -1): 1}
    assert fake_curses.colors == {}
-    assert fake_curses.pairs == {}
-    assert stdscr.attr == 0
+    assert fake_curses.pairs == {1: (-1, -1)}
+    assert stdscr.attr == 1 << 8


 def test_init_screen_256_color(stdscr, syntax):
@@ -131,7 +135,7 @@ def test_lazily_instantiated_pairs(stdscr, syntax):
        assert len(fake_curses.pairs) == 1

        style = THEME.select(('string.python',))
-        attr = syntax.get_blank_file_highlighter().attr(style)
+        attr = syntax.blank_file_highlighter().attr(style)
        assert attr == 2 << 8

        assert len(syntax.color_manager.raw_pairs) == 2
@@ -143,5 +147,22 @@ def test_style_attributes_applied(stdscr, syntax):
        syntax._init_screen(stdscr)

        style = THEME.select(('keyword.python',))
-        attr = syntax.get_blank_file_highlighter().attr(style)
+        attr = syntax.blank_file_highlighter().attr(style)
        assert attr == 2 << 8 | curses.A_BOLD
+
+
+def test_syntax_highlight_cache_first_line(stdscr, make_grammars):
+    with FakeCurses.patch(n_colors=256, can_change_color=False):
+        grammars = make_grammars({
+            'scopeName': 'source.demo',
+            'fileTypes': ['demo'],
+            'patterns': [{'match': r'\Aint', 'name': 'keyword'}],
+        })
+        syntax = Syntax(grammars, THEME, ColorManager.make())
+        syntax._init_screen(stdscr)
+        file_hl = syntax.file_highlighter('foo.demo', '')
+        file_hl.highlight_until(['int', 'int'], 2)
+        assert file_hl.regions == [
+            (HL(0, 3, curses.A_BOLD | 2 << 8),),
+            (),
+        ]
--- a/tests/textmate_demo_test.py
+++ b/tests/textmate_demo_test.py
@@ -0,0 +1,84 @@
+import json
+
+import pytest
+
+from babi.textmate_demo import main
+
+THEME = {
+    'colors': {'foreground': '#ffffff', 'background': '#000000'},
+    'tokenColors': [
+        {'scope': 'bold', 'settings': {'fontStyle': 'bold'}},
+        {'scope': 'italic', 'settings': {'fontStyle': 'italic'}},
+        {'scope': 'underline', 'settings': {'fontStyle': 'underline'}},
+        {'scope': 'comment', 'settings': {'foreground': '#1e77d3'}},
+    ],
+}
+
+GRAMMAR = {
+    'scopeName': 'source.demo',
+    'fileTypes': ['demo'],
+    'patterns': [
+        {'match': r'\*[^*]*\*', 'name': 'bold'},
+        {'match': '/[^/]*/', 'name': 'italic'},
+        {'match': '_[^_]*_', 'name': 'underline'},
+        {'match': '#.*', 'name': 'comment'},
+    ],
+}
+
+
+@pytest.fixture
+def theme_grammars(tmpdir):
+    theme = tmpdir.join('config/theme.json').ensure()
+    theme.write(json.dumps(THEME))
+    grammars = tmpdir.join('grammar_v1').ensure_dir()
+    grammars.join('source.demo.json').write(json.dumps(GRAMMAR))
+    return theme, grammars
+
+
+def test_basic(theme_grammars, tmpdir, capsys):
+    theme, grammars = theme_grammars
+
+    f = tmpdir.join('f.demo')
+    f.write('*bold*/italic/_underline_# comment\n')
+
+    assert not main((
+        '--theme', str(theme), '--grammar-dir', str(grammars),
+        str(f),
+    ))
+
+    out, _ = capsys.readouterr()
+
+    assert out == (
+        '\x1b[48;2;0;0;0m\n'
+        '\x1b[38;2;255;255;255m\x1b[48;2;0;0;0m\x1b[1m'
+        '*bold*'
+        '\x1b[39m\x1b[49m\x1b[22m'
+        '\x1b[38;2;255;255;255m\x1b[48;2;0;0;0m\x1b[3m'
+        '/italic/'
+        '\x1b[39m\x1b[49m\x1b[23m'
+        '\x1b[38;2;255;255;255m\x1b[48;2;0;0;0m\x1b[4m'
+        '_underline_'
+        '\x1b[39m\x1b[49m\x1b[24m'
+        '\x1b[38;2;30;119;211m\x1b[48;2;0;0;0m'
+        '# comment'
+        '\x1b[39m\x1b[49m\x1b'
+        '[38;2;255;255;255m\x1b[48;2;0;0;0m\n\x1b[39m\x1b[49m'
+        '\x1b[m'
+    )
+
+
+def test_basic_with_blank_theme(theme_grammars, tmpdir, capsys):
+    theme, grammars = theme_grammars
+    theme.write('{}')
+
+    f = tmpdir.join('f.demo')
+    f.write('*bold*/italic/_underline_# comment\n')
+
+    assert not main((
+        '--theme', str(theme), '--grammar-dir', str(grammars),
+        str(f),
+    ))
+
+    out, _ = capsys.readouterr()
+
+    assert out == '*bold*/italic/_underline_# comment\n\x1b[m'
--- a/tests/theme_test.py
+++ b/tests/theme_test.py
@@ -72,6 +72,31 @@ def test_theme_scope_split_by_commas():
    assert theme.select(('c',)).i is True


+def test_theme_scope_comma_at_beginning_and_end():
+    theme = Theme.from_dct({
+        'colors': {'foreground': '#cccccc', 'background': '#333333'},
+        'tokenColors': [
+            {'scope': '\n,a,b,\n', 'settings': {'fontStyle': 'italic'}},
+        ],
+    })
+    assert theme.select(('d',)).i is False
+    assert theme.select(('a',)).i is True
+    assert theme.select(('b',)).i is True
+
+
+def test_theme_scope_internal_newline_commas():
+    # this is arguably malformed, but `cobalt2` in the wild has this issue
+    theme = Theme.from_dct({
+        'colors': {'foreground': '#cccccc', 'background': '#333333'},
+        'tokenColors': [
+            {'scope': '\n,a,\n,b,\n', 'settings': {'fontStyle': 'italic'}},
+        ],
+    })
+    assert theme.select(('d',)).i is False
+    assert theme.select(('a',)).i is True
+    assert theme.select(('b',)).i is True
+
+
 def test_theme_scope_as_A_list():
    theme = Theme.from_dct({
        'colors': {'foreground': '#cccccc', 'background': '#333333'},
Author	SHA1	Message	Date
Anthony Sottile	7850481565	v0.0.4	2020-03-28 08:01:02 -07:00
Anthony Sottile	b536291989	Fix replacing with embedded newline characters Resolves #39	2020-03-27 20:32:43 -07:00
Anthony Sottile	f8737557d3	Add a sample theme to the README	2020-03-27 19:29:52 -07:00
Anthony Sottile	d597b4087d	add dist and build to gitignore	2020-03-27 19:10:11 -07:00
Anthony Sottile	41aa025d3d	Fix edge highlighting for 1-lenght highlights	2020-03-27 19:06:50 -07:00
Anthony Sottile	de956b7bab	fix saving files with windows newlines	2020-03-27 18:42:37 -07:00
Anthony Sottile	1d3d413b93	Fix grammars which include \z	2020-03-27 18:18:16 -07:00
Anthony Sottile	50ad1e06f9	Add demo for showing vs code's tokenization	2020-03-27 17:59:35 -07:00
Anthony Sottile	032c3d78fc	v0.0.3	2020-03-26 20:38:52 -07:00
Anthony Sottile	a197645087	merge the textmate demo into babi	2020-03-26 20:26:57 -07:00
Anthony Sottile	9f8e400d32	switch to babi-grammars for syntax	2020-03-26 19:43:01 -07:00
Anthony Sottile	2123e6ee84	improve performance by ~.8% apparently contextlib.suppress is enough to show up in profiles	2020-03-23 20:57:53 -07:00
Anthony Sottile	b529dde91a	Fix incorrect caching in syntax highlighter the concrete broken case was for markdown with yaml ```md --- x: y --- (this one shouldn't be yaml highlighted) --- x: y --- ```	2020-03-23 20:05:47 -07:00
Anthony Sottile	c4e2f8e9cf	this is unused	2020-03-22 20:12:04 -07:00
Anthony Sottile	e7d4fa1a07	v0.0.2	2020-03-22 19:57:00 -07:00
Anthony Sottile	c186adcc6c	partial windows support	2020-03-22 19:54:52 -07:00
Anthony Sottile	bdf07b8cb3	fix expansion of regexes with regex-special characters	2020-03-22 12:43:34 -07:00
Anthony Sottile	bf1c3d1ee1	Fix highlight color in replace/selection	2020-03-21 21:08:43 -07:00
Anthony Sottile	f1772ec829	Merge pull request #43 from pganssle/add_version Pull version from system metadata	2020-03-21 16:42:08 -07:00
Paul Ganssle	84b489bb9b	Pull version from system metadata	2020-03-21 16:09:20 -07:00
Anthony Sottile	175fd61119	Add secret --key-debug to debug keypresses	2020-03-21 15:57:23 -07:00
Anthony Sottile	01bb6d91b9	add highlighting for makefiles	2020-03-21 15:27:07 -07:00
Anthony Sottile	ffd5c87118	Identify grammars by filename conventions	2020-03-21 15:25:27 -07:00
Anthony Sottile	87f3e32f36	More lazily instanatiate grammars	2020-03-21 14:19:51 -07:00
Anthony Sottile	d20be693d2	Add docker syntax	2020-03-21 11:47:37 -07:00
Anthony Sottile	d826b8b472	bump hecate for @DanielChabrowski's fix	2020-03-19 21:25:20 -07:00
Daniel Chabrowski	25173c5dca	Add "open" functionality with ^P	2020-03-19 20:57:01 -07:00
Anthony Sottile	b2ebfa7b48	Improve quick prompt appearance	2020-03-19 20:37:39 -07:00
Anthony Sottile	efa6561200	improve multiple file close behaviour	2020-03-19 20:05:57 -07:00
Anthony Sottile	b683657f23	Support `babi -` for reading from stdin Resolves #42	2020-03-19 18:52:24 -07:00
Anthony Sottile	b59d03858c	Improve comments-json parsing	2020-03-18 14:04:51 -07:00
Anthony Sottile	6ec1da061b	Fix for begin-but-no-end rules (xml)	2020-03-18 11:56:36 -07:00
Anthony Sottile	c08557b6ca	remove un-commenting as it's handled by bin/download-theme	2020-03-17 13:13:46 -07:00
Anthony Sottile	006c2bc8e4	Add script for downloading themes	2020-03-17 12:41:52 -07:00
Anthony Sottile	080f6e1d54	Add support for shorthand hex colors	2020-03-17 12:37:31 -07:00
Anthony Sottile	e77a660029	fix for internal extra commas in theme scopes	2020-03-17 12:13:36 -07:00
Anthony Sottile	e32e5b8c05	Fix one edge case with comma scopes	2020-03-17 11:53:23 -07:00
Anthony Sottile	08638f990c	Add limited support for named colors Resolves #41	2020-03-17 11:00:59 -07:00
Anthony Sottile	414adffa9b	Fix highlighting edges and unify highlighting code	2020-03-16 15:19:21 -07:00
Anthony Sottile	8d77d5792a	use a mapping interface for FileHL.regions	2020-03-15 20:10:44 -07:00
Anthony Sottile	c85c50c207	Move find/replace highlighting to a highlighter	2020-03-15 19:54:13 -07:00
Anthony Sottile	d5376ca6f2	properly detect hidden (.extension-only) files	2020-03-15 19:23:46 -07:00
Anthony Sottile	31e7c9345b	Remove this cache, it is essentially a memory leak	2020-03-15 18:09:12 -07:00
Anthony Sottile	41543f8d6c	Use default hash for some highlighting primitives - this improves performance by ~13% - a lot of time was spent in `tuple.__hash__` for these particular types - the types that were changed are: - constructed once and then kept forever - act as "singletons"	2020-03-15 15:45:34 -07:00
Anthony Sottile	1be4e80edd	Add syntax highlight for puppet	2020-03-14 15:39:10 -07:00