v0.0.7

for example: ruby

.coveragerc

@@ -1,37 +0,0 @@
-[run]
-branch = True
-parallel = True
-source = .
-omit =
-    .tox/*
-    /usr/*
-    setup.py
-    # Don't complain if non-runnable code isn't run
-    */__main__.py
-
-[report]
-show_missing = True
-skip_covered = True
-exclude_lines =
-    # Have to re-enable the standard pragma
-    \#\s*pragma: no cover
-    # We optionally substitute this
-    ${COVERAGE_IGNORE_WINDOWS}
-
-    # Don't complain if tests don't hit defensive assertion code:
-    ^\s*raise AssertionError\b
-    ^\s*raise NotImplementedError\b
-    ^\s*return NotImplemented\b
-    ^\s*raise$
-
-    # Ignore typing-related things
-    ^if (False|TYPE_CHECKING):
-    : \.\.\.$
-
-    # Don't complain if non-runnable code isn't run:
-    ^if __name__ == ['"]__main__['"]:$
-
-[html]
-directory = coverage-html
-
-# vim:ft=dosini

.gitignore

@@ -5,4 +5,6 @@
 /.mypy_cache
 /.pytest_cache
 /.tox
+/build
+/dist
 /venv*

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.3.0
+    rev: v2.5.0
     hooks:
     -   id: trailing-whitespace
     -   id: end-of-file-fixer
@@ -11,29 +11,34 @@ repos:
     -   id: name-tests-test
     -   id: requirements-txt-fixer
 -   repo: https://gitlab.com/pycqa/flake8
-    rev: 3.7.8
+    rev: 3.7.9
     hooks:
     -   id: flake8
+        additional_dependencies: [flake8-typing-imports==1.7.0]
 -   repo: https://github.com/pre-commit/mirrors-autopep8
-    rev: v1.4.4
+    rev: v1.5
     hooks:
     -   id: autopep8
 -   repo: https://github.com/asottile/reorder_python_imports
-    rev: v1.7.0
+    rev: v2.1.0
     hooks:
     -   id: reorder-python-imports
         args: [--py3-plus]
 -   repo: https://github.com/asottile/add-trailing-comma
-    rev: v1.4.1
+    rev: v2.0.1
     hooks:
     -   id: add-trailing-comma
         args: [--py36-plus]
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v1.24.0
+    rev: v2.1.0
     hooks:
     -   id: pyupgrade
         args: [--py36-plus]
+-   repo: https://github.com/asottile/setup-cfg-fmt
+    rev: v1.7.0
+    hooks:
+    -   id: setup-cfg-fmt
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.730
+    rev: v0.770
     hooks:
     -   id: mypy

README.md

@@ -6,6 +6,10 @@ babi
 
 a text editor, eventually...
 
+### installation
+
+`pip install babi`
+
 ### why is it called babi?
 
 I usually use the text editor `nano`, frequently I typo this.  on a qwerty
@@ -13,39 +17,90 @@ keyboard, when the right hand is shifted left by one, `nano` becomes `babi`.
 
 ### quitting babi
 
-currently you can quit `babi` by using `^X` (or `^C` which triggers a
-backtrace).
+currently you can quit `babi` by using <kbd>^X</kbd> (or via <kbd>esc</kbd> +
+<kbd>:q</kbd>).
+
+### key combinations
+
+these are all of the current key bindings in babi
+
+- <kbd>^S</kbd>: save
+- <kbd>^O</kbd>: save as
+- <kbd>^X</kbd>: quit
+- <kbd>^P</kbd>: open file
+- arrow keys: movement
+- <kbd>^A</kbd> / <kbd>home</kbd>: move to beginning of line
+- <kbd>^E</kbd> / <kbd>end</kbd>: move to end of line
+- <kbd>^Y</kbd> / <kbd>pageup</kbd>: move up one page
+- <kbd>^V</kbd> / <kbd>pagedown</kbd>: move down one page
+- <kbd>^-left</kbd> / <kbd>^-right</kbd>: jump by word
+- <kbd>^-home</kbd> / <kbd>^-end</kbd>: jump to beginning / end of file
+- <kbd>^_</kbd>: jump to line number
+- selection: <kbd>shift</kbd> + ...: extend the current selection
+    - arrow keys
+    - <kbd>home</kbd> / <kbd>end</kdb>
+    - <kbd>pageup</kbd> / <kbd>pagedown</kbd>
+    - <kbd>^-left</kbd> / <kbd>^-right</kbd>
+    - <kbd>^-end</kbd> / <kbd>^-home</kbd>
+- <kbd>tab</kbd> / <kbd>shift-tab</kbd>: indent or dedent current line (or
+  selection)
+- <kbd>^K</kbd> / <kbd>^U</kbd>: cut and uncut the current line (or selection)
+- <kbd>M-u</kbd> / <kbd>M-U</kbd>: undo / redo
+- <kbd>^W</kbd>: search
+- <kbd>^\\</kbd>: search and replace
+- <kbd>^C</kbd>: show the current position in the file
+- <kbd>^-up</kbd> / <kbd>^-down</kbd>: scroll screen by a single line
+- <kbd>M-left</kbd> / <kbd>M-right</kbd>: go to previous / next file
+- <kbd>^Z</kbd>: background
+- <kbd>esc</kbd>: open the command mode
+    - <kbd>:q</kbd>: quit
+    - <kbd>:w</kbd>: write the file
+    - <kbd>:wq</kbd>: write the file and quit
+    - <kbd>:sort</kbd>: sort the file (or selection)
+
+in prompts (search, search replace, command):
+- <kbd>^C</kbd>: cancel
+- <kbd>^K</kbd>: cut to end
+- <kbd>^R</kbd>: reverse search
+
+### setting up syntax highlighting
+
+the syntax highlighting setup is a bit manual right now
+
+1. find a visual studio code theme, convert it to json (if it is not already
+   json) and put it at `~/.config/babi/theme.json`.  a helper script is
+   provided to make this easier: `./bin/download-theme NAME URL`
+
+here's a modified vs dark plus theme that works:
+
+```bash
+./bin/download-theme vs-dark-asottile https://gist.github.com/asottile/b465856c82b1aaa4ba8c7c6314a72e13/raw/22d602fb355fb12b04f176a733941ba5713bc36c/vs_dark_asottile.json
+```
 
 ## demos
 
-not much works yet, here's a few things
-
-### color test (`babi --color-test`)
-
-this is just to demo color support, this test mode will probably be deleted
-eventually.  it uses a little trick to invert foreground and background to
-get all of the color combinations.  there's one additional color not in this
-grid which is the "inverted default"
-
-![](https://i.fluffy.cc/rwdVdMsmZGDZrsT2qVlZHL5Z0XGj9v5v.png)
+most things work!  here's a few screenshots
 
 ### file view
 
-this opens the file, displays it, and can be edited in some ways and can save!
-movement is currently enabled through the arrow keys, home + `^A`, end + `^E`,
-and some key combinations are detected.  unknown keys are displayed as errors
-in the status bar.  babi will scroll if the cursor goes off screen either from
-resize events or from movement.  babi can edit multiple files.  babi has a
-command mode (so you can quit it like vim `:q`!).
+this opens the file, displays it, and can be edited and can save! unknown keys
+are displayed as errors in the status bar.  babi will scroll if the cursor
+goes off screen either from resize events or from movement.  babi can edit
+multiple files.  babi has a command mode (so you can quit it like vim
+<kbd>:q</kbd>!).  babi also support syntax highlighting
 
-![](https://i.fluffy.cc/14Xc4hZg87CBnRBPGgFTKWbQFXFDmmwx.png)
+![](https://i.fluffy.cc/5WFZBJ4mWs7wtThD9strQnGlJqw4Z9KS.png)
 
-![](https://i.fluffy.cc/wLvTm86lbLnjBgF0WtVQpsxW90QbJwz5.png)
+![](https://i.fluffy.cc/qrNhgCK34qKQ6tw4GHLSGs4984Qqnqh7.png)
 
-![](https://i.fluffy.cc/RhVmwb8MQkZZbC399GtV99RSH3SB6FTZ.png)
+![](https://i.fluffy.cc/DKlkjnZ4tgfnxH7cxjnLcB7GkBVdW35v.png)
 
-![](https://i.fluffy.cc/dKDd9rBm7hsXVsgZfvXM63gC8QQxJdhk.png)
+![](https://i.fluffy.cc/VqHWHfWNW73sppZlHv0C4lw63TVczZfZ.png)
 
-![](https://i.fluffy.cc/PQq1sqpcx59tWNFGF4nThQH1gSVHjVCn.png)
+![](https://i.fluffy.cc/p8lv61TCql1MJfpBDqbNPWPf27lmGWFN.png)
 
-![](https://i.fluffy.cc/KfGg7NhNTTH5X4ZsxdsMt72RVg5nR79H.png)
+![](https://i.fluffy.cc/ZH5sswB4FSbpW8FfcXL1KZWdJnjxRkbW.png)
+
+![](https://i.fluffy.cc/Rw8nZKFC3R36mNrV01fL2gk4rfwWn7wX.png)
+
+![](https://i.fluffy.cc/FSD92ZVN4xcMFPv1V7gc0Xzk8TCQTgdg.png)

SCREENS.md

@@ -53,3 +53,24 @@ see the progress of babi over time
 - babi can be quit using `:q` and can save using `:w`
 
 ![](https://i.fluffy.cc/KfGg7NhNTTH5X4ZsxdsMt72RVg5nR79H.png)
+
+### 2020-03-14
+
+- a lot of stuff has changed, there's now syntax highlighting and other things
+- sorry I haven't updated in a while
+
+![](https://i.fluffy.cc/5WFZBJ4mWs7wtThD9strQnGlJqw4Z9KS.png)
+
+![](https://i.fluffy.cc/qrNhgCK34qKQ6tw4GHLSGs4984Qqnqh7.png)
+
+![](https://i.fluffy.cc/DKlkjnZ4tgfnxH7cxjnLcB7GkBVdW35v.png)
+
+![](https://i.fluffy.cc/VqHWHfWNW73sppZlHv0C4lw63TVczZfZ.png)
+
+![](https://i.fluffy.cc/p8lv61TCql1MJfpBDqbNPWPf27lmGWFN.png)
+
+![](https://i.fluffy.cc/ZH5sswB4FSbpW8FfcXL1KZWdJnjxRkbW.png)
+
+![](https://i.fluffy.cc/Rw8nZKFC3R36mNrV01fL2gk4rfwWn7wX.png)
+
+![](https://i.fluffy.cc/FSD92ZVN4xcMFPv1V7gc0Xzk8TCQTgdg.png)

babi.py

@@ -1,971 +0,0 @@
-import argparse
-import collections
-import contextlib
-import curses
-import enum
-import functools
-import hashlib
-import io
-import os
-import signal
-import sys
-from typing import Any
-from typing import Callable
-from typing import cast
-from typing import Dict
-from typing import Generator
-from typing import IO
-from typing import Iterator
-from typing import List
-from typing import NamedTuple
-from typing import Optional
-from typing import Tuple
-from typing import TYPE_CHECKING
-from typing import TypeVar
-from typing import Union
-
-if TYPE_CHECKING:
-    from typing import Protocol  # python3.8+
-else:
-    Protocol = object
-
-VERSION_STR = 'babi v0'
-TCallable = TypeVar('TCallable', bound=Callable[..., Any])
-
-
-def _line_x(x: int, width: int) -> int:
-    margin = min(width - 3, 6)
-    if x + 1 < width:
-        return 0
-    elif width == 1:
-        return x
-    else:
-        return (
-            width - margin - 2 +
-            (x + 1 - width) //
-            (width - margin - 2) *
-            (width - margin - 2)
-        )
-
-
-def _scrolled_line(s: str, x: int, width: int, *, current: bool) -> str:
-    line_x = _line_x(x, width)
-    if current and line_x:
-        s = f'«{s[line_x + 1:]}'
-        if line_x and len(s) > width:
-            return f'{s[:width - 1]}»'
-        else:
-            return s.ljust(width)
-    elif len(s) > width:
-        return f'{s[:width - 1]}»'
-    else:
-        return s.ljust(width)
-
-
-class MutableSequenceNoSlice(Protocol):
-    def __len__(self) -> int: ...
-    def __getitem__(self, idx: int) -> str: ...
-    def __setitem__(self, idx: int, val: str) -> None: ...
-    def __delitem__(self, idx: int) -> None: ...
-    def insert(self, idx: int, val: str) -> None: ...
-
-    def __iter__(self) -> Iterator[str]:
-        for i in range(len(self)):
-            yield self[i]
-
-    def append(self, val: str) -> None:
-        self.insert(len(self), val)
-
-    def pop(self, idx: int = -1) -> str:
-        victim = self[idx]
-        del self[idx]
-        return victim
-
-
-def _del(lst: MutableSequenceNoSlice, *, idx: int) -> None:
-    del lst[idx]
-
-
-def _set(lst: MutableSequenceNoSlice, *, idx: int, val: str) -> None:
-    lst[idx] = val
-
-
-def _ins(lst: MutableSequenceNoSlice, *, idx: int, val: str) -> None:
-    lst.insert(idx, val)
-
-
-class ListSpy(MutableSequenceNoSlice):
-    def __init__(self, lst: MutableSequenceNoSlice) -> None:
-        self._lst = lst
-        self._undo: List[Callable[[MutableSequenceNoSlice], None]] = []
-
-    def __repr__(self) -> str:
-        return f'{type(self).__name__}({self._lst})'
-
-    def __len__(self) -> int:
-        return len(self._lst)
-
-    def __getitem__(self, idx: int) -> str:
-        return self._lst[idx]
-
-    def __setitem__(self, idx: int, val: str) -> None:
-        self._undo.append(functools.partial(_set, idx=idx, val=self._lst[idx]))
-        self._lst[idx] = val
-
-    def __delitem__(self, idx: int) -> None:
-        if idx < 0:
-            idx %= len(self)
-        self._undo.append(functools.partial(_ins, idx=idx, val=self._lst[idx]))
-        del self._lst[idx]
-
-    def insert(self, idx: int, val: str) -> None:
-        if idx < 0:
-            idx %= len(self)
-        self._undo.append(functools.partial(_del, idx=idx))
-        self._lst.insert(idx, val)
-
-    def undo(self, lst: MutableSequenceNoSlice) -> None:
-        for fn in reversed(self._undo):
-            fn(lst)
-
-    @property
-    def has_modifications(self) -> bool:
-        return bool(self._undo)
-
-
-class Margin(NamedTuple):
-    header: bool
-    footer: bool
-
-    @property
-    def body_lines(self) -> int:
-        return curses.LINES - self.header - self.footer
-
-    @property
-    def page_size(self) -> int:
-        if self.body_lines <= 2:
-            return 1
-        else:
-            return self.body_lines - 2
-
-    @classmethod
-    def from_screen(cls, screen: 'curses._CursesWindow') -> 'Margin':
-        if curses.LINES == 1:
-            return cls(header=False, footer=False)
-        elif curses.LINES == 2:
-            return cls(header=False, footer=True)
-        else:
-            return cls(header=True, footer=True)
-
-
-def _get_color_pair_mapping() -> Dict[Tuple[int, int], int]:
-    ret = {}
-    i = 0
-    for bg in range(-1, 16):
-        for fg in range(bg, 16):
-            ret[(fg, bg)] = i
-            i += 1
-    return ret
-
-
-COLORS = _get_color_pair_mapping()
-del _get_color_pair_mapping
-
-
-def _has_colors() -> bool:
-    return curses.has_colors and curses.COLORS >= 16
-
-
-def _color(fg: int, bg: int) -> int:
-    if _has_colors():
-        if bg > fg:
-            return curses.A_REVERSE | curses.color_pair(COLORS[(bg, fg)])
-        else:
-            return curses.color_pair(COLORS[(fg, bg)])
-    else:
-        if bg > fg:
-            return curses.A_REVERSE | curses.color_pair(0)
-        else:
-            return curses.color_pair(0)
-
-
-def _init_colors(stdscr: 'curses._CursesWindow') -> None:
-    curses.use_default_colors()
-    if not _has_colors():
-        return
-    for (fg, bg), pair in COLORS.items():
-        if pair == 0:  # cannot reset pair 0
-            continue
-        curses.init_pair(pair, fg, bg)
-
-
-class Status:
-    def __init__(self) -> None:
-        self._status = ''
-        self._action_counter = -1
-
-    def update(self, status: str) -> None:
-        self._status = status
-        self._action_counter = 25
-
-    def draw(self, stdscr: 'curses._CursesWindow', margin: Margin) -> None:
-        if margin.footer or self._status:
-            stdscr.insstr(curses.LINES - 1, 0, ' ' * curses.COLS)
-            if self._status:
-                status = f' {self._status} '
-                x = (curses.COLS - len(status)) // 2
-                if x < 0:
-                    x = 0
-                    status = status.strip()
-                stdscr.insstr(curses.LINES - 1, x, status, curses.A_REVERSE)
-
-    def tick(self, margin: Margin) -> None:
-        # when the window is only 1-tall, hide the status quicker
-        if margin.footer:
-            self._action_counter -= 1
-        else:
-            self._action_counter -= 24
-        if self._action_counter < 0:
-            self._status = ''
-
-    def prompt(self, screen: 'Screen', prompt: str) -> str:
-        pos = 0
-        buf = ''
-        while True:
-            width = curses.COLS - len(prompt)
-            cmd = f'{prompt}{_scrolled_line(buf, pos, width, current=True)}'
-            screen.stdscr.insstr(curses.LINES - 1, 0, cmd, curses.A_REVERSE)
-            line_x = _line_x(pos, width)
-            screen.stdscr.move(curses.LINES - 1, pos - line_x)
-            key = _get_char(screen.stdscr)
-
-            if key.key == curses.KEY_RESIZE:
-                screen.resize()
-            elif key.key == curses.KEY_LEFT:
-                pos = max(0, pos - 1)
-            elif key.key == curses.KEY_RIGHT:
-                pos = min(len(buf), pos + 1)
-            elif key.key == curses.KEY_HOME or key.keyname == b'^A':
-                pos = 0
-            elif key.key == curses.KEY_END or key.keyname == b'^E':
-                pos = len(buf)
-            elif key.key == curses.KEY_BACKSPACE:
-                if pos > 0:
-                    buf = buf[:pos - 1] + buf[pos:]
-                    pos -= 1
-            elif key.key == curses.KEY_DC:
-                if pos < len(buf):
-                    buf = buf[:pos] + buf[pos + 1:]
-            elif isinstance(key.wch, str) and key.wch.isprintable():
-                buf = buf[:pos] + key.wch + buf[pos:]
-                pos += 1
-            elif key.keyname == b'^C':
-                return ''
-            elif key.key == ord('\r'):
-                return buf
-
-
-def _restore_lines_eof_invariant(lines: MutableSequenceNoSlice) -> None:
-    """The file lines will always contain a blank empty string at the end to
-    simplify rendering.  This should be called whenever the end of the file
-    might change.
-    """
-    if not lines or lines[-1] != '':
-        lines.append('')
-
-
-def _get_lines(sio: IO[str]) -> Tuple[List[str], str, bool, str]:
-    sha256 = hashlib.sha256()
-    lines = []
-    newlines = collections.Counter({'\n': 0})  # default to `\n`
-    for line in sio:
-        sha256.update(line.encode())
-        for ending in ('\r\n', '\n'):
-            if line.endswith(ending):
-                lines.append(line[:-1 * len(ending)])
-                newlines[ending] += 1
-                break
-        else:
-            lines.append(line)
-    _restore_lines_eof_invariant(lines)
-    (nl, _), = newlines.most_common(1)
-    mixed = len({k for k, v in newlines.items() if v}) > 1
-    return lines, nl, mixed, sha256.hexdigest()
-
-
-class Action:
-    def __init__(
-            self, *, name: str, spy: ListSpy,
-            start_x: int, start_line: int, start_modified: bool,
-            end_x: int, end_line: int, end_modified: bool,
-    ):
-        self.name = name
-        self.spy = spy
-        self.start_x = start_x
-        self.start_line = start_line
-        self.start_modified = start_modified
-        self.end_x = end_x
-        self.end_line = end_line
-        self.end_modified = end_modified
-        self.final = False
-
-    def apply(self, file: 'File') -> 'Action':
-        spy = ListSpy(file.lines)
-        action = Action(
-            name=self.name, spy=spy,
-            start_x=self.end_x, start_line=self.end_line,
-            start_modified=self.end_modified,
-            end_x=self.start_x, end_line=self.start_line,
-            end_modified=self.start_modified,
-        )
-
-        self.spy.undo(spy)
-        file.x = self.start_x
-        file.cursor_line = self.start_line
-        file.modified = self.start_modified
-
-        return action
-
-
-def action(func: TCallable) -> TCallable:
-    @functools.wraps(func)
-    def action_inner(self: 'File', *args: Any, **kwargs: Any) -> Any:
-        assert not isinstance(self.lines, ListSpy), 'nested edit/movement'
-        if self.undo_stack:
-            self.undo_stack[-1].final = True
-        return func(self, *args, **kwargs)
-    return cast(TCallable, action_inner)
-
-
-def edit_action(name: str) -> Callable[[TCallable], TCallable]:
-    def edit_action_decorator(func: TCallable) -> TCallable:
-        @functools.wraps(func)
-        def edit_action_inner(self: 'File', *args: Any, **kwargs: Any) -> Any:
-            continue_last = (
-                self.undo_stack and
-                self.undo_stack[-1].name == name and
-                not self.undo_stack[-1].final
-            )
-            if continue_last:
-                spy = self.undo_stack[-1].spy
-            else:
-                if self.undo_stack:
-                    self.undo_stack[-1].final = True
-                spy = ListSpy(self.lines)
-
-            before_x, before_line = self.x, self.cursor_line
-            before_modified = self.modified
-            assert not isinstance(self.lines, ListSpy), 'recursive action?'
-            orig, self.lines = self.lines, spy
-            try:
-                return func(self, *args, **kwargs)
-            finally:
-                self.lines = orig
-                self.redo_stack.clear()
-                if continue_last:
-                    self.undo_stack[-1].end_x = self.x
-                    self.undo_stack[-1].end_line = self.cursor_line
-                    self.undo_stack[-1].end_modified = self.modified
-                elif spy.has_modifications:
-                    action = Action(
-                        name=name, spy=spy,
-                        start_x=before_x, start_line=before_line,
-                        start_modified=before_modified,
-                        end_x=self.x, end_line=self.cursor_line,
-                        end_modified=self.modified,
-                    )
-                    self.undo_stack.append(action)
-        return cast(TCallable, edit_action_inner)
-    return edit_action_decorator
-
-
-class File:
-    def __init__(self, filename: Optional[str]) -> None:
-        self.filename = filename
-        self.modified = False
-        self.lines: MutableSequenceNoSlice = []
-        self.nl = '\n'
-        self.file_line = self.cursor_line = self.x = self.x_hint = 0
-        self.sha256: Optional[str] = None
-        self.undo_stack: List[Action] = []
-        self.redo_stack: List[Action] = []
-
-    def ensure_loaded(self, status: Status) -> None:
-        if self.lines:
-            return
-
-        if self.filename is not None and os.path.isfile(self.filename):
-            with open(self.filename, newline='') as f:
-                self.lines, self.nl, mixed, self.sha256 = _get_lines(f)
-        else:
-            if self.filename is not None:
-                if os.path.lexists(self.filename):
-                    status.update(f'{self.filename!r} is not a file')
-                    self.filename = None
-                else:
-                    status.update('(new file)')
-            sio = io.StringIO('')
-            self.lines, self.nl, mixed, self.sha256 = _get_lines(sio)
-
-        if mixed:
-            status.update(f'mixed newlines will be converted to {self.nl!r}')
-            self.modified = True
-
-    def __repr__(self) -> str:
-        attrs = ',\n    '.join(f'{k}={v!r}' for k, v in self.__dict__.items())
-        return f'{type(self).__name__}(\n    {attrs},\n)'
-
-    # movement
-
-    def _scroll_screen_if_needed(self, margin: Margin) -> None:
-        # if the `cursor_line` is not on screen, make it so
-        if (
-                self.file_line <=
-                self.cursor_line <
-                self.file_line + margin.body_lines
-        ):
-            return
-
-        self.file_line = max(self.cursor_line - margin.body_lines // 2, 0)
-
-    def _scroll_amount(self) -> int:
-        return int(curses.LINES / 2 + .5)
-
-    def _set_x_after_vertical_movement(self) -> None:
-        self.x = min(len(self.lines[self.cursor_line]), self.x_hint)
-
-    def maybe_scroll_down(self, margin: Margin) -> None:
-        if self.cursor_line >= self.file_line + margin.body_lines:
-            self.file_line += self._scroll_amount()
-
-    @action
-    def down(self, margin: Margin) -> None:
-        if self.cursor_line < len(self.lines) - 1:
-            self.cursor_line += 1
-            self.maybe_scroll_down(margin)
-            self._set_x_after_vertical_movement()
-
-    def _maybe_scroll_up(self, margin: Margin) -> None:
-        if self.cursor_line < self.file_line:
-            self.file_line -= self._scroll_amount()
-            self.file_line = max(self.file_line, 0)
-
-    @action
-    def up(self, margin: Margin) -> None:
-        if self.cursor_line > 0:
-            self.cursor_line -= 1
-            self._maybe_scroll_up(margin)
-            self._set_x_after_vertical_movement()
-
-    @action
-    def right(self, margin: Margin) -> None:
-        if self.x >= len(self.lines[self.cursor_line]):
-            if self.cursor_line < len(self.lines) - 1:
-                self.x = 0
-                self.cursor_line += 1
-                self.maybe_scroll_down(margin)
-        else:
-            self.x += 1
-        self.x_hint = self.x
-
-    @action
-    def left(self, margin: Margin) -> None:
-        if self.x == 0:
-            if self.cursor_line > 0:
-                self.cursor_line -= 1
-                self.x = len(self.lines[self.cursor_line])
-                self._maybe_scroll_up(margin)
-        else:
-            self.x -= 1
-        self.x_hint = self.x
-
-    @action
-    def home(self, margin: Margin) -> None:
-        self.x = self.x_hint = 0
-
-    @action
-    def end(self, margin: Margin) -> None:
-        self.x = self.x_hint = len(self.lines[self.cursor_line])
-
-    @action
-    def ctrl_home(self, margin: Margin) -> None:
-        self.x = self.x_hint = 0
-        self.cursor_line = self.file_line = 0
-
-    @action
-    def ctrl_end(self, margin: Margin) -> None:
-        self.x = self.x_hint = 0
-        self.cursor_line = len(self.lines) - 1
-        self._scroll_screen_if_needed(margin)
-
-    @action
-    def page_up(self, margin: Margin) -> None:
-        if self.cursor_line < margin.body_lines:
-            self.cursor_line = self.file_line = 0
-        else:
-            pos = max(self.file_line - margin.page_size, 0)
-            self.cursor_line = self.file_line = pos
-        self._set_x_after_vertical_movement()
-
-    @action
-    def page_down(self, margin: Margin) -> None:
-        if self.file_line + margin.body_lines >= len(self.lines):
-            self.cursor_line = len(self.lines) - 1
-        else:
-            pos = self.file_line + margin.page_size
-            self.cursor_line = self.file_line = pos
-        self._set_x_after_vertical_movement()
-
-    # editing
-
-    @edit_action('backspace text')
-    def backspace(self, margin: Margin) -> None:
-        # backspace at the beginning of the file does nothing
-        if self.cursor_line == 0 and self.x == 0:
-            pass
-        # at the beginning of the line, we join the current line and
-        # the previous line
-        elif self.x == 0:
-            victim = self.lines.pop(self.cursor_line)
-            new_x = len(self.lines[self.cursor_line - 1])
-            self.lines[self.cursor_line - 1] += victim
-            self.cursor_line -= 1
-            self._maybe_scroll_up(margin)
-            self.x = self.x_hint = new_x
-            # deleting the fake end-of-file doesn't cause modification
-            self.modified |= self.cursor_line < len(self.lines) - 1
-            _restore_lines_eof_invariant(self.lines)
-        else:
-            s = self.lines[self.cursor_line]
-            self.lines[self.cursor_line] = s[:self.x - 1] + s[self.x:]
-            self.x = self.x_hint = self.x - 1
-            self.modified = True
-
-    @edit_action('delete text')
-    def delete(self, margin: Margin) -> None:
-        # noop at end of the file
-        if self.cursor_line == len(self.lines) - 1:
-            pass
-        # if we're at the end of the line, collapse the line afterwards
-        elif self.x == len(self.lines[self.cursor_line]):
-            victim = self.lines.pop(self.cursor_line + 1)
-            self.lines[self.cursor_line] += victim
-            self.modified = True
-        else:
-            s = self.lines[self.cursor_line]
-            self.lines[self.cursor_line] = s[:self.x] + s[self.x + 1:]
-            self.modified = True
-
-    @edit_action('line break')
-    def enter(self, margin: Margin) -> None:
-        s = self.lines[self.cursor_line]
-        self.lines[self.cursor_line] = s[:self.x]
-        self.lines.insert(self.cursor_line + 1, s[self.x:])
-        self.cursor_line += 1
-        self.maybe_scroll_down(margin)
-        self.x = self.x_hint = 0
-        self.modified = True
-
-    @edit_action('cut')
-    def cut(self, cut_buffer: Tuple[str, ...]) -> Tuple[str, ...]:
-        if self.cursor_line == len(self.lines) - 1:
-            return ()
-        else:
-            victim = self.lines.pop(self.cursor_line)
-            self.x = self.x_hint = 0
-            self.modified = True
-            return cut_buffer + (victim,)
-
-    @edit_action('uncut')
-    def uncut(self, cut_buffer: Tuple[str, ...], margin: Margin) -> None:
-        for cut_line in cut_buffer:
-            line = self.lines[self.cursor_line]
-            before, after = line[:self.x], line[self.x:]
-            self.lines[self.cursor_line] = before + cut_line
-            self.lines.insert(self.cursor_line + 1, after)
-            self.cursor_line += 1
-            self.x = self.x_hint = 0
-            self.maybe_scroll_down(margin)
-
-    DISPATCH = {
-        # movement
-        curses.KEY_DOWN: down,
-        curses.KEY_UP: up,
-        curses.KEY_LEFT: left,
-        curses.KEY_RIGHT: right,
-        curses.KEY_HOME: home,
-        curses.KEY_END: end,
-        curses.KEY_PPAGE: page_up,
-        curses.KEY_NPAGE: page_down,
-        # editing
-        curses.KEY_BACKSPACE: backspace,
-        curses.KEY_DC: delete,
-        ord('\r'): enter,
-    }
-    DISPATCH_KEY = {
-        # movement
-        b'^A': home,
-        b'^E': end,
-        b'^Y': page_up,
-        b'^V': page_down,
-        b'kHOM5': ctrl_home,
-        b'kEND5': ctrl_end,
-    }
-
-    @edit_action('text')
-    def c(self, wch: str, margin: Margin) -> None:
-        s = self.lines[self.cursor_line]
-        self.lines[self.cursor_line] = s[:self.x] + wch + s[self.x:]
-        self.x = self.x_hint = self.x + 1
-        self.modified = True
-        _restore_lines_eof_invariant(self.lines)
-
-    def _undo_redo(
-            self,
-            op: str,
-            from_stack: List[Action],
-            to_stack: List[Action],
-            status: Status,
-            margin: Margin,
-    ) -> None:
-        if not from_stack:
-            status.update(f'nothing to {op}!')
-        else:
-            action = from_stack.pop()
-            to_stack.append(action.apply(self))
-            self._scroll_screen_if_needed(margin)
-            status.update(f'{op}: {action.name}')
-
-    def undo(self, status: Status, margin: Margin) -> None:
-        self._undo_redo(
-            'undo', self.undo_stack, self.redo_stack, status, margin,
-        )
-
-    def redo(self, status: Status, margin: Margin) -> None:
-        self._undo_redo(
-            'redo', self.redo_stack, self.undo_stack, status, margin,
-        )
-
-    @action
-    def save(self, status: Status) -> None:
-        # TODO: make directories if they don't exist
-        # TODO: maybe use mtime / stat as a shortcut for hashing below
-        # TODO: strip trailing whitespace?
-        # TODO: save atomically?
-        if self.filename is None:
-            status.update('(no filename, not implemented)')
-            return
-
-        if os.path.isfile(self.filename):
-            with open(self.filename) as f:
-                *_, sha256 = _get_lines(f)
-        else:
-            sha256 = hashlib.sha256(b'').hexdigest()
-
-        contents = self.nl.join(self.lines)
-        sha256_to_save = hashlib.sha256(contents.encode()).hexdigest()
-
-        # the file on disk is the same as when we opened it
-        if sha256 not in (self.sha256, sha256_to_save):
-            status.update('(file changed on disk, not implemented)')
-            return
-
-        with open(self.filename, 'w') as f:
-            f.write(contents)
-
-        self.modified = False
-        self.sha256 = sha256_to_save
-        num_lines = len(self.lines) - 1
-        lines = 'lines' if num_lines != 1 else 'line'
-        status.update(f'saved! ({num_lines} {lines} written)')
-
-        # fix up modified state in undo / redo stacks
-        for stack in (self.undo_stack, self.redo_stack):
-            first = True
-            for action in reversed(stack):
-                action.end_modified = not first
-                action.start_modified = True
-                first = False
-
-    # positioning
-
-    def cursor_y(self, margin: Margin) -> int:
-        return self.cursor_line - self.file_line + margin.header
-
-    def line_x(self) -> int:
-        return _line_x(self.x, curses.COLS)
-
-    def cursor_x(self) -> int:
-        return self.x - self.line_x()
-
-    def move_cursor(
-            self,
-            stdscr: 'curses._CursesWindow',
-            margin: Margin,
-    ) -> None:
-        stdscr.move(self.cursor_y(margin), self.cursor_x())
-
-    def draw(self, stdscr: 'curses._CursesWindow', margin: Margin) -> None:
-        to_display = min(len(self.lines) - self.file_line, margin.body_lines)
-        for i in range(to_display):
-            line_idx = self.file_line + i
-            line = self.lines[line_idx]
-            current = line_idx == self.cursor_line
-            line = _scrolled_line(line, self.x, curses.COLS, current=current)
-            stdscr.insstr(i + margin.header, 0, line)
-        blankline = ' ' * curses.COLS
-        for i in range(to_display, margin.body_lines):
-            stdscr.insstr(i + margin.header, 0, blankline)
-
-
-class Screen:
-    def __init__(
-            self,
-            stdscr: 'curses._CursesWindow',
-            files: List[File],
-    ) -> None:
-        self.stdscr = stdscr
-        self.files = files
-        self.i = 0
-        self.status = Status()
-        self.margin = Margin.from_screen(self.stdscr)
-        self.cut_buffer: Tuple[str, ...] = ()
-
-    @property
-    def file(self) -> File:
-        return self.files[self.i]
-
-    def _draw_header(self) -> None:
-        filename = self.file.filename or '<<new file>>'
-        if self.file.modified:
-            filename += ' *'
-        if len(self.files) > 1:
-            files = f'[{self.i + 1}/{len(self.files)}] '
-            version_width = len(VERSION_STR) + 2 + len(files)
-        else:
-            files = ''
-            version_width = len(VERSION_STR) + 2
-        centered = filename.center(curses.COLS)[version_width:]
-        s = f' {VERSION_STR} {files}{centered}{files}'
-        self.stdscr.insstr(0, 0, s, curses.A_REVERSE)
-
-    def draw(self) -> None:
-        if self.margin.header:
-            self._draw_header()
-        self.file.draw(self.stdscr, self.margin)
-        self.status.draw(self.stdscr, self.margin)
-
-    def resize(self) -> None:
-        curses.update_lines_cols()
-        self.margin = Margin.from_screen(self.stdscr)
-        self.file.maybe_scroll_down(self.margin)
-        self.draw()
-
-
-def _color_test(stdscr: 'curses._CursesWindow') -> None:
-    header = f' {VERSION_STR}'
-    header += '<< color test >>'.center(curses.COLS)[len(header):]
-    stdscr.insstr(0, 0, header, curses.A_REVERSE)
-
-    maxy, maxx = stdscr.getmaxyx()
-    if maxy < 19 or maxx < 68:  # pragma: no cover (will be deleted)
-        raise SystemExit('--color-test needs a window of at least 68 x 19')
-
-    y = 1
-    for fg in range(-1, 16):
-        x = 0
-        for bg in range(-1, 16):
-            if bg > fg:
-                s = f'*{COLORS[bg, fg]:3}'
-            else:
-                s = f' {COLORS[fg, bg]:3}'
-            stdscr.addstr(y, x, s, _color(fg, bg))
-            x += 4
-        y += 1
-    stdscr.get_wch()
-
-
-class Key(NamedTuple):
-    wch: Union[int, str]
-    key: int
-    keyname: bytes
-
-
-# TODO: find a place to populate these, surely there's a database somewhere
-SEQUENCE_KEY = {
-    '\x1bOH': curses.KEY_HOME,
-    '\x1bOF': curses.KEY_END,
-}
-SEQUENCE_KEYNAME = {
-    '\x1b[1;5H': b'kHOM5',  # C-Home
-    '\x1b[1;5F': b'kEND5',  # C-End
-    '\x1bOH': b'KEY_HOME',
-    '\x1bOF': b'KEY_END',
-    '\x1b[1;3A': b'kUP3',  # M-Up
-    '\x1b[1;3B': b'kDN3',  # M-Down
-    '\x1b[1;3C': b'kRIT3',  # M-Right
-    '\x1b[1;3D': b'kLFT3',  # M-Left
-}
-
-
-def _get_char(stdscr: 'curses._CursesWindow') -> Key:
-    wch = stdscr.get_wch()
-    if isinstance(wch, str) and wch == '\x1b':
-        stdscr.nodelay(True)
-        try:
-            while True:
-                try:
-                    new_wch = stdscr.get_wch()
-                    if isinstance(new_wch, str):
-                        wch += new_wch
-                    else:  # pragma: no cover (impossible?)
-                        curses.unget_wch(new_wch)
-                        break
-                except curses.error:
-                    break
-        finally:
-            stdscr.nodelay(False)
-
-        if len(wch) == 2:
-            return Key(wch, -1, f'M-{wch[1]}'.encode())
-        elif len(wch) > 1:
-            key = SEQUENCE_KEY.get(wch, -1)
-            keyname = SEQUENCE_KEYNAME.get(wch, b'unknown')
-            return Key(wch, key, keyname)
-    elif wch == '\x7f':  # pragma: no cover (macos)
-        key = curses.KEY_BACKSPACE
-        keyname = curses.keyname(key)
-        return Key(wch, key, keyname)
-
-    key = wch if isinstance(wch, int) else ord(wch)
-    keyname = curses.keyname(key)
-    return Key(wch, key, keyname)
-
-
-EditResult = enum.Enum('EditResult', 'EXIT NEXT PREV')
-
-
-def _edit(screen: Screen) -> EditResult:
-    prevkey = Key('', 0, b'')
-    screen.file.ensure_loaded(screen.status)
-
-    while True:
-        screen.status.tick(screen.margin)
-
-        screen.draw()
-        screen.file.move_cursor(screen.stdscr, screen.margin)
-
-        key = _get_char(screen.stdscr)
-
-        if key.key == curses.KEY_RESIZE:
-            screen.resize()
-        elif key.key in File.DISPATCH:
-            screen.file.DISPATCH[key.key](screen.file, screen.margin)
-        elif key.keyname in File.DISPATCH_KEY:
-            screen.file.DISPATCH_KEY[key.keyname](screen.file, screen.margin)
-        elif key.keyname == b'^K':
-            if prevkey.keyname == b'^K':
-                cut_buffer = screen.cut_buffer
-            else:
-                cut_buffer = ()
-            screen.cut_buffer = screen.file.cut(cut_buffer)
-        elif key.keyname == b'^U':
-            screen.file.uncut(screen.cut_buffer, screen.margin)
-        elif key.keyname == b'M-u':
-            screen.file.undo(screen.status, screen.margin)
-        elif key.keyname == b'M-U':
-            screen.file.redo(screen.status, screen.margin)
-        elif key.keyname == b'^[':  # escape
-            response = screen.status.prompt(screen, '')
-            if response == ':q':
-                return EditResult.EXIT
-            elif response == ':w':
-                screen.file.save(screen.status)
-            elif response == ':wq':
-                screen.file.save(screen.status)
-                return EditResult.EXIT
-            elif response == '':  # noop / cancel
-                screen.status.update('')
-            else:
-                screen.status.update(f'invalid command: {response}')
-        elif key.keyname == b'^S':
-            screen.file.save(screen.status)
-        elif key.keyname == b'^X':
-            return EditResult.EXIT
-        elif key.keyname == b'kLFT3':
-            return EditResult.PREV
-        elif key.keyname == b'kRIT3':
-            return EditResult.NEXT
-        elif key.keyname == b'^Z':
-            curses.endwin()
-            os.kill(os.getpid(), signal.SIGSTOP)
-            screen.stdscr = _init_screen()
-            screen.resize()
-        elif isinstance(key.wch, str) and key.wch.isprintable():
-            screen.file.c(key.wch, screen.margin)
-        else:
-            screen.status.update(f'unknown key: {key}')
-
-        prevkey = key
-
-
-def c_main(stdscr: 'curses._CursesWindow', args: argparse.Namespace) -> None:
-    if args.color_test:
-        return _color_test(stdscr)
-    screen = Screen(stdscr, [File(f) for f in args.filenames or [None]])
-    while screen.files:
-        screen.i = screen.i % len(screen.files)
-        res = _edit(screen)
-        if res == EditResult.EXIT:
-            del screen.files[screen.i]
-        elif res == EditResult.NEXT:
-            screen.i += 1
-        elif res == EditResult.PREV:
-            screen.i -= 1
-        else:
-            raise AssertionError(f'unreachable {res}')
-
-
-def _init_screen() -> 'curses._CursesWindow':
-    # set the escape delay so curses does not pause waiting for sequences
-    if sys.version_info >= (3, 9):  # pragma: no cover
-        curses.set_escdelay(25)
-    else:  # pragma: no cover
-        os.environ.setdefault('ESCDELAY', '25')
-
-    stdscr = curses.initscr()
-    curses.noecho()
-    curses.cbreak()
-    # <enter> is not transformed into '\n' so it can be differentiated from ^J
-    curses.nonl()
-    # ^S / ^Q / ^Z / ^\ are passed through
-    curses.raw()
-    stdscr.keypad(True)
-    with contextlib.suppress(curses.error):
-        curses.start_color()
-    _init_colors(stdscr)
-    return stdscr
-
-
-@contextlib.contextmanager
-def make_stdscr() -> Generator['curses._CursesWindow', None, None]:
-    """essentially `curses.wrapper` but split out to implement ^Z"""
-    stdscr = _init_screen()
-    try:
-        yield stdscr
-    finally:
-        curses.endwin()
-
-
-def main() -> int:
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--color-test', action='store_true')
-    parser.add_argument('filenames', metavar='filename', nargs='*')
-    args = parser.parse_args()
-    with make_stdscr() as stdscr:
-        c_main(stdscr, args)
-    return 0
-
-
-if __name__ == '__main__':
-    exit(main())

babi/__main__.py

@@ -0,0 +1,4 @@
+from babi.main import main
+
+if __name__ == '__main__':
+    exit(main())

babi/_types.py

@@ -0,0 +1,6 @@
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from typing import Protocol  # python3.8+
+else:
+    Protocol = object

babi/buf.py

@@ -0,0 +1,300 @@
+import bisect
+import contextlib
+from typing import Callable
+from typing import Generator
+from typing import Iterator
+from typing import List
+from typing import NamedTuple
+from typing import Optional
+from typing import Tuple
+
+from babi._types import Protocol
+from babi.horizontal_scrolling import line_x
+from babi.horizontal_scrolling import scrolled_line
+from babi.horizontal_scrolling import wcwidth
+from babi.margin import Margin
+
+SetCallback = Callable[['Buf', int, str], None]
+DelCallback = Callable[['Buf', int, str], None]
+InsCallback = Callable[['Buf', int], None]
+
+
+def _offsets(s: str) -> Tuple[int, ...]:
+    ret = [0]
+    for c in s:
+        if c == '\t':
+            ret.append(ret[-1] + (4 - ret[-1] % 4))
+        else:
+            ret.append(ret[-1] + wcwidth(c))
+    return tuple(ret)
+
+
+class Modification(Protocol):
+    def __call__(self, buf: 'Buf') -> None: ...
+
+
+class SetModification(NamedTuple):
+    idx: int
+    s: str
+
+    def __call__(self, buf: 'Buf') -> None:
+        buf[self.idx] = self.s
+
+
+class InsModification(NamedTuple):
+    idx: int
+    s: str
+
+    def __call__(self, buf: 'Buf') -> None:
+        buf.insert(self.idx, self.s)
+
+
+class DelModification(NamedTuple):
+    idx: int
+
+    def __call__(self, buf: 'Buf') -> None:
+        del buf[self.idx]
+
+
+class Buf:
+    def __init__(self, lines: List[str]) -> None:
+        self._lines = lines
+        self.file_y = self.y = self._x = self._x_hint = 0
+
+        self._set_callbacks: List[SetCallback] = [self._set_cb]
+        self._del_callbacks: List[DelCallback] = [self._del_cb]
+        self._ins_callbacks: List[InsCallback] = [self._ins_cb]
+
+        self._positions: List[Optional[Tuple[int, ...]]] = []
+
+    # read only interface
+
+    def __repr__(self) -> str:
+        return (
+            f'{type(self).__name__}('
+            f'{self._lines!r}, x={self.x}, y={self.y}, file_y={self.file_y}'
+            f')'
+        )
+
+    def __bool__(self) -> bool:
+        return bool(self._lines)
+
+    def __getitem__(self, idx: int) -> str:
+        return self._lines[idx]
+
+    def __iter__(self) -> Iterator[str]:
+        yield from self._lines
+
+    def __len__(self) -> int:
+        return len(self._lines)
+
+    # mutators
+
+    def __setitem__(self, idx: int, val: str) -> None:
+        if idx < 0:
+            idx %= len(self)
+        victim = self._lines[idx]
+
+        self._lines[idx] = val
+
+        for set_callback in self._set_callbacks:
+            set_callback(self, idx, victim)
+
+    def __delitem__(self, idx: int) -> None:
+        if idx < 0:
+            idx %= len(self)
+        victim = self._lines[idx]
+
+        del self._lines[idx]
+
+        for del_callback in self._del_callbacks:
+            del_callback(self, idx, victim)
+
+    def insert(self, idx: int, val: str) -> None:
+        if idx < 0:
+            idx %= len(self)
+
+        self._lines.insert(idx, val)
+
+        for ins_callback in self._ins_callbacks:
+            ins_callback(self, idx)
+
+    # also mutators, but implemented using above functions
+
+    def append(self, val: str) -> None:
+        self.insert(len(self), val)
+
+    def pop(self, idx: int = -1) -> str:
+        victim = self[idx]
+        del self[idx]
+        return victim
+
+    def restore_eof_invariant(self) -> None:
+        """the file lines will always contain a blank empty string at the end'
+        to simplify rendering.  call this whenever the last line may change
+        """
+        if self[-1] != '':
+            self.append('')
+
+    # event handling
+
+    def add_set_callback(self, cb: SetCallback) -> None:
+        self._set_callbacks.append(cb)
+
+    def remove_set_callback(self, cb: SetCallback) -> None:
+        self._set_callbacks.remove(cb)
+
+    def add_del_callback(self, cb: DelCallback) -> None:
+        self._del_callbacks.append(cb)
+
+    def remove_del_callback(self, cb: DelCallback) -> None:
+        self._del_callbacks.remove(cb)
+
+    def add_ins_callback(self, cb: InsCallback) -> None:
+        self._ins_callbacks.append(cb)
+
+    def remove_ins_callback(self, cb: InsCallback) -> None:
+        self._ins_callbacks.remove(cb)
+
+    @contextlib.contextmanager
+    def record(self) -> Generator[List[Modification], None, None]:
+        modifications: List[Modification] = []
+
+        def set_cb(buf: 'Buf', idx: int, victim: str) -> None:
+            modifications.append(SetModification(idx, victim))
+
+        def del_cb(buf: 'Buf', idx: int, victim: str) -> None:
+            modifications.append(InsModification(idx, victim))
+
+        def ins_cb(buf: 'Buf', idx: int) -> None:
+            modifications.append(DelModification(idx))
+
+        self.add_set_callback(set_cb)
+        self.add_del_callback(del_cb)
+        self.add_ins_callback(ins_cb)
+        try:
+            yield modifications
+        finally:
+            self.remove_ins_callback(ins_cb)
+            self.remove_del_callback(del_cb)
+            self.remove_set_callback(set_cb)
+
+    def apply(self, modifications: List[Modification]) -> List[Modification]:
+        with self.record() as ret_modifications:
+            for modification in reversed(modifications):
+                modification(self)
+        return ret_modifications
+
+    # position properties
+
+    @property
+    def displayable_count(self) -> int:
+        return len(self._lines) - self.file_y
+
+    @property
+    def x(self) -> int:
+        return self._x
+
+    @x.setter
+    def x(self, x: int) -> None:
+        self._x = x
+        self._x_hint = self._cursor_x
+
+    def _extend_positions(self, idx: int) -> None:
+        self._positions.extend([None] * (1 + idx - len(self._positions)))
+
+    def _set_cb(self, buf: 'Buf', idx: int, victim: str) -> None:
+        self._extend_positions(idx)
+        self._positions[idx] = None
+
+    def _del_cb(self, buf: 'Buf', idx: int, victim: str) -> None:
+        self._extend_positions(idx)
+        del self._positions[idx]
+
+    def _ins_cb(self, buf: 'Buf', idx: int) -> None:
+        self._extend_positions(idx)
+        self._positions.insert(idx, None)
+
+    def line_positions(self, idx: int) -> Tuple[int, ...]:
+        self._extend_positions(idx)
+        value = self._positions[idx]
+        if value is None:
+            value = self._positions[idx] = _offsets(self._lines[idx])
+        return value
+
+    def line_x(self, margin: Margin) -> int:
+        return line_x(self._cursor_x, margin.cols)
+
+    @property
+    def _cursor_x(self) -> int:
+        return self.line_positions(self.y)[self.x]
+
+    def cursor_position(self, margin: Margin) -> Tuple[int, int]:
+        y = self.y - self.file_y + margin.header
+        x = self._cursor_x - self.line_x(margin)
+        return y, x
+
+    # rendered lines
+
+    def rendered_line(self, idx: int, margin: Margin) -> str:
+        x = self._cursor_x if idx == self.y else 0
+        return scrolled_line(self._lines[idx].expandtabs(4), x, margin.cols)
+
+    # movement
+
+    def scroll_screen_if_needed(self, margin: Margin) -> None:
+        # if the `y` is not on screen, make it so
+        if not (self.file_y <= self.y < self.file_y + margin.body_lines):
+            self.file_y = max(self.y - margin.body_lines // 2, 0)
+
+    def _set_x_after_vertical_movement(self) -> None:
+        positions = self.line_positions(self.y)
+        x = bisect.bisect_left(positions, self._x_hint)
+        x = min(len(self._lines[self.y]), x)
+        if positions[x] > self._x_hint:
+            x -= 1
+        self._x = x
+
+    def up(self, margin: Margin) -> None:
+        if self.y > 0:
+            self.y -= 1
+            if self.y < self.file_y:
+                self.file_y = max(self.file_y - margin.scroll_amount, 0)
+            self._set_x_after_vertical_movement()
+
+    def down(self, margin: Margin) -> None:
+        if self.y < len(self._lines) - 1:
+            self.y += 1
+            if self.y >= self.file_y + margin.body_lines:
+                self.file_y += margin.scroll_amount
+            self._set_x_after_vertical_movement()
+
+    def right(self, margin: Margin) -> None:
+        if self.x >= len(self._lines[self.y]):
+            if self.y < len(self._lines) - 1:
+                self.down(margin)
+                self.x = 0
+        else:
+            self.x += 1
+
+    def left(self, margin: Margin) -> None:
+        if self.x == 0:
+            if self.y > 0:
+                self.up(margin)
+                self.x = len(self._lines[self.y])
+        else:
+            self.x -= 1
+
+    # screen movement
+
+    def file_up(self, margin: Margin) -> None:
+        if self.file_y > 0:
+            self.file_y -= 1
+            if self.y > self.file_y + margin.body_lines - 1:
+                self.up(margin)
+
+    def file_down(self, margin: Margin) -> None:
+        if self.file_y < len(self._lines) - 1:
+            self.file_y += 1
+            if self.y < self.file_y:
+                self.down(margin)

babi/cached_property.py

@@ -0,0 +1,26 @@
+import sys
+
+if sys.version_info >= (3, 8):  # pragma: no cover (>=py38)
+    from functools import cached_property
+else:  # pragma: no cover (<py38)
+    from typing import Callable
+    from typing import Generic
+    from typing import Optional
+    from typing import Type
+    from typing import TypeVar
+
+    TSelf = TypeVar('TSelf')
+    TRet = TypeVar('TRet')
+
+    class cached_property(Generic[TSelf, TRet]):
+        def __init__(self, func: Callable[[TSelf], TRet]) -> None:
+            self._func = func
+
+        def __get__(
+                self,
+                instance: Optional[TSelf],
+                owner: Optional[Type[TSelf]] = None,
+        ) -> TRet:
+            assert instance is not None
+            ret = instance.__dict__[self._func.__name__] = self._func(instance)
+            return ret

babi/color.py

@@ -0,0 +1,20 @@
+from typing import NamedTuple
+
+# TODO: find a standard which defines these
+# limited number of "named" colors
+NAMED_COLORS = {'white': '#ffffff', 'black': '#000000'}
+
+
+class Color(NamedTuple):
+    r: int
+    g: int
+    b: int
+
+    @classmethod
+    def parse(cls, s: str) -> 'Color':
+        if s.startswith('#') and len(s) >= 7:
+            return cls(r=int(s[1:3], 16), g=int(s[3:5], 16), b=int(s[5:7], 16))
+        elif s.startswith('#'):
+            return cls.parse(f'#{s[1] * 2}{s[2] * 2}{s[3] * 2}')
+        else:
+            return cls.parse(NAMED_COLORS[s])

babi/color_kd.py

@@ -0,0 +1,90 @@
+import functools
+import itertools
+from typing import List
+from typing import NamedTuple
+from typing import Optional
+from typing import Tuple
+
+from babi._types import Protocol
+from babi.color import Color
+
+
+def _square_distance(c1: Color, c2: Color) -> int:
+    return (c1.r - c2.r) ** 2 + (c1.g - c2.g) ** 2 + (c1.b - c2.b) ** 2
+
+
+class KD(Protocol):
+    @property
+    def color(self) -> Color: ...
+    @property
+    def n(self) -> int: ...
+    @property
+    def left(self) -> Optional['KD']: ...
+    @property
+    def right(self) -> Optional['KD']: ...
+
+
+class _KD(NamedTuple):
+    color: Color
+    n: int
+    left: Optional[KD]
+    right: Optional[KD]
+
+
+def _build(colors: List[Tuple[Color, int]], depth: int = 0) -> Optional[KD]:
+    if not colors:
+        return None
+
+    axis = depth % 3
+    colors.sort(key=lambda kv: kv[0][axis])
+    pivot = len(colors) // 2
+
+    return _KD(
+        *colors[pivot],
+        _build(colors[:pivot], depth=depth + 1),
+        _build(colors[pivot + 1:], depth=depth + 1),
+    )
+
+
+def nearest(color: Color, colors: Optional[KD]) -> int:
+    best = 0
+    dist = 2 ** 32
+
+    def _search(kd: Optional[KD], *, depth: int) -> None:
+        nonlocal best
+        nonlocal dist
+
+        if kd is None:
+            return
+
+        cand_dist = _square_distance(color, kd.color)
+        if cand_dist < dist:
+            best, dist = kd.n, cand_dist
+
+        axis = depth % 3
+        diff = color[axis] - kd.color[axis]
+        if diff > 0:
+            _search(kd.right, depth=depth + 1)
+            if diff ** 2 < dist:
+                _search(kd.left, depth=depth + 1)
+        else:
+            _search(kd.left, depth=depth + 1)
+            if diff ** 2 < dist:
+                _search(kd.right, depth=depth + 1)
+
+    _search(colors, depth=0)
+    return best
+
+
+@functools.lru_cache(maxsize=1)
+def make_256() -> Optional[KD]:
+    vals = (0, 95, 135, 175, 215, 255)
+    colors = [
+        (Color(r, g, b), i)
+        for i, (r, g, b) in enumerate(itertools.product(vals, vals, vals), 16)
+    ]
+    for i in range(24):
+        v = 10 * i + 8
+        colors.append((Color(v, v, v), 232 + i))
+
+    return _build(colors)

babi/color_manager.py

@@ -0,0 +1,47 @@
+import curses
+from typing import Dict
+from typing import NamedTuple
+from typing import Optional
+from typing import Tuple
+
+from babi import color_kd
+from babi.color import Color
+
+
+def _color_to_curses(color: Color) -> Tuple[int, int, int]:
+    factor = 1000 / 255
+    return int(color.r * factor), int(color.g * factor), int(color.b * factor)
+
+
+class ColorManager(NamedTuple):
+    colors: Dict[Color, int]
+    raw_pairs: Dict[Tuple[int, int], int]
+
+    def init_color(self, color: Color) -> None:
+        if curses.can_change_color():
+            n = min(self.colors.values(), default=256) - 1
+            self.colors[color] = n
+            curses.init_color(n, *_color_to_curses(color))
+        elif curses.COLORS >= 256:
+            self.colors[color] = color_kd.nearest(color, color_kd.make_256())
+        else:
+            self.colors[color] = -1
+
+    def color_pair(self, fg: Optional[Color], bg: Optional[Color]) -> int:
+        fg_i = self.colors[fg] if fg is not None else -1
+        bg_i = self.colors[bg] if bg is not None else -1
+        return self.raw_color_pair(fg_i, bg_i)
+
+    def raw_color_pair(self, fg: int, bg: int) -> int:
+        try:
+            return self.raw_pairs[(fg, bg)]
+        except KeyError:
+            pass
+
+        n = self.raw_pairs[(fg, bg)] = len(self.raw_pairs) + 1
+        curses.init_pair(n, fg, bg)
+        return n
+
+    @classmethod
+    def make(cls) -> 'ColorManager':
+        return cls({}, {})

babi/fdict.py

@@ -0,0 +1,44 @@
+from typing import Generic
+from typing import Iterable
+from typing import Mapping
+from typing import TypeVar
+
+from babi._types import Protocol
+
+TKey = TypeVar('TKey', contravariant=True)
+TValue = TypeVar('TValue', covariant=True)
+
+
+class FDict(Generic[TKey, TValue]):
+    def __init__(self, dct: Mapping[TKey, TValue]) -> None:
+        self._dct = dct
+
+    def __getitem__(self, k: TKey) -> TValue:
+        return self._dct[k]
+
+    def __contains__(self, k: TKey) -> bool:
+        return k in self._dct
+
+    def __repr__(self) -> str:
+        return f'{type(self).__name__}({self._dct})'
+
+    def values(self) -> Iterable[TValue]:
+        return self._dct.values()
+
+
+class Indexable(Generic[TKey, TValue], Protocol):
+    def __getitem__(self, key: TKey) -> TValue: ...
+
+
+class FChainMap(Generic[TKey, TValue]):
+    def __init__(self, *mappings: Indexable[TKey, TValue]) -> None:
+        self._mappings = mappings
+
+    def __getitem__(self, key: TKey) -> TValue:
+        for mapping in reversed(self._mappings):
+            try:
+                return mapping[key]
+            except KeyError:
+                pass
+        else:
+            raise KeyError(key)

babi/file.py

@@ -0,0 +1,826 @@
+import collections
+import contextlib
+import curses
+import functools
+import hashlib
+import io
+import itertools
+import os.path
+from typing import Any
+from typing import Callable
+from typing import cast
+from typing import Generator
+from typing import IO
+from typing import List
+from typing import Match
+from typing import NamedTuple
+from typing import Optional
+from typing import Pattern
+from typing import Tuple
+from typing import TYPE_CHECKING
+from typing import TypeVar
+from typing import Union
+
+from babi.buf import Buf
+from babi.buf import Modification
+from babi.color_manager import ColorManager
+from babi.hl.interface import FileHL
+from babi.hl.interface import HLFactory
+from babi.hl.replace import Replace
+from babi.hl.selection import Selection
+from babi.hl.trailing_whitespace import TrailingWhitespace
+from babi.margin import Margin
+from babi.prompt import PromptResult
+from babi.status import Status
+
+if TYPE_CHECKING:
+    from babi.main import Screen  # XXX: circular
+
+TCallable = TypeVar('TCallable', bound=Callable[..., Any])
+
+
+def get_lines(sio: IO[str]) -> Tuple[List[str], str, bool, str]:
+    sha256 = hashlib.sha256()
+    lines = []
+    newlines = collections.Counter({'\n': 0})  # default to `\n`
+    for line in sio:
+        sha256.update(line.encode())
+        for ending in ('\r\n', '\n'):
+            if line.endswith(ending):
+                lines.append(line[:-1 * len(ending)])
+                newlines[ending] += 1
+                break
+        else:
+            lines.append(line)
+    # always make sure we end in a newline
+    lines.append('')
+    (nl, _), = newlines.most_common(1)
+    mixed = len({k for k, v in newlines.items() if v}) > 1
+    return lines, nl, mixed, sha256.hexdigest()
+
+
+class Action:
+    def __init__(
+            self, *, name: str, modifications: List[Modification],
+            start_x: int, start_y: int, start_modified: bool,
+            end_x: int, end_y: int, end_modified: bool,
+            final: bool,
+    ):
+        self.name = name
+        self.modifications = modifications
+        self.start_x = start_x
+        self.start_y = start_y
+        self.start_modified = start_modified
+        self.end_x = end_x
+        self.end_y = end_y
+        self.end_modified = end_modified
+        self.final = final
+
+    def apply(self, file: 'File') -> 'Action':
+        action = Action(
+            name=self.name, modifications=file.buf.apply(self.modifications),
+            start_x=self.end_x, start_y=self.end_y,
+            start_modified=self.end_modified,
+            end_x=self.start_x, end_y=self.start_y,
+            end_modified=self.start_modified,
+            final=True,
+        )
+
+        file.buf.y = self.start_y
+        file.buf.x = self.start_x
+        file.modified = self.start_modified
+
+        return action
+
+
+def action(func: TCallable) -> TCallable:
+    @functools.wraps(func)
+    def action_inner(self: 'File', *args: Any, **kwargs: Any) -> Any:
+        self.finalize_previous_action()
+        return func(self, *args, **kwargs)
+    return cast(TCallable, action_inner)
+
+
+def edit_action(
+        name: str,
+        *,
+        final: bool,
+) -> Callable[[TCallable], TCallable]:
+    def edit_action_decorator(func: TCallable) -> TCallable:
+        @functools.wraps(func)
+        def edit_action_inner(self: 'File', *args: Any, **kwargs: Any) -> Any:
+            with self.edit_action_context(name, final=final):
+                return func(self, *args, **kwargs)
+        return cast(TCallable, edit_action_inner)
+    return edit_action_decorator
+
+
+def keep_selection(func: TCallable) -> TCallable:
+    @functools.wraps(func)
+    def keep_selection_inner(self: 'File', *args: Any, **kwargs: Any) -> Any:
+        with self.select():
+            return func(self, *args, **kwargs)
+    return cast(TCallable, keep_selection_inner)
+
+
+def clear_selection(func: TCallable) -> TCallable:
+    @functools.wraps(func)
+    def clear_selection_inner(self: 'File', *args: Any, **kwargs: Any) -> Any:
+        ret = func(self, *args, **kwargs)
+        self.selection.clear()
+        return ret
+    return cast(TCallable, clear_selection_inner)
+
+
+class Found(NamedTuple):
+    y: int
+    match: Match[str]
+
+
+class _SearchIter:
+    def __init__(
+            self,
+            file: 'File',
+            reg: Pattern[str],
+            *,
+            offset: int,
+    ) -> None:
+        self.file = file
+        self.reg = reg
+        self.offset = offset
+        self.wrapped = False
+        self._start_x = file.buf.x + offset
+        self._start_y = file.buf.y
+
+    def __iter__(self) -> '_SearchIter':
+        return self
+
+    def _stop_if_past_original(self, y: int, match: Match[str]) -> Found:
+        if (
+                self.wrapped and (
+                    y > self._start_y or
+                    y == self._start_y and match.start() >= self._start_x
+                )
+        ):
+            raise StopIteration()
+        return Found(y, match)
+
+    def __next__(self) -> Tuple[int, Match[str]]:
+        x = self.file.buf.x + self.offset
+        y = self.file.buf.y
+
+        match = self.reg.search(self.file.buf[y], x)
+        if match:
+            return self._stop_if_past_original(y, match)
+
+        if self.wrapped:
+            for line_y in range(y + 1, self._start_y + 1):
+                match = self.reg.search(self.file.buf[line_y])
+                if match:
+                    return self._stop_if_past_original(line_y, match)
+        else:
+            for line_y in range(y + 1, len(self.file.buf)):
+                match = self.reg.search(self.file.buf[line_y])
+                if match:
+                    return self._stop_if_past_original(line_y, match)
+
+            self.wrapped = True
+
+            for line_y in range(0, self._start_y + 1):
+                match = self.reg.search(self.file.buf[line_y])
+                if match:
+                    return self._stop_if_past_original(line_y, match)
+
+        raise StopIteration()
+
+
+class File:
+    def __init__(
+            self,
+            filename: Optional[str],
+            color_manager: ColorManager,
+            hl_factories: Tuple[HLFactory, ...],
+    ) -> None:
+        self.filename = filename
+        self.modified = False
+        self.buf = Buf([])
+        self.nl = '\n'
+        self.sha256: Optional[str] = None
+        self._in_edit_action = False
+        self.undo_stack: List[Action] = []
+        self.redo_stack: List[Action] = []
+        self._hl_factories = hl_factories
+        self._trailing_whitespace = TrailingWhitespace(color_manager)
+        self._replace_hl = Replace()
+        self.selection = Selection()
+        self._file_hls: Tuple[FileHL, ...] = ()
+
+    def ensure_loaded(self, status: Status, stdin: str) -> None:
+        if self.buf:
+            return
+
+        if self.filename == '-':
+            status.update('(from stdin)')
+            self.filename = None
+            self.modified = True
+            sio = io.StringIO(stdin)
+            lines, self.nl, mixed, self.sha256 = get_lines(sio)
+        elif self.filename is not None and os.path.isfile(self.filename):
+            with open(self.filename, newline='') as f:
+                lines, self.nl, mixed, self.sha256 = get_lines(f)
+        else:
+            if self.filename is not None:
+                if os.path.lexists(self.filename):
+                    status.update(f'{self.filename!r} is not a file')
+                    self.filename = None
+                else:
+                    status.update('(new file)')
+            lines, self.nl, mixed, self.sha256 = get_lines(io.StringIO(''))
+
+        self.buf = Buf(lines)
+
+        if mixed:
+            status.update(f'mixed newlines will be converted to {self.nl!r}')
+            self.modified = True
+
+        file_hls = []
+        for factory in self._hl_factories:
+            if self.filename is not None:
+                hl = factory.file_highlighter(self.filename, self.buf[0])
+                file_hls.append(hl)
+            else:
+                file_hls.append(factory.blank_file_highlighter())
+        self._file_hls = (
+            *file_hls,
+            self._trailing_whitespace, self._replace_hl, self.selection,
+        )
+        for file_hl in self._file_hls:
+            file_hl.register_callbacks(self.buf)
+
+    def __repr__(self) -> str:
+        return f'<{type(self).__name__} {self.filename!r}>'
+
+    # movement
+
+    @action
+    def up(self, margin: Margin) -> None:
+        self.buf.up(margin)
+
+    @action
+    def down(self, margin: Margin) -> None:
+        self.buf.down(margin)
+
+    @action
+    def right(self, margin: Margin) -> None:
+        self.buf.right(margin)
+
+    @action
+    def left(self, margin: Margin) -> None:
+        self.buf.left(margin)
+
+    @action
+    def home(self, margin: Margin) -> None:
+        self.buf.x = 0
+
+    @action
+    def end(self, margin: Margin) -> None:
+        self.buf.x = len(self.buf[self.buf.y])
+
+    @action
+    def ctrl_up(self, margin: Margin) -> None:
+        self.buf.file_up(margin)
+
+    @action
+    def ctrl_down(self, margin: Margin) -> None:
+        self.buf.file_down(margin)
+
+    @action
+    def ctrl_right(self, margin: Margin) -> None:
+        line = self.buf[self.buf.y]
+        # if we're at the second to last character, jump to end of line
+        if self.buf.x == len(line) - 1:
+            self.buf.right(margin)
+        # if we're at the end of the line, jump forward to the next non-ws
+        elif self.buf.x == len(line):
+            while (
+                    self.buf.y < len(self.buf) - 1 and (
+                        self.buf.x == len(self.buf[self.buf.y]) or
+                        self.buf[self.buf.y][self.buf.x].isspace()
+                    )
+            ):
+                self.buf.right(margin)
+        # if we're inside the line, jump to next position that's not our type
+        else:
+            self.buf.right(margin)
+            tp = line[self.buf.x].isalnum()
+            while self.buf.x < len(line) and tp == line[self.buf.x].isalnum():
+                self.buf.right(margin)
+
+    @action
+    def ctrl_left(self, margin: Margin) -> None:
+        line = self.buf[self.buf.y]
+        # if we're at position 1 and it's not a space, go to the beginning
+        if self.buf.x == 1 and not line[:self.buf.x].isspace():
+            self.buf.left(margin)
+        # if we're at the beginning or it's all space up to here jump to the
+        # end of the previous non-space line
+        elif self.buf.x == 0 or line[:self.buf.x].isspace():
+            self.buf.x = 0
+            while self.buf.y > 0 and self.buf.x == 0:
+                self.buf.left(margin)
+        else:
+            self.buf.left(margin)
+            tp = line[self.buf.x - 1].isalnum()
+            while self.buf.x > 0 and tp == line[self.buf.x - 1].isalnum():
+                self.buf.left(margin)
+
+    @action
+    def ctrl_home(self, margin: Margin) -> None:
+        self.buf.x = 0
+        self.buf.y = self.buf.file_y = 0
+
+    @action
+    def ctrl_end(self, margin: Margin) -> None:
+        self.buf.x = 0
+        self.buf.y = len(self.buf) - 1
+        self.buf.scroll_screen_if_needed(margin)
+
+    @action
+    def go_to_line(self, lineno: int, margin: Margin) -> None:
+        self.buf.x = 0
+        if lineno == 0:
+            self.buf.y = 0
+        elif lineno > len(self.buf):
+            self.buf.y = len(self.buf) - 1
+        elif lineno < 0:
+            self.buf.y = max(0, lineno + len(self.buf))
+        else:
+            self.buf.y = lineno - 1
+        self.buf.scroll_screen_if_needed(margin)
+
+    @action
+    def search(
+            self,
+            reg: Pattern[str],
+            status: Status,
+            margin: Margin,
+    ) -> None:
+        search = _SearchIter(self, reg, offset=1)
+        try:
+            line_y, match = next(iter(search))
+        except StopIteration:
+            status.update('no matches')
+        else:
+            if line_y == self.buf.y and match.start() == self.buf.x:
+                status.update('this is the only occurrence')
+            else:
+                if search.wrapped:
+                    status.update('search wrapped')
+                self.buf.y = line_y
+                self.buf.x = match.start()
+                self.buf.scroll_screen_if_needed(margin)
+
+    @clear_selection
+    def replace(
+            self,
+            screen: 'Screen',
+            reg: Pattern[str],
+            replace: str,
+    ) -> None:
+        self.finalize_previous_action()
+
+        count = 0
+        res: Union[str, PromptResult] = ''
+        search = _SearchIter(self, reg, offset=0)
+        for line_y, match in search:
+            end = match.end()
+            self.buf.y = line_y
+            self.buf.x = match.start()
+            self.buf.scroll_screen_if_needed(screen.margin)
+            if res != 'a':  # make `a` replace the rest of them
+                with self._replace_hl.region(self.buf.y, self.buf.x, end):
+                    screen.draw()
+                    res = screen.quick_prompt('replace', ('yes', 'no', 'all'))
+            if res in {'y', 'a'}:
+                count += 1
+                with self.edit_action_context('replace', final=True):
+                    replaced = match.expand(replace)
+                    line = screen.file.buf[line_y]
+                    if '\n' in replaced:
+                        replaced_lines = replaced.split('\n')
+                        self.buf[line_y] = (
+                            f'{line[:match.start()]}{replaced_lines[0]}'
+                        )
+                        for i, ins_line in enumerate(replaced_lines[1:-1], 1):
+                            self.buf.insert(line_y + i, ins_line)
+                        last_insert = line_y + len(replaced_lines) - 1
+                        self.buf.insert(
+                            last_insert, f'{replaced_lines[-1]}{line[end:]}',
+                        )
+                        self.buf.y = last_insert
+                        self.buf.x = 0
+                        search.offset = len(replaced_lines[-1])
+                    else:
+                        self.buf[line_y] = (
+                            f'{line[:match.start()]}{replaced}{line[end:]}'
+                        )
+                        search.offset = len(replaced)
+            elif res == 'n':
+                search.offset = 1
+            else:
+                assert res is PromptResult.CANCELLED
+                return
+
+        if res == '':  # we never went through the loop
+            screen.status.update('no matches')
+        else:
+            occurrences = 'occurrence' if count == 1 else 'occurrences'
+            screen.status.update(f'replaced {count} {occurrences}')
+
+    @action
+    def page_up(self, margin: Margin) -> None:
+        if self.buf.y < margin.body_lines:
+            self.buf.y = self.buf.file_y = 0
+        else:
+            pos = max(self.buf.file_y - margin.page_size, 0)
+            self.buf.y = self.buf.file_y = pos
+        self.buf.x = 0
+
+    @action
+    def page_down(self, margin: Margin) -> None:
+        if self.buf.file_y + margin.body_lines >= len(self.buf):
+            self.buf.y = len(self.buf) - 1
+        else:
+            pos = self.buf.file_y + margin.page_size
+            self.buf.y = self.buf.file_y = pos
+        self.buf.x = 0
+
+    # editing
+
+    @edit_action('backspace text', final=False)
+    @clear_selection
+    def backspace(self, margin: Margin) -> None:
+        # backspace at the beginning of the file does nothing
+        if self.buf.y == 0 and self.buf.x == 0:
+            pass
+        # backspace at the end of the file does not change the contents
+        elif self.buf.y == len(self.buf) - 1:
+            self.buf.left(margin)
+        # at the beginning of the line, we join the current line and
+        # the previous line
+        elif self.buf.x == 0:
+            y, victim = self.buf.y, self.buf.pop(self.buf.y)
+            self.buf.left(margin)
+            self.buf[y - 1] += victim
+        else:
+            s = self.buf[self.buf.y]
+            self.buf[self.buf.y] = s[:self.buf.x - 1] + s[self.buf.x:]
+            self.buf.left(margin)
+
+    @edit_action('delete text', final=False)
+    @clear_selection
+    def delete(self, margin: Margin) -> None:
+        if (
+            # noop at end of the file
+            self.buf.y == len(self.buf) - 1 or
+            # noop at end of last real line
+            (
+                self.buf.y == len(self.buf) - 2 and
+                self.buf.x == len(self.buf[self.buf.y])
+            )
+        ):
+            pass
+        # if we're at the end of the line, collapse the line afterwards
+        elif self.buf.x == len(self.buf[self.buf.y]):
+            victim = self.buf.pop(self.buf.y + 1)
+            self.buf[self.buf.y] += victim
+        else:
+            s = self.buf[self.buf.y]
+            self.buf[self.buf.y] = s[:self.buf.x] + s[self.buf.x + 1:]
+
+    @edit_action('line break', final=False)
+    @clear_selection
+    def enter(self, margin: Margin) -> None:
+        s = self.buf[self.buf.y]
+        self.buf[self.buf.y] = s[:self.buf.x]
+        self.buf.insert(self.buf.y + 1, s[self.buf.x:])
+        self.buf.down(margin)
+        self.buf.x = 0
+
+    @edit_action('indent selection', final=True)
+    def _indent_selection(self, margin: Margin) -> None:
+        assert self.selection.start is not None
+        sel_y, sel_x = self.selection.start
+        (s_y, _), (e_y, _) = self.selection.get()
+        for l_y in range(s_y, e_y + 1):
+            if self.buf[l_y]:
+                self.buf[l_y] = ' ' * 4 + self.buf[l_y]
+                if l_y == self.buf.y:
+                    self.buf.x += 4
+                if l_y == sel_y and sel_x != 0:
+                    sel_x += 4
+        self.selection.set(sel_y, sel_x, self.buf.y, self.buf.x)
+
+    @edit_action('insert tab', final=False)
+    def _tab(self, margin: Margin) -> None:
+        n = 4 - self.buf.x % 4
+        line = self.buf[self.buf.y]
+        self.buf[self.buf.y] = line[:self.buf.x] + n * ' ' + line[self.buf.x:]
+        self.buf.x += n
+        self.buf.restore_eof_invariant()
+
+    def tab(self, margin: Margin) -> None:
+        if self.selection.start is not None:
+            self._indent_selection(margin)
+        else:
+            self._tab(margin)
+
+    @staticmethod
+    def _dedent_line(s: str) -> int:
+        bound = min(len(s), 4)
+        i = 0
+        while i < bound and s[i] == ' ':
+            i += 1
+        return i
+
+    @edit_action('dedent selection', final=True)
+    def _dedent_selection(self, margin: Margin) -> None:
+        assert self.selection.start is not None
+        sel_y, sel_x = self.selection.start
+        (s_y, _), (e_y, _) = self.selection.get()
+        for l_y in range(s_y, e_y + 1):
+            n = self._dedent_line(self.buf[l_y])
+            if n:
+                self.buf[l_y] = self.buf[l_y][n:]
+                if l_y == self.buf.y:
+                    self.buf.x = max(self.buf.x - n, 0)
+                if l_y == sel_y:
+                    sel_x = max(sel_x - n, 0)
+        self.selection.set(sel_y, sel_x, self.buf.y, self.buf.x)
+
+    @edit_action('dedent', final=True)
+    def _dedent(self, margin: Margin) -> None:
+        n = self._dedent_line(self.buf[self.buf.y])
+        if n:
+            self.buf[self.buf.y] = self.buf[self.buf.y][n:]
+            self.buf.x = max(self.buf.x - n, 0)
+
+    def shift_tab(self, margin: Margin) -> None:
+        if self.selection.start is not None:
+            self._dedent_selection(margin)
+        else:
+            self._dedent(margin)
+
+    @edit_action('cut selection', final=True)
+    @clear_selection
+    def cut_selection(self, margin: Margin) -> Tuple[str, ...]:
+        ret = []
+        (s_y, s_x), (e_y, e_x) = self.selection.get()
+        if s_y == e_y:
+            ret.append(self.buf[s_y][s_x:e_x])
+            self.buf[s_y] = self.buf[s_y][:s_x] + self.buf[s_y][e_x:]
+        else:
+            ret.append(self.buf[s_y][s_x:])
+            for l_y in range(s_y + 1, e_y):
+                ret.append(self.buf[l_y])
+            ret.append(self.buf[e_y][:e_x])
+
+            self.buf[s_y] = self.buf[s_y][:s_x] + self.buf[e_y][e_x:]
+            for _ in range(s_y + 1, e_y + 1):
+                self.buf.pop(s_y + 1)
+        self.buf.y = s_y
+        self.buf.x = s_x
+        self.buf.scroll_screen_if_needed(margin)
+        return tuple(ret)
+
+    def cut(self, cut_buffer: Tuple[str, ...]) -> Tuple[str, ...]:
+        # only continue a cut if the last action is a non-final cut
+        if not self._continue_last_action('cut'):
+            cut_buffer = ()
+
+        with self.edit_action_context('cut', final=False):
+            if self.buf.y == len(self.buf) - 1:
+                return cut_buffer
+            else:
+                victim = self.buf.pop(self.buf.y)
+                self.buf.x = 0
+                return cut_buffer + (victim,)
+
+    def _uncut(self, cut_buffer: Tuple[str, ...], margin: Margin) -> None:
+        for cut_line in cut_buffer:
+            line = self.buf[self.buf.y]
+            before, after = line[:self.buf.x], line[self.buf.x:]
+            self.buf[self.buf.y] = before + cut_line
+            self.buf.insert(self.buf.y + 1, after)
+            self.buf.down(margin)
+            self.buf.x = 0
+
+    @edit_action('uncut', final=True)
+    @clear_selection
+    def uncut(self, cut_buffer: Tuple[str, ...], margin: Margin) -> None:
+        self._uncut(cut_buffer, margin)
+
+    @edit_action('uncut selection', final=True)
+    @clear_selection
+    def uncut_selection(
+            self,
+            cut_buffer: Tuple[str, ...], margin: Margin,
+    ) -> None:
+        self._uncut(cut_buffer, margin)
+        self.buf.up(margin)
+        self.buf.x = len(self.buf[self.buf.y])
+        self.buf[self.buf.y] += self.buf.pop(self.buf.y + 1)
+        self.buf.restore_eof_invariant()
+
+    def _sort(self, margin: Margin, s_y: int, e_y: int) -> None:
+        # self.buf intentionally does not support slicing so we use islice
+        lines = sorted(itertools.islice(self.buf, s_y, e_y))
+        for i, line in zip(range(s_y, e_y), lines):
+            self.buf[i] = line
+
+        self.buf.y = s_y
+        self.buf.x = 0
+        self.buf.scroll_screen_if_needed(margin)
+
+    @edit_action('sort', final=True)
+    def sort(self, margin: Margin) -> None:
+        self._sort(margin, 0, len(self.buf) - 1)
+
+    @edit_action('sort selection', final=True)
+    @clear_selection
+    def sort_selection(self, margin: Margin) -> None:
+        (s_y, _), (e_y, _) = self.selection.get()
+        e_y = min(e_y + 1, len(self.buf) - 1)
+        if self.buf[e_y - 1] == '':
+            e_y -= 1
+        self._sort(margin, s_y, e_y)
+
+    DISPATCH = {
+        # movement
+        b'KEY_UP': up,
+        b'KEY_DOWN': down,
+        b'KEY_RIGHT': right,
+        b'KEY_LEFT': left,
+        b'KEY_HOME': home,
+        b'^A': home,
+        b'KEY_END': end,
+        b'^E': end,
+        b'KEY_PPAGE': page_up,
+        b'^Y': page_up,
+        b'KEY_NPAGE': page_down,
+        b'^V': page_down,
+        b'kUP5': ctrl_up,
+        b'kDN5': ctrl_down,
+        b'kRIT5': ctrl_right,
+        b'kLFT5': ctrl_left,
+        b'kHOM5': ctrl_home,
+        b'kEND5': ctrl_end,
+        # editing
+        b'KEY_BACKSPACE': backspace,
+        b'KEY_DC': delete,
+        b'^M': enter,
+        b'^I': tab,
+        b'KEY_BTAB': shift_tab,
+        # selection (shift + movement)
+        b'KEY_SR': keep_selection(up),
+        b'KEY_SF': keep_selection(down),
+        b'KEY_SLEFT': keep_selection(left),
+        b'KEY_SRIGHT': keep_selection(right),
+        b'KEY_SHOME': keep_selection(home),
+        b'KEY_SEND': keep_selection(end),
+        b'KEY_SPREVIOUS': keep_selection(page_up),
+        b'KEY_SNEXT': keep_selection(page_down),
+        b'kRIT6': keep_selection(ctrl_right),
+        b'kLFT6': keep_selection(ctrl_left),
+        b'kHOM6': keep_selection(ctrl_home),
+        b'kEND6': keep_selection(ctrl_end),
+    }
+
+    @edit_action('text', final=False)
+    @clear_selection
+    def c(self, wch: str, margin: Margin) -> None:
+        s = self.buf[self.buf.y]
+        self.buf[self.buf.y] = s[:self.buf.x] + wch + s[self.buf.x:]
+        self.buf.x += len(wch)
+        self.buf.restore_eof_invariant()
+
+    def finalize_previous_action(self) -> None:
+        assert not self._in_edit_action, 'nested edit/movement'
+        self.selection.clear()
+        if self.undo_stack:
+            self.undo_stack[-1].final = True
+
+    def _continue_last_action(self, name: str) -> bool:
+        return (
+            bool(self.undo_stack) and
+            self.undo_stack[-1].name == name and
+            not self.undo_stack[-1].final
+        )
+
+    @contextlib.contextmanager
+    def edit_action_context(
+            self, name: str,
+            *,
+            final: bool,
+    ) -> Generator[None, None, None]:
+        continue_last = self._continue_last_action(name)
+        if not continue_last and self.undo_stack:
+            self.undo_stack[-1].final = True
+
+        before_x, before_line = self.buf.x, self.buf.y
+        before_modified = self.modified
+        assert not self._in_edit_action, f'recursive action? {name}'
+        self._in_edit_action = True
+        try:
+            with self.buf.record() as modifications:
+                yield
+        finally:
+            self._in_edit_action = False
+            self.redo_stack.clear()
+            if continue_last:
+                self.undo_stack[-1].end_x = self.buf.x
+                self.undo_stack[-1].end_y = self.buf.y
+                self.undo_stack[-1].modifications.extend(modifications)
+            elif modifications:
+                self.modified = True
+                action = Action(
+                    name=name, modifications=modifications,
+                    start_x=before_x, start_y=before_line,
+                    start_modified=before_modified,
+                    end_x=self.buf.x, end_y=self.buf.y,
+                    end_modified=True,
+                    final=final,
+                )
+                self.undo_stack.append(action)
+
+    @contextlib.contextmanager
+    def select(self) -> Generator[None, None, None]:
+        if self.selection.start is None:
+            start = (self.buf.y, self.buf.x)
+        else:
+            start = self.selection.start
+        try:
+            yield
+        finally:
+            self.selection.set(*start, self.buf.y, self.buf.x)
+
+    # positioning
+
+    def move_cursor(
+            self,
+            stdscr: 'curses._CursesWindow',
+            margin: Margin,
+    ) -> None:
+        stdscr.move(*self.buf.cursor_position(margin))
+
+    def draw(self, stdscr: 'curses._CursesWindow', margin: Margin) -> None:
+        to_display = min(self.buf.displayable_count, margin.body_lines)
+
+        for file_hl in self._file_hls:
+            # XXX: this will go away?
+            file_hl.highlight_until(self.buf, self.buf.file_y + to_display)
+
+        for i in range(to_display):
+            draw_y = i + margin.header
+            l_y = self.buf.file_y + i
+            stdscr.insstr(draw_y, 0, self.buf.rendered_line(l_y, margin))
+
+            l_x = self.buf.line_x(margin) if l_y == self.buf.y else 0
+            l_x_max = l_x + margin.cols
+            for file_hl in self._file_hls:
+                for region in file_hl.regions[l_y]:
+                    l_positions = self.buf.line_positions(l_y)
+                    r_x = l_positions[region.x]
+                    # the selection highlight intentionally extends one past
+                    # the end of the line, which won't have a position
+                    if region.end == len(l_positions):
+                        r_end = l_positions[-1] + 1
+                    else:
+                        r_end = l_positions[region.end]
+
+                    if r_x >= l_x_max:
+                        break
+                    elif r_end <= l_x:
+                        continue
+
+                    if l_x and r_x <= l_x:
+                        if file_hl.include_edge:
+                            h_s_x = 0
+                        else:
+                            h_s_x = 1
+                    else:
+                        h_s_x = r_x - l_x
+
+                    if r_end >= l_x_max and l_x_max < l_positions[-1]:
+                        if file_hl.include_edge:
+                            h_e_x = margin.cols
+                        else:
+                            h_e_x = margin.cols - 1
+                    else:
+                        h_e_x = r_end - l_x
+
+                    stdscr.chgat(draw_y, h_s_x, h_e_x - h_s_x, region.attr)
+
+        for i in range(to_display, margin.body_lines):
+            stdscr.move(i + margin.header, 0)
+            stdscr.clrtoeol()

babi/highlight.py

@@ -0,0 +1,776 @@
+import functools
+import json
+import os.path
+from typing import Any
+from typing import Dict
+from typing import FrozenSet
+from typing import List
+from typing import Match
+from typing import NamedTuple
+from typing import Optional
+from typing import Tuple
+from typing import TypeVar
+
+from identify.identify import tags_from_filename
+
+from babi._types import Protocol
+from babi.fdict import FChainMap
+from babi.reg import _Reg
+from babi.reg import _RegSet
+from babi.reg import ERR_REG
+from babi.reg import expand_escaped
+from babi.reg import make_reg
+from babi.reg import make_regset
+
+T = TypeVar('T')
+Scope = Tuple[str, ...]
+Regions = Tuple['Region', ...]
+Captures = Tuple[Tuple[int, '_Rule'], ...]
+
+
+def uniquely_constructed(t: T) -> T:
+    """avoid tuple.__hash__ for "singleton" constructed objects"""
+    t.__hash__ = object.__hash__  # type: ignore
+    return t
+
+
+def _split_name(s: Optional[str]) -> Tuple[str, ...]:
+    if s is None:
+        return ()
+    else:
+        return tuple(s.split())
+
+
+class _Rule(Protocol):
+    """hax for recursive types python/mypy#731"""
+    @property
+    def name(self) -> Tuple[str, ...]: ...
+    @property
+    def match(self) -> Optional[str]: ...
+    @property
+    def begin(self) -> Optional[str]: ...
+    @property
+    def end(self) -> Optional[str]: ...
+    @property
+    def while_(self) -> Optional[str]: ...
+    @property
+    def content_name(self) -> Tuple[str, ...]: ...
+    @property
+    def captures(self) -> Captures: ...
+    @property
+    def begin_captures(self) -> Captures: ...
+    @property
+    def end_captures(self) -> Captures: ...
+    @property
+    def while_captures(self) -> Captures: ...
+    @property
+    def include(self) -> Optional[str]: ...
+    @property
+    def patterns(self) -> 'Tuple[_Rule, ...]': ...
+    @property
+    def repository(self) -> 'FChainMap[str, _Rule]': ...
+
+
+@uniquely_constructed
+class Rule(NamedTuple):
+    name: Tuple[str, ...]
+    match: Optional[str]
+    begin: Optional[str]
+    end: Optional[str]
+    while_: Optional[str]
+    content_name: Tuple[str, ...]
+    captures: Captures
+    begin_captures: Captures
+    end_captures: Captures
+    while_captures: Captures
+    include: Optional[str]
+    patterns: Tuple[_Rule, ...]
+    repository: FChainMap[str, _Rule]
+
+    @classmethod
+    def make(
+            cls,
+            dct: Dict[str, Any],
+            parent_repository: FChainMap[str, _Rule],
+    ) -> _Rule:
+        if 'repository' in dct:
+            # this looks odd, but it's so we can have a self-referential
+            # immutable-after-construction chain map
+            repository_dct: Dict[str, _Rule] = {}
+            repository = FChainMap(parent_repository, repository_dct)
+            for k, sub_dct in dct['repository'].items():
+                repository_dct[k] = Rule.make(sub_dct, repository)
+        else:
+            repository = parent_repository
+
+        name = _split_name(dct.get('name'))
+        match = dct.get('match')
+        begin = dct.get('begin')
+        end = dct.get('end')
+        while_ = dct.get('while')
+        content_name = _split_name(dct.get('contentName'))
+
+        if 'captures' in dct:
+            captures = tuple(
+                (int(k), Rule.make(v, repository))
+                for k, v in dct['captures'].items()
+            )
+        else:
+            captures = ()
+
+        if 'beginCaptures' in dct:
+            begin_captures = tuple(
+                (int(k), Rule.make(v, repository))
+                for k, v in dct['beginCaptures'].items()
+            )
+        else:
+            begin_captures = ()
+
+        if 'endCaptures' in dct:
+            end_captures = tuple(
+                (int(k), Rule.make(v, repository))
+                for k, v in dct['endCaptures'].items()
+            )
+        else:
+            end_captures = ()
+
+        if 'whileCaptures' in dct:
+            while_captures = tuple(
+                (int(k), Rule.make(v, repository))
+                for k, v in dct['whileCaptures'].items()
+            )
+        else:
+            while_captures = ()
+
+        # some grammars (at least xml) have begin rules with no end
+        if begin is not None and end is None and while_ is None:
+            end = '$impossible^'
+
+        # Using the captures key for a begin/end/while rule is short-hand for
+        # giving both beginCaptures and endCaptures with same values
+        if begin and end and captures:
+            begin_captures = end_captures = captures
+            captures = ()
+        elif begin and while_ and captures:
+            begin_captures = while_captures = captures
+            captures = ()
+
+        include = dct.get('include')
+
+        if 'patterns' in dct:
+            patterns = tuple(Rule.make(d, repository) for d in dct['patterns'])
+        else:
+            patterns = ()
+
+        return cls(
+            name=name,
+            match=match,
+            begin=begin,
+            end=end,
+            while_=while_,
+            content_name=content_name,
+            captures=captures,
+            begin_captures=begin_captures,
+            end_captures=end_captures,
+            while_captures=while_captures,
+            include=include,
+            patterns=patterns,
+            repository=repository,
+        )
+
+
+@uniquely_constructed
+class Grammar(NamedTuple):
+    scope_name: str
+    repository: FChainMap[str, _Rule]
+    patterns: Tuple[_Rule, ...]
+
+    @classmethod
+    def make(cls, data: Dict[str, Any]) -> 'Grammar':
+        scope_name = data['scopeName']
+        if 'repository' in data:
+            # this looks odd, but it's so we can have a self-referential
+            # immutable-after-construction chain map
+            repository_dct: Dict[str, _Rule] = {}
+            repository = FChainMap(repository_dct)
+            for k, dct in data['repository'].items():
+                repository_dct[k] = Rule.make(dct, repository)
+        else:
+            repository = FChainMap()
+        patterns = tuple(Rule.make(d, repository) for d in data['patterns'])
+        return cls(
+            scope_name=scope_name,
+            repository=repository,
+            patterns=patterns,
+        )
+
+
+class Region(NamedTuple):
+    start: int
+    end: int
+    scope: Scope
+
+
+class State(NamedTuple):
+    entries: Tuple['Entry', ...]
+    while_stack: Tuple[Tuple['WhileRule', int], ...]
+
+    @classmethod
+    def root(cls, entry: 'Entry') -> 'State':
+        return cls((entry,), ())
+
+    @property
+    def cur(self) -> 'Entry':
+        return self.entries[-1]
+
+    def push(self, entry: 'Entry') -> 'State':
+        return self._replace(entries=(*self.entries, entry))
+
+    def pop(self) -> 'State':
+        return self._replace(entries=self.entries[:-1])
+
+    def push_while(self, rule: 'WhileRule', entry: 'Entry') -> 'State':
+        entries = (*self.entries, entry)
+        while_stack = (*self.while_stack, (rule, len(entries)))
+        return self._replace(entries=entries, while_stack=while_stack)
+
+    def pop_while(self) -> 'State':
+        entries, while_stack = self.entries[:-1], self.while_stack[:-1]
+        return self._replace(entries=entries, while_stack=while_stack)
+
+
+class CompiledRule(Protocol):
+    @property
+    def name(self) -> Tuple[str, ...]: ...
+
+    def start(
+            self,
+            compiler: 'Compiler',
+            match: Match[str],
+            state: State,
+    ) -> Tuple[State, bool, Regions]:
+        ...
+
+    def search(
+            self,
+            compiler: 'Compiler',
+            state: State,
+            line: str,
+            pos: int,
+            first_line: bool,
+            boundary: bool,
+    ) -> Optional[Tuple[State, int, bool, Regions]]:
+        ...
+
+
+class CompiledRegsetRule(CompiledRule, Protocol):
+    @property
+    def regset(self) -> _RegSet: ...
+    @property
+    def u_rules(self) -> Tuple[_Rule, ...]: ...
+
+
+class Entry(NamedTuple):
+    scope: Tuple[str, ...]
+    rule: CompiledRule
+    reg: _Reg = ERR_REG
+    boundary: bool = False
+
+
+def _inner_capture_parse(
+        compiler: 'Compiler',
+        start: int,
+        s: str,
+        scope: Scope,
+        rule: CompiledRule,
+) -> Regions:
+    state = State.root(Entry(scope + rule.name, rule))
+    _, regions = highlight_line(compiler, state, s, first_line=False)
+    return tuple(
+        r._replace(start=r.start + start, end=r.end + start) for r in regions
+    )
+
+
+def _captures(
+        compiler: 'Compiler',
+        scope: Scope,
+        match: Match[str],
+        captures: Captures,
+) -> Regions:
+    ret: List[Region] = []
+    pos, pos_end = match.span()
+    for i, u_rule in captures:
+        try:
+            group_s = match[i]
+        except IndexError:  # some grammars are malformed here?
+            continue
+        if not group_s:
+            continue
+
+        rule = compiler.compile_rule(u_rule)
+        start, end = match.span(i)
+        if start < pos:
+            # TODO: could maybe bisect but this is probably fast enough
+            j = len(ret) - 1
+            while j > 0 and start < ret[j - 1].end:
+                j -= 1
+
+            oldtok = ret[j]
+            newtok = []
+            if start > oldtok.start:
+                newtok.append(oldtok._replace(end=start))
+
+            newtok.extend(
+                _inner_capture_parse(
+                    compiler, start, match[i], oldtok.scope, rule,
+                ),
+            )
+
+            if end < oldtok.end:
+                newtok.append(oldtok._replace(start=end))
+            ret[j:j + 1] = newtok
+        else:
+            if start > pos:
+                ret.append(Region(pos, start, scope))
+
+            ret.extend(
+                _inner_capture_parse(compiler, start, match[i], scope, rule),
+            )
+
+            pos = end
+
+    if pos < pos_end:
+        ret.append(Region(pos, pos_end, scope))
+    return tuple(ret)
+
+
+def _do_regset(
+        idx: int,
+        match: Optional[Match[str]],
+        rule: CompiledRegsetRule,
+        compiler: 'Compiler',
+        state: State,
+        pos: int,
+) -> Optional[Tuple[State, int, bool, Regions]]:
+    if match is None:
+        return None
+
+    ret = []
+    if match.start() > pos:
+        ret.append(Region(pos, match.start(), state.cur.scope))
+
+    target_rule = compiler.compile_rule(rule.u_rules[idx])
+    state, boundary, regions = target_rule.start(compiler, match, state)
+    ret.extend(regions)
+
+    return state, match.end(), boundary, tuple(ret)
+
+
+@uniquely_constructed
+class PatternRule(NamedTuple):
+    name: Tuple[str, ...]
+    regset: _RegSet
+    u_rules: Tuple[_Rule, ...]
+
+    def start(
+            self,
+            compiler: 'Compiler',
+            match: Match[str],
+            state: State,
+    ) -> Tuple[State, bool, Regions]:
+        raise AssertionError(f'unreachable {self}')
+
+    def search(
+            self,
+            compiler: 'Compiler',
+            state: State,
+            line: str,
+            pos: int,
+            first_line: bool,
+            boundary: bool,
+    ) -> Optional[Tuple[State, int, bool, Regions]]:
+        idx, match = self.regset.search(line, pos, first_line, boundary)
+        return _do_regset(idx, match, self, compiler, state, pos)
+
+
+@uniquely_constructed
+class MatchRule(NamedTuple):
+    name: Tuple[str, ...]
+    captures: Captures
+
+    def start(
+            self,
+            compiler: 'Compiler',
+            match: Match[str],
+            state: State,
+    ) -> Tuple[State, bool, Regions]:
+        scope = state.cur.scope + self.name
+        return state, False, _captures(compiler, scope, match, self.captures)
+
+    def search(
+            self,
+            compiler: 'Compiler',
+            state: State,
+            line: str,
+            pos: int,
+            first_line: bool,
+            boundary: bool,
+    ) -> Optional[Tuple[State, int, bool, Regions]]:
+        raise AssertionError(f'unreachable {self}')
+
+
+@uniquely_constructed
+class EndRule(NamedTuple):
+    name: Tuple[str, ...]
+    content_name: Tuple[str, ...]
+    begin_captures: Captures
+    end_captures: Captures
+    end: str
+    regset: _RegSet
+    u_rules: Tuple[_Rule, ...]
+
+    def start(
+            self,
+            compiler: 'Compiler',
+            match: Match[str],
+            state: State,
+    ) -> Tuple[State, bool, Regions]:
+        scope = state.cur.scope + self.name
+        next_scope = scope + self.content_name
+
+        boundary = match.end() == len(match.string)
+        reg = make_reg(expand_escaped(match, self.end))
+        state = state.push(Entry(next_scope, self, reg, boundary))
+        regions = _captures(compiler, scope, match, self.begin_captures)
+        return state, True, regions
+
+    def _end_ret(
+            self,
+            compiler: 'Compiler',
+            state: State,
+            pos: int,
+            m: Match[str],
+    ) -> Tuple[State, int, bool, Regions]:
+        ret = []
+        if m.start() > pos:
+            ret.append(Region(pos, m.start(), state.cur.scope))
+        ret.extend(_captures(compiler, state.cur.scope, m, self.end_captures))
+        return state.pop(), m.end(), False, tuple(ret)
+
+    def search(
+            self,
+            compiler: 'Compiler',
+            state: State,
+            line: str,
+            pos: int,
+            first_line: bool,
+            boundary: bool,
+    ) -> Optional[Tuple[State, int, bool, Regions]]:
+        end_match = state.cur.reg.search(line, pos, first_line, boundary)
+        if end_match is not None and end_match.start() == pos:
+            return self._end_ret(compiler, state, pos, end_match)
+        elif end_match is None:
+            idx, match = self.regset.search(line, pos, first_line, boundary)
+            return _do_regset(idx, match, self, compiler, state, pos)
+        else:
+            idx, match = self.regset.search(line, pos, first_line, boundary)
+            if match is None or end_match.start() <= match.start():
+                return self._end_ret(compiler, state, pos, end_match)
+            else:
+                return _do_regset(idx, match, self, compiler, state, pos)
+
+
+@uniquely_constructed
+class WhileRule(NamedTuple):
+    name: Tuple[str, ...]
+    content_name: Tuple[str, ...]
+    begin_captures: Captures
+    while_captures: Captures
+    while_: str
+    regset: _RegSet
+    u_rules: Tuple[_Rule, ...]
+
+    def start(
+            self,
+            compiler: 'Compiler',
+            match: Match[str],
+            state: State,
+    ) -> Tuple[State, bool, Regions]:
+        scope = state.cur.scope + self.name
+        next_scope = scope + self.content_name
+
+        boundary = match.end() == len(match.string)
+        reg = make_reg(expand_escaped(match, self.while_))
+        state = state.push_while(self, Entry(next_scope, self, reg, boundary))
+        regions = _captures(compiler, scope, match, self.begin_captures)
+        return state, True, regions
+
+    def continues(
+            self,
+            compiler: 'Compiler',
+            state: State,
+            line: str,
+            pos: int,
+            first_line: bool,
+            boundary: bool,
+    ) -> Optional[Tuple[int, bool, Regions]]:
+        match = state.cur.reg.match(line, pos, first_line, boundary)
+        if match is None:
+            return None
+
+        ret = _captures(compiler, state.cur.scope, match, self.while_captures)
+        return match.end(), True, ret
+
+    def search(
+            self,
+            compiler: 'Compiler',
+            state: State,
+            line: str,
+            pos: int,
+            first_line: bool,
+            boundary: bool,
+    ) -> Optional[Tuple[State, int, bool, Regions]]:
+        idx, match = self.regset.search(line, pos, first_line, boundary)
+        return _do_regset(idx, match, self, compiler, state, pos)
+
+
+class Compiler:
+    def __init__(self, grammar: Grammar, grammars: 'Grammars') -> None:
+        self._root_scope = grammar.scope_name
+        self._grammars = grammars
+        self._rule_to_grammar: Dict[_Rule, Grammar] = {}
+        self._c_rules: Dict[_Rule, CompiledRule] = {}
+        root = self._compile_root(grammar)
+        self.root_state = State.root(Entry(root.name, root))
+
+    def _visit_rule(self, grammar: Grammar, rule: _Rule) -> _Rule:
+        self._rule_to_grammar[rule] = grammar
+        return rule
+
+    @functools.lru_cache(maxsize=None)
+    def _include(
+            self,
+            grammar: Grammar,
+            repository: FChainMap[str, _Rule],
+            s: str,
+    ) -> Tuple[List[str], Tuple[_Rule, ...]]:
+        if s == '$self':
+            return self._patterns(grammar, grammar.patterns)
+        elif s == '$base':
+            grammar = self._grammars.grammar_for_scope(self._root_scope)
+            return self._include(grammar, grammar.repository, '$self')
+        elif s.startswith('#'):
+            return self._patterns(grammar, (repository[s[1:]],))
+        elif '#' not in s:
+            grammar = self._grammars.grammar_for_scope(s)
+            return self._include(grammar, grammar.repository, '$self')
+        else:
+            scope, _, s = s.partition('#')
+            grammar = self._grammars.grammar_for_scope(scope)
+            return self._include(grammar, grammar.repository, f'#{s}')
+
+    @functools.lru_cache(maxsize=None)
+    def _patterns(
+            self,
+            grammar: Grammar,
+            rules: Tuple[_Rule, ...],
+    ) -> Tuple[List[str], Tuple[_Rule, ...]]:
+        ret_regs = []
+        ret_rules: List[_Rule] = []
+        for rule in rules:
+            if rule.include is not None:
+                tmp_regs, tmp_rules = self._include(
+                    grammar, rule.repository, rule.include,
+                )
+                ret_regs.extend(tmp_regs)
+                ret_rules.extend(tmp_rules)
+            elif rule.match is None and rule.begin is None and rule.patterns:
+                tmp_regs, tmp_rules = self._patterns(grammar, rule.patterns)
+                ret_regs.extend(tmp_regs)
+                ret_rules.extend(tmp_rules)
+            elif rule.match is not None:
+                ret_regs.append(rule.match)
+                ret_rules.append(self._visit_rule(grammar, rule))
+            elif rule.begin is not None:
+                ret_regs.append(rule.begin)
+                ret_rules.append(self._visit_rule(grammar, rule))
+            else:
+                raise AssertionError(f'unreachable {rule}')
+        return ret_regs, tuple(ret_rules)
+
+    def _captures_ref(
+            self,
+            grammar: Grammar,
+            captures: Captures,
+    ) -> Captures:
+        return tuple((n, self._visit_rule(grammar, r)) for n, r in captures)
+
+    def _compile_root(self, grammar: Grammar) -> PatternRule:
+        regs, rules = self._patterns(grammar, grammar.patterns)
+        return PatternRule((grammar.scope_name,), make_regset(*regs), rules)
+
+    def _compile_rule(self, grammar: Grammar, rule: _Rule) -> CompiledRule:
+        assert rule.include is None, rule
+        if rule.match is not None:
+            captures_ref = self._captures_ref(grammar, rule.captures)
+            return MatchRule(rule.name, captures_ref)
+        elif rule.begin is not None and rule.end is not None:
+            regs, rules = self._patterns(grammar, rule.patterns)
+            return EndRule(
+                rule.name,
+                rule.content_name,
+                self._captures_ref(grammar, rule.begin_captures),
+                self._captures_ref(grammar, rule.end_captures),
+                rule.end,
+                make_regset(*regs),
+                rules,
+            )
+        elif rule.begin is not None and rule.while_ is not None:
+            regs, rules = self._patterns(grammar, rule.patterns)
+            return WhileRule(
+                rule.name,
+                rule.content_name,
+                self._captures_ref(grammar, rule.begin_captures),
+                self._captures_ref(grammar, rule.while_captures),
+                rule.while_,
+                make_regset(*regs),
+                rules,
+            )
+        else:
+            regs, rules = self._patterns(grammar, rule.patterns)
+            return PatternRule(rule.name, make_regset(*regs), rules)
+
+    def compile_rule(self, rule: _Rule) -> CompiledRule:
+        try:
+            return self._c_rules[rule]
+        except KeyError:
+            pass
+
+        grammar = self._rule_to_grammar[rule]
+        ret = self._c_rules[rule] = self._compile_rule(grammar, rule)
+        return ret
+
+
+class Grammars:
+    def __init__(self, *directories: str) -> None:
+        self._scope_to_files = {
+            os.path.splitext(filename)[0]: os.path.join(directory, filename)
+            for directory in directories
+            if os.path.exists(directory)
+            for filename in sorted(os.listdir(directory))
+            if filename.endswith('.json')
+        }
+
+        unknown_grammar = {'scopeName': 'source.unknown', 'patterns': []}
+        self._raw = {'source.unknown': unknown_grammar}
+        self._file_types: List[Tuple[FrozenSet[str], str]] = []
+        self._first_line: List[Tuple[_Reg, str]] = []
+        self._parsed: Dict[str, Grammar] = {}
+        self._compiled: Dict[str, Compiler] = {}
+
+    def _raw_for_scope(self, scope: str) -> Dict[str, Any]:
+        try:
+            return self._raw[scope]
+        except KeyError:
+            pass
+
+        grammar_path = self._scope_to_files.pop(scope)
+        with open(grammar_path) as f:
+            ret = self._raw[scope] = json.load(f)
+
+        file_types = frozenset(ret.get('fileTypes', ()))
+        first_line = make_reg(ret.get('firstLineMatch', '$impossible^'))
+
+        self._file_types.append((file_types, scope))
+        self._first_line.append((first_line, scope))
+
+        return ret
+
+    def grammar_for_scope(self, scope: str) -> Grammar:
+        try:
+            return self._parsed[scope]
+        except KeyError:
+            pass
+
+        raw = self._raw_for_scope(scope)
+        ret = self._parsed[scope] = Grammar.make(raw)
+        return ret
+
+    def compiler_for_scope(self, scope: str) -> Compiler:
+        try:
+            return self._compiled[scope]
+        except KeyError:
+            pass
+
+        grammar = self.grammar_for_scope(scope)
+        ret = self._compiled[scope] = Compiler(grammar, self)
+        return ret
+
+    def blank_compiler(self) -> Compiler:
+        return self.compiler_for_scope('source.unknown')
+
+    def compiler_for_file(self, filename: str, first_line: str) -> Compiler:
+        for tag in tags_from_filename(filename) - {'text'}:
+            try:
+                # TODO: this doesn't always match even if we detect it
+                return self.compiler_for_scope(f'source.{tag}')
+            except KeyError:
+                pass
+
+        # didn't find it in the fast path, need to read all the json
+        for k in tuple(self._scope_to_files):
+            self._raw_for_scope(k)
+
+        _, _, ext = os.path.basename(filename).rpartition('.')
+        for extensions, scope in self._file_types:
+            if ext in extensions:
+                return self.compiler_for_scope(scope)
+
+        for reg, scope in self._first_line:
+            if reg.match(first_line, 0, first_line=True, boundary=True):
+                return self.compiler_for_scope(scope)
+
+        return self.compiler_for_scope('source.unknown')
+
+
+def highlight_line(
+        compiler: 'Compiler',
+        state: State,
+        line: str,
+        first_line: bool,
+) -> Tuple[State, Regions]:
+    ret: List[Region] = []
+    pos = 0
+    boundary = state.cur.boundary
+
+    # TODO: this is still a little wasteful
+    while_stack = []
+    for while_rule, idx in state.while_stack:
+        while_stack.append((while_rule, idx))
+        while_state = State(state.entries[:idx], tuple(while_stack))
+
+        while_res = while_rule.continues(
+            compiler, while_state, line, pos, first_line, boundary,
+        )
+        if while_res is None:
+            state = while_state.pop_while()
+            break
+        else:
+            pos, boundary, regions = while_res
+            ret.extend(regions)
+
+    search_res = state.cur.rule.search(
+        compiler, state, line, pos, first_line, boundary,
+    )
+    while search_res is not None:
+        state, pos, boundary, regions = search_res
+        ret.extend(regions)
+
+        search_res = state.cur.rule.search(
+            compiler, state, line, pos, first_line, boundary,
+        )
+
+    if pos < len(line):
+        ret.append(Region(pos, len(line), state.cur.scope))
+
+    return state, tuple(ret)

babi/history.py

@@ -0,0 +1,32 @@
+import collections
+import contextlib
+import os.path
+from typing import Dict
+from typing import Generator
+from typing import List
+
+from babi.user_data import xdg_data
+
+
+class History:
+    def __init__(self) -> None:
+        self._orig_len: Dict[str, int] = collections.defaultdict(int)
+        self.data: Dict[str, List[str]] = collections.defaultdict(list)
+        self.prev: Dict[str, str] = {}
+
+    @contextlib.contextmanager
+    def save(self) -> Generator[None, None, None]:
+        history_dir = xdg_data('history')
+        os.makedirs(history_dir, exist_ok=True)
+        for filename in os.listdir(history_dir):
+            with open(os.path.join(history_dir, filename)) as f:
+                self.data[filename] = f.read().splitlines()
+                self._orig_len[filename] = len(self.data[filename])
+        try:
+            yield
+        finally:
+            for k, v in self.data.items():
+                new_history = v[self._orig_len[k]:]
+                if new_history:
+                    with open(os.path.join(history_dir, k), 'a+') as f:
+                        f.write('\n'.join(new_history) + '\n')

babi/hl/interface.py

@@ -0,0 +1,32 @@
+from typing import NamedTuple
+from typing import Tuple
+
+from babi._types import Protocol
+from babi.buf import Buf
+
+
+class HL(NamedTuple):
+    x: int
+    end: int
+    attr: int
+
+
+HLs = Tuple[HL, ...]
+
+
+class RegionsMapping(Protocol):
+    def __getitem__(self, idx: int) -> HLs: ...
+
+
+class FileHL(Protocol):
+    @property
+    def include_edge(self) -> bool: ...
+    @property
+    def regions(self) -> RegionsMapping: ...
+    def highlight_until(self, lines: Buf, idx: int) -> None: ...
+    def register_callbacks(self, buf: Buf) -> None: ...
+
+
+class HLFactory(Protocol):
+    def file_highlighter(self, filename: str, first_line: str) -> FileHL: ...
+    def blank_file_highlighter(self) -> FileHL: ...

babi/hl/replace.py

@@ -0,0 +1,32 @@
+import collections
+import contextlib
+import curses
+from typing import Dict
+from typing import Generator
+
+from babi.buf import Buf
+from babi.hl.interface import HL
+from babi.hl.interface import HLs
+
+
+class Replace:
+    include_edge = True
+
+    def __init__(self) -> None:
+        self.regions: Dict[int, HLs] = collections.defaultdict(tuple)
+
+    def highlight_until(self, lines: Buf, idx: int) -> None:
+        """our highlight regions are populated in other ways"""
+
+    def register_callbacks(self, buf: Buf) -> None:
+        """our highlight regions are populated in other ways"""
+
+    @contextlib.contextmanager
+    def region(self, y: int, x: int, end: int) -> Generator[None, None, None]:
+        # XXX: this assumes pair 1 is the background
+        attr = curses.A_REVERSE | curses.A_DIM | curses.color_pair(1)
+        self.regions[y] = (HL(x=x, end=end, attr=attr),)
+        try:
+            yield
+        finally:
+            del self.regions[y]

babi/hl/selection.py

@@ -0,0 +1,58 @@
+import collections
+import curses
+from typing import Dict
+from typing import Optional
+from typing import Tuple
+
+from babi.buf import Buf
+from babi.hl.interface import HL
+from babi.hl.interface import HLs
+
+
+class Selection:
+    include_edge = True
+
+    def __init__(self) -> None:
+        self.regions: Dict[int, HLs] = collections.defaultdict(tuple)
+        self.start: Optional[Tuple[int, int]] = None
+        self.end: Optional[Tuple[int, int]] = None
+
+    def register_callbacks(self, buf: Buf) -> None:
+        """our highlight regions are populated in other ways"""
+
+    def highlight_until(self, lines: Buf, idx: int) -> None:
+        if self.start is None or self.end is None:
+            return
+
+        # XXX: this assumes pair 1 is the background
+        attr = curses.A_REVERSE | curses.A_DIM | curses.color_pair(1)
+        (s_y, s_x), (e_y, e_x) = self.get()
+        if s_y == e_y:
+            self.regions[s_y] = (HL(x=s_x, end=e_x, attr=attr),)
+        else:
+            self.regions[s_y] = (
+                HL(x=s_x, end=len(lines[s_y]) + 1, attr=attr),
+            )
+            for l_y in range(s_y + 1, e_y):
+                self.regions[l_y] = (
+                    HL(x=0, end=len(lines[l_y]) + 1, attr=attr),
+                )
+            self.regions[e_y] = (HL(x=0, end=e_x, attr=attr),)
+
+    def get(self) -> Tuple[Tuple[int, int], Tuple[int, int]]:
+        assert self.start is not None and self.end is not None
+        if self.start < self.end:
+            return self.start, self.end
+        else:
+            return self.end, self.start
+
+    def clear(self) -> None:
+        if self.start is not None and self.end is not None:
+            (s_y, _), (e_y, _) = self.get()
+            for l_y in range(s_y, e_y + 1):
+                del self.regions[l_y]
+        self.start = self.end = None
+
+    def set(self, s_y: int, s_x: int, e_y: int, e_x: int) -> None:
+        self.clear()
+        self.start, self.end = (s_y, s_x), (e_y, e_x)

babi/hl/syntax.py

@@ -0,0 +1,165 @@
+import curses
+import functools
+import math
+from typing import Callable
+from typing import List
+from typing import NamedTuple
+from typing import Optional
+from typing import Tuple
+
+from babi.buf import Buf
+from babi.color_manager import ColorManager
+from babi.highlight import Compiler
+from babi.highlight import Grammars
+from babi.highlight import highlight_line
+from babi.highlight import State
+from babi.hl.interface import HL
+from babi.hl.interface import HLs
+from babi.theme import Style
+from babi.theme import Theme
+from babi.user_data import prefix_data
+from babi.user_data import xdg_config
+from babi.user_data import xdg_data
+
+A_ITALIC = getattr(curses, 'A_ITALIC', 0x80000000)  # new in py37
+
+
+class FileSyntax:
+    include_edge = False
+
+    def __init__(
+            self,
+            compiler: Compiler,
+            theme: Theme,
+            color_manager: ColorManager,
+    ) -> None:
+        self._compiler = compiler
+        self._theme = theme
+        self._color_manager = color_manager
+
+        self.regions: List[HLs] = []
+        self._states: List[State] = []
+
+        # this will be assigned a functools.lru_cache per instance for
+        # better hit rate and memory usage
+        self._hl: Optional[Callable[[State, str, bool], Tuple[State, HLs]]]
+        self._hl = None
+
+    def attr(self, style: Style) -> int:
+        pair = self._color_manager.color_pair(style.fg, style.bg)
+        return (
+            curses.color_pair(pair) |
+            curses.A_BOLD * style.b |
+            A_ITALIC * style.i |
+            curses.A_UNDERLINE * style.u
+        )
+
+    def _hl_uncached(
+            self,
+            state: State,
+            line: str,
+            first_line: bool,
+    ) -> Tuple[State, HLs]:
+        new_state, regions = highlight_line(
+            self._compiler, state, f'{line}\n', first_line=first_line,
+        )
+
+        # remove the trailing newline
+        new_end = regions[-1]._replace(end=regions[-1].end - 1)
+        regions = regions[:-1] + (new_end,)
+
+        regs: List[HL] = []
+        for r in regions:
+            style = self._theme.select(r.scope)
+            if style == self._theme.default:
+                continue
+
+            attr = self.attr(style)
+            if (
+                    regs and
+                    regs[-1].attr == attr and
+                    regs[-1].end == r.start
+            ):
+                regs[-1] = regs[-1]._replace(end=r.end)
+            else:
+                regs.append(HL(x=r.start, end=r.end, attr=attr))
+
+        return new_state, tuple(regs)
+
+    def _set_cb(self, lines: Buf, idx: int, victim: str) -> None:
+        del self.regions[idx:]
+        del self._states[idx:]
+
+    def _del_cb(self, lines: Buf, idx: int, victim: str) -> None:
+        del self.regions[idx:]
+        del self._states[idx:]
+
+    def _ins_cb(self, lines: Buf, idx: int) -> None:
+        del self.regions[idx:]
+        del self._states[idx:]
+
+    def register_callbacks(self, buf: Buf) -> None:
+        buf.add_set_callback(self._set_cb)
+        buf.add_del_callback(self._del_cb)
+        buf.add_ins_callback(self._ins_cb)
+
+    def highlight_until(self, lines: Buf, idx: int) -> None:
+        if self._hl is None:
+            # the docs claim better performance with power of two sizing
+            size = max(4096, 2 ** (int(math.log(len(lines), 2)) + 2))
+            self._hl = functools.lru_cache(maxsize=size)(self._hl_uncached)
+
+        if not self._states:
+            state = self._compiler.root_state
+        else:
+            state = self._states[-1]
+
+        for i in range(len(self._states), idx):
+            # https://github.com/python/mypy/issues/8579
+            state, regions = self._hl(state, lines[i], i == 0)  # type: ignore
+            self._states.append(state)
+            self.regions.append(regions)
+
+
+class Syntax(NamedTuple):
+    grammars: Grammars
+    theme: Theme
+    color_manager: ColorManager
+
+    def file_highlighter(self, filename: str, first_line: str) -> FileSyntax:
+        compiler = self.grammars.compiler_for_file(filename, first_line)
+        return FileSyntax(compiler, self.theme, self.color_manager)
+
+    def blank_file_highlighter(self) -> FileSyntax:
+        compiler = self.grammars.blank_compiler()
+        return FileSyntax(compiler, self.theme, self.color_manager)
+
+    def _init_screen(self, stdscr: 'curses._CursesWindow') -> None:
+        default_fg, default_bg = self.theme.default.fg, self.theme.default.bg
+        all_colors = {c for c in (default_fg, default_bg) if c is not None}
+        todo = list(self.theme.rules.children.values())
+        while todo:
+            rule = todo.pop()
+            if rule.style.fg is not None:
+                all_colors.add(rule.style.fg)
+            if rule.style.bg is not None:
+                all_colors.add(rule.style.bg)
+            todo.extend(rule.children.values())
+
+        for color in sorted(all_colors):
+            self.color_manager.init_color(color)
+
+        pair = self.color_manager.color_pair(default_fg, default_bg)
+        stdscr.bkgd(' ', curses.color_pair(pair))
+
+    @classmethod
+    def from_screen(
+            cls,
+            stdscr: 'curses._CursesWindow',
+            color_manager: ColorManager,
+    ) -> 'Syntax':
+        grammars = Grammars(prefix_data('grammar_v1'), xdg_data('grammar_v1'))
+        theme = Theme.from_filename(xdg_config('theme.json'))
+        ret = cls(grammars, theme, color_manager)
+        ret._init_screen(stdscr)
+        return ret

babi/hl/trailing_whitespace.py

@@ -0,0 +1,52 @@
+import curses
+from typing import List
+
+from babi.buf import Buf
+from babi.color_manager import ColorManager
+from babi.hl.interface import HL
+from babi.hl.interface import HLs
+
+
+class TrailingWhitespace:
+    include_edge = False
+
+    def __init__(self, color_manager: ColorManager) -> None:
+        self._color_manager = color_manager
+
+        self.regions: List[HLs] = []
+
+    def _trailing_ws(self, line: str) -> HLs:
+        if not line:
+            return ()
+
+        i = len(line)
+        while i > 0 and line[i - 1].isspace():
+            i -= 1
+
+        if i == len(line):
+            return ()
+        else:
+            pair = self._color_manager.raw_color_pair(-1, curses.COLOR_RED)
+            attr = curses.color_pair(pair)
+            return (HL(x=i, end=len(line), attr=attr),)
+
+    def _set_cb(self, lines: Buf, idx: int, victim: str) -> None:
+        if idx < len(self.regions):
+            self.regions[idx] = self._trailing_ws(lines[idx])
+
+    def _del_cb(self, lines: Buf, idx: int, victim: str) -> None:
+        if idx < len(self.regions):
+            del self.regions[idx]
+
+    def _ins_cb(self, lines: Buf, idx: int) -> None:
+        if idx < len(self.regions):
+            self.regions.insert(idx, self._trailing_ws(lines[idx]))
+
+    def register_callbacks(self, buf: Buf) -> None:
+        buf.add_set_callback(self._set_cb)
+        buf.add_del_callback(self._del_cb)
+        buf.add_ins_callback(self._ins_cb)
+
+    def highlight_until(self, lines: Buf, idx: int) -> None:
+        for i in range(len(self.regions), idx):
+            self.regions.append(self._trailing_ws(lines[i]))

babi/horizontal_scrolling.py

@@ -0,0 +1,46 @@
+import curses
+
+from babi.cached_property import cached_property
+
+
+def line_x(x: int, width: int) -> int:
+    if x + 1 < width:
+        return 0
+    elif width == 1:
+        return x
+    else:
+        margin = min(width - 3, 6)
+        return (
+            width - margin - 2 +
+            (x + 1 - width) //
+            (width - margin - 2) *
+            (width - margin - 2)
+        )
+
+
+def scrolled_line(s: str, x: int, width: int) -> str:
+    l_x = line_x(x, width)
+    if l_x:
+        s = f'«{s[l_x + 1:]}'
+        if len(s) > width:
+            return f'{s[:width - 1]}»'
+        else:
+            return s.ljust(width)
+    elif len(s) > width:
+        return f'{s[:width - 1]}»'
+    else:
+        return s.ljust(width)
+
+
+class _CalcWidth:
+    @cached_property
+    def _window(self) -> 'curses._CursesWindow':
+        return curses.newwin(1, 10)
+
+    def wcwidth(self, c: str) -> int:
+        self._window.addstr(0, 0, c)
+        return self._window.getyx()[1]
+
+
+wcwidth = _CalcWidth().wcwidth
+del _CalcWidth

babi/main.py

@@ -0,0 +1,113 @@
+import argparse
+import curses
+import os
+import sys
+from typing import Optional
+from typing import Sequence
+
+from babi.buf import Buf
+from babi.file import File
+from babi.perf import Perf
+from babi.perf import perf_log
+from babi.screen import EditResult
+from babi.screen import make_stdscr
+from babi.screen import Screen
+
+CONSOLE = 'CONIN$' if sys.platform == 'win32' else '/dev/tty'
+
+
+def _edit(screen: Screen, stdin: str) -> EditResult:
+    screen.file.ensure_loaded(screen.status, stdin)
+
+    while True:
+        screen.status.tick(screen.margin)
+        screen.draw()
+        screen.file.move_cursor(screen.stdscr, screen.margin)
+
+        key = screen.get_char()
+        if key.keyname in File.DISPATCH:
+            File.DISPATCH[key.keyname](screen.file, screen.margin)
+        elif key.keyname in Screen.DISPATCH:
+            ret = Screen.DISPATCH[key.keyname](screen)
+            if isinstance(ret, EditResult):
+                return ret
+        elif key.keyname == b'STRING':
+            assert isinstance(key.wch, str), key.wch
+            screen.file.c(key.wch, screen.margin)
+        else:
+            screen.status.update(f'unknown key: {key}')
+
+
+def c_main(
+        stdscr: 'curses._CursesWindow',
+        args: argparse.Namespace,
+        stdin: str,
+) -> int:
+    with perf_log(args.perf_log) as perf:
+        screen = Screen(stdscr, args.filenames or [None], perf)
+        with screen.history.save():
+            while screen.files:
+                screen.i = screen.i % len(screen.files)
+                res = _edit(screen, stdin)
+                if res == EditResult.EXIT:
+                    del screen.files[screen.i]
+                    # always go to the next file except at the end
+                    screen.i = min(screen.i, len(screen.files) - 1)
+                    screen.status.clear()
+                elif res == EditResult.NEXT:
+                    screen.i += 1
+                    screen.status.clear()
+                elif res == EditResult.PREV:
+                    screen.i -= 1
+                    screen.status.clear()
+                elif res == EditResult.OPEN:
+                    screen.i = len(screen.files) - 1
+                else:
+                    raise AssertionError(f'unreachable {res}')
+    return 0
+
+
+def _key_debug(stdscr: 'curses._CursesWindow') -> int:
+    screen = Screen(stdscr, ['<<key debug>>'], Perf())
+    screen.file.buf = Buf([''])
+
+    while True:
+        screen.status.update('press q to quit')
+        screen.draw()
+        screen.file.move_cursor(screen.stdscr, screen.margin)
+
+        key = screen.get_char()
+        screen.file.buf.insert(-1, f'{key.wch!r} {key.keyname.decode()!r}')
+        screen.file.down(screen.margin)
+        if key.wch == curses.KEY_RESIZE:
+            screen.resize()
+        if key.wch == 'q':
+            return 0
+
+
+def main(argv: Optional[Sequence[str]] = None) -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument('filenames', metavar='filename', nargs='*')
+    parser.add_argument('--perf-log')
+    parser.add_argument(
+        '--key-debug', action='store_true', help=argparse.SUPPRESS,
+    )
+    args = parser.parse_args(argv)
+
+    if '-' in args.filenames:
+        print('reading stdin...', file=sys.stderr)
+        stdin = sys.stdin.read()
+        tty = os.open(CONSOLE, os.O_RDONLY)
+        os.dup2(tty, sys.stdin.fileno())
+    else:
+        stdin = ''
+
+    with make_stdscr() as stdscr:
+        if args.key_debug:
+            return _key_debug(stdscr)
+        else:
+            return c_main(stdscr, args, stdin)
+
+
+if __name__ == '__main__':
+    exit(main())

babi/margin.py

@@ -0,0 +1,35 @@
+import curses
+from typing import NamedTuple
+
+
+class Margin(NamedTuple):
+    lines: int
+    cols: int
+
+    @property
+    def header(self) -> bool:
+        return self.lines > 2
+
+    @property
+    def footer(self) -> bool:
+        return self.lines > 1
+
+    @property
+    def body_lines(self) -> int:
+        return self.lines - self.header - self.footer
+
+    @property
+    def page_size(self) -> int:
+        if self.body_lines <= 2:
+            return 1
+        else:
+            return self.body_lines - 2
+
+    @property
+    def scroll_amount(self) -> int:
+        # integer round up without banker's rounding (so 1/2 => 1 instead of 0)
+        return int(self.lines / 2 + .5)
+
+    @classmethod
+    def from_current_screen(cls) -> 'Margin':
+        return cls(curses.LINES, curses.COLS)

babi/perf.py

@@ -0,0 +1,56 @@
+import contextlib
+import cProfile
+import time
+from typing import Generator
+from typing import List
+from typing import Optional
+from typing import Tuple
+
+
+class Perf:
+    def __init__(self) -> None:
+        self._prof: Optional[cProfile.Profile] = None
+        self._records: List[Tuple[str, float]] = []
+        self._name: Optional[str] = None
+        self._time: Optional[float] = None
+
+    def start(self, name: str) -> None:
+        if self._prof:
+            assert self._name is None, self._name
+            self._name = name
+            self._time = time.monotonic()
+            self._prof.enable()
+
+    def end(self) -> None:
+        if self._prof:
+            assert self._name is not None
+            assert self._time is not None
+            self._prof.disable()
+            self._records.append((self._name, time.monotonic() - self._time))
+            self._name = self._time = None
+
+    def init_profiling(self) -> None:
+        self._prof = cProfile.Profile()
+        self.start('startup')
+
+    def save_profiles(self, filename: str) -> None:
+        assert self._prof is not None
+        self._prof.dump_stats(f'{filename}.pstats')
+        with open(filename, 'w') as f:
+            f.write('μs\tevent\n')
+            for name, duration in self._records:
+                f.write(f'{int(duration * 1000 * 1000)}\t{name}\n')
+
+
+@contextlib.contextmanager
+def perf_log(filename: Optional[str]) -> Generator[Perf, None, None]:
+    perf = Perf()
+    if filename is None:
+        yield perf
+    else:
+        perf.init_profiling()
+        try:
+            yield perf
+        finally:
+            perf.end()
+            perf.save_profiles(filename)

babi/prompt.py

@@ -0,0 +1,191 @@
+import curses
+import enum
+from typing import List
+from typing import Optional
+from typing import Tuple
+from typing import TYPE_CHECKING
+from typing import Union
+
+from babi.horizontal_scrolling import line_x
+from babi.horizontal_scrolling import scrolled_line
+
+if TYPE_CHECKING:
+    from babi.main import Screen  # XXX: circular
+
+PromptResult = enum.Enum('PromptResult', 'CANCELLED')
+
+
+class Prompt:
+    def __init__(self, screen: 'Screen', prompt: str, lst: List[str]) -> None:
+        self._screen = screen
+        self._prompt = prompt
+        self._lst = lst
+        self._y = len(lst) - 1
+        self._x = len(self._s)
+
+    @property
+    def _s(self) -> str:
+        return self._lst[self._y]
+
+    @_s.setter
+    def _s(self, s: str) -> None:
+        self._lst[self._y] = s
+
+    def _render_prompt(self, *, base: Optional[str] = None) -> None:
+        base = base or self._prompt
+        if not base or self._screen.margin.cols < 7:
+            prompt_s = ''
+        elif len(base) > self._screen.margin.cols - 6:
+            prompt_s = f'{base[:self._screen.margin.cols - 7]}…: '
+        else:
+            prompt_s = f'{base}: '
+        width = self._screen.margin.cols - len(prompt_s)
+        line = scrolled_line(self._s, self._x, width)
+        cmd = f'{prompt_s}{line}'
+        prompt_line = self._screen.margin.lines - 1
+        self._screen.stdscr.insstr(prompt_line, 0, cmd, curses.A_REVERSE)
+        x = len(prompt_s) + self._x - line_x(self._x, width)
+        self._screen.stdscr.move(prompt_line, x)
+
+    def _up(self) -> None:
+        self._y = max(0, self._y - 1)
+        self._x = len(self._s)
+
+    def _down(self) -> None:
+        self._y = min(len(self._lst) - 1, self._y + 1)
+        self._x = len(self._s)
+
+    def _right(self) -> None:
+        self._x = min(len(self._s), self._x + 1)
+
+    def _left(self) -> None:
+        self._x = max(0, self._x - 1)
+
+    def _home(self) -> None:
+        self._x = 0
+
+    def _end(self) -> None:
+        self._x = len(self._s)
+
+    def _ctrl_left(self) -> None:
+        if self._x <= 1:
+            self._x = 0
+        else:
+            self._x -= 1
+            tp = self._s[self._x - 1].isalnum()
+            while self._x > 0 and tp == self._s[self._x - 1].isalnum():
+                self._x -= 1
+
+    def _ctrl_right(self) -> None:
+        if self._x >= len(self._s) - 1:
+            self._x = len(self._s)
+        else:
+            self._x += 1
+            tp = self._s[self._x].isalnum()
+            while self._x < len(self._s) and tp == self._s[self._x].isalnum():
+                self._x += 1
+
+    def _backspace(self) -> None:
+        if self._x > 0:
+            self._s = self._s[:self._x - 1] + self._s[self._x:]
+            self._x -= 1
+
+    def _delete(self) -> None:
+        if self._x < len(self._s):
+            self._s = self._s[:self._x] + self._s[self._x + 1:]
+
+    def _cut_to_end(self) -> None:
+        self._s = self._s[:self._x]
+
+    def _resize(self) -> None:
+        self._screen.resize()
+
+    def _check_failed(self, idx: int, s: str) -> Tuple[bool, int]:
+        failed = False
+        for search_idx in range(idx, -1, -1):
+            if s in self._lst[search_idx]:
+                idx = self._y = search_idx
+                self._x = self._lst[search_idx].index(s)
+                break
+        else:
+            failed = True
+        return failed, idx
+
+    def _reverse_search(self) -> Union[None, str, PromptResult]:
+        reverse_s = ''
+        idx = self._y
+        while True:
+            fail, idx = self._check_failed(idx, reverse_s)
+
+            if fail:
+                base = f'{self._prompt}(failed reverse-search)`{reverse_s}`'
+            else:
+                base = f'{self._prompt}(reverse-search)`{reverse_s}`'
+
+            self._render_prompt(base=base)
+
+            key = self._screen.get_char()
+            if key.keyname == b'KEY_RESIZE':
+                self._screen.resize()
+            elif key.keyname == b'KEY_BACKSPACE':
+                reverse_s = reverse_s[:-1]
+            elif key.keyname == b'^R':
+                idx = max(0, idx - 1)
+            elif key.keyname == b'^C':
+                return self._screen.status.cancelled()
+            elif key.keyname == b'^M':
+                return self._s
+            elif key.keyname == b'STRING':
+                assert isinstance(key.wch, str), key.wch
+                for c in key.wch:
+                    reverse_s += c
+                    failed, idx = self._check_failed(idx, reverse_s)
+            else:
+                self._x = len(self._s)
+                return None
+
+    def _cancel(self) -> PromptResult:
+        return self._screen.status.cancelled()
+
+    def _submit(self) -> str:
+        return self._s
+
+    DISPATCH = {
+        # movement
+        b'KEY_UP': _up,
+        b'KEY_DOWN': _down,
+        b'KEY_RIGHT': _right,
+        b'KEY_LEFT': _left,
+        b'KEY_HOME': _home,
+        b'^A': _home,
+        b'KEY_END': _end,
+        b'^E': _end,
+        b'kRIT5': _ctrl_right,
+        b'kLFT5': _ctrl_left,
+        # editing
+        b'KEY_BACKSPACE': _backspace,
+        b'KEY_DC': _delete,
+        b'^K': _cut_to_end,
+        # misc
+        b'KEY_RESIZE': _resize,
+        b'^R': _reverse_search,
+        b'^M': _submit,
+        b'^C': _cancel,
+    }
+
+    def _c(self, c: str) -> None:
+        self._s = self._s[:self._x] + c + self._s[self._x:]
+        self._x += len(c)
+
+    def run(self) -> Union[PromptResult, str]:
+        while True:
+            self._render_prompt()
+
+            key = self._screen.get_char()
+            if key.keyname in Prompt.DISPATCH:
+                ret = Prompt.DISPATCH[key.keyname](self)
+                if ret is not None:
+                    return ret
+            elif key.keyname == b'STRING':
+                assert isinstance(key.wch, str), key.wch
+                self._c(key.wch)

babi/reg.py

@@ -0,0 +1,154 @@
+import functools
+import re
+from typing import Match
+from typing import Optional
+from typing import Tuple
+
+import onigurumacffi
+
+from babi.cached_property import cached_property
+
+_BACKREF_RE = re.compile(r'((?<!\\)(?:\\\\)*)\\([0-9]+)')
+
+
+def _replace_esc(s: str, chars: str) -> str:
+    """replace the given escape sequences of `chars` with \\uffff"""
+    for c in chars:
+        if f'\\{c}' in s:
+            break
+    else:
+        return s
+
+    b = []
+    i = 0
+    length = len(s)
+    while i < length:
+        try:
+            sbi = s.index('\\', i)
+        except ValueError:
+            b.append(s[i:])
+            break
+        if sbi > i:
+            b.append(s[i:sbi])
+        b.append('\\')
+        i = sbi + 1
+        if i < length:
+            if s[i] in chars:
+                b.append('\uffff')
+            else:
+                b.append(s[i])
+        i += 1
+    return ''.join(b)
+
+
+class _Reg:
+    def __init__(self, s: str) -> None:
+        self._pattern = s
+
+    def __repr__(self) -> str:
+        return f'{type(self).__name__}({self._pattern!r})'
+
+    @cached_property
+    def _reg(self) -> onigurumacffi._Pattern:
+        return onigurumacffi.compile(_replace_esc(self._pattern, 'z'))
+
+    @cached_property
+    def _reg_no_A(self) -> onigurumacffi._Pattern:
+        return onigurumacffi.compile(_replace_esc(self._pattern, 'Az'))
+
+    @cached_property
+    def _reg_no_G(self) -> onigurumacffi._Pattern:
+        return onigurumacffi.compile(_replace_esc(self._pattern, 'Gz'))
+
+    @cached_property
+    def _reg_no_A_no_G(self) -> onigurumacffi._Pattern:
+        return onigurumacffi.compile(_replace_esc(self._pattern, 'AGz'))
+
+    def _get_reg(
+            self,
+            first_line: bool,
+            boundary: bool,
+    ) -> onigurumacffi._Pattern:
+        if boundary:
+            if first_line:
+                return self._reg
+            else:
+                return self._reg_no_A
+        else:
+            if first_line:
+                return self._reg_no_G
+            else:
+                return self._reg_no_A_no_G
+
+    def search(
+            self,
+            line: str,
+            pos: int,
+            first_line: bool,
+            boundary: bool,
+    ) -> Optional[Match[str]]:
+        return self._get_reg(first_line, boundary).search(line, pos)
+
+    def match(
+            self,
+            line: str,
+            pos: int,
+            first_line: bool,
+            boundary: bool,
+    ) -> Optional[Match[str]]:
+        return self._get_reg(first_line, boundary).match(line, pos)
+
+
+class _RegSet:
+    def __init__(self, *s: str) -> None:
+        self._patterns = s
+
+    def __repr__(self) -> str:
+        args = ', '.join(repr(s) for s in self._patterns)
+        return f'{type(self).__name__}({args})'
+
+    @cached_property
+    def _set(self) -> onigurumacffi._RegSet:
+        return onigurumacffi.compile_regset(*self._patterns)
+
+    @cached_property
+    def _set_no_A(self) -> onigurumacffi._RegSet:
+        patterns = (_replace_esc(p, 'A') for p in self._patterns)
+        return onigurumacffi.compile_regset(*patterns)
+
+    @cached_property
+    def _set_no_G(self) -> onigurumacffi._RegSet:
+        patterns = (_replace_esc(p, 'G') for p in self._patterns)
+        return onigurumacffi.compile_regset(*patterns)
+
+    @cached_property
+    def _set_no_A_no_G(self) -> onigurumacffi._RegSet:
+        patterns = (_replace_esc(p, 'AG') for p in self._patterns)
+        return onigurumacffi.compile_regset(*patterns)
+
+    def search(
+            self,
+            line: str,
+            pos: int,
+            first_line: bool,
+            boundary: bool,
+    ) -> Tuple[int, Optional[Match[str]]]:
+        if boundary:
+            if first_line:
+                return self._set.search(line, pos)
+            else:
+                return self._set_no_A.search(line, pos)
+        else:
+            if first_line:
+                return self._set_no_G.search(line, pos)
+            else:
+                return self._set_no_A_no_G.search(line, pos)
+
+
+def expand_escaped(match: Match[str], s: str) -> str:
+    return _BACKREF_RE.sub(lambda m: f'{m[1]}{re.escape(match[int(m[2])])}', s)
+
+
+make_reg = functools.lru_cache(maxsize=None)(_Reg)
+make_regset = functools.lru_cache(maxsize=None)(_RegSet)
+ERR_REG = make_reg(')this pattern always triggers an error when used(')

babi/screen.py

@@ -0,0 +1,570 @@
+import contextlib
+import curses
+import enum
+import hashlib
+import os
+import re
+import signal
+import sys
+from typing import Generator
+from typing import List
+from typing import NamedTuple
+from typing import Optional
+from typing import Pattern
+from typing import Tuple
+from typing import Union
+
+from babi.color_manager import ColorManager
+from babi.file import Action
+from babi.file import File
+from babi.file import get_lines
+from babi.history import History
+from babi.hl.syntax import Syntax
+from babi.margin import Margin
+from babi.perf import Perf
+from babi.prompt import Prompt
+from babi.prompt import PromptResult
+from babi.status import Status
+
+if sys.version_info >= (3, 8):  # pragma: no cover (py38+)
+    import importlib.metadata as importlib_metadata
+else:  # pragma: no cover (<py38)
+    import importlib_metadata
+
+VERSION_STR = f'babi v{importlib_metadata.version("babi")}'
+EditResult = enum.Enum('EditResult', 'EXIT NEXT PREV OPEN')
+
+# TODO: find a place to populate these, surely there's a database somewhere
+SEQUENCE_KEYNAME = {
+    '\x1bOH': b'KEY_HOME',
+    '\x1bOF': b'KEY_END',
+    '\x1b[1;2A': b'KEY_SR',
+    '\x1b[1;2B': b'KEY_SF',
+    '\x1b[1;2C': b'KEY_SRIGHT',
+    '\x1b[1;2D': b'KEY_SLEFT',
+    '\x1b[1;2H': b'KEY_SHOME',
+    '\x1b[1;2F': b'KEY_SEND',
+    '\x1b[5;2~': b'KEY_SPREVIOUS',
+    '\x1b[6;2~': b'KEY_SNEXT',
+    '\x1b[1;3A': b'kUP3',  # M-Up
+    '\x1b[1;3B': b'kDN3',  # M-Down
+    '\x1b[1;3C': b'kRIT3',  # M-Right
+    '\x1b[1;3D': b'kLFT3',  # M-Left
+    '\x1b[1;5A': b'kUP5',  # ^Up
+    '\x1b[1;5B': b'kDN5',  # ^Down
+    '\x1b[1;5C': b'kRIT5',  # ^Right
+    '\x1b[1;5D': b'kLFT5',  # ^Left
+    '\x1b[1;5H': b'kHOM5',  # ^Home
+    '\x1b[1;5F': b'kEND5',  # ^End
+    '\x1b[1;6C': b'kRIT6',  # Shift + ^Right
+    '\x1b[1;6D': b'kLFT6',  # Shift + ^Left
+    '\x1b[1;6H': b'kHOM6',  # Shift + ^Home
+    '\x1b[1;6F': b'kEND6',  # Shift + ^End
+}
+KEYNAME_REWRITE = {
+    # windows-curses: numeric pad arrow keys
+    # - some overlay keyboards pick these as well
+    # - in xterm it seems these are mapped automatically
+    b'KEY_A2': b'KEY_UP',
+    b'KEY_C2': b'KEY_DOWN',
+    b'KEY_B3': b'KEY_RIGHT',
+    b'KEY_B1': b'KEY_LEFT',
+    b'PADSTOP': b'KEY_DC',
+    b'KEY_A3': b'KEY_PPAGE',
+    b'KEY_C3': b'KEY_NPAGE',
+    b'KEY_A1': b'KEY_HOME',
+    b'KEY_C1': b'KEY_END',
+    # windows-curses: map to our M- names
+    b'ALT_U': b'M-u',
+    # windows-curses: arguably these names are better than the xterm names
+    b'CTL_UP': b'kUP5',
+    b'CTL_DOWN': b'kDN5',
+    b'CTL_RIGHT': b'kRIT5',
+    b'CTL_LEFT': b'kLFT5',
+    b'ALT_RIGHT': b'kRIT3',
+    b'ALT_LEFT': b'kLFT3',
+    # windows-curses: idk why these are different
+    b'KEY_SUP': b'KEY_SR',
+    b'KEY_SDOWN': b'KEY_SF',
+    # macos: (sends this for backspace key, others interpret this as well)
+    b'^?': b'KEY_BACKSPACE',
+    # linux, perhaps others
+    b'^H': b'KEY_BACKSPACE',  # ^Backspace on my keyboard
+}
+
+
+class Key(NamedTuple):
+    wch: Union[int, str]
+    keyname: bytes
+
+
+class Screen:
+    def __init__(
+            self,
+            stdscr: 'curses._CursesWindow',
+            filenames: List[Optional[str]],
+            perf: Perf,
+    ) -> None:
+        self.stdscr = stdscr
+        self.color_manager = ColorManager.make()
+        self.hl_factories = (Syntax.from_screen(stdscr, self.color_manager),)
+        self.files = [
+            File(filename, self.color_manager, self.hl_factories)
+            for filename in filenames
+        ]
+        self.i = 0
+        self.history = History()
+        self.perf = perf
+        self.status = Status()
+        self.margin = Margin.from_current_screen()
+        self.cut_buffer: Tuple[str, ...] = ()
+        self.cut_selection = False
+        self._buffered_input: Union[int, str, None] = None
+
+    @property
+    def file(self) -> File:
+        return self.files[self.i]
+
+    def _draw_header(self) -> None:
+        filename = self.file.filename or '<<new file>>'
+        if self.file.modified:
+            filename += ' *'
+        if len(self.files) > 1:
+            files = f'[{self.i + 1}/{len(self.files)}] '
+            version_width = len(VERSION_STR) + 2 + len(files)
+        else:
+            files = ''
+            version_width = len(VERSION_STR) + 2
+        centered = filename.center(self.margin.cols)[version_width:]
+        s = f' {VERSION_STR} {files}{centered}{files}'
+        self.stdscr.insstr(0, 0, s, curses.A_REVERSE)
+
+    def _get_sequence_home_end(self, wch: str) -> str:
+        try:
+            c = self.stdscr.get_wch()
+        except curses.error:
+            return wch
+        else:
+            if isinstance(c, int) or c not in 'HF':
+                self._buffered_input = c
+                return wch
+            else:
+                return f'{wch}{c}'
+
+    def _get_sequence_bracketed(self, wch: str) -> str:
+        for _ in range(3):  # [0-9]{1,2};
+            try:
+                c = self.stdscr.get_wch()
+            except curses.error:
+                return wch
+            else:
+                if isinstance(c, int):
+                    self._buffered_input = c
+                    return wch
+                else:
+                    wch += c
+                    if c == ';':
+                        break
+        else:
+            return wch  # unexpected input while searching for `;`
+
+        for _ in range(2):  # [0-9].
+            try:
+                c = self.stdscr.get_wch()
+            except curses.error:
+                return wch
+            else:
+                if isinstance(c, int):
+                    self._buffered_input = c
+                    return wch
+                else:
+                    wch += c
+
+        return wch
+
+    def _get_sequence(self, wch: str) -> str:
+        self.stdscr.nodelay(True)
+        try:
+            c = self.stdscr.get_wch()
+        except curses.error:
+            return wch
+        else:
+            if isinstance(c, int):  # M-BSpace
+                return f'{wch}({c})'  # TODO
+            elif c == 'O':
+                return self._get_sequence_home_end(f'{wch}O')
+            elif c == '[':
+                return self._get_sequence_bracketed(f'{wch}[')
+            else:
+                return f'{wch}{c}'
+        finally:
+            self.stdscr.nodelay(False)
+
+    def _get_string(self, wch: str) -> str:
+        self.stdscr.nodelay(True)
+        try:
+            while True:
+                try:
+                    c = self.stdscr.get_wch()
+                    if isinstance(c, str) and c.isprintable():
+                        wch += c
+                    else:
+                        self._buffered_input = c
+                        break
+                except curses.error:
+                    break
+        finally:
+            self.stdscr.nodelay(False)
+        return wch
+
+    def _get_char(self) -> Key:
+        if self._buffered_input is not None:
+            wch, self._buffered_input = self._buffered_input, None
+        else:
+            wch = self.stdscr.get_wch()
+        if isinstance(wch, str) and wch == '\x1b':
+            wch = self._get_sequence(wch)
+            if len(wch) == 2:
+                return Key(wch, f'M-{wch[1]}'.encode())
+            elif len(wch) > 1:
+                keyname = SEQUENCE_KEYNAME.get(wch, b'unknown')
+                return Key(wch, keyname)
+        elif isinstance(wch, str) and wch.isprintable():
+            wch = self._get_string(wch)
+            return Key(wch, b'STRING')
+
+        key = wch if isinstance(wch, int) else ord(wch)
+        keyname = curses.keyname(key)
+        keyname = KEYNAME_REWRITE.get(keyname, keyname)
+        return Key(wch, keyname)
+
+    def get_char(self) -> Key:
+        self.perf.end()
+        ret = self._get_char()
+        self.perf.start(ret.keyname.decode())
+        return ret
+
+    def draw(self) -> None:
+        if self.margin.header:
+            self._draw_header()
+        self.file.draw(self.stdscr, self.margin)
+        self.status.draw(self.stdscr, self.margin)
+
+    def resize(self) -> None:
+        curses.update_lines_cols()
+        self.margin = Margin.from_current_screen()
+        self.file.buf.scroll_screen_if_needed(self.margin)
+        self.draw()
+
+    def quick_prompt(
+            self,
+            prompt: str,
+            opt_strs: Tuple[str, ...],
+    ) -> Union[str, PromptResult]:
+        opts = [opt[0] for opt in opt_strs]
+        while True:
+            x = 0
+            prompt_line = self.margin.lines - 1
+
+            def _write(s: str, *, attr: int = curses.A_REVERSE) -> None:
+                nonlocal x
+
+                if x >= self.margin.cols:
+                    return
+                self.stdscr.insstr(prompt_line, x, s, attr)
+                x += len(s)
+
+            _write(prompt)
+            _write(' [')
+            for i, opt_str in enumerate(opt_strs):
+                _write(opt_str[0], attr=curses.A_REVERSE | curses.A_BOLD)
+                _write(opt_str[1:])
+                if i != len(opt_strs) - 1:
+                    _write(', ')
+            _write(']?')
+
+            if x < self.margin.cols - 1:
+                s = ' ' * (self.margin.cols - x)
+                self.stdscr.insstr(prompt_line, x, s, curses.A_REVERSE)
+                x += 1
+            else:
+                x = self.margin.cols - 1
+                self.stdscr.insstr(prompt_line, x, '…', curses.A_REVERSE)
+
+            self.stdscr.move(prompt_line, x)
+
+            key = self.get_char()
+            if key.keyname == b'KEY_RESIZE':
+                self.resize()
+            elif key.keyname == b'^C':
+                return self.status.cancelled()
+            elif isinstance(key.wch, str) and key.wch in opts:
+                return key.wch
+
+    def prompt(
+            self,
+            prompt: str,
+            *,
+            allow_empty: bool = False,
+            history: Optional[str] = None,
+            default_prev: bool = False,
+            default: Optional[str] = None,
+    ) -> Union[str, PromptResult]:
+        default = default or ''
+        self.status.clear()
+        if history is not None:
+            history_data = [*self.history.data[history], default]
+            if default_prev and history in self.history.prev:
+                prompt = f'{prompt} [{self.history.prev[history]}]'
+        else:
+            history_data = [default]
+
+        ret = Prompt(self, prompt, history_data).run()
+
+        if ret is not PromptResult.CANCELLED and history is not None:
+            if ret:  # only put non-empty things in history
+                history_lst = self.history.data[history]
+                if not history_lst or history_lst[-1] != ret:
+                    history_lst.append(ret)
+                self.history.prev[history] = ret
+            elif default_prev and history in self.history.prev:
+                return self.history.prev[history]
+
+        if not allow_empty and not ret:
+            return self.status.cancelled()
+        else:
+            return ret
+
+    def go_to_line(self) -> None:
+        response = self.prompt('enter line number')
+        if response is not PromptResult.CANCELLED:
+            try:
+                lineno = int(response)
+            except ValueError:
+                self.status.update(f'not an integer: {response!r}')
+            else:
+                self.file.go_to_line(lineno, self.margin)
+
+    def current_position(self) -> None:
+        line = f'line {self.file.buf.y + 1}'
+        col = f'col {self.file.buf.x + 1}'
+        line_count = max(len(self.file.buf) - 1, 1)
+        lines_word = 'line' if line_count == 1 else 'lines'
+        self.status.update(f'{line}, {col} (of {line_count} {lines_word})')
+
+    def cut(self) -> None:
+        if self.file.selection.start:
+            self.cut_buffer = self.file.cut_selection(self.margin)
+            self.cut_selection = True
+        else:
+            self.cut_buffer = self.file.cut(self.cut_buffer)
+            self.cut_selection = False
+
+    def uncut(self) -> None:
+        if self.cut_selection:
+            self.file.uncut_selection(self.cut_buffer, self.margin)
+        else:
+            self.file.uncut(self.cut_buffer, self.margin)
+
+    def _get_search_re(self, prompt: str) -> Union[Pattern[str], PromptResult]:
+        response = self.prompt(prompt, history='search', default_prev=True)
+        if response is PromptResult.CANCELLED:
+            return response
+        try:
+            return re.compile(response)
+        except re.error:
+            self.status.update(f'invalid regex: {response!r}')
+            return PromptResult.CANCELLED
+
+    def _undo_redo(
+            self,
+            op: str,
+            from_stack: List[Action],
+            to_stack: List[Action],
+    ) -> None:
+        if not from_stack:
+            self.status.update(f'nothing to {op}!')
+        else:
+            action = from_stack.pop()
+            to_stack.append(action.apply(self.file))
+            self.file.buf.scroll_screen_if_needed(self.margin)
+            self.status.update(f'{op}: {action.name}')
+            self.file.selection.clear()
+
+    def undo(self) -> None:
+        self._undo_redo('undo', self.file.undo_stack, self.file.redo_stack)
+
+    def redo(self) -> None:
+        self._undo_redo('redo', self.file.redo_stack, self.file.undo_stack)
+
+    def search(self) -> None:
+        response = self._get_search_re('search')
+        if response is not PromptResult.CANCELLED:
+            self.file.search(response, self.status, self.margin)
+
+    def replace(self) -> None:
+        search_response = self._get_search_re('search (to replace)')
+        if search_response is not PromptResult.CANCELLED:
+            response = self.prompt(
+                'replace with', history='replace', allow_empty=True,
+            )
+            if response is not PromptResult.CANCELLED:
+                self.file.replace(self, search_response, response)
+
+    def command(self) -> Optional[EditResult]:
+        response = self.prompt('', history='command')
+        if response == ':q':
+            return self.quit_save_modified()
+        elif response == ':q!':
+            return EditResult.EXIT
+        elif response == ':w':
+            self.save()
+        elif response == ':wq':
+            self.save()
+            return EditResult.EXIT
+        elif response == ':sort':
+            if self.file.selection.start:
+                self.file.sort_selection(self.margin)
+            else:
+                self.file.sort(self.margin)
+            self.status.update('sorted!')
+        elif response is not PromptResult.CANCELLED:
+            self.status.update(f'invalid command: {response}')
+        return None
+
+    def save(self) -> Optional[PromptResult]:
+        self.file.finalize_previous_action()
+
+        # TODO: make directories if they don't exist
+        # TODO: maybe use mtime / stat as a shortcut for hashing below
+        # TODO: strip trailing whitespace?
+        # TODO: save atomically?
+        if self.file.filename is None:
+            filename = self.prompt('enter filename')
+            if filename is PromptResult.CANCELLED:
+                return PromptResult.CANCELLED
+            else:
+                self.file.filename = filename
+
+        if os.path.isfile(self.file.filename):
+            with open(self.file.filename, newline='') as f:
+                *_, sha256 = get_lines(f)
+        else:
+            sha256 = hashlib.sha256(b'').hexdigest()
+
+        contents = self.file.nl.join(self.file.buf)
+        sha256_to_save = hashlib.sha256(contents.encode()).hexdigest()
+
+        # the file on disk is the same as when we opened it
+        if sha256 not in (self.file.sha256, sha256_to_save):
+            self.status.update('(file changed on disk, not implemented)')
+            return PromptResult.CANCELLED
+
+        with open(self.file.filename, 'w', newline='') as f:
+            f.write(contents)
+
+        self.file.modified = False
+        self.file.sha256 = sha256_to_save
+        num_lines = len(self.file.buf) - 1
+        lines = 'lines' if num_lines != 1 else 'line'
+        self.status.update(f'saved! ({num_lines} {lines} written)')
+
+        # fix up modified state in undo / redo stacks
+        for stack in (self.file.undo_stack, self.file.redo_stack):
+            first = True
+            for action in reversed(stack):
+                action.end_modified = not first
+                action.start_modified = True
+                first = False
+        return None
+
+    def save_filename(self) -> Optional[PromptResult]:
+        response = self.prompt('enter filename', default=self.file.filename)
+        if response is PromptResult.CANCELLED:
+            return PromptResult.CANCELLED
+        else:
+            self.file.filename = response
+            return self.save()
+
+    def open_file(self) -> Optional[EditResult]:
+        response = self.prompt('enter filename', history='open')
+        if response is not PromptResult.CANCELLED:
+            opened = File(response, self.color_manager, self.hl_factories)
+            self.files.append(opened)
+            return EditResult.OPEN
+        else:
+            return None
+
+    def quit_save_modified(self) -> Optional[EditResult]:
+        if self.file.modified:
+            response = self.quick_prompt(
+                'file is modified - save', ('yes', 'no'),
+            )
+            if response == 'y':
+                if self.save_filename() is not PromptResult.CANCELLED:
+                    return EditResult.EXIT
+                else:
+                    return None
+            elif response == 'n':
+                return EditResult.EXIT
+            else:
+                assert response is PromptResult.CANCELLED
+                return None
+        return EditResult.EXIT
+
+    def background(self) -> None:
+        curses.endwin()
+        os.kill(os.getpid(), signal.SIGSTOP)
+        self.stdscr = _init_screen()
+        self.resize()
+
+    DISPATCH = {
+        b'KEY_RESIZE': resize,
+        b'^_': go_to_line,
+        b'^C': current_position,
+        b'^K': cut,
+        b'^U': uncut,
+        b'M-u': undo,
+        b'M-U': redo,
+        b'^W': search,
+        b'^\\': replace,
+        b'^[': command,
+        b'^S': save,
+        b'^O': save_filename,
+        b'^X': quit_save_modified,
+        b'^P': open_file,
+        b'kLFT3': lambda screen: EditResult.PREV,
+        b'kRIT3': lambda screen: EditResult.NEXT,
+        b'^Z': background,
+    }
+
+
+def _init_screen() -> 'curses._CursesWindow':
+    # set the escape delay so curses does not pause waiting for sequences
+    if sys.version_info >= (3, 9):  # pragma: no cover
+        curses.set_escdelay(25)
+    else:  # pragma: no cover
+        os.environ.setdefault('ESCDELAY', '25')
+
+    stdscr = curses.initscr()
+    curses.noecho()
+    curses.cbreak()
+    # <enter> is not transformed into '\n' so it can be differentiated from ^J
+    curses.nonl()
+    # ^S / ^Q / ^Z / ^\ are passed through
+    curses.raw()
+    stdscr.keypad(True)
+
+    with contextlib.suppress(curses.error):
+        curses.start_color()
+        curses.use_default_colors()
+    return stdscr
+
+
+@contextlib.contextmanager
+def make_stdscr() -> Generator['curses._CursesWindow', None, None]:
+    """essentially `curses.wrapper` but split out to implement ^Z"""
+    try:
+        yield _init_screen()
+    finally:
+        curses.endwin()

babi/status.py

@@ -0,0 +1,41 @@
+import curses
+
+from babi.margin import Margin
+from babi.prompt import PromptResult
+
+
+class Status:
+    def __init__(self) -> None:
+        self._status = ''
+        self._action_counter = -1
+
+    def update(self, status: str) -> None:
+        self._status = status
+        self._action_counter = 25
+
+    def clear(self) -> None:
+        self._status = ''
+
+    def draw(self, stdscr: 'curses._CursesWindow', margin: Margin) -> None:
+        if margin.footer or self._status:
+            stdscr.insstr(margin.lines - 1, 0, ' ' * margin.cols)
+            if self._status:
+                status = f' {self._status} '
+                x = (margin.cols - len(status)) // 2
+                if x < 0:
+                    x = 0
+                    status = status.strip()
+                stdscr.insstr(margin.lines - 1, x, status, curses.A_REVERSE)
+
+    def tick(self, margin: Margin) -> None:
+        # when the window is only 1-tall, hide the status quicker
+        if margin.footer:
+            self._action_counter -= 1
+        else:
+            self._action_counter -= 24
+        if self._action_counter < 0:
+            self.clear()
+
+    def cancelled(self) -> PromptResult:
+        self.update('cancelled')
+        return PromptResult.CANCELLED

babi/textmate_demo.py

@@ -0,0 +1,69 @@
+import argparse
+from typing import Optional
+from typing import Sequence
+
+from babi.highlight import Compiler
+from babi.highlight import Grammars
+from babi.highlight import highlight_line
+from babi.theme import Style
+from babi.theme import Theme
+from babi.user_data import prefix_data
+from babi.user_data import xdg_config
+
+
+def print_styled(s: str, style: Style) -> None:
+    color_s = ''
+    undo_s = ''
+    if style.fg is not None:
+        color_s += '\x1b[38;2;{r};{g};{b}m'.format(**style.fg._asdict())
+        undo_s += '\x1b[39m'
+    if style.bg is not None:
+        color_s += '\x1b[48;2;{r};{g};{b}m'.format(**style.bg._asdict())
+        undo_s += '\x1b[49m'
+    if style.b:
+        color_s += '\x1b[1m'
+        undo_s += '\x1b[22m'
+    if style.i:
+        color_s += '\x1b[3m'
+        undo_s += '\x1b[23m'
+    if style.u:
+        color_s += '\x1b[4m'
+        undo_s += '\x1b[24m'
+    print(f'{color_s}{s}{undo_s}', end='', flush=True)
+
+
+def _highlight_output(theme: Theme, compiler: Compiler, filename: str) -> int:
+    state = compiler.root_state
+
+    if theme.default.bg is not None:
+        print('\x1b[48;2;{r};{g};{b}m'.format(**theme.default.bg._asdict()))
+    with open(filename) as f:
+        for line_idx, line in enumerate(f):
+            first_line = line_idx == 0
+            state, regions = highlight_line(compiler, state, line, first_line)
+            for start, end, scope in regions:
+                print_styled(line[start:end], theme.select(scope))
+    print('\x1b[m', end='')
+    return 0
+
+
+def main(argv: Optional[Sequence[str]] = None) -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--theme', default=xdg_config('theme.json'))
+    parser.add_argument('--grammar-dir', default=prefix_data('grammar_v1'))
+    parser.add_argument('filename')
+    args = parser.parse_args(argv)
+
+    with open(args.filename) as f:
+        first_line = next(f, '')
+
+    theme = Theme.from_filename(args.theme)
+
+    grammars = Grammars(args.grammar_dir)
+    compiler = grammars.compiler_for_file(args.filename, first_line)
+
+    return _highlight_output(theme, compiler, args.filename)
+
+
+if __name__ == '__main__':
+    exit(main())

babi/theme.py

@@ -0,0 +1,151 @@
+import functools
+import json
+import os.path
+from typing import Any
+from typing import Dict
+from typing import NamedTuple
+from typing import Optional
+from typing import Tuple
+
+from babi._types import Protocol
+from babi.color import Color
+from babi.fdict import FDict
+
+
+class Style(NamedTuple):
+    fg: Optional[Color]
+    bg: Optional[Color]
+    b: bool
+    i: bool
+    u: bool
+
+    @classmethod
+    def blank(cls) -> 'Style':
+        return cls(fg=None, bg=None, b=False, i=False, u=False)
+
+
+class PartialStyle(NamedTuple):
+    fg: Optional[Color] = None
+    bg: Optional[Color] = None
+    b: Optional[bool] = None
+    i: Optional[bool] = None
+    u: Optional[bool] = None
+
+    def overlay_on(self, dct: Dict[str, Any]) -> None:
+        for attr in self._fields:
+            value = getattr(self, attr)
+            if value is not None:
+                dct[attr] = value
+
+    @classmethod
+    def from_dct(cls, dct: Dict[str, Any]) -> 'PartialStyle':
+        kv = cls()._asdict()
+        if 'foreground' in dct:
+            kv['fg'] = Color.parse(dct['foreground'])
+        if 'background' in dct:
+            kv['bg'] = Color.parse(dct['background'])
+        if dct.get('fontStyle') == 'bold':
+            kv['b'] = True
+        elif dct.get('fontStyle') == 'italic':
+            kv['i'] = True
+        elif dct.get('fontStyle') == 'underline':
+            kv['u'] = True
+        return cls(**kv)
+
+
+class _TrieNode(Protocol):
+    @property
+    def style(self) -> PartialStyle: ...
+    @property
+    def children(self) -> FDict[str, '_TrieNode']: ...
+
+
+class TrieNode(NamedTuple):
+    style: PartialStyle
+    children: FDict[str, _TrieNode]
+
+    @classmethod
+    def from_dct(cls, dct: Dict[str, Any]) -> _TrieNode:
+        children = FDict({
+            k: TrieNode.from_dct(v) for k, v in dct['children'].items()
+        })
+        return cls(PartialStyle.from_dct(dct), children)
+
+
+class Theme(NamedTuple):
+    default: Style
+    rules: _TrieNode
+
+    @functools.lru_cache(maxsize=None)
+    def select(self, scope: Tuple[str, ...]) -> Style:
+        if not scope:
+            return self.default
+        else:
+            style = self.select(scope[:-1])._asdict()
+            node = self.rules
+            for part in scope[-1].split('.'):
+                if part not in node.children:
+                    break
+                else:
+                    node = node.children[part]
+                    node.style.overlay_on(style)
+            return Style(**style)
+
+    @classmethod
+    def from_dct(cls, data: Dict[str, Any]) -> 'Theme':
+        default = Style.blank()._asdict()
+
+        for k in ('foreground', 'editor.foreground'):
+            if k in data.get('colors', {}):
+                default['fg'] = Color.parse(data['colors'][k])
+                break
+
+        for k in ('background', 'editor.background'):
+            if k in data.get('colors', {}):
+                default['bg'] = Color.parse(data['colors'][k])
+                break
+
+        root: Dict[str, Any] = {'children': {}}
+        rules = data.get('tokenColors', []) + data.get('settings', [])
+        for rule in rules:
+            if 'scope' not in rule:
+                scopes = ['']
+            elif rule['scope'] == '':
+                scopes = ['']
+            elif isinstance(rule['scope'], str):
+                scopes = [
+                    s.strip()
+                    # some themes have a buggy trailing/leading comma
+                    for s in rule['scope'].strip().strip(',').split(',')
+                    if s.strip()
+                ]
+            else:
+                scopes = rule['scope']
+
+            for scope in scopes:
+                if ' ' in scope:
+                    # TODO: implement parent scopes
+                    continue
+                elif scope == '':
+                    PartialStyle.from_dct(rule['settings']).overlay_on(default)
+                    continue
+
+                cur = root
+                for part in scope.split('.'):
+                    cur = cur['children'].setdefault(part, {'children': {}})
+
+                cur.update(rule['settings'])
+
+        return cls(Style(**default), TrieNode.from_dct(root))
+
+    @classmethod
+    def blank(cls) -> 'Theme':
+        return cls(Style.blank(), TrieNode.from_dct({'children': {}}))
+
+    @classmethod
+    def from_filename(cls, filename: str) -> 'Theme':
+        if not os.path.exists(filename):
+            return cls.blank()
+        else:
+            with open(filename) as f:
+                return cls.from_dct(json.load(f))

babi/user_data.py

@@ -0,0 +1,21 @@
+import os.path
+import sys
+
+
+def _xdg(*path: str, env: str, default: str) -> str:
+    return os.path.join(
+        os.environ.get(env) or os.path.expanduser(default),
+        'babi', *path,
+    )
+
+
+def xdg_data(*path: str) -> str:
+    return _xdg(*path, env='XDG_DATA_HOME', default='~/.local/share')
+
+
+def xdg_config(*path: str) -> str:
+    return _xdg(*path, env='XDG_CONFIG_HOME', default='~/.config')
+
+
+def prefix_data(*path: str) -> str:
+    return os.path.join(sys.prefix, 'share/babi', *path)

bin/download-theme

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+import argparse
+import io
+import json
+import os.path
+import plistlib
+import re
+import urllib.request
+from typing import Any
+
+import cson  # pip install cson
+
+TOKEN = re.compile(br'(\\\\|\\"|"|//|\n)')
+
+
+def json_with_comments(s: bytes) -> Any:
+    bio = io.BytesIO()
+
+    idx = 0
+    in_string = False
+    in_comment = False
+
+    match = TOKEN.search(s, idx)
+    while match:
+        if not in_comment:
+            bio.write(s[idx:match.start()])
+
+        tok = match[0]
+        if not in_comment and tok == b'"':
+            in_string = not in_string
+        elif in_comment and tok == b'\n':
+            in_comment = False
+        elif not in_string and tok == b'//':
+            in_comment = True
+
+        if not in_comment:
+            bio.write(tok)
+
+        idx = match.end()
+        match = TOKEN.search(s, idx)
+
+    bio.seek(0)
+    return json.load(bio)
+
+
+STRATEGIES = (json.loads, plistlib.loads, cson.loads, json_with_comments)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument('name')
+    parser.add_argument('url')
+    args = parser.parse_args()
+
+    if '/blob/' in args.url:
+        url = args.url.replace('/blob/', '/raw/')
+    else:
+        url = args.url
+
+    contents = urllib.request.urlopen(url).read()
+
+    errors = []
+    for strategy in STRATEGIES:
+        try:
+            loaded = strategy(contents)
+        except Exception as e:
+            errors.append((f'{strategy.__module__}.{strategy.__name__}', e))
+        else:
+            break
+    else:
+        errors_s = '\n'.join(f'\t{name}: {error}' for name, error in errors)
+        raise AssertionError(f'could not load as json/plist/cson:\n{errors_s}')
+
+    config_dir = os.path.expanduser('~/.config/babi')
+    os.makedirs(config_dir, exist_ok=True)
+    dest = os.path.join(config_dir, f'{args.name}.json')
+    with open(dest, 'w') as f:
+        json.dump(loaded, f)
+
+    theme_json = os.path.join(config_dir, 'theme.json')
+    if os.path.lexists(theme_json):
+        os.remove(theme_json)
+    os.symlink(dest, theme_json)
+    return 0
+
+
+if __name__ == '__main__':
+    exit(main())

requirements-dev.txt

@@ -1,4 +1,5 @@
+covdefaults
 coverage
-git+https://github.com/mjsir911/hecate@092f811
+git+https://github.com/asottile/hecate@875567f
 pytest
 remote-pdb

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = babi
-version = 0.0.0
+version = 0.0.7
 description = a text editor
 long_description = file: README.md
 long_description_content_type = text/markdown
@@ -20,16 +20,32 @@ classifiers =
     Programming Language :: Python :: Implementation :: PyPy
 
 [options]
-py_modules = babi
-python_requires = >=3.6
+packages = find:
+install_requires =
+    babi-grammars
+    identify
+    onigurumacffi>=0.0.10
+    importlib_metadata>=1;python_version<"3.8"
+    windows-curses;sys_platform=="win32"
+python_requires = >=3.6.1
 
 [options.entry_points]
 console_scripts =
-    babi = babi:main
+    babi = babi.main:main
+    babi-textmate-demo = babi.textmate_demo:main
+
+[options.packages.find]
+exclude =
+    tests*
+    testing*
 
 [bdist_wheel]
 universal = True
 
+[coverage:run]
+plugins = covdefaults
+parallel = true
+
 [mypy]
 check_untyped_defs = true
 disallow_any_generics = true

testing/runner.py

@@ -0,0 +1,230 @@
+import contextlib
+import curses
+import enum
+import re
+from typing import List
+from typing import Tuple
+
+from hecate import Runner
+
+
+class Token(enum.Enum):
+    FG_ESC = re.compile(r'\x1b\[38;5;(\d+)m')
+    BG_ESC = re.compile(r'\x1b\[48;5;(\d+)m')
+    RESET = re.compile(r'\x1b\[0?m')
+    ESC = re.compile(r'\x1b\[(\d+)m')
+    NL = re.compile(r'\n')
+    CHAR = re.compile('.')
+
+
+def tokenize_colors(s):
+    i = 0
+    while i < len(s):
+        for tp in Token:
+            match = tp.value.match(s, i)
+            if match is not None:
+                yield tp, match
+                i = match.end()
+                break
+        else:
+            raise AssertionError(f'unreachable: not matched at {i}?')
+
+
+def to_attrs(screen, width):
+    fg = bg = -1
+    attr = 0
+    idx = 0
+    ret: List[List[Tuple[int, int, int]]]
+    ret = [[] for _ in range(len(screen.splitlines()))]
+
+    for tp, match in tokenize_colors(screen):
+        if tp is Token.FG_ESC:
+            fg = int(match[1])
+        elif tp is Token.BG_ESC:
+            bg = int(match[1])
+        elif tp is Token.RESET:
+            fg = bg = -1
+            attr = 0
+        elif tp is Token.ESC:
+            if match[1] == '7':
+                attr |= curses.A_REVERSE
+            elif match[1] == '39':
+                fg = -1
+            elif match[1] == '49':
+                bg = -1
+            elif 40 <= int(match[1]) <= 47:
+                bg = int(match[1]) - 40
+            else:
+                raise AssertionError(f'unknown escape {match[1]}')
+        elif tp is Token.NL:
+            ret[idx].extend([(fg, bg, attr)] * (width - len(ret[idx])))
+            idx += 1
+        elif tp is Token.CHAR:
+            ret[idx].append((fg, bg, attr))
+        else:
+            raise AssertionError(f'unreachable {tp} {match}')
+
+    return ret
+
+
+class PrintsErrorRunner(Runner):
+    def __init__(self, *args, **kwargs):
+        self._prev_screenshot = None
+        super().__init__(*args, **kwargs)
+
+    def screenshot(self, *args, **kwargs):
+        ret = super().screenshot(*args, **kwargs)
+        if ret != self._prev_screenshot:
+            print('=' * 79, flush=True)
+            print(ret, end='', flush=True)
+            print('=' * 79, flush=True)
+            self._prev_screenshot = ret
+        return ret
+
+    def color_screenshot(self):
+        ret = self.tmux.execute_command('capture-pane', '-ept0')
+        if ret != self._prev_screenshot:
+            print('=' * 79, flush=True)
+            print(ret, end='\x1b[m', flush=True)
+            print('=' * 79, flush=True)
+            self._prev_screenshot = ret
+        return ret
+
+    def get_attrs(self):
+        width, _ = self.get_pane_size()
+        return to_attrs(self.color_screenshot(), width)
+
+    def await_text(self, text, timeout=None):
+        """copied from the base implementation but doesn't munge newlines"""
+        for _ in self.poll_until_timeout(timeout):
+            screen = self.screenshot()
+            if text in screen:  # pragma: no branch
+                return
+        raise AssertionError(
+            f'Timeout while waiting for text {text!r} to appear',
+        )
+
+    def await_text_missing(self, s):
+        """largely based on await_text"""
+        for _ in self.poll_until_timeout():
+            screen = self.screenshot()
+            munged = screen.replace('\n', '')
+            if s not in munged:  # pragma: no branch
+                return
+        raise AssertionError(
+            f'Timeout while waiting for text {s!r} to disappear',
+        )
+
+    def assert_cursor_line_equals(self, s):
+        cursor_line = self._get_cursor_line()
+        assert cursor_line == s, (cursor_line, s)
+
+    def assert_screen_line_equals(self, n, s):
+        screen_line = self._get_screen_line(n)
+        assert screen_line == s, (screen_line, s)
+
+    def assert_screen_attr_equals(self, n, attr):
+        attr_line = self.get_attrs()[n]
+        assert attr_line == attr, (n, attr_line, attr)
+
+    def assert_full_contents(self, s):
+        contents = self.screenshot()
+        assert contents == s
+
+    def get_pane_size(self):
+        cmd = ('display', '-t0', '-p', '#{pane_width}\t#{pane_height}')
+        w, h = self.tmux.execute_command(*cmd).split()
+        return int(w), int(h)
+
+    def _get_cursor_position(self):
+        cmd = ('display', '-t0', '-p', '#{cursor_x}\t#{cursor_y}')
+        x, y = self.tmux.execute_command(*cmd).split()
+        return int(x), int(y)
+
+    def await_cursor_position(self, *, x, y):
+        for _ in self.poll_until_timeout():
+            pos = self._get_cursor_position()
+            if pos == (x, y):  # pragma: no branch
+                return
+
+        raise AssertionError(
+            f'Timeout while waiting for cursor to reach {(x, y)}\n'
+            f'Last cursor position: {pos}',
+        )
+
+    def _get_screen_line(self, n):
+        return self.screenshot().splitlines()[n]
+
+    def _get_cursor_line(self):
+        _, y = self._get_cursor_position()
+        return self._get_screen_line(y)
+
+    @contextlib.contextmanager
+    def resize(self, width, height):
+        current_w, current_h = self.get_pane_size()
+        sleep_cmd = (
+            'bash', '-c',
+            f'echo {"*" * (current_w * current_h)} && '
+            f'exec sleep infinity',
+        )
+
+        panes = 0
+
+        hsplit_w = current_w - width - 1
+        if hsplit_w > 0:
+            cmd = ('split-window', '-ht0', '-l', hsplit_w, *sleep_cmd)
+            self.tmux.execute_command(*cmd)
+            panes += 1
+
+        vsplit_h = current_h - height - 1
+        if vsplit_h > 0:  # pragma: no branch  # TODO
+            cmd = ('split-window', '-vt0', '-l', vsplit_h, *sleep_cmd)
+            self.tmux.execute_command(*cmd)
+            panes += 1
+
+        assert self.get_pane_size() == (width, height)
+        try:
+            yield
+        finally:
+            for _ in range(panes):
+                self.tmux.execute_command('kill-pane', '-t1')
+
+    def press_and_enter(self, s):
+        self.press(s)
+        self.press('Enter')
+
+    def answer_no_if_modified(self):
+        if '*' in self._get_screen_line(0):
+            self.press('n')
+
+    def run(self, callback):
+        # this is a bit of a hack, the in-process fake defers all execution
+        callback()
+
+    @contextlib.contextmanager
+    def on_error(self):
+        try:
+            yield
+        except AssertionError:  # pragma: no cover (only on failure)
+            self.screenshot()
+            raise
+
+
+@contextlib.contextmanager
+def and_exit(h):
+    yield
+    # only try and exit in non-exceptional cases
+    h.press('^X')
+    h.answer_no_if_modified()
+    h.await_exit()
+
+
+def trigger_command_mode(h):
+    # in order to enter a steady state, trigger an unknown key first and then
+    # press escape to open the command mode.  this is necessary as `Escape` is
+    # the start of "escape sequences" and sending characters too quickly will
+    # be interpreted as a single keypress
+    h.press('^J')
+    h.await_text('unknown key')
+    h.press('Escape')
+    h.await_text_missing('unknown key')

testing/vsc_test/.gitignore

@@ -0,0 +1,2 @@
+/node_modules
+/package-lock.json

testing/vsc_test/package.json

@@ -0,0 +1,5 @@
+{
+  "dependencies": [
+    "vscode-textmate"
+  ]
+}

testing/vsc_test/vsc.js

@@ -0,0 +1,51 @@
+const fs = require('fs');
+const vsctm = require('vscode-textmate');
+
+if (process.argv.length < 4) {
+    console.log('usage: t.js GRAMMAR FILE');
+    process.exit(1);
+}
+
+const grammar = process.argv[2];
+const file = process.argv[3];
+
+const scope = JSON.parse(fs.readFileSync(grammar, {encoding: 'UTF-8'})).scopeName;
+
+/**
+ * Utility to read a file as a promise
+ */
+function readFile(path) {
+    return new Promise((resolve, reject) => {
+        fs.readFile(path, (error, data) => error ? reject(error) : resolve(data));
+    })
+}
+
+// Create a registry that can create a grammar from a scope name.
+const registry = new vsctm.Registry({
+    loadGrammar: (scopeName) => {
+        if (scopeName === scope) {
+            return readFile(grammar).then(data => vsctm.parseRawGrammar(data.toString(), grammar))
+        }
+        console.log(`Unknown scope name: ${scopeName}`);
+        return null;
+    }
+});
+
+// Load the JavaScript grammar and any other grammars included by it async.
+registry.loadGrammar(scope).then(grammar => {
+    const text = fs.readFileSync(file, {encoding: 'UTF-8'}).trimEnd('\n').split(/\n/);
+    let ruleStack = vsctm.INITIAL;
+    for (let i = 0; i < text.length; i++) {
+        const line = text[i];
+        const lineTokens = grammar.tokenizeLine(line, ruleStack);
+        console.log(`\nTokenizing line: ${line}`);
+        for (let j = 0; j < lineTokens.tokens.length; j++) {
+            const token = lineTokens.tokens[j];
+            console.log(` - token from ${token.startIndex} to ${token.endIndex} ` +
+              `(${line.substring(token.startIndex, token.endIndex)}) ` +
+              `with scopes ${token.scopes.join(', ')}`
+            );
+        }
+        ruleStack = lineTokens.ruleStack;
+    }
+});

tests/buf_test.py

@@ -0,0 +1,178 @@
+import pytest
+
+from babi.buf import Buf
+
+
+def test_buf_repr():
+    ret = repr(Buf(['a', 'b', 'c']))
+    assert ret == "Buf(['a', 'b', 'c'], x=0, y=0, file_y=0)"
+
+
+def test_buf_item_retrieval():
+    buf = Buf(['a', 'b', 'c'])
+    assert buf[1] == 'b'
+    assert buf[-1] == 'c'
+    with pytest.raises(IndexError):
+        buf[3]
+
+
+def test_buf_del():
+    lst = ['a', 'b', 'c']
+
+    buf = Buf(lst)
+
+    with buf.record() as modifications:
+        del buf[1]
+
+    assert lst == ['a', 'c']
+
+    buf.apply(modifications)
+
+    assert lst == ['a', 'b', 'c']
+
+
+def test_buf_del_with_negative():
+    lst = ['a', 'b', 'c']
+
+    buf = Buf(lst)
+
+    with buf.record() as modifications:
+        del buf[-1]
+
+    assert lst == ['a', 'b']
+
+    buf.apply(modifications)
+
+    assert lst == ['a', 'b', 'c']
+
+
+def test_buf_insert():
+    lst = ['a', 'b', 'c']
+
+    buf = Buf(lst)
+
+    with buf.record() as modifications:
+        buf.insert(1, 'q')
+
+    assert lst == ['a', 'q', 'b', 'c']
+
+    buf.apply(modifications)
+
+    assert lst == ['a', 'b', 'c']
+
+
+def test_buf_insert_with_negative():
+    lst = ['a', 'b', 'c']
+
+    buf = Buf(lst)
+
+    with buf.record() as modifications:
+        buf.insert(-1, 'q')
+
+    assert lst == ['a', 'b', 'q', 'c']
+
+    buf.apply(modifications)
+
+    assert lst == ['a', 'b', 'c']
+
+
+def test_buf_set_value():
+    lst = ['a', 'b', 'c']
+
+    buf = Buf(lst)
+
+    with buf.record() as modifications:
+        buf[1] = 'hello'
+
+    assert lst == ['a', 'hello', 'c']
+
+    buf.apply(modifications)
+
+    assert lst == ['a', 'b', 'c']
+
+
+def test_buf_set_value_idx_negative():
+    lst = ['a', 'b', 'c']
+
+    buf = Buf(lst)
+
+    with buf.record() as modifications:
+        buf[-1] = 'hello'
+
+    assert lst == ['a', 'b', 'hello']
+
+    buf.apply(modifications)
+
+    assert lst == ['a', 'b', 'c']
+
+
+def test_buf_multiple_modifications():
+    lst = ['a', 'b', 'c']
+
+    buf = Buf(lst)
+
+    with buf.record() as modifications:
+        buf[1] = 'hello'
+        buf.insert(1, 'ohai')
+        del buf[0]
+
+    assert lst == ['ohai', 'hello', 'c']
+
+    buf.apply(modifications)
+
+    assert lst == ['a', 'b', 'c']
+
+
+def test_buf_iter():
+    buf = Buf(['a', 'b', 'c'])
+    buf_iter = iter(buf)
+    assert next(buf_iter) == 'a'
+    assert next(buf_iter) == 'b'
+    assert next(buf_iter) == 'c'
+    with pytest.raises(StopIteration):
+        next(buf_iter)
+
+
+def test_buf_append():
+    lst = ['a', 'b', 'c']
+
+    buf = Buf(lst)
+
+    with buf.record() as modifications:
+        buf.append('q')
+
+    assert lst == ['a', 'b', 'c', 'q']
+
+    buf.apply(modifications)
+
+    assert lst == ['a', 'b', 'c']
+
+
+def test_buf_pop_default():
+    lst = ['a', 'b', 'c']
+
+    buf = Buf(lst)
+
+    with buf.record() as modifications:
+        buf.pop()
+
+    assert lst == ['a', 'b']
+
+    buf.apply(modifications)
+
+    assert lst == ['a', 'b', 'c']
+
+
+def test_buf_pop_idx():
+    lst = ['a', 'b', 'c']
+
+    buf = Buf(lst)
+
+    with buf.record() as modifications:
+        buf.pop(1)
+
+    assert lst == ['a', 'c']
+
+    buf.apply(modifications)
+
+    assert lst == ['a', 'b', 'c']

tests/color_kd_test.py

@@ -0,0 +1,63 @@
+from babi import color_kd
+from babi.color import Color
+
+
+def test_build_trivial():
+    assert color_kd._build([]) is None
+
+
+def test_build_single_node():
+    kd = color_kd._build([(Color(0, 0, 0), 255)])
+    assert kd == color_kd._KD(Color(0, 0, 0), 255, left=None, right=None)
+
+
+def test_build_many_colors():
+    kd = color_kd._build([
+        (Color(0, 106, 200), 255),
+        (Color(1, 105, 201), 254),
+        (Color(2, 104, 202), 253),
+        (Color(3, 103, 203), 252),
+        (Color(4, 102, 204), 251),
+        (Color(5, 101, 205), 250),
+        (Color(6, 100, 206), 249),
+    ])
+
+    # each level is sorted by the next dimension
+    assert kd == color_kd._KD(
+        Color(3, 103, 203),
+        252,
+        left=color_kd._KD(
+            Color(1, 105, 201), 254,
+            left=color_kd._KD(Color(2, 104, 202), 253, None, None),
+            right=color_kd._KD(Color(0, 106, 200), 255, None, None),
+        ),
+        right=color_kd._KD(
+            Color(5, 101, 205), 250,
+            left=color_kd._KD(Color(6, 100, 206), 249, None, None),
+            right=color_kd._KD(Color(4, 102, 204), 251, None, None),
+        ),
+    )
+
+
+def test_nearest_trivial():
+    assert color_kd.nearest(Color(0, 0, 0), None) == 0
+
+
+def test_nearest_one_node():
+    kd = color_kd._build([(Color(100, 100, 100), 99)])
+    assert color_kd.nearest(Color(0, 0, 0), kd) == 99
+
+
+def test_nearest_on_square_distance():
+    kd = color_kd._build([
+        (Color(50, 50, 50), 255),
+        (Color(50, 51, 50), 254),
+    ])
+    assert color_kd.nearest(Color(0, 0, 0), kd) == 255
+    assert color_kd.nearest(Color(52, 52, 52), kd) == 254
+
+
+def test_smoke_kd_256():
+    kd_256 = color_kd.make_256()
+    assert color_kd.nearest(Color(0, 0, 0), kd_256) == 16
+    assert color_kd.nearest(Color(0x1e, 0x77, 0xd3), kd_256) == 32

tests/color_manager_test.py

@@ -0,0 +1,16 @@
+import pytest
+
+from babi.color import Color
+from babi.color_manager import _color_to_curses
+
+
+@pytest.mark.parametrize(
+    ('color', 'expected'),
+    (
+        (Color(0x00, 0x00, 0x00), (0, 0, 0)),
+        (Color(0xff, 0xff, 0xff), (1000, 1000, 1000)),
+        (Color(0x1e, 0x77, 0xd3), (117, 466, 827)),
+    ),
+)
+def test_color_to_curses(color, expected):
+    assert _color_to_curses(color) == expected

tests/color_test.py

@@ -0,0 +1,16 @@
+import pytest
+
+from babi.color import Color
+
+
+@pytest.mark.parametrize(
+    ('s', 'expected'),
+    (
+        ('#1e77d3', Color(0x1e, 0x77, 0xd3)),
+        ('white', Color(0xff, 0xff, 0xff)),
+        ('black', Color(0x00, 0x00, 0x00)),
+        ('#ccc', Color(0xcc, 0xcc, 0xcc)),
+    ),
+)
+def test_color_parse(s, expected):
+    assert Color.parse(s) == expected

tests/conftest.py

@@ -0,0 +1,17 @@
+import json
+
+import pytest
+
+from babi.highlight import Grammars
+
+
+@pytest.fixture
+def make_grammars(tmpdir):
+    grammar_dir = tmpdir.join('grammars').ensure_dir()
+
+    def make_grammars(*grammar_dcts):
+        for grammar in grammar_dcts:
+            filename = f'{grammar["scopeName"]}.json'
+            grammar_dir.join(filename).write(json.dumps(grammar))
+        return Grammars(grammar_dir)
+    return make_grammars

tests/fdict_test.py

@@ -0,0 +1,28 @@
+import pytest
+
+from babi.fdict import FChainMap
+from babi.fdict import FDict
+
+
+def test_fdict_repr():
+    # mostly because this shouldn't get hit elsewhere but is uesful for
+    # debugging purposes
+    assert repr(FDict({1: 2, 3: 4})) == 'FDict({1: 2, 3: 4})'
+
+
+def test_f_chain_map():
+    chain_map = FChainMap({1: 2}, {3: 4}, FDict({1: 5}))
+    assert chain_map[1] == 5
+    assert chain_map[3] == 4
+
+    with pytest.raises(KeyError) as excinfo:
+        chain_map[2]
+    k, = excinfo.value.args
+    assert k == 2
+
+
+def test_f_chain_map_extend():
+    chain_map = FChainMap({1: 2})
+    assert chain_map[1] == 2
+    chain_map = FChainMap(chain_map, {1: 5})
+    assert chain_map[1] == 5

tests/features/command_mode_test.py

@@ -0,0 +1,196 @@
+import pytest
+
+from testing.runner import and_exit
+from testing.runner import trigger_command_mode
+
+
+def test_quit_via_colon_q(run):
+    with run() as h:
+        trigger_command_mode(h)
+        h.press_and_enter(':q')
+        h.await_exit()
+
+
+def test_key_navigation_in_command_mode(run):
+    with run() as h, and_exit(h):
+        trigger_command_mode(h)
+        h.press('hello world')
+        h.await_cursor_position(x=11, y=23)
+        h.press('Left')
+        h.await_cursor_position(x=10, y=23)
+        h.press('Right')
+        h.await_cursor_position(x=11, y=23)
+        h.press('Home')
+        h.await_cursor_position(x=0, y=23)
+        h.press('End')
+        h.await_cursor_position(x=11, y=23)
+        h.press('^A')
+        h.await_cursor_position(x=0, y=23)
+        h.press('^E')
+        h.await_cursor_position(x=11, y=23)
+
+        h.press('DC')  # does nothing at end
+        h.await_cursor_position(x=11, y=23)
+        h.await_text('\nhello world\n')
+
+        h.press('Home')
+
+        h.press('DC')
+        h.await_cursor_position(x=0, y=23)
+        h.await_text('\nello world\n')
+
+        # unknown keys don't do anything
+        h.press('^J')
+        h.await_text('\nello world\n')
+
+        h.press('Enter')
+
+
+@pytest.mark.parametrize('key', ('BSpace', '^H'))
+def test_command_mode_backspace(run, key):
+    with run() as h, and_exit(h):
+        trigger_command_mode(h)
+        h.press('hello world')
+        h.await_text('\nhello world\n')
+
+        h.press(key)
+        h.await_text('\nhello worl\n')
+
+        h.press('Home')
+        h.press(key)  # does nothing at beginning
+        h.await_cursor_position(x=0, y=23)
+        h.await_text('\nhello worl\n')
+
+        h.press('^C')
+
+
+def test_command_mode_ctrl_k(run):
+    with run() as h, and_exit(h):
+        trigger_command_mode(h)
+        h.press('hello world')
+        h.await_text('\nhello world\n')
+        h.press('^Left')
+        h.press('Left')
+        h.press('^K')
+        h.await_text('\nhello\n')
+        h.press('Enter')
+
+
+def test_command_mode_control_left(run):
+    with run() as h, and_exit(h):
+        trigger_command_mode(h)
+        h.press('hello world')
+        h.await_cursor_position(x=11, y=23)
+        h.press('^Left')
+        h.await_cursor_position(x=6, y=23)
+        h.press('^Left')
+        h.await_cursor_position(x=0, y=23)
+        h.press('^Left')
+        h.await_cursor_position(x=0, y=23)
+        h.press('Right')
+        h.await_cursor_position(x=1, y=23)
+        h.press('^Left')
+        h.await_cursor_position(x=0, y=23)
+        h.press('^C')
+
+
+def test_command_mode_control_right(run):
+    with run() as h, and_exit(h):
+        trigger_command_mode(h)
+        h.press('hello world')
+        h.await_cursor_position(x=11, y=23)
+        h.press('^Right')
+        h.await_cursor_position(x=11, y=23)
+        h.press('Left')
+        h.await_cursor_position(x=10, y=23)
+        h.press('^Right')
+        h.await_cursor_position(x=11, y=23)
+        h.press('^A')
+        h.await_cursor_position(x=0, y=23)
+        h.press('^Right')
+        h.await_cursor_position(x=5, y=23)
+        h.press('^Right')
+        h.await_cursor_position(x=11, y=23)
+        h.press('^C')
+
+
+def test_save_via_command_mode(run, tmpdir):
+    f = tmpdir.join('f')
+
+    with run(str(f)) as h, and_exit(h):
+        h.press('hello world')
+        trigger_command_mode(h)
+        h.press_and_enter(':w')
+
+    assert f.read() == 'hello world\n'
+
+
+def test_repeated_command_mode_does_not_show_previous_command(run, tmpdir):
+    f = tmpdir.join('f')
+
+    with run(str(f)) as h, and_exit(h):
+        h.press('ohai')
+        trigger_command_mode(h)
+        h.press_and_enter(':w')
+        trigger_command_mode(h)
+        h.await_text_missing(':w')
+        h.press('Enter')
+
+
+def test_write_and_quit(run, tmpdir):
+    f = tmpdir.join('f')
+
+    with run(str(f)) as h, and_exit(h):
+        h.press('hello world')
+        trigger_command_mode(h)
+        h.press_and_enter(':wq')
+        h.await_exit()
+
+    assert f.read() == 'hello world\n'
+
+
+def test_resizing_and_scrolling_in_command_mode(run):
+    with run(width=20) as h, and_exit(h):
+        h.press('a' * 15)
+        h.await_text(f'\n{"a" * 15}\n')
+        trigger_command_mode(h)
+        h.press('b' * 15)
+        h.await_text(f'\n{"b" * 15}\n')
+
+        with h.resize(width=16, height=24):
+            h.await_text('\n«aaaaaa\n')  # the text contents
+            h.await_text('\n«bbbbbb\n')  # the text contents
+            h.await_cursor_position(x=7, y=23)
+            h.press('Left')
+            h.await_cursor_position(x=14, y=23)
+            h.await_text(f'\n{"b" * 15}\n')
+
+        h.press('Enter')
+
+
+def test_invalid_command(run):
+    with run() as h, and_exit(h):
+        trigger_command_mode(h)
+        h.press_and_enter(':fake')
+        h.await_text('invalid command: :fake')
+
+
+def test_empty_command_is_noop(run):
+    with run() as h, and_exit(h):
+        h.press('hello ')
+        trigger_command_mode(h)
+        h.press('Enter')
+        h.press('world')
+        h.await_text('hello world')
+        h.await_text_missing('invalid command')
+
+
+def test_cancel_command_mode(run):
+    with run() as h, and_exit(h):
+        h.press('hello ')
+        trigger_command_mode(h)
+        h.press(':q')
+        h.press('^C')
+        h.press('world')
+        h.await_text('hello world')
+        h.await_text_missing('invalid command')

tests/features/conftest.py

@@ -0,0 +1,475 @@
+import contextlib
+import curses
+import os
+import sys
+from typing import List
+from typing import NamedTuple
+from typing import Tuple
+from typing import Union
+from unittest import mock
+
+import pytest
+
+from babi._types import Protocol
+from babi.main import main
+from babi.screen import VERSION_STR
+from testing.runner import PrintsErrorRunner
+
+
+@pytest.fixture(autouse=True)
+def prefix_home(tmpdir):
+    prefix_home = tmpdir.join('prefix_home')
+    with mock.patch.object(sys, 'prefix', str(prefix_home)):
+        yield prefix_home
+
+
+@pytest.fixture(autouse=True)
+def xdg_data_home(tmpdir):
+    data_home = tmpdir.join('data_home')
+    with mock.patch.dict(os.environ, {'XDG_DATA_HOME': str(data_home)}):
+        yield data_home
+
+
+@pytest.fixture(autouse=True)
+def xdg_config_home(tmpdir):
+    config_home = tmpdir.join('config_home')
+    with mock.patch.dict(os.environ, {'XDG_CONFIG_HOME': str(config_home)}):
+        yield config_home
+
+
+@pytest.fixture
+def ten_lines(tmpdir):
+    f = tmpdir.join('f')
+    f.write('\n'.join(f'line_{i}' for i in range(10)))
+    return f
+
+
+class Screen:
+    def __init__(self, width, height):
+        self.nodelay = False
+        self.width = width
+        self.height = height
+        self.lines = [' ' * self.width for _ in range(self.height)]
+        self.attrs = [[(0, 0, 0)] * self.width for _ in range(self.height)]
+        self.x = self.y = 0
+        self._prev_screenshot = None
+
+    def screenshot(self):
+        ret = ''.join(f'{line.rstrip()}\n' for line in self.lines)
+        if ret != self._prev_screenshot:
+            print('=' * 79)
+            print(ret, end='')
+            print('=' * 79)
+            self._prev_screenshot = ret
+        return ret
+
+    def addstr(self, y, x, s, attr):
+        self.lines[y] = self.lines[y][:x] + s + self.lines[y][x + len(s):]
+
+        line_attr = self.attrs[y]
+        new = [attr] * len(s)
+        self.attrs[y] = line_attr[:x] + new + line_attr[x + len(s):]
+
+        self.y = y
+        self.x = x + len(s)
+
+    def insstr(self, y, x, s, attr):
+        line = self.lines[y]
+        self.lines[y] = (line[:x] + s + line[x:])[:self.width]
+
+        line_attr = self.attrs[y]
+        new = [attr] * len(s)
+        self.attrs[y] = (line_attr[:x] + new + line_attr[x:])[:self.width]
+
+    def chgat(self, y, x, n, attr):
+        assert n >= 0  # TODO: switch to > 0, we should never do 0-length
+        self.attrs[y][x:x + n] = [attr] * n
+
+    def move(self, y, x):
+        assert 0 <= y < self.height
+        assert 0 <= x < self.width
+        print(f'MOVE: y: {y}, x: {x}')
+        self.y, self.x = y, x
+
+    def resize(self, *, width, height):
+        if height > self.height:
+            self.lines.extend([''] * (height - self.height))
+        else:
+            self.lines = self.lines[:height]
+        if width > self.width:
+            self.lines[:] = [line.ljust(width) for line in self.lines]
+        else:
+            self.lines[:] = [line[:width] for line in self.lines]
+        self.width, self.height = width, height
+
+
+class Op(Protocol):
+    def __call__(self, screen: Screen) -> None: ...
+
+
+class AwaitText(NamedTuple):
+    text: str
+
+    def __call__(self, screen: Screen) -> None:
+        if self.text not in screen.screenshot():
+            raise AssertionError(f'expected: {self.text!r}')
+
+
+class AwaitTextMissing(NamedTuple):
+    text: str
+
+    def __call__(self, screen: Screen) -> None:
+        if self.text in screen.screenshot():
+            raise AssertionError(f'expected missing: {self.text!r}')
+
+
+class AwaitCursorPosition(NamedTuple):
+    x: int
+    y: int
+
+    def __call__(self, screen: Screen) -> None:
+        assert (self.x, self.y) == (screen.x, screen.y)
+
+
+class AssertCursorLineEquals(NamedTuple):
+    line: str
+
+    def __call__(self, screen: Screen) -> None:
+        assert screen.lines[screen.y].rstrip() == self.line
+
+
+class AssertScreenLineEquals(NamedTuple):
+    n: int
+    line: str
+
+    def __call__(self, screen: Screen) -> None:
+        assert screen.lines[self.n].rstrip() == self.line
+
+
+class AssertScreenAttrEquals(NamedTuple):
+    n: int
+    attr: List[Tuple[int, int, int]]
+
+    def __call__(self, screen: Screen) -> None:
+        assert screen.attrs[self.n] == self.attr
+
+
+class AssertFullContents(NamedTuple):
+    contents: str
+
+    def __call__(self, screen: Screen) -> None:
+        assert screen.screenshot() == self.contents
+
+
+class Resize(NamedTuple):
+    width: int
+    height: int
+
+    def __call__(self, screen: Screen) -> None:
+        screen.resize(width=self.width, height=self.height)
+
+
+class KeyPress(NamedTuple):
+    wch: Union[int, str]
+
+    def __call__(self, screen: Screen) -> None:
+        raise AssertionError('unreachable')
+
+
+class CursesError(NamedTuple):
+    def __call__(self, screen: Screen) -> None:
+        if screen.nodelay:
+            raise curses.error()
+
+
+class CursesScreen:
+    def __init__(self, screen, runner):
+        self._screen = screen
+        self._runner = runner
+        self._bkgd_attr = (-1, -1, 0)
+
+    def _to_attr(self, attr):
+        if attr == 0:
+            return self._bkgd_attr
+        else:
+            pair = (attr & (0xff << 8)) >> 8
+            if pair == 0:
+                fg, bg, _ = self._bkgd_attr
+            else:
+                fg, bg = self._runner.color_pairs[pair]
+            attr = attr & ~(0xff << 8)
+            return (fg, bg, attr)
+
+    def bkgd(self, c, attr):
+        assert c == ' '
+        self._bkgd_attr = self._to_attr(attr)
+
+    def keypad(self, val):
+        pass
+
+    def nodelay(self, val):
+        self._screen.nodelay = val
+
+    def addstr(self, y, x, s, attr=0):
+        self._screen.addstr(y, x, s, self._to_attr(attr))
+
+    def insstr(self, y, x, s, attr=0):
+        self._screen.insstr(y, x, s, self._to_attr(attr))
+
+    def clrtoeol(self):
+        s = self._screen.width * ' '
+        self.insstr(self._screen.y, self._screen.x, s)
+
+    def chgat(self, y, x, n, attr):
+        self._screen.chgat(y, x, n, self._to_attr(attr))
+
+    def move(self, y, x):
+        self._screen.move(y, x)
+
+    def getyx(self):
+        return self._screen.y, self._screen.x
+
+    def get_wch(self):
+        return self._runner._get_wch()
+
+
+class Key(NamedTuple):
+    tmux: str
+    curses: bytes
+    wch: Union[int, str]
+
+    @property
+    def value(self) -> int:
+        return self.wch if isinstance(self.wch, int) else ord(self.wch)
+
+
+KEYS = [
+    Key('Enter', b'^M', '\r'),
+    Key('Tab', b'^I', '\t'),
+    Key('BTab', b'KEY_BTAB', curses.KEY_BTAB),
+    Key('DC', b'KEY_DC', curses.KEY_DC),
+    Key('BSpace', b'KEY_BACKSPACE', curses.KEY_BACKSPACE),
+    Key('Up', b'KEY_UP', curses.KEY_UP),
+    Key('Down', b'KEY_DOWN', curses.KEY_DOWN),
+    Key('Right', b'KEY_RIGHT', curses.KEY_RIGHT),
+    Key('Left', b'KEY_LEFT', curses.KEY_LEFT),
+    Key('Home', b'KEY_HOME', curses.KEY_HOME),
+    Key('End', b'KEY_END', curses.KEY_END),
+    Key('PageUp', b'KEY_PPAGE', curses.KEY_PPAGE),
+    Key('PageDown', b'KEY_NPAGE', curses.KEY_NPAGE),
+    Key('^Up', b'kUP5', 566),
+    Key('^Down', b'kDN5', 525),
+    Key('^Right', b'kRIT5', 560),
+    Key('^Left', b'kLFT5', 545),
+    Key('^Home', b'kHOM5', 535),
+    Key('^End', b'kEND5', 530),
+    Key('M-Right', b'kRIT3', 558),
+    Key('M-Left', b'kLFT3', 543),
+    Key('S-Up', b'KEY_SR', curses.KEY_SR),
+    Key('S-Down', b'KEY_SF', curses.KEY_SF),
+    Key('S-Right', b'KEY_SRIGHT', curses.KEY_SRIGHT),
+    Key('S-Left', b'KEY_SLEFT', curses.KEY_SLEFT),
+    Key('S-Home', b'KEY_SHOME', curses.KEY_SHOME),
+    Key('S-End', b'KEY_SEND', curses.KEY_SEND),
+    Key('^A', b'^A', '\x01'),
+    Key('^C', b'^C', '\x03'),
+    Key('^H', b'^H', '\x08'),
+    Key('^K', b'^K', '\x0b'),
+    Key('^E', b'^E', '\x05'),
+    Key('^J', b'^J', '\n'),
+    Key('^O', b'^O', '\x0f'),
+    Key('^P', b'^P', '\x10'),
+    Key('^R', b'^R', '\x12'),
+    Key('^S', b'^S', '\x13'),
+    Key('^U', b'^U', '\x15'),
+    Key('^V', b'^V', '\x16'),
+    Key('^W', b'^W', '\x17'),
+    Key('^X', b'^X', '\x18'),
+    Key('^Y', b'^Y', '\x19'),
+    Key('^[', b'^[', '\x1b'),
+    Key('^_', b'^_', '\x1f'),
+    Key('^\\', b'^\\', '\x1c'),
+    Key('!resize', b'KEY_RESIZE', curses.KEY_RESIZE),
+]
+KEYS_TMUX = {k.tmux: k.wch for k in KEYS}
+KEYS_CURSES = {k.value: k.curses for k in KEYS}
+
+
+class DeferredRunner:
+    def __init__(self, command, width=80, height=24, term='screen'):
+        self.command = command
+        self._i = 0
+        self._ops: List[Op] = []
+        self.color_pairs = {0: (7, 0)}
+        self.screen = Screen(width, height)
+        self._n_colors, self._can_change_color = {
+            'screen': (8, False),
+            'screen-256color': (256, False),
+            'xterm-256color': (256, True),
+        }[term]
+
+    def _get_wch(self):
+        while not isinstance(self._ops[self._i], KeyPress):
+            self._i += 1
+            try:
+                self._ops[self._i - 1](self.screen)
+            except AssertionError:  # pragma: no cover (only on failures)
+                self.screen.screenshot()
+                raise
+        self._i += 1
+        keypress_event = self._ops[self._i - 1]
+        assert isinstance(keypress_event, KeyPress)
+        print(f'KEY: {keypress_event.wch!r}')
+        return keypress_event.wch
+
+    def await_text(self, text, timeout=1):
+        self._ops.append(AwaitText(text))
+
+    def await_text_missing(self, text):
+        self._ops.append(AwaitTextMissing(text))
+
+    def await_cursor_position(self, *, x, y):
+        self._ops.append(AwaitCursorPosition(x, y))
+
+    def assert_cursor_line_equals(self, line):
+        self._ops.append(AssertCursorLineEquals(line))
+
+    def assert_screen_line_equals(self, n, line):
+        self._ops.append(AssertScreenLineEquals(n, line))
+
+    def assert_screen_attr_equals(self, n, attr):
+        self._ops.append(AssertScreenAttrEquals(n, attr))
+
+    def assert_full_contents(self, contents):
+        self._ops.append(AssertFullContents(contents))
+
+    def run(self, callback):
+        self._ops.append(lambda screen: callback())
+
+    def _expand_key(self, s):
+        if s == 'Escape':
+            return [KeyPress('\x1b'), CursesError()]
+        elif s in KEYS_TMUX:
+            return [KeyPress(KEYS_TMUX[s])]
+        elif s.startswith('^') and len(s) > 1 and s[1].isupper():
+            raise AssertionError(f'unknown key {s}')
+        elif s.startswith('M-'):
+            return [KeyPress('\x1b'), KeyPress(s[2:]), CursesError()]
+        else:
+            return [*(KeyPress(k) for k in s), CursesError()]
+
+    def press(self, s):
+        self._ops.extend(self._expand_key(s))
+
+    def press_and_enter(self, s):
+        self.press(s)
+        self.press('Enter')
+
+    def press_sequence(self, *ks):
+        for k in ks:
+            for op in self._expand_key(k):
+                if not isinstance(op, CursesError):
+                    self._ops.append(op)
+        self._ops.append(CursesError())
+
+    def answer_no_if_modified(self):
+        self.press('n')
+
+    @contextlib.contextmanager
+    def resize(self, *, width, height):
+        orig_width, orig_height = self.screen.width, self.screen.height
+        self._ops.append(Resize(width, height))
+        self._ops.append(KeyPress(curses.KEY_RESIZE))
+        try:
+            yield
+        finally:
+            self._ops.append(Resize(orig_width, orig_height))
+            self._ops.append(KeyPress(curses.KEY_RESIZE))
+
+    def _curses__noop(self, *_, **__):
+        pass
+
+    _curses_cbreak = _curses_endwin = _curses_noecho = _curses__noop
+    _curses_nonl = _curses_raw = _curses_use_default_colors = _curses__noop
+
+    _curses_error = curses.error  # so we don't mock the exception
+
+    def _curses_keyname(self, k):
+        return KEYS_CURSES.get(k, b'')
+
+    def _curses_update_lines_cols(self):
+        curses.LINES = self.screen.height
+        curses.COLS = self.screen.width
+
+    def _curses_start_color(self):
+        curses.COLORS = self._n_colors
+
+    def _curses_can_change_color(self):
+        return self._can_change_color
+
+    def _curses_init_pair(self, pair, fg, bg):
+        self.color_pairs[pair] = (fg, bg)
+
+    def _curses_color_pair(self, pair):
+        assert pair in self.color_pairs
+        return pair << 8
+
+    def _curses_initscr(self):
+        self._curses_update_lines_cols()
+        return CursesScreen(self.screen, self)
+
+    def _curses_newwin(self, height, width):
+        return CursesScreen(Screen(width, height), self)
+
+    def _curses_not_implemented(self, fn):
+        def fn_inner(*args, **kwargs):
+            raise NotImplementedError(fn)
+        return fn_inner
+
+    def _patch_curses(self):
+        patches = {
+            k: getattr(self, f'_curses_{k}', self._curses_not_implemented(k))
+            for k in dir(curses)
+            if not k.startswith('_') and callable(getattr(curses, k))
+        }
+        return mock.patch.multiple(curses, **patches)
+
+    def await_exit(self):
+        with self._patch_curses():
+            main(self.command)
+        # we have already exited -- check remaining things
+        # KeyPress with failing condition or error
+        for i in range(self._i, len(self._ops)):
+            if self._ops[i] not in {KeyPress('n'), CursesError()}:
+                raise AssertionError(self._ops[i:])
+
+
+@contextlib.contextmanager
+def run_fake(*cmd, **kwargs):
+    h = DeferredRunner(cmd, **kwargs)
+    h.await_text(VERSION_STR)
+    yield h
+
+
+@contextlib.contextmanager
+def run_tmux(*args, term='screen', **kwargs):
+    cmd = (sys.executable, '-mcoverage', 'run', '-m', 'babi', *args)
+    cmd = ('env', f'TERM={term}', *cmd)
+    with PrintsErrorRunner(*cmd, **kwargs) as h, h.on_error():
+        # startup with coverage can be slow
+        h.await_text(VERSION_STR, timeout=2)
+        yield h
+
+
+@pytest.fixture(
+    scope='session',
+    params=[run_fake, run_tmux],
+    ids=['fake', 'tmux'],
+)
+def run(request):
+    return request.param
+
+
+@pytest.fixture(scope='session', params=[run_fake], ids=['fake'])
+def run_only_fake(request):
+    return request.param

tests/features/current_position_test.py

@@ -0,0 +1,18 @@
+from testing.runner import and_exit
+
+
+def test_current_position(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('^C')
+        h.await_text('line 1, col 1 (of 10 lines)')
+        h.press('Right')
+        h.press('^C')
+        h.await_text('line 1, col 2 (of 10 lines)')
+        h.press('Down')
+        h.press('^C')
+        h.await_text('line 2, col 2 (of 10 lines)')
+        h.press('Up')
+        for i in range(10):
+            h.press('^K')
+        h.press('^C')
+        h.await_text('line 1, col 1 (of 1 line)')

tests/features/cut_uncut_test.py

@@ -0,0 +1,152 @@
+from testing.runner import and_exit
+
+
+def test_cut_and_uncut(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('^K')
+        h.await_text_missing('line_0')
+        h.await_text(' *')
+        h.press('^U')
+        h.await_text('line_0')
+
+        h.press('^Home')
+        h.press('^K')
+        h.press('^K')
+        h.await_text_missing('line_1')
+        h.press('^U')
+        h.await_text('line_0')
+
+
+def test_cut_at_beginning_of_file(run):
+    with run() as h, and_exit(h):
+        h.press('^K')
+        h.press('^K')
+        h.press('^K')
+        h.await_text_missing('*')
+
+
+def test_cut_end_of_file(run):
+    with run() as h, and_exit(h):
+        h.press('hi')
+        h.press('Down')
+        h.press('^K')
+        h.press('hi')
+
+
+def test_cut_end_of_file_noop_extra_cut(run):
+    with run() as h, and_exit(h):
+        h.press('hi')
+        h.press('^K')
+        h.press('^K')
+        h.press('^U')
+        h.await_text('hi')
+
+
+def test_cut_uncut_multiple_file_buffers(run, tmpdir):
+    f1 = tmpdir.join('f1')
+    f1.write('hello\nworld\n')
+    f2 = tmpdir.join('f2')
+    f2.write('good\nbye\n')
+
+    with run(str(f1), str(f2)) as h, and_exit(h):
+        h.press('^K')
+        h.await_text_missing('hello')
+        h.press('^X')
+        h.press('n')
+        h.await_text_missing('world')
+        h.press('^U')
+        h.await_text('hello\ngood\nbye\n')
+
+
+def test_selection_cut_uncut(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('Right')
+        h.press('S-Right')
+        h.press('S-Down')
+        h.press('^K')
+        h.await_cursor_position(x=1, y=1)
+        h.await_text('lne_1\n')
+        h.await_text_missing('line_0')
+        h.await_text(' *')
+        h.press('^U')
+        h.await_cursor_position(x=2, y=2)
+        h.await_text('line_0\nline_1')
+
+
+def test_selection_cut_uncut_backwards_select(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        for _ in range(3):
+            h.press('Down')
+
+        h.press('Right')
+        h.press('S-Up')
+        h.press('S-Up')
+        h.press('S-Right')
+
+        h.press('^K')
+        h.await_text('line_0\nliine_3\nline_4\n')
+        h.await_cursor_position(x=2, y=2)
+        h.await_text(' *')
+
+        h.press('^U')
+        h.await_text('line_0\nline_1\nline_2\nline_3\nline_4\n')
+        h.await_cursor_position(x=1, y=4)
+
+
+def test_selection_cut_uncut_within_line(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('Right')
+        h.press('S-Right')
+        h.press('S-Right')
+
+        h.press('^K')
+        h.await_text('le_0\n')
+        h.await_cursor_position(x=1, y=1)
+        h.await_text(' *')
+
+        h.press('^U')
+        h.await_text('line_0\n')
+        h.await_cursor_position(x=3, y=1)
+
+
+def test_selection_cut_uncut_selection_offscreen_y(run, ten_lines):
+    with run(str(ten_lines), height=4) as h, and_exit(h):
+        for _ in range(3):
+            h.press('S-Down')
+        h.await_text_missing('line_0')
+        h.await_text('line_3')
+        h.press('^K')
+        h.await_text_missing('line_2')
+        h.await_cursor_position(x=0, y=1)
+
+
+def test_selection_cut_uncut_selection_offscreen_x(run):
+    with run() as h, and_exit(h):
+        h.press(f'hello{"o" * 100}')
+        h.await_text_missing('hello')
+        h.press('Home')
+        h.await_text('hello')
+        for _ in range(5):
+            h.press('Right')
+        h.press('S-End')
+        h.await_text_missing('hello')
+        h.press('^K')
+        h.await_text('hello\n')
+
+
+def test_selection_cut_uncut_at_end_of_file(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('S-Down')
+        h.press('S-Right')
+        h.press('^K')
+        h.await_text_missing('line_0')
+        h.await_text_missing('line_1')
+        h.await_text('ine_1')
+
+        h.press('^End')
+        h.press('^U')
+        h.await_text('line_0\nl\n')
+        h.await_cursor_position(x=1, y=11)
+
+        h.press('Down')
+        h.await_cursor_position(x=0, y=12)

tests/features/go_to_line_test.py

@@ -0,0 +1,68 @@
+import pytest
+
+from testing.runner import and_exit
+
+
+def test_prompt_window_width(run):
+    with run() as h, and_exit(h):
+        h.press('^_')
+        h.await_text('enter line number:')
+        h.press('123')
+        with h.resize(width=23, height=24):
+            h.await_text('\nenter line number: «3')
+        with h.resize(width=22, height=24):
+            h.await_text('\nenter line numb…: «3')
+        with h.resize(width=7, height=24):
+            h.await_text('\n…: «3')
+        with h.resize(width=6, height=24):
+            h.await_text('\n123')
+        h.press('Enter')
+
+
+def test_go_to_line_line(run, ten_lines):
+    def _jump_to_line(n):
+        h.press('^_')
+        h.await_text('enter line number:')
+        h.press_and_enter(str(n))
+        h.await_text_missing('enter line number:')
+
+    with run(str(ten_lines), height=9) as h, and_exit(h):
+        # still on screen
+        _jump_to_line(3)
+        h.await_cursor_position(x=0, y=3)
+        # should go to beginning of file
+        _jump_to_line(0)
+        h.await_cursor_position(x=0, y=1)
+        # should go to end of the file
+        _jump_to_line(999)
+        h.await_cursor_position(x=0, y=4)
+        h.assert_screen_line_equals(3, 'line_9')
+        # should also go to the end of the file
+        _jump_to_line(-1)
+        h.await_cursor_position(x=0, y=4)
+        h.assert_screen_line_equals(3, 'line_9')
+        # should go to beginning of file
+        _jump_to_line(-999)
+        h.await_cursor_position(x=0, y=1)
+        h.assert_cursor_line_equals('line_0')
+
+
+@pytest.mark.parametrize('key', ('Enter', '^C'))
+def test_go_to_line_cancel(run, ten_lines, key):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('Down')
+        h.await_cursor_position(x=0, y=2)
+
+        h.press('^_')
+        h.await_text('enter line number:')
+        h.press(key)
+        h.await_cursor_position(x=0, y=2)
+        h.await_text('cancelled')
+
+
+def test_go_to_line_not_an_integer(run):
+    with run() as h, and_exit(h):
+        h.press('^_')
+        h.await_text('enter line number:')
+        h.press_and_enter('asdf')
+        h.await_text("not an integer: 'asdf'")

tests/features/indent_test.py

@@ -0,0 +1,108 @@
+from testing.runner import and_exit
+
+
+def test_indent_at_beginning_of_line(run):
+    with run() as h, and_exit(h):
+        h.press('hello')
+        h.press('Home')
+        h.press('Tab')
+        h.await_text('\n    hello\n')
+        h.await_cursor_position(x=4, y=1)
+
+
+def test_indent_not_full_tab(run):
+    with run() as h, and_exit(h):
+        h.press('h')
+        h.press('Tab')
+        h.press('ello')
+        h.await_text('h   ello')
+        h.await_cursor_position(x=8, y=1)
+
+
+def test_indent_fixes_eof(run):
+    with run() as h, and_exit(h):
+        h.press('Tab')
+        h.press('Down')
+        h.await_cursor_position(x=0, y=2)
+
+
+def test_indent_selection(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('S-Right')
+        h.press('Tab')
+        h.await_text('\n    line_0\n')
+        h.await_cursor_position(x=5, y=1)
+        h.press('^K')
+        h.await_text('\nine_0\n')
+
+
+def test_indent_selection_does_not_extend_mid_line_selection(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('Right')
+        h.press('S-Right')
+        h.press('Tab')
+        h.await_text('\n    line_0\n')
+        h.await_cursor_position(x=6, y=1)
+        h.press('^K')
+        h.await_text('\n    lne_0\n')
+
+
+def test_indent_selection_leaves_blank_lines(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('1\n\n2\n\n3\n')
+    with run(str(f)) as h, and_exit(h):
+        for _ in range(3):
+            h.press('S-Down')
+        h.press('Tab')
+        h.press('^S')
+    assert f.read() == '    1\n\n    2\n\n3\n'
+
+
+def test_dedent_no_indentation(run):
+    with run() as h, and_exit(h):
+        h.press('a')
+        h.press('BTab')
+        h.await_text('\na\n')
+        h.await_cursor_position(x=1, y=1)
+
+
+def test_dedent_exactly_one_indent(run):
+    with run() as h, and_exit(h):
+        h.press('Tab')
+        h.press('a')
+        h.await_text('\n    a\n')
+        h.press('BTab')
+        h.await_text('\na\n')
+        h.await_cursor_position(x=1, y=1)
+
+
+def test_dedent_selection(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('1\n  2\n        3\n')
+    with run(str(f)) as h, and_exit(h):
+        for _ in range(3):
+            h.press('S-Down')
+        h.press('BTab')
+        h.await_text('\n1\n2\n    3\n')
+
+
+def test_dedent_beginning_of_line(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('    hi\n')
+    with run(str(f)) as h, and_exit(h):
+        h.press('BTab')
+        h.await_text('\nhi\n')
+
+
+def test_dedent_selection_does_not_make_selection_negative(run):
+    with run() as h, and_exit(h):
+        h.press('Tab')
+        h.press('hello')
+        h.press('Home')
+        h.press('Right')
+        h.press('S-Right')
+        h.press('BTab')
+        h.await_text('\nhello\n')
+        h.press('S-Right')
+        h.press('^K')
+        h.await_text('\nello\n')

tests/features/key_debug_test.py

@@ -0,0 +1,22 @@
+import curses
+
+from babi.screen import VERSION_STR
+
+
+def test_key_debug(run):
+    with run('--key-debug') as h:
+        h.await_text(VERSION_STR, timeout=2)
+
+        h.await_text('press q to quit')
+
+        h.press('a')
+        h.await_text("'a' 'STRING'")
+
+        h.press('^X')
+        h.await_text(r"'\x18' '^X'")
+
+        with h.resize(width=20, height=20):
+            h.await_text(f"{curses.KEY_RESIZE} 'KEY_RESIZE'")
+
+        h.press('q')
+        h.await_exit()

tests/features/movement_test.py

@@ -0,0 +1,440 @@
+import pytest
+
+from testing.runner import and_exit
+
+
+def test_arrow_key_movement(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write(
+        'short\n'
+        '\n'
+        'long long long long\n',
+    )
+    with run(str(f)) as h, and_exit(h):
+        h.await_text('short')
+        h.await_cursor_position(x=0, y=1)
+        # should not go off the beginning of the file
+        h.press('Left')
+        h.await_cursor_position(x=0, y=1)
+        h.press('Up')
+        h.await_cursor_position(x=0, y=1)
+        # left and right should work
+        h.press('Right')
+        h.press('Right')
+        h.await_cursor_position(x=2, y=1)
+        h.press('Left')
+        h.await_cursor_position(x=1, y=1)
+        # up should still be a noop on line 1
+        h.press('Up')
+        h.await_cursor_position(x=1, y=1)
+        # down once should put it on the beginning of the second line
+        h.press('Down')
+        h.await_cursor_position(x=0, y=2)
+        # down again should restore the x positon on the next line
+        h.press('Down')
+        h.await_cursor_position(x=1, y=3)
+        # down once more should put it on the special end-of-file line
+        h.press('Down')
+        h.await_cursor_position(x=0, y=4)
+        # should not go off the end of the file
+        h.press('Down')
+        h.await_cursor_position(x=0, y=4)
+        h.press('Right')
+        h.await_cursor_position(x=0, y=4)
+        # left should put it at the end of the line
+        h.press('Left')
+        h.await_cursor_position(x=19, y=3)
+        # right should put it to the next line
+        h.press('Right')
+        h.await_cursor_position(x=0, y=4)
+        # if the hint-x is too high it should not go past the end of line
+        h.press('Left')
+        h.press('Up')
+        h.press('Up')
+        h.await_cursor_position(x=5, y=1)
+        # and moving back down should still retain the hint-x
+        h.press('Down')
+        h.press('Down')
+        h.await_cursor_position(x=19, y=3)
+
+
+@pytest.mark.parametrize(
+    ('page_up', 'page_down'),
+    (('PageUp', 'PageDown'), ('^Y', '^V')),
+)
+def test_page_up_and_page_down(run, ten_lines, page_up, page_down):
+    with run(str(ten_lines), height=10) as h, and_exit(h):
+        h.press('Down')
+        h.press('Down')
+        h.press(page_up)
+        h.await_cursor_position(x=0, y=1)
+
+        h.press(page_down)
+        h.await_text('line_8')
+        h.await_cursor_position(x=0, y=1)
+        h.assert_cursor_line_equals('line_6')
+
+        h.press(page_up)
+        h.await_text_missing('line_8')
+        h.await_cursor_position(x=0, y=1)
+        h.assert_cursor_line_equals('line_0')
+
+        h.press(page_down)
+        h.press(page_down)
+        h.await_cursor_position(x=0, y=5)
+        h.assert_cursor_line_equals('')
+        h.press('Up')
+        h.await_cursor_position(x=0, y=4)
+        h.assert_cursor_line_equals('line_9')
+
+
+def test_page_up_and_page_down_x_0(run, ten_lines):
+    with run(str(ten_lines), height=10) as h, and_exit(h):
+        h.press('Right')
+        h.press('PageDown')
+        h.await_cursor_position(x=0, y=1)
+        h.assert_cursor_line_equals('line_6')
+
+        h.press('Right')
+        h.press('PageUp')
+        h.await_cursor_position(x=0, y=1)
+        h.assert_cursor_line_equals('line_0')
+
+
+def test_page_up_page_down_size_small_window(run, ten_lines):
+    with run(str(ten_lines), height=4) as h, and_exit(h):
+        h.press('PageDown')
+        h.await_text('line_2')
+        h.await_cursor_position(x=0, y=1)
+        h.assert_cursor_line_equals('line_1')
+
+        h.press('Down')
+        h.press('PageUp')
+        h.await_text_missing('line_2')
+        h.await_cursor_position(x=0, y=1)
+        h.assert_cursor_line_equals('line_0')
+
+
+def test_ctrl_home(run, ten_lines):
+    with run(str(ten_lines), height=4) as h, and_exit(h):
+        for _ in range(3):
+            h.press('PageDown')
+        h.await_text_missing('line_0')
+
+        h.press('^Home')
+        h.await_text('line_0')
+        h.await_cursor_position(x=0, y=1)
+
+
+def test_ctrl_end(run, ten_lines):
+    with run(str(ten_lines), height=6) as h, and_exit(h):
+        h.press('^End')
+        h.await_cursor_position(x=0, y=3)
+        h.assert_screen_line_equals(2, 'line_9')
+
+
+def test_ctrl_end_already_on_last_page(run, ten_lines):
+    with run(str(ten_lines), height=9) as h, and_exit(h):
+        h.press('PageDown')
+        h.await_cursor_position(x=0, y=1)
+        h.await_text('line_9')
+
+        h.press('^End')
+        h.await_cursor_position(x=0, y=6)
+        h.assert_screen_line_equals(5, 'line_9')
+
+
+def test_scrolling_arrow_key_movement(run, ten_lines):
+    with run(str(ten_lines), height=10) as h, and_exit(h):
+        h.await_text('line_7')
+        # we should not have scrolled after 7 presses
+        for _ in range(7):
+            h.press('Down')
+        h.await_text('line_0')
+        h.await_cursor_position(x=0, y=8)
+        # but this should scroll down
+        h.press('Down')
+        h.await_text('line_8')
+        h.await_cursor_position(x=0, y=4)
+        h.assert_cursor_line_equals('line_8')
+        # we should not have scrolled after 3 up presses
+        for _ in range(3):
+            h.press('Up')
+        h.await_text('line_9')
+        # but this should scroll up
+        h.press('Up')
+        h.await_text('line_0')
+
+
+def test_ctrl_down_beginning_of_file(run, ten_lines):
+    with run(str(ten_lines), height=5) as h, and_exit(h):
+        h.press('^Down')
+        h.await_text('line_3')
+        h.await_cursor_position(x=0, y=1)
+        h.assert_cursor_line_equals('line_1')
+
+
+def test_ctrl_up_moves_screen_up_one_line(run, ten_lines):
+    with run(str(ten_lines), height=5) as h, and_exit(h):
+        h.press('^Down')
+        h.press('^Up')
+        h.await_text('line_0')
+        h.await_text('line_2')
+        h.await_cursor_position(x=0, y=2)
+
+
+def test_ctrl_up_at_beginning_of_file_does_nothing(run, ten_lines):
+    with run(str(ten_lines), height=5) as h, and_exit(h):
+        h.press('^Up')
+        h.await_text('line_0')
+        h.await_text('line_2')
+        h.await_cursor_position(x=0, y=1)
+
+
+def test_ctrl_up_at_bottom_of_screen(run, ten_lines):
+    with run(str(ten_lines), height=5) as h, and_exit(h):
+        h.press('^Down')
+        h.press('Down')
+        h.press('Down')
+        h.await_text('line_1')
+        h.await_text('line_3')
+        h.await_cursor_position(x=0, y=3)
+        h.press('^Up')
+        h.await_text('line_0')
+        h.await_cursor_position(x=0, y=3)
+
+
+def test_ctrl_down_at_end_of_file(run, ten_lines):
+    with run(str(ten_lines), height=5) as h, and_exit(h):
+        h.press('^End')
+        for i in range(4):
+            h.press('^Down')
+        h.press('Up')
+        h.await_text('line_9')
+        h.assert_cursor_line_equals('line_9')
+
+
+def test_ctrl_down_causing_cursor_movement_should_fix_x(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('line_1\n\nline_2\n\nline_3\n\nline_4\n')
+
+    with run(str(f), height=5) as h, and_exit(h):
+        h.press('Right')
+        h.press('^Down')
+        h.await_text_missing('\nline_1\n')
+        h.await_cursor_position(x=0, y=1)
+
+
+def test_ctrl_up_causing_cursor_movement_should_fix_x(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('line_1\n\nline_2\n\nline_3\n\nline_4\n')
+
+    with run(str(f), height=5) as h, and_exit(h):
+        h.press('^Down')
+        h.press('^Down')
+        h.press('Down')
+        h.press('Down')
+        h.press('Right')
+        h.await_text('line_3')
+        h.press('^Up')
+        h.await_text_missing('line_3')
+        h.await_cursor_position(x=0, y=3)
+
+
+@pytest.mark.parametrize('k', ('End', '^E'))
+def test_end_key(run, tmpdir, k):
+    f = tmpdir.join('f')
+    f.write('hello world\nhello world\n')
+
+    with run(str(f)) as h, and_exit(h):
+        h.await_text('hello world')
+        h.await_cursor_position(x=0, y=1)
+        h.press(k)
+        h.await_cursor_position(x=11, y=1)
+        h.press('Down')
+        h.await_cursor_position(x=11, y=2)
+
+
+@pytest.mark.parametrize('k', ('Home', '^A'))
+def test_home_key(run, tmpdir, k):
+    f = tmpdir.join('f')
+    f.write('hello world\nhello world\n')
+
+    with run(str(f)) as h, and_exit(h):
+        h.await_text('hello world')
+        h.press('Down')
+        h.press('Left')
+        h.await_cursor_position(x=11, y=1)
+        h.press(k)
+        h.await_cursor_position(x=0, y=1)
+        h.press('Down')
+        h.await_cursor_position(x=0, y=2)
+
+
+def test_page_up_does_not_go_negative(run, ten_lines):
+    with run(str(ten_lines), height=10) as h, and_exit(h):
+        for _ in range(8):
+            h.press('Down')
+        h.await_cursor_position(x=0, y=4)
+        h.press('^Y')
+        h.await_cursor_position(x=0, y=1)
+        h.assert_cursor_line_equals('line_0')
+
+
+@pytest.fixture
+def jump_word_file(tmpdir):
+    f = tmpdir.join('f')
+    contents = '''\
+hello world
+
+hi
+
+    this(is_some_code)  # comment
+'''
+    f.write(contents)
+    yield f
+
+
+def test_ctrl_right_jump_by_word(run, jump_word_file):
+    with run(str(jump_word_file)) as h, and_exit(h):
+        h.press('^Right')
+        h.await_cursor_position(x=5, y=1)
+        h.press('^Right')
+        h.await_cursor_position(x=11, y=1)
+        h.press('Left')
+        h.await_cursor_position(x=10, y=1)
+        h.press('^Right')
+        h.await_cursor_position(x=11, y=1)
+        h.press('^Right')
+        h.await_cursor_position(x=0, y=3)
+        h.press('^Right')
+        h.await_cursor_position(x=2, y=3)
+        h.press('^Right')
+        h.await_cursor_position(x=4, y=5)
+        h.press('^Right')
+        h.await_cursor_position(x=8, y=5)
+        h.press('^Right')
+        h.await_cursor_position(x=11, y=5)
+        h.press('Down')
+        h.press('^Right')
+        h.await_cursor_position(x=0, y=6)
+
+
+def test_ctrl_left_jump_by_word(run, jump_word_file):
+    with run(str(jump_word_file)) as h, and_exit(h):
+        h.press('^Left')
+        h.await_cursor_position(x=0, y=1)
+        h.press('Right')
+        h.await_cursor_position(x=1, y=1)
+        h.press('^Left')
+        h.await_cursor_position(x=0, y=1)
+        h.press('PageDown')
+        h.await_cursor_position(x=0, y=6)
+        h.press('^Left')
+        h.await_cursor_position(x=33, y=5)
+        h.press('^Left')
+        h.await_cursor_position(x=26, y=5)
+        h.press('Home')
+        h.press('Right')
+        h.await_cursor_position(x=1, y=5)
+        h.press('^Left')
+        h.await_cursor_position(x=2, y=3)
+
+
+def test_ctrl_right_triggering_scroll(run, jump_word_file):
+    with run(str(jump_word_file), height=4) as h, and_exit(h):
+        h.press('Down')
+        h.await_cursor_position(x=0, y=2)
+        h.press('^Right')
+        h.await_cursor_position(x=0, y=1)
+        h.assert_cursor_line_equals('hi')
+
+
+def test_ctrl_left_triggering_scroll(run, jump_word_file):
+    with run(str(jump_word_file)) as h, and_exit(h):
+        h.press('Down')
+        h.await_cursor_position(x=0, y=2)
+        h.press('^Down')
+        h.await_cursor_position(x=0, y=1)
+        h.press('^Left')
+        h.await_cursor_position(x=11, y=1)
+        h.assert_cursor_line_equals('hello world')
+
+
+def test_sequence_handling(run_only_fake):
+    # this test is run with the fake runner since it simulates some situations
+    # that are either impossible or due to race conditions (that we can only
+    # force with the fake runner)
+    with run_only_fake() as h, and_exit(h):
+        h.press_sequence('\x1b[1;5C\x1b[1;5D test1')  # ^Left + ^Right
+        h.await_text('test1')
+        h.await_text_missing('unknown key')
+
+        h.press_sequence('\x1bOH', '\x1bOF', ' test2')  # Home + End
+        h.await_text('test1 test2')
+        h.await_text_missing('unknown key')
+
+        h.press_sequence(' tq', 'M-O', 'BSpace', 'est3')
+        h.await_text('test1 test2 test3')
+        h.await_text('unknown key')
+        h.await_text('M-O')
+
+        h.press('M-[')
+        h.await_text_missing('M-O')
+        h.await_text('M-[')
+
+        h.press('M-O')
+        h.await_text_missing('M-[')
+        h.await_text('M-O')
+
+        h.press_sequence(' tq', 'M-[', 'BSpace', 'est4')
+        h.await_text('test1 test2 test3 test4')
+        h.await_text_missing('M-O')
+        h.await_text('M-[')
+
+        # TODO: this is broken for now, not quite sure what to do with it
+        h.press_sequence('\x1b', 'BSpace')
+        h.await_text(r'\x1b(263)')
+
+        # the sequences after here are "wrong" but I don't think a human
+        # could type them
+
+        h.press_sequence(' tq', '\x1b[1;', 'BSpace', 'est5')
+        h.await_text('test1 test2 test3 test4 test5')
+        h.await_text(r'\x1b[1;')
+
+        h.press_sequence('\x1b[111', ' test6')
+        h.await_text('test1 test2 test3 test4 test5 test6')
+        h.await_text(r'\x1b[111')
+
+        h.press('\x1b[1;')
+        h.press(' test7')
+        h.await_text('test1 test2 test3 test4 test5 test6 test7')
+        h.await_text(r'\x1b[1;')
+
+
+def test_indentation_using_tabs(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write(f'123456789\n\t12\t{"x" * 20}\n')
+
+    with run(str(f), width=20) as h, and_exit(h):
+        h.await_text('123456789\n    12  xxxxxxxxxxx»\n')
+
+        h.press('Down')
+        h.await_cursor_position(x=0, y=2)
+        h.press('Up')
+        h.await_cursor_position(x=0, y=1)
+
+        h.press('Right')
+        h.await_cursor_position(x=1, y=1)
+        h.press('Down')
+        h.await_cursor_position(x=0, y=2)
+        h.press('Up')
+        h.await_cursor_position(x=1, y=1)
+
+        h.press('Down')
+        h.await_cursor_position(x=0, y=2)
+        h.press('Right')
+        h.await_cursor_position(x=4, y=2)
+        h.press('Up')
+        h.await_cursor_position(x=4, y=1)

tests/features/multiple_files_test.py

@@ -0,0 +1,88 @@
+import pytest
+
+
+@pytest.fixture
+def abc(tmpdir):
+    a = tmpdir.join('file_a')
+    a.write('a text')
+    b = tmpdir.join('file_b')
+    b.write('b text')
+    c = tmpdir.join('file_c')
+    c.write('c text')
+    yield a, b, c
+
+
+def test_multiple_files(run, abc):
+    a, b, c = abc
+
+    with run(str(a), str(b), str(c)) as h:
+        h.await_text('file_a')
+        h.await_text('[1/3]')
+        h.await_text('a text')
+        h.press('Right')
+        h.await_cursor_position(x=1, y=1)
+
+        h.press('M-Right')
+        h.await_text('file_b')
+        h.await_text('[2/3]')
+        h.await_text('b text')
+        h.await_cursor_position(x=0, y=1)
+
+        h.press('M-Left')
+        h.await_text('file_a')
+        h.await_text('[1/3]')
+        h.await_cursor_position(x=1, y=1)
+
+        # wrap around
+        h.press('M-Left')
+        h.await_text('file_c')
+        h.await_text('[3/3]')
+        h.await_text('c text')
+
+        # make sure to clear statuses when switching files
+        h.press('^J')
+        h.await_text('unknown key')
+        h.press('M-Right')
+        h.await_text_missing('unknown key')
+        h.press('^J')
+        h.await_text('unknown key')
+        h.press('M-Left')
+        h.await_text_missing('unknown key')
+
+        # also make sure to clear statuses when exiting files
+        h.press('^J')
+        h.await_text('unknown key')
+        h.press('^X')
+        h.await_text('file_b')
+        h.await_text_missing('unknown key')
+        h.press('^X')
+        h.await_text('file_a')
+        h.press('^X')
+        h.await_exit()
+
+
+def test_multiple_files_close_from_beginning(run, abc):
+    a, b, c = abc
+
+    with run(str(a), str(b), str(c)) as h:
+        h.press('^X')
+        h.await_text('file_b')
+        h.press('^X')
+        h.await_text('file_c')
+        h.press('^X')
+        h.await_exit()
+
+
+def test_multiple_files_close_from_end(run, abc):
+    a, b, c = abc
+
+    with run(str(a), str(b), str(c)) as h:
+        h.press('M-Right')
+        h.await_text('file_b')
+
+        h.press('^X')
+        h.await_text('file_c')
+        h.press('^X')
+        h.await_text('file_a')
+        h.press('^X')
+        h.await_exit()

tests/features/open_test.py

@@ -0,0 +1,38 @@
+from testing.runner import and_exit
+
+
+def test_open_cancelled(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('hello world')
+
+    with run(str(f)) as h, and_exit(h):
+        h.await_text('hello world')
+
+        h.press('^P')
+        h.await_text('enter filename:')
+        h.press('^C')
+
+        h.await_text('cancelled')
+        h.await_text('hello world')
+
+
+def test_open(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('hello world')
+    g = tmpdir.join('g')
+    g.write('goodbye world')
+
+    with run(str(f)) as h:
+        h.await_text('hello world')
+
+        h.press('^P')
+        h.press_and_enter(str(g))
+
+        h.await_text('[2/2]')
+        h.await_text('goodbye world')
+
+        h.press('^X')
+        h.await_text('hello world')
+
+        h.press('^X')
+        h.await_exit()

tests/features/perf_log_test.py

@@ -0,0 +1,13 @@
+from testing.runner import and_exit
+
+
+def test(run, tmpdir, ten_lines):
+    f = tmpdir.join('f.log')
+    with run(str(ten_lines), '--perf-log', str(f)) as h, and_exit(h):
+        h.press('Right')
+        h.press('Down')
+    lines = f.read().splitlines()
+    assert lines[0] == 'μs\tevent'
+    expected = ['startup', 'KEY_RIGHT', 'KEY_DOWN', '^X']
+    assert [line.split()[-1] for line in lines[1:]] == expected
+    assert tmpdir.join('f.log.pstats').exists()

tests/features/replace_test.py

@@ -0,0 +1,302 @@
+import pytest
+
+from testing.runner import and_exit
+
+
+@pytest.mark.parametrize('key', ('^C', 'Enter'))
+def test_replace_cancel(run, key):
+    with run() as h, and_exit(h):
+        h.press('^\\')
+        h.await_text('search (to replace):')
+        h.press(key)
+        h.await_text('cancelled')
+
+
+def test_replace_invalid_regex(run):
+    with run() as h, and_exit(h):
+        h.press('^\\')
+        h.await_text('search (to replace):')
+        h.press_and_enter('(')
+        h.await_text("invalid regex: '('")
+
+
+def test_replace_cancel_at_replace_string(run):
+    with run() as h, and_exit(h):
+        h.press('^\\')
+        h.await_text('search (to replace):')
+        h.press_and_enter('hello')
+        h.await_text('replace with:')
+        h.press('^C')
+        h.await_text('cancelled')
+
+
+def test_replace_actual_contents(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('^\\')
+        h.await_text('search (to replace):')
+        h.press_and_enter('line_0')
+        h.await_text('replace with:')
+        h.press_and_enter('ohai')
+        h.await_text('replace [yes, no, all]?')
+        h.press('y')
+        h.await_text_missing('line_0')
+        h.await_text('ohai')
+        h.await_text(' *')
+        h.await_text('replaced 1 occurrence')
+
+
+def test_replace_sets_x_hint_properly(run, tmpdir):
+    f = tmpdir.join('f')
+    contents = '''\
+beginning_line
+
+match me!
+'''
+    f.write(contents)
+    with run(str(f)) as h, and_exit(h):
+        h.press('^\\')
+        h.await_text('search (to replace):')
+        h.press_and_enter('me!')
+        h.await_text('replace with:')
+        h.press_and_enter('youuuu')
+        h.await_text('replace [yes, no, all]?')
+        h.press('y')
+        h.await_cursor_position(x=6, y=3)
+        h.press('Up')
+        h.press('Up')
+        h.await_cursor_position(x=6, y=1)
+
+
+def test_replace_cancel_at_individual_replace(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('^\\')
+        h.await_text('search (to replace):')
+        h.press_and_enter(r'line_\d')
+        h.await_text('replace with:')
+        h.press_and_enter('ohai')
+        h.await_text('replace [yes, no, all]?')
+        h.press('^C')
+        h.await_text('cancelled')
+
+
+def test_replace_unknown_characters_at_individual_replace(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('^\\')
+        h.await_text('search (to replace):')
+        h.press_and_enter(r'line_\d')
+        h.await_text('replace with:')
+        h.press_and_enter('ohai')
+        h.await_text('replace [yes, no, all]?')
+        h.press('?')
+        h.press('^C')
+        h.await_text('cancelled')
+
+
+def test_replace_say_no_to_individual_replace(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('^\\')
+        h.await_text('search (to replace):')
+        h.press_and_enter('line_[135]')
+        h.await_text('replace with:')
+        h.press_and_enter('ohai')
+        h.await_text('replace [yes, no, all]?')
+        h.press('y')
+        h.await_text_missing('line_1')
+        h.press('n')
+        h.await_text('line_3')
+        h.press('y')
+        h.await_text_missing('line_5')
+        h.await_text('replaced 2 occurrences')
+
+
+def test_replace_all(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('^\\')
+        h.await_text('search (to replace):')
+        h.press_and_enter(r'line_(\d)')
+        h.await_text('replace with:')
+        h.press_and_enter(r'ohai+\1')
+        h.await_text('replace [yes, no, all]?')
+        h.press('a')
+        h.await_text_missing('line')
+        h.await_text('ohai+1')
+        h.await_text('replaced 10 occurrences')
+
+
+def test_replace_with_empty_string(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('^\\')
+        h.await_text('search (to replace):')
+        h.press_and_enter('line_1')
+        h.await_text('replace with:')
+        h.press('Enter')
+        h.await_text('replace [yes, no, all]?')
+        h.press('y')
+        h.await_text_missing('line_1')
+
+
+def test_replace_search_not_found(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('^\\')
+        h.await_text('search (to replace):')
+        h.press_and_enter('wat')
+        # TODO: would be nice to not prompt for a replace string in this case
+        h.await_text('replace with:')
+        h.press('Enter')
+        h.await_text('no matches')
+
+
+def test_replace_small_window_size(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('^\\')
+        h.await_text('search (to replace):')
+        h.press_and_enter('line')
+        h.await_text('replace with:')
+        h.press_and_enter('wat')
+        h.await_text('replace [yes, no, all]?')
+
+        with h.resize(width=8, height=24):
+            h.await_text('replace…')
+
+        h.press('^C')
+
+
+def test_replace_height_1_highlight(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('x' * 90)
+    with run(str(f)) as h, and_exit(h):
+        h.press('^\\')
+        h.await_text('search (to replace):')
+        h.press_and_enter('^x+$')
+        h.await_text('replace with:')
+        h.press('Enter')
+        h.await_text('replace [yes, no, all]?')
+
+        with h.resize(width=80, height=1):
+            h.await_text_missing('xxxxx')
+        h.await_text('xxxxx')
+
+        h.press('^C')
+
+
+def test_replace_line_goes_off_screen(run):
+    with run() as h, and_exit(h):
+        h.press(f'{"a" * 20}{"b" * 90}')
+        h.press('^A')
+        h.await_text(f'{"a" * 20}{"b" * 59}»')
+        h.press('^\\')
+        h.await_text('search (to replace):')
+        h.press_and_enter('b+')
+        h.await_text('replace with:')
+        h.press_and_enter('wat')
+        h.await_text('replace [yes, no, all]?')
+        h.await_text(f'{"a" * 20}{"b" * 59}»')
+        h.press('y')
+        h.await_text(f'{"a" * 20}wat')
+        h.await_text('replaced 1 occurrence')
+
+
+def test_replace_undo_undoes_only_one(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('^\\')
+        h.await_text('search (to replace):')
+        h.press_and_enter('line')
+        h.await_text('replace with:')
+        h.press_and_enter('wat')
+        h.press('y')
+        h.await_text_missing('line_0')
+        h.press('y')
+        h.await_text_missing('line_1')
+        h.press('^C')
+        h.press('M-u')
+        h.await_text('line_1')
+        h.await_text_missing('line_0')
+
+
+def test_replace_multiple_occurrences_in_line(run):
+    with run() as h, and_exit(h):
+        h.press('baaaaabaaaaa')
+        h.press('^\\')
+        h.await_text('search (to replace):')
+        h.press_and_enter('a+')
+        h.await_text('replace with:')
+        h.press_and_enter('q')
+        h.await_text('replace [yes, no, all]?')
+        h.press('a')
+        h.await_text('bqbq')
+
+
+def test_replace_after_wrapping(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('Down')
+        h.press('^\\')
+        h.await_text('search (to replace):')
+        h.press_and_enter('line_[02]')
+        h.await_text('replace with:')
+        h.press_and_enter('ohai')
+        h.await_text('replace [yes, no, all]?')
+        h.press('y')
+        h.await_text_missing('line_2')
+        h.press('y')
+        h.await_text_missing('line_0')
+        h.await_text('replaced 2 occurrences')
+
+
+def test_replace_after_cursor_after_wrapping(run):
+    with run() as h, and_exit(h):
+        h.press('baaab')
+        h.press('Left')
+        h.press('^\\')
+        h.await_text('search (to replace):')
+        h.press_and_enter('b')
+        h.await_text('replace with:')
+        h.press_and_enter('q')
+        h.await_text('replace [yes, no, all]?')
+        h.press('n')
+        h.press('y')
+        h.await_text('replaced 1 occurrence')
+        h.await_text('qaaab')
+
+
+def test_replace_separate_line_after_wrapping(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('Down')
+        h.press('Down')
+        h.press('^\\')
+        h.await_text('search (to replace):')
+        h.press_and_enter('line_[01]')
+        h.await_text('replace with:')
+        h.press_and_enter('_')
+        h.await_text('replace [yes, no, all]?')
+        h.press('y')
+        h.await_text_missing('line_0')
+        h.press('y')
+        h.await_text_missing('line_1')
+
+
+def test_replace_with_newline_characters(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('^\\')
+        h.await_text('search (to replace):')
+        h.press_and_enter('(line)_([01])')
+        h.await_text('replace with:')
+        h.press_and_enter(r'\1\n\2')
+        h.await_text('replace [yes, no, all]?')
+        h.press('a')
+        h.await_text_missing('line_0')
+        h.await_text_missing('line_1')
+        h.await_text('line\n0\nline\n1\n')
+
+
+def test_replace_with_multiple_newline_characters(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('^\\')
+        h.await_text('search (to replace):')
+        h.press_and_enter('(li)(ne)_(1)')
+        h.await_text('replace with:')
+        h.press_and_enter(r'\1\n\2\n\3\n')
+        h.await_text('replace [yes, no, all]?')
+        h.press('a')
+
+        h.await_text_missing('line_1')
+        h.await_text('li\nne\n1\n\nline_2')

tests/features/resize_test.py

@@ -0,0 +1,170 @@
+from babi.screen import VERSION_STR
+from testing.runner import and_exit
+
+
+def test_window_height_2(run, tmpdir):
+    # 2 tall:
+    # - header is hidden, otherwise behaviour is normal
+    f = tmpdir.join('f.txt')
+    f.write('hello world')
+
+    with run(str(f)) as h, and_exit(h):
+        h.await_text('hello world')
+
+        with h.resize(width=80, height=2):
+            h.await_text_missing(VERSION_STR)
+            h.assert_full_contents('hello world\n\n')
+            h.press('^J')
+            h.await_text('unknown key')
+
+        h.await_text(VERSION_STR)
+
+
+def test_window_height_1(run, tmpdir):
+    # 1 tall:
+    # - only file contents as body
+    # - status takes precedence over body, but cleared after single action
+    f = tmpdir.join('f.txt')
+    f.write('hello world')
+
+    with run(str(f)) as h, and_exit(h):
+        h.await_text('hello world')
+
+        with h.resize(width=80, height=1):
+            h.await_text_missing(VERSION_STR)
+            h.assert_full_contents('hello world\n')
+            h.press('^J')
+            h.await_text('unknown key')
+            h.press('Right')
+            h.await_text_missing('unknown key')
+            h.press('Down')
+
+
+def test_reacts_to_resize(run):
+    with run() as h, and_exit(h):
+        h.await_text('<<new file>>')
+        with h.resize(width=10, height=20):
+            h.await_text_missing('<<new file>>')
+        h.await_text('<<new file>>')
+
+
+def test_resize_scrolls_up(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.await_text('line_9')
+
+        for _ in range(7):
+            h.press('Down')
+        h.await_cursor_position(x=0, y=8)
+
+        # a resize to a height of 10 should not scroll
+        with h.resize(width=80, height=10):
+            h.await_text_missing('line_8')
+            h.await_cursor_position(x=0, y=8)
+
+        h.await_text('line_8')
+
+        # but a resize to smaller should
+        with h.resize(width=80, height=9):
+            h.await_text_missing('line_0')
+            h.await_cursor_position(x=0, y=4)
+            # make sure we're still on the same line
+            h.assert_cursor_line_equals('line_7')
+
+
+def test_resize_scroll_does_not_go_negative(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        for _ in range(5):
+            h.press('Down')
+        h.await_cursor_position(x=0, y=6)
+
+        with h.resize(width=80, height=7):
+            h.await_text_missing('line_9')
+        h.await_text('line_9')
+
+        for _ in range(3):
+            h.press('Up')
+
+        h.assert_screen_line_equals(1, 'line_0')
+
+
+def test_horizontal_scrolling(run, tmpdir):
+    f = tmpdir.join('f')
+    lots_of_text = ''.join(
+        ''.join(str(i) * 10 for i in range(10))
+        for _ in range(10)
+    )
+    f.write(f'line1\n{lots_of_text}\n')
+
+    with run(str(f)) as h, and_exit(h):
+        h.await_text('6777777777»')
+        h.press('Down')
+        for _ in range(78):
+            h.press('Right')
+        h.await_text('6777777777»')
+        h.press('Right')
+        h.await_text('«77777778')
+        h.await_text('344444444445»')
+        h.await_cursor_position(x=7, y=2)
+        for _ in range(71):
+            h.press('Right')
+        h.await_text('«77777778')
+        h.await_text('344444444445»')
+        h.press('Right')
+        h.await_text('«444445')
+        h.await_text('1222»')
+
+
+def test_horizontal_scrolling_exact_width(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('0' * 80)
+
+    with run(str(f)) as h, and_exit(h):
+        h.await_text('000')
+        for _ in range(78):
+            h.press('Right')
+        h.await_text_missing('»')
+        h.await_cursor_position(x=78, y=1)
+        h.press('Right')
+        h.await_text('«0000000')
+        h.await_cursor_position(x=7, y=1)
+
+
+def test_horizontal_scrolling_narrow_window(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write(''.join(str(i) * 10 for i in range(10)))
+
+    with run(str(f)) as h, and_exit(h):
+        with h.resize(width=5, height=24):
+            h.await_text('0000»')
+            for _ in range(3):
+                h.press('Right')
+            h.await_text('0000»')
+            h.press('Right')
+            h.await_cursor_position(x=3, y=1)
+            h.await_text('«000»')
+            for _ in range(6):
+                h.press('Right')
+            h.await_text('«001»')
+
+
+def test_window_width_1(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('hello')
+
+    with run(str(f)) as h, and_exit(h):
+        with h.resize(width=1, height=24):
+            h.await_text('»')
+            for _ in range(3):
+                h.press('Right')
+        h.await_text('hello')
+        h.await_cursor_position(x=3, y=1)
+
+
+def test_resize_while_cursor_at_bottom(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('x\n' * 35)
+    with run(str(f), height=40) as h, and_exit(h):
+        h.press('^End')
+        h.await_cursor_position(x=0, y=36)
+        with h.resize(width=80, height=5):
+            h.await_cursor_position(x=0, y=2)

tests/features/save_test.py

@@ -0,0 +1,252 @@
+import pytest
+
+from testing.runner import and_exit
+from testing.runner import trigger_command_mode
+
+
+def test_mixed_newlines(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write_binary(b'foo\nbar\r\n')
+    with run(str(f)) as h, and_exit(h):
+        # should start as modified
+        h.await_text('f *')
+        h.await_text(r"mixed newlines will be converted to '\n'")
+
+
+def test_modify_file_with_windows_newlines(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write_binary(b'foo\r\nbar\r\n')
+    with run(str(f)) as h, and_exit(h):
+        # should not start modified
+        h.await_text_missing('*')
+        h.press('Enter')
+        h.await_text('*')
+        h.press('^S')
+        h.await_text('saved!')
+    assert f.read_binary() == b'\r\nfoo\r\nbar\r\n'
+
+
+def test_saving_file_with_multiple_lines_at_end_maintains_those(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('foo\n\n')
+    with run(str(f)) as h, and_exit(h):
+        h.press('a')
+        h.await_text('*')
+        h.press('^S')
+        h.await_text('saved!')
+
+    assert f.read() == 'afoo\n\n'
+
+
+def test_new_file(run):
+    with run('this_is_a_new_file') as h, and_exit(h):
+        h.await_text('this_is_a_new_file')
+        h.await_text('(new file)')
+
+
+def test_not_a_file(run, tmpdir):
+    d = tmpdir.join('d').ensure_dir()
+    with run(str(d)) as h, and_exit(h):
+        h.await_text('<<new file>>')
+        h.await_text("d' is not a file")
+
+
+def test_save_no_filename_specified(run, tmpdir):
+    f = tmpdir.join('f')
+
+    with run() as h, and_exit(h):
+        h.press('hello world')
+        h.press('^S')
+        h.await_text('enter filename:')
+        h.press_and_enter(str(f))
+        h.await_text('saved! (1 line written)')
+        h.await_text_missing('*')
+    assert f.read() == 'hello world\n'
+
+
+@pytest.mark.parametrize('k', ('Enter', '^C'))
+def test_save_no_filename_specified_cancel(run, k):
+    with run() as h, and_exit(h):
+        h.press('hello world')
+        h.press('^S')
+        h.await_text('enter filename:')
+        h.press(k)
+        h.await_text('cancelled')
+
+
+def test_saving_file_on_disk_changes(run, tmpdir):
+    # TODO: this should show some sort of diffing thing or just allow overwrite
+    f = tmpdir.join('f')
+
+    with run(str(f)) as h, and_exit(h):
+        h.run(lambda: f.write('hello world'))
+
+        h.press('^S')
+        h.await_text('file changed on disk, not implemented')
+
+
+def test_allows_saving_same_contents_as_modified_contents(run, tmpdir):
+    f = tmpdir.join('f')
+
+    with run(str(f)) as h, and_exit(h):
+        h.run(lambda: f.write('hello world\n'))
+        h.press('hello world')
+        h.await_text('hello world')
+
+        h.press('^S')
+        h.await_text('saved! (1 line written)')
+        h.await_text_missing('*')
+
+    assert f.read() == 'hello world\n'
+
+
+def test_allows_saving_if_file_on_disk_does_not_change(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('hello world\n')
+
+    with run(str(f)) as h, and_exit(h):
+        h.await_text('hello world')
+        h.press('ohai')
+        h.press('Enter')
+
+        h.press('^S')
+        h.await_text('saved! (2 lines written)')
+        h.await_text_missing('*')
+
+    assert f.read() == 'ohai\nhello world\n'
+
+
+def test_save_file_when_it_did_not_exist(run, tmpdir):
+    f = tmpdir.join('f')
+
+    with run(str(f)) as h, and_exit(h):
+        h.press('hello world')
+        h.press('^S')
+        h.await_text('saved! (1 line written)')
+        h.await_text_missing('*')
+
+    assert f.read() == 'hello world\n'
+
+
+def test_save_via_ctrl_o(run, tmpdir):
+    f = tmpdir.join('f')
+    with run(str(f)) as h, and_exit(h):
+        h.press('hello world')
+        h.press('^O')
+        h.await_text(f'enter filename: ')
+        h.press('Enter')
+        h.await_text('saved! (1 line written)')
+    assert f.read() == 'hello world\n'
+
+
+def test_save_via_ctrl_o_set_filename(run, tmpdir):
+    f = tmpdir.join('f')
+    with run() as h, and_exit(h):
+        h.press('hello world')
+        h.press('^O')
+        h.await_text('enter filename:')
+        h.press_and_enter(str(f))
+        h.await_text('saved! (1 line written)')
+    assert f.read() == 'hello world\n'
+
+
+@pytest.mark.parametrize('key', ('^C', 'Enter'))
+def test_save_via_ctrl_o_cancelled(run, key):
+    with run() as h, and_exit(h):
+        h.press('hello world')
+        h.press('^O')
+        h.await_text('enter filename:')
+        h.press(key)
+        h.await_text('cancelled')
+
+
+def test_save_via_ctrl_o_position(run):
+    with run('filename') as h, and_exit(h):
+        h.press('hello world')
+        h.press('^O')
+        h.await_text('enter filename: filename')
+        h.await_cursor_position(x=24, y=23)
+        h.press('^C')
+
+
+def test_save_on_exit_cancel_yn(run):
+    with run() as h, and_exit(h):
+        h.press('hello')
+        h.await_text('hello')
+        h.press('^X')
+        h.await_text('file is modified - save [yes, no]?')
+        h.press('^C')
+        h.await_text('cancelled')
+
+
+def test_save_on_exit_cancel_filename(run):
+    with run() as h, and_exit(h):
+        h.press('hello')
+        h.await_text('hello')
+        h.press('^X')
+        h.await_text('file is modified - save [yes, no]?')
+        h.press('y')
+        h.await_text('enter filename:')
+        h.press('^C')
+        h.await_text('cancelled')
+
+
+def test_save_on_exit(run, tmpdir):
+    f = tmpdir.join('f')
+    with run(str(f)) as h:
+        h.press('hello')
+        h.await_text('hello')
+        h.press('^X')
+        h.await_text('file is modified - save [yes, no]?')
+        h.press('y')
+        h.await_text(f'enter filename: {f}')
+        h.press('Enter')
+        h.await_exit()
+
+
+def test_save_on_exit_resize(run, tmpdir):
+    with run() as h, and_exit(h):
+        h.press('hello')
+        h.await_text('hello')
+        h.press('^X')
+        h.await_text('file is modified - save [yes, no]?')
+        with h.resize(width=10, height=24):
+            h.await_text('file is m…')
+        h.await_text('file is modified - save [yes, no]?')
+        h.press('^C')
+        h.await_text('cancelled')
+
+
+def test_vim_save_on_exit_cancel_yn(run):
+    with run() as h, and_exit(h):
+        h.press('hello')
+        h.await_text('hello')
+        trigger_command_mode(h)
+        h.press_and_enter(':q')
+        h.await_text('file is modified - save [yes, no]?')
+        h.press('^C')
+        h.await_text('cancelled')
+
+
+def test_vim_save_on_exit(run, tmpdir):
+    f = tmpdir.join('f')
+    with run(str(f)) as h:
+        h.press('hello')
+        h.await_text('hello')
+        trigger_command_mode(h)
+        h.press_and_enter(':q')
+        h.await_text('file is modified - save [yes, no]?')
+        h.press('y')
+        h.await_text(f'enter filename: ')
+        h.press('Enter')
+        h.await_exit()
+
+
+def test_vim_force_exit(run, tmpdir):
+    f = tmpdir.join('f')
+    with run(str(f)) as h:
+        h.press('hello')
+        h.await_text('hello')
+        trigger_command_mode(h)
+        h.press_and_enter(':q!')
+        h.await_exit()

tests/features/search_test.py

@@ -0,0 +1,365 @@
+import pytest
+
+from testing.runner import and_exit
+
+
+def test_search_wraps(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('Down')
+        h.press('Down')
+        h.await_cursor_position(x=0, y=3)
+        h.press('^W')
+        h.await_text('search:')
+        h.press_and_enter('^line_0$')
+        h.await_text('search wrapped')
+        h.await_cursor_position(x=0, y=1)
+
+
+def test_search_find_next_line(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.await_cursor_position(x=0, y=1)
+        h.press('^W')
+        h.await_text('search:')
+        h.press_and_enter('^line_')
+        h.await_cursor_position(x=0, y=2)
+
+
+def test_search_find_later_in_line(run):
+    with run() as h, and_exit(h):
+        h.press_and_enter('lol')
+        h.press('Up')
+        h.press('Right')
+        h.await_cursor_position(x=1, y=1)
+
+        h.press('^W')
+        h.await_text('search:')
+        h.press_and_enter('l')
+        h.await_cursor_position(x=2, y=1)
+
+
+def test_search_only_one_match_already_at_that_match(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('Down')
+        h.await_cursor_position(x=0, y=2)
+        h.press('^W')
+        h.await_text('search:')
+        h.press_and_enter('^line_1$')
+        h.await_text('this is the only occurrence')
+        h.await_cursor_position(x=0, y=2)
+
+
+def test_search_sets_x_hint_properly(run, tmpdir):
+    f = tmpdir.join('f')
+    contents = '''\
+beginning_line
+
+match me!
+'''
+    f.write(contents)
+    with run(str(f)) as h, and_exit(h):
+        h.press('^W')
+        h.await_text('search:')
+        h.press_and_enter('me!')
+        h.await_cursor_position(x=6, y=3)
+        h.press('Up')
+        h.press('Up')
+        h.await_cursor_position(x=6, y=1)
+
+
+def test_search_not_found(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('^W')
+        h.await_text('search:')
+        h.press_and_enter('this will not match')
+        h.await_text('no matches')
+        h.await_cursor_position(x=0, y=1)
+
+
+def test_search_invalid_regex(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('^W')
+        h.await_text('search:')
+        h.press_and_enter('invalid(regex')
+        h.await_text("invalid regex: 'invalid(regex'")
+
+
+@pytest.mark.parametrize('key', ('Enter', '^C'))
+def test_search_cancel(run, ten_lines, key):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('^W')
+        h.await_text('search:')
+        h.press(key)
+        h.await_text('cancelled')
+
+
+def test_search_repeated_search(run, ten_lines):
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('^W')
+        h.press('line')
+        h.await_text('search: line')
+        h.press('Enter')
+        h.await_cursor_position(x=0, y=2)
+
+        h.press('^W')
+        h.await_text('search [line]:')
+        h.press('Enter')
+        h.await_cursor_position(x=0, y=3)
+
+
+def test_search_history_recorded(run):
+    with run() as h, and_exit(h):
+        h.press('^W')
+        h.await_text('search:')
+        h.press_and_enter('asdf')
+        h.await_text('no matches')
+
+        h.press('^W')
+        h.press('Up')
+        h.await_text('search [asdf]: asdf')
+        h.press('BSpace')
+        h.press('test')
+        h.await_text('search [asdf]: asdtest')
+        h.press('Down')
+        h.await_text_missing('asdtest')
+        h.press('Down')  # can't go past the end
+        h.press('Up')
+        h.await_text('asdtest')
+        h.press('Up')  # can't go past the beginning
+        h.await_text('asdtest')
+        h.press('Enter')
+        h.await_text('no matches')
+
+        h.press('^W')
+        h.press('Up')
+        h.await_text('search [asdtest]: asdtest')
+        h.press('Up')
+        h.await_text('search [asdtest]: asdf')
+        h.press('^C')
+
+
+def test_search_history_duplicates_dont_repeat(run):
+    with run() as h, and_exit(h):
+        h.press('^W')
+        h.await_text('search:')
+        h.press_and_enter('search1')
+        h.await_text('no matches')
+
+        h.press('^W')
+        h.await_text('search [search1]:')
+        h.press_and_enter('search2')
+        h.await_text('no matches')
+
+        h.press('^W')
+        h.await_text('search [search2]:')
+        h.press_and_enter('search2')
+        h.await_text('no matches')
+
+        h.press('^W')
+        h.press('Up')
+        h.await_text('search2')
+        h.press('Up')
+        h.await_text('search1')
+        h.press('Enter')
+
+
+def test_search_history_is_saved_between_sessions(run, xdg_data_home):
+    with run() as h, and_exit(h):
+        h.press('^W')
+        h.press_and_enter('search1')
+        h.press('^W')
+        h.press_and_enter('search2')
+
+    contents = xdg_data_home.join('babi/history/search').read()
+    assert contents == 'search1\nsearch2\n'
+
+    with run() as h, and_exit(h):
+        h.press('^W')
+        h.press('Up')
+        h.await_text('search: search2')
+        h.press('Up')
+        h.await_text('search: search1')
+        h.press('Enter')
+
+
+def test_search_multiple_sessions_append_to_history(run, xdg_data_home):
+    xdg_data_home.join('babi/history/search').ensure().write(
+        'orig\n'
+        'history\n',
+    )
+
+    with run() as h1, and_exit(h1):
+        with run() as h2, and_exit(h2):
+            h2.press('^W')
+            h2.press_and_enter('h2 history')
+        h1.press('^W')
+        h1.press_and_enter('h1 history')
+
+    contents = xdg_data_home.join('babi/history/search').read()
+    assert contents == (
+        'orig\n'
+        'history\n'
+        'h2 history\n'
+        'h1 history\n'
+    )
+
+
+def test_search_default_same_as_prev_history(run, xdg_data_home, ten_lines):
+    xdg_data_home.join('babi/history/search').ensure().write('line\n')
+
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('^W')
+        h.press_and_enter('line')
+        h.await_cursor_position(x=0, y=2)
+        h.press('^W')
+        h.await_text('search [line]:')
+        h.press('Enter')
+        h.await_cursor_position(x=0, y=3)
+
+
+@pytest.mark.parametrize('key', ('BSpace', '^H'))
+def test_search_reverse_search_history_backspace(run, xdg_data_home, key):
+    xdg_data_home.join('babi/history/search').ensure().write(
+        'line_5\n'
+        'line_3\n'
+        'line_1\n',
+    )
+    with run() as h, and_exit(h):
+        h.press('^W')
+        h.press('^R')
+        h.await_text('search(reverse-search)``:')
+        h.press('linea')
+        h.await_text('search(failed reverse-search)`linea`: line_1')
+        h.press(key)
+        h.await_text('search(reverse-search)`line`: line_1')
+        h.press('^C')
+
+
+def test_search_reverse_search_history(run, xdg_data_home, ten_lines):
+    xdg_data_home.join('babi/history/search').ensure().write(
+        'line_5\n'
+        'line_3\n'
+        'line_1\n',
+    )
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('^W')
+        h.press('^R')
+        h.await_text('search(reverse-search)``:')
+        h.press('line')
+        h.await_text('search(reverse-search)`line`: line_1')
+        h.press('^R')
+        h.await_text('search(reverse-search)`line`: line_3')
+        h.press('Enter')
+        h.await_cursor_position(x=0, y=4)
+
+
+def test_search_reverse_search_pos_during(run, xdg_data_home, ten_lines):
+    xdg_data_home.join('babi/history/search').ensure().write(
+        'line_3\n',
+    )
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('^W')
+        h.press('^R')
+        h.press('ne')
+        h.await_text('search(reverse-search)`ne`: line_3')
+        h.await_cursor_position(y=23, x=30)
+        h.press('^C')
+
+
+def test_search_reverse_search_pos_after(run, xdg_data_home, ten_lines):
+    xdg_data_home.join('babi/history/search').ensure().write(
+        'line_3\n',
+    )
+    with run(str(ten_lines), height=20) as h, and_exit(h):
+        h.press('^W')
+        h.press('^R')
+        h.press('line')
+        h.await_text('search(reverse-search)`line`: line_3')
+        h.press('Right')
+        h.await_text('search: line_3')
+        h.await_cursor_position(y=19, x=14)
+        h.press('^C')
+
+
+def test_search_reverse_search_enter_appends(run, xdg_data_home, ten_lines):
+    xdg_data_home.join('babi/history/search').ensure().write(
+        'line_1\n'
+        'line_3\n',
+    )
+    with run(str(ten_lines)) as h, and_exit(h):
+        h.press('^W')
+        h.press('^R')
+        h.press('1')
+        h.await_text('search(reverse-search)`1`: line_1')
+        h.press('Enter')
+        h.press('^W')
+        h.press('Up')
+        h.await_text('search [line_1]: line_1')
+        h.press('^C')
+
+
+def test_search_reverse_search_history_cancel(run):
+    with run() as h, and_exit(h):
+        h.press('^W')
+        h.press('^R')
+        h.await_text('search(reverse-search)``:')
+        h.press('^C')
+        h.await_text('cancelled')
+
+
+def test_search_reverse_search_resizing(run):
+    with run() as h, and_exit(h):
+        h.press('^W')
+        h.press('^R')
+        with h.resize(width=24, height=24):
+            h.await_text('search(reverse-se…:')
+            h.press('^C')
+
+
+def test_search_reverse_search_does_not_wrap_around(run, xdg_data_home):
+    xdg_data_home.join('babi/history/search').ensure().write(
+        'line_1\n'
+        'line_3\n',
+    )
+    with run() as h, and_exit(h):
+        h.press('^W')
+        h.press('^R')
+        # this should not wrap around
+        for i in range(6):
+            h.press('^R')
+        h.await_text('search(reverse-search)``: line_1')
+        h.press('^C')
+
+
+def test_search_reverse_search_ctrl_r_on_failed_match(run, xdg_data_home):
+    xdg_data_home.join('babi/history/search').ensure().write(
+        'nomatch\n'
+        'line_1\n',
+    )
+    with run() as h, and_exit(h):
+        h.press('^W')
+        h.press('^R')
+        h.press('line')
+        h.await_text('search(reverse-search)`line`: line_1')
+        h.press('^R')
+        h.await_text('search(failed reverse-search)`line`: line_1')
+        h.press('^C')
+
+
+def test_search_reverse_search_keeps_current_text_displayed(run):
+    with run() as h, and_exit(h):
+        h.press('^W')
+        h.press('ohai')
+        h.await_text('search: ohai')
+        h.press('^R')
+        h.await_text('search(reverse-search)``: ohai')
+        h.press('^C')
+
+
+def test_search_history_extra_blank_lines(run, xdg_data_home):
+    with run() as h, and_exit(h):
+        h.press('^W')
+        h.press_and_enter('hello')
+    with run() as h, and_exit(h):
+        pass
+    contents = xdg_data_home.join('babi/history/search').read()
+    assert contents == 'hello\n'

tests/features/sort_test.py

@@ -0,0 +1,59 @@
+import pytest
+
+from testing.runner import and_exit
+from testing.runner import trigger_command_mode
+
+
+@pytest.fixture
+def unsorted(tmpdir):
+    f = tmpdir.join('f')
+    f.write('d\nb\nc\na\n')
+    return f
+
+
+def test_sort_entire_file(run, unsorted):
+    with run(str(unsorted)) as h, and_exit(h):
+        trigger_command_mode(h)
+        h.press_and_enter(':sort')
+        h.await_text('sorted!')
+        h.await_cursor_position(x=0, y=1)
+        h.press('^S')
+    assert unsorted.read() == 'a\nb\nc\nd\n'
+
+
+def test_sort_selection(run, unsorted):
+    with run(str(unsorted)) as h, and_exit(h):
+        h.press('S-Down')
+        trigger_command_mode(h)
+        h.press_and_enter(':sort')
+        h.await_text('sorted!')
+        h.await_cursor_position(x=0, y=1)
+        h.press('^S')
+    assert unsorted.read() == 'b\nd\nc\na\n'
+
+
+def test_sort_selection_does_not_include_eof(run, unsorted):
+    with run(str(unsorted)) as h, and_exit(h):
+        for _ in range(5):
+            h.press('S-Down')
+        trigger_command_mode(h)
+        h.press_and_enter(':sort')
+        h.await_text('sorted!')
+        h.await_cursor_position(x=0, y=1)
+        h.press('^S')
+    assert unsorted.read() == 'a\nb\nc\nd\n'
+
+
+def test_sort_does_not_include_blank_line_after(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('b\na\n\nd\nc\n')
+
+    with run(str(f)) as h, and_exit(h):
+        h.press('S-Down')
+        h.press('S-Down')
+        trigger_command_mode(h)
+        h.press_and_enter(':sort')
+        h.await_text('sorted!')
+        h.await_cursor_position(x=0, y=1)
+        h.press('^S')
+    assert f.read() == 'a\nb\n\nd\nc\n'

tests/features/status_test.py

@@ -0,0 +1,19 @@
+from testing.runner import and_exit
+
+
+def test_status_clearing_behaviour(run):
+    with run() as h, and_exit(h):
+        h.press('^J')
+        h.await_text('unknown key')
+        for i in range(24):
+            h.press('Left')
+        h.await_text('unknown key')
+        h.press('Left')
+        h.await_text_missing('unknown key')
+
+
+def test_very_narrow_window_status(run):
+    with run(height=50) as h, and_exit(h):
+        with h.resize(width=5, height=50):
+            h.press('^J')
+            h.await_text('unkno')

tests/features/stdin_test.py

@@ -0,0 +1,22 @@
+import shlex
+import sys
+
+from babi.screen import VERSION_STR
+from testing.runner import PrintsErrorRunner
+
+
+def test_open_from_stdin():
+    with PrintsErrorRunner('env', 'TERM=screen', 'bash', '--norc') as h:
+        cmd = (sys.executable, '-mcoverage', 'run', '-m', 'babi', '-')
+        babi_cmd = ' '.join(shlex.quote(part) for part in cmd)
+        h.press_and_enter(fr"echo $'hello\nworld' | {babi_cmd}")
+
+        h.await_text(VERSION_STR, timeout=2)
+        h.await_text('<<new file>> *')
+        h.await_text('hello\nworld')
+
+        h.press('^X')
+        h.press('n')
+        h.await_text_missing('<<new file>>')
+        h.press_and_enter('exit')
+        h.await_exit()

tests/features/suspend_test.py

@@ -0,0 +1,48 @@
+import shlex
+import sys
+
+from babi.screen import VERSION_STR
+from testing.runner import PrintsErrorRunner
+
+
+def test_suspend(tmpdir):
+    f = tmpdir.join('f')
+    f.write('hello')
+
+    with PrintsErrorRunner('env', 'TERM=screen', 'bash', '--norc') as h:
+        cmd = (sys.executable, '-mcoverage', 'run', '-m', 'babi', str(f))
+        h.press_and_enter(' '.join(shlex.quote(part) for part in cmd))
+        h.await_text(VERSION_STR, timeout=2)
+        h.await_text('hello')
+
+        h.press('^Z')
+        h.await_text_missing('hello')
+
+        h.press_and_enter('fg')
+        h.await_text('hello')
+
+        h.press('^X')
+        h.press_and_enter('exit')
+        h.await_exit()
+
+
+def test_suspend_with_resize(tmpdir):
+    f = tmpdir.join('f')
+    f.write('hello')
+
+    with PrintsErrorRunner('env', 'TERM=screen', 'bash', '--norc') as h:
+        cmd = (sys.executable, '-mcoverage', 'run', '-m', 'babi', str(f))
+        h.press_and_enter(' '.join(shlex.quote(part) for part in cmd))
+        h.await_text(VERSION_STR, timeout=2)
+        h.await_text('hello')
+
+        h.press('^Z')
+        h.await_text_missing('hello')
+
+        with h.resize(80, 10):
+            h.press_and_enter('fg')
+            h.await_text('hello')
+
+        h.press('^X')
+        h.press_and_enter('exit')
+        h.await_exit()

tests/features/syntax_highlight_test.py

@@ -0,0 +1,155 @@
+import curses
+import json
+
+import pytest
+
+from testing.runner import and_exit
+
+
+THEME = json.dumps({
+    'colors': {'background': '#00d700', 'foreground': '#303030'},
+    'tokenColors': [
+        {'scope': 'comment', 'settings': {'foreground': '#767676'}},
+        {
+            'scope': 'diffremove',
+            'settings': {'foreground': '#5f0000', 'background': '#ff5f5f'},
+        },
+        {'scope': 'tqs', 'settings': {'foreground': '#00005f'}},
+        {'scope': 'qmark', 'settings': {'foreground': '#5f0000'}},
+        {'scope': 'b', 'settings': {'fontStyle': 'bold'}},
+        {'scope': 'i', 'settings': {'fontStyle': 'italic'}},
+        {'scope': 'u', 'settings': {'fontStyle': 'underline'}},
+    ],
+})
+SYNTAX = json.dumps({
+    'scopeName': 'source.demo',
+    'fileTypes': ['demo'],
+    'firstLineMatch': '^#!/usr/bin/(env demo|demo)$',
+    'patterns': [
+        {'match': r'#.*$\n?', 'name': 'comment'},
+        {'match': r'^-.*$\n?', 'name': 'diffremove'},
+        {'begin': '"""', 'end': '"""', 'name': 'tqs'},
+        {'match': r'\?', 'name': 'qmark'},
+    ],
+})
+DEMO_S = '''\
+- foo
+# comment here
+uncolored
+"""tqs!
+still more
+"""
+'''
+
+
+@pytest.fixture(autouse=True)
+def theme_and_grammar(xdg_data_home, xdg_config_home):
+    xdg_config_home.join('babi/theme.json').ensure().write(THEME)
+    xdg_data_home.join('babi/grammar_v1/demo.json').ensure().write(SYNTAX)
+
+
+@pytest.fixture
+def demo(tmpdir):
+    f = tmpdir.join('f.demo')
+    f.write(DEMO_S)
+    yield f
+
+
+def test_syntax_highlighting(run, demo):
+    with run(str(demo), term='screen-256color', width=20) as h, and_exit(h):
+        h.await_text('still more')
+        for i, attr in enumerate([
+                [(236, 40, curses.A_REVERSE)] * 20,        # header
+                [(52, 203, 0)] * 5 + [(236, 40, 0)] * 15,  # - foo
+                [(243, 40, 0)] * 14 + [(236, 40, 0)] * 6,  # # comment here
+                [(236, 40, 0)] * 20,                       # uncolored
+                [(17, 40, 0)] * 7 + [(236, 40, 0)] * 13,   # """tqs!
+                [(17, 40, 0)] * 10 + [(236, 40, 0)] * 10,  # still more
+                [(17, 40, 0)] * 3 + [(236, 40, 0)] * 17,   # """
+        ]):
+            h.assert_screen_attr_equals(i, attr)
+
+
+def test_syntax_highlighting_does_not_highlight_arrows(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write(
+        f'#!/usr/bin/env demo\n'
+        f'# l{"o" * 15}ng comment\n',
+    )
+
+    with run(str(f), term='screen-256color', width=20) as h, and_exit(h):
+        h.await_text('loooo')
+        h.assert_screen_attr_equals(2, [(243, 40, 0)] * 19 + [(236, 40, 0)])
+
+        h.press('Down')
+        h.press('^E')
+        h.await_text_missing('loooo')
+        expected = [(236, 40, 0)] + [(243, 40, 0)] * 15 + [(236, 40, 0)] * 4
+        h.assert_screen_attr_equals(2, expected)
+
+
+def test_syntax_highlighting_off_screen_does_not_crash(run, tmpdir):
+    f = tmpdir.join('f.demo')
+    f.write(f'"""a"""{"x" * 40}"""b"""')
+
+    with run(str(f), term='screen-256color', width=20) as h, and_exit(h):
+        h.await_text('"""a"""')
+        h.assert_screen_attr_equals(1, [(17, 40, 0)] * 7 + [(236, 40, 0)] * 13)
+        h.press('^E')
+        h.await_text('"""b"""')
+        expected = [(236, 40, 0)] * 11 + [(17, 40, 0)] * 7 + [(236, 40, 0)] * 2
+        h.assert_screen_attr_equals(1, expected)
+
+
+def test_syntax_highlighting_one_off_left_of_screen(run, tmpdir):
+    f = tmpdir.join('f.demo')
+    f.write(f'{"x" * 11}?123456789')
+
+    with run(str(f), term='screen-256color', width=20) as h, and_exit(h):
+        h.await_text('xxx?123')
+        expected = [(236, 40, 0)] * 11 + [(52, 40, 0)] + [(236, 40, 0)] * 8
+        h.assert_screen_attr_equals(1, expected)
+
+        h.press('End')
+        h.await_text_missing('?')
+        h.assert_screen_attr_equals(1, [(236, 40, 0)] * 20)
+
+
+def test_syntax_highlighting_to_edge_of_screen(run, tmpdir):
+    f = tmpdir.join('f.demo')
+    f.write(f'# {"x" * 18}')
+
+    with run(str(f), term='screen-256color', width=20) as h, and_exit(h):
+        h.await_text('# xxx')
+        h.assert_screen_attr_equals(1, [(243, 40, 0)] * 20)
+
+
+def test_syntax_highlighting_with_tabs(run, tmpdir):
+    f = tmpdir.join('f.demo')
+    f.write('\t# 12345678901234567890\n')
+
+    with run(str(f), term='screen-256color', width=20) as h, and_exit(h):
+        h.await_text('1234567890')
+        expected = 4 * [(236, 40, 0)] + 15 * [(243, 40, 0)] + [(236, 40, 0)]
+        h.assert_screen_attr_equals(1, expected)
+
+
+def test_syntax_highlighting_tabs_after_line_creation(run, tmpdir):
+    f = tmpdir.join('f')
+    # trailing whitespace is used to trigger highlighting
+    f.write('foo\n\txx \ny    \n')
+
+    with run(str(f), term='screen-256color') as h, and_exit(h):
+        # this looks weird, but it populates the width cache
+        h.press('Down')
+        h.press('Down')
+        h.press('Down')
+
+        # press enter after the tab
+        h.press('Up')
+        h.press('Up')
+        h.press('Right')
+        h.press('Right')
+        h.press('Enter')
+
+        h.await_text('foo\n    x\nx\ny\n')

tests/features/text_editing_test.py

@@ -0,0 +1,149 @@
+import pytest
+
+from testing.runner import and_exit
+
+
+def test_basic_text_editing(run, tmpdir):
+    with run() as h, and_exit(h):
+        h.press('hello world')
+        h.await_text('hello world')
+        h.press('Down')
+        h.press('bye!')
+        h.await_text('bye!')
+        h.await_text('hello world\nbye!\n')
+
+
+def test_backspace_at_beginning_of_file(run):
+    with run() as h, and_exit(h):
+        h.press('BSpace')
+        h.await_text_missing('unknown key')
+        h.assert_cursor_line_equals('')
+        h.await_text_missing('*')
+
+
+def test_backspace_joins_lines(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('foo\nbar\nbaz\n')
+
+    with run(str(f)) as h, and_exit(h):
+        h.await_text('foo')
+        h.press('Down')
+        h.press('BSpace')
+        h.await_text('foobar')
+        h.await_text('f *')
+        h.await_cursor_position(x=3, y=1)
+        # pressing down should retain the X position
+        h.press('Down')
+        h.await_cursor_position(x=3, y=2)
+
+
+def test_backspace_at_end_of_file_still_allows_scrolling_down(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('hello world')
+
+    with run(str(f)) as h, and_exit(h):
+        h.await_text('hello world')
+        h.press('Down')
+        h.press('BSpace')
+        h.press('Down')
+        h.await_cursor_position(x=0, y=2)
+        h.await_text_missing('*')
+
+
+@pytest.mark.parametrize('key', ('BSpace', '^H'))
+def test_backspace_deletes_text(run, tmpdir, key):
+    f = tmpdir.join('f')
+    f.write('ohai there')
+
+    with run(str(f)) as h, and_exit(h):
+        h.await_text('ohai there')
+        for _ in range(3):
+            h.press('Right')
+        h.press(key)
+        h.await_text('ohi')
+        h.await_text('f *')
+        h.await_cursor_position(x=2, y=1)
+
+
+def test_delete_at_end_of_file(run, tmpdir):
+    with run() as h, and_exit(h):
+        h.press('DC')
+        h.await_text_missing('unknown key')
+        h.await_text_missing('*')
+
+
+def test_delete_removes_character_afterwards(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('hello world')
+
+    with run(str(f)) as h, and_exit(h):
+        h.await_text('hello world')
+        h.press('Right')
+        h.press('DC')
+        h.await_text('hllo world')
+        h.await_text('f *')
+
+
+def test_delete_at_end_of_line(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('hello\nworld\n')
+
+    with run(str(f)) as h, and_exit(h):
+        h.await_text('hello')
+        h.press('Down')
+        h.press('Left')
+        h.press('DC')
+        h.await_text('helloworld')
+        h.await_text('f *')
+
+
+def test_delete_at_end_of_last_line(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('hello\n')
+
+    with run(str(f)) as h, and_exit(h):
+        h.await_text('hello')
+        h.press('End')
+        h.press('DC')
+        # should not make the file modified
+        h.await_text_missing('*')
+
+        # delete should still be functional
+        h.press('Left')
+        h.press('Left')
+        h.press('DC')
+        h.await_text('helo')
+
+
+def test_press_enter_beginning_of_file(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('hello world')
+
+    with run(str(f)) as h, and_exit(h):
+        h.await_text('hello world')
+        h.press('Enter')
+        h.await_text('\n\nhello world')
+        h.await_cursor_position(x=0, y=2)
+        h.await_text('f *')
+
+
+def test_press_enter_mid_line(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('hello world')
+
+    with run(str(f)) as h, and_exit(h):
+        h.await_text('hello world')
+        for _ in range(5):
+            h.press('Right')
+        h.press('Enter')
+        h.await_text('hello\n world')
+        h.await_cursor_position(x=0, y=2)
+        h.press('Up')
+        h.await_cursor_position(x=0, y=1)
+
+
+def test_press_string_sequence(run):
+    with run() as h, and_exit(h):
+        h.press('hello world\x1bOH')
+        h.await_text('hello world')
+        h.await_cursor_position(x=0, y=1)

tests/features/trailing_whitespace_test.py

@@ -0,0 +1,23 @@
+import curses
+
+from testing.runner import and_exit
+
+
+def test_trailing_whitespace_highlighting(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write('0123456789     \n')
+
+    with run(str(f), term='screen-256color', width=20) as h, and_exit(h):
+        h.await_text('123456789')
+        h.assert_screen_attr_equals(0, [(-1, -1, curses.A_REVERSE)] * 20)
+        attrs = [(-1, -1, 0)] * 10 + [(-1, 1, 0)] * 5 + [(-1, -1, 0)] * 5
+        h.assert_screen_attr_equals(1, attrs)
+
+
+def test_trailing_whitespace_does_not_highlight_line_continuation(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write(f'{" " * 30}\nhello\n')
+
+    with run(str(f), term='screen-256color', width=20) as h, and_exit(h):
+        h.await_text('hello')
+        h.assert_screen_attr_equals(1, [(-1, 1, 0)] * 19 + [(-1, -1, 0)])

tests/features/undo_redo_test.py

@@ -0,0 +1,142 @@
+from testing.runner import and_exit
+
+
+def test_nothing_to_undo_redo(run):
+    with run() as h, and_exit(h):
+        h.press('M-u')
+        h.await_text('nothing to undo!')
+        h.press('M-U')
+        h.await_text('nothing to redo!')
+
+
+def test_undo_redo(run):
+    with run() as h, and_exit(h):
+        h.press('hello')
+        h.await_text('hello')
+        h.press('M-u')
+        h.await_text('undo: text')
+        h.await_text_missing('hello')
+        h.await_text_missing(' *')
+        h.press('M-U')
+        h.await_text('redo: text')
+        h.await_text('hello')
+        h.await_text(' *')
+
+
+def test_undo_redo_movement_interrupts_actions(run):
+    with run() as h, and_exit(h):
+        h.press('hello')
+        h.press('Left')
+        h.press('Right')
+        h.press('world')
+        h.press('M-u')
+        h.await_text('undo: text')
+        h.await_text('hello')
+
+
+def test_undo_redo_action_interrupts_actions(run):
+    with run() as h, and_exit(h):
+        h.press('hello')
+        h.await_text('hello')
+        h.press('BSpace')
+        h.await_text_missing('hello')
+        h.press('M-u')
+        h.await_text('hello')
+        h.press('world')
+        h.await_text('helloworld')
+        h.press('M-u')
+        h.await_text_missing('world')
+        h.await_text('hello')
+
+
+def test_undo_redo_mixed_newlines(run, tmpdir):
+    f = tmpdir.join('f')
+    f.write_binary(b'foo\nbar\r\n')
+
+    with run(str(f)) as h, and_exit(h):
+        h.press('hello')
+        h.press('M-u')
+        h.await_text('undo: text')
+        h.await_text(' *')
+
+
+def test_undo_redo_with_save(run, tmpdir):
+    f = tmpdir.join('f').ensure()
+
+    with run(str(f)) as h, and_exit(h):
+        h.press('hello')
+        h.press('^S')
+        h.await_text_missing(' *')
+        h.press('M-u')
+        h.await_text(' *')
+        h.press('M-U')
+        h.await_text_missing(' *')
+        h.press('M-u')
+        h.await_text(' *')
+        h.press('^S')
+        h.await_text_missing(' *')
+        h.press('M-U')
+        h.await_text(' *')
+
+
+def test_undo_redo_implicit_linebreak(run, tmpdir):
+    f = tmpdir.join('f')
+
+    def _assert_contents(s):
+        assert f.read() == s
+
+    with run(str(f)) as h, and_exit(h):
+        h.press('hello')
+        h.press('M-u')
+        h.press('^S')
+        h.await_text('saved!')
+        h.run(lambda: _assert_contents(''))
+        h.press('M-U')
+        h.press('^S')
+        h.await_text('saved!')
+        h.run(lambda: _assert_contents('hello\n'))
+
+
+def test_redo_cleared_after_action(run, tmpdir):
+    with run() as h, and_exit(h):
+        h.press('hello')
+        h.press('M-u')
+        h.press('world')
+        h.press('M-U')
+        h.await_text('nothing to redo!')
+
+
+def test_undo_no_action_when_noop(run):
+    with run() as h, and_exit(h):
+        h.press('hello')
+        h.press('Enter')
+        h.press('world')
+        h.press('Down')
+        h.press('^K')
+        h.press('M-u')
+        h.await_text('undo: text')
+        h.await_cursor_position(x=0, y=2)
+
+
+def test_undo_redo_causes_scroll(run):
+    with run(height=8) as h, and_exit(h):
+        for i in range(10):
+            h.press('Enter')
+        h.await_cursor_position(x=0, y=3)
+        h.press('M-u')
+        h.await_cursor_position(x=0, y=1)
+        h.press('M-U')
+        h.await_cursor_position(x=0, y=4)
+
+
+def test_undo_redo_clears_selection(run, ten_lines):
+    # maintaining the selection across undo/redo is both difficult and not all
+    # that useful.  prior to this it was buggy anyway (a negative selection
+    # indented and then undone would highlight out of bounds)
+    with run(str(ten_lines), width=20) as h, and_exit(h):
+        h.press('S-Down')
+        h.press('Tab')
+        h.await_cursor_position(x=4, y=2)
+        h.press('M-u')
+        h.await_cursor_position(x=0, y=2)
+        h.assert_screen_attr_equals(1, [(-1, -1, 0)] * 20)

tests/file_test.py

@@ -0,0 +1,34 @@
+import io
+
+import pytest
+
+from babi.color_manager import ColorManager
+from babi.file import File
+from babi.file import get_lines
+
+
+def test_position_repr():
+    ret = repr(File('f.txt', ColorManager.make(), ()))
+    assert ret == "<File 'f.txt'>"
+
+
+@pytest.mark.parametrize(
+    ('s', 'lines', 'nl', 'mixed'),
+    (
+        pytest.param('', [''], '\n', False, id='trivial'),
+        pytest.param('1\n2\n', ['1', '2', ''], '\n', False, id='lf'),
+        pytest.param('1\r\n2\r\n', ['1', '2', ''], '\r\n', False, id='crlf'),
+        pytest.param('1\r\n2\n', ['1', '2', ''], '\n', True, id='mixed'),
+        pytest.param('1\n2', ['1', '2', ''], '\n', False, id='noeol'),
+    ),
+)
+def test_get_lines(s, lines, nl, mixed):
+    # sha256 tested below
+    ret_lines, ret_nl, ret_mixed, _ = get_lines(io.StringIO(s))
+    assert (ret_lines, ret_nl, ret_mixed) == (lines, nl, mixed)
+
+
+def test_get_lines_sha256_checksum():
+    ret = get_lines(io.StringIO(''))
+    sha256 = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'
+    assert ret == ([''], '\n', False, sha256)

tests/highlight_test.py

@@ -0,0 +1,639 @@
+import pytest
+
+from babi.highlight import highlight_line
+from babi.highlight import Region
+
+
+def test_grammar_matches_extension_only_name(make_grammars):
+    data = {'scopeName': 'shell', 'patterns': [], 'fileTypes': ['bashrc']}
+    grammars = make_grammars(data)
+    compiler = grammars.compiler_for_file('.bashrc', 'alias nano=babi')
+    assert compiler.root_state.entries[0].scope[0] == 'shell'
+
+
+def test_grammar_matches_via_identify_tag(make_grammars):
+    grammars = make_grammars({'scopeName': 'source.ini', 'patterns': []})
+    compiler = grammars.compiler_for_file('setup.cfg', '')
+    assert compiler.root_state.entries[0].scope[0] == 'source.ini'
+
+
+@pytest.fixture
+def compiler_state(make_grammars):
+    def _compiler_state(*grammar_dcts):
+        grammars = make_grammars(*grammar_dcts)
+        compiler = grammars.compiler_for_scope(grammar_dcts[0]['scopeName'])
+        return compiler, compiler.root_state
+    return _compiler_state
+
+
+def test_backslash_a(compiler_state):
+    grammar = {
+        'scopeName': 'test',
+        'patterns': [{'name': 'aaa', 'match': r'\Aa+'}],
+    }
+    compiler, state = compiler_state(grammar)
+
+    state, (region_0,) = highlight_line(compiler, state, 'aaa', True)
+    state, (region_1,) = highlight_line(compiler, state, 'aaa', False)
+
+    # \A should only match at the beginning of the file
+    assert region_0 == Region(0, 3, ('test', 'aaa'))
+    assert region_1 == Region(0, 3, ('test',))
+
+
+BEGIN_END_NO_NL = {
+    'scopeName': 'test',
+    'patterns': [{
+        'begin': 'x',
+        'end': 'x',
+        'patterns': [
+            {'match': r'\Ga', 'name': 'ga'},
+            {'match': 'a', 'name': 'noga'},
+        ],
+    }],
+}
+
+
+def test_backslash_g_inline(compiler_state):
+    compiler, state = compiler_state(BEGIN_END_NO_NL)
+
+    _, regions = highlight_line(compiler, state, 'xaax', True)
+    assert regions == (
+        Region(0, 1, ('test',)),
+        Region(1, 2, ('test', 'ga')),
+        Region(2, 3, ('test', 'noga')),
+        Region(3, 4, ('test',)),
+    )
+
+
+def test_backslash_g_next_line(compiler_state):
+    compiler, state = compiler_state(BEGIN_END_NO_NL)
+
+    state, regions1 = highlight_line(compiler, state, 'x\n', True)
+    state, regions2 = highlight_line(compiler, state, 'aax\n', False)
+
+    assert regions1 == (
+        Region(0, 1, ('test',)),
+        Region(1, 2, ('test',)),
+    )
+    assert regions2 == (
+        Region(0, 1, ('test', 'noga')),
+        Region(1, 2, ('test', 'noga')),
+        Region(2, 3, ('test',)),
+        Region(3, 4, ('test',)),
+    )
+
+
+def test_end_before_other_match(compiler_state):
+    compiler, state = compiler_state(BEGIN_END_NO_NL)
+
+    state, regions = highlight_line(compiler, state, 'xazzx', True)
+
+    assert regions == (
+        Region(0, 1, ('test',)),
+        Region(1, 2, ('test', 'ga')),
+        Region(2, 4, ('test',)),
+        Region(4, 5, ('test',)),
+    )
+
+
+BEGIN_END_NL = {
+    'scopeName': 'test',
+    'patterns': [{
+        'begin': r'x$\n?',
+        'end': 'x',
+        'patterns': [
+            {'match': r'\Ga', 'name': 'ga'},
+            {'match': 'a', 'name': 'noga'},
+        ],
+    }],
+}
+
+
+def test_backslash_g_captures_nl(compiler_state):
+    compiler, state = compiler_state(BEGIN_END_NL)
+
+    state, regions1 = highlight_line(compiler, state, 'x\n', True)
+    state, regions2 = highlight_line(compiler, state, 'aax\n', False)
+
+    assert regions1 == (
+        Region(0, 2, ('test',)),
+    )
+    assert regions2 == (
+        Region(0, 1, ('test', 'ga')),
+        Region(1, 2, ('test', 'noga')),
+        Region(2, 3, ('test',)),
+        Region(3, 4, ('test',)),
+    )
+
+
+def test_backslash_g_captures_nl_next_line(compiler_state):
+    compiler, state = compiler_state(BEGIN_END_NL)
+
+    state, regions1 = highlight_line(compiler, state, 'x\n', True)
+    state, regions2 = highlight_line(compiler, state, 'aa\n', False)
+    state, regions3 = highlight_line(compiler, state, 'aax\n', False)
+
+    assert regions1 == (
+        Region(0, 2, ('test',)),
+    )
+    assert regions2 == (
+        Region(0, 1, ('test', 'ga')),
+        Region(1, 2, ('test', 'noga')),
+        Region(2, 3, ('test',)),
+    )
+    assert regions3 == (
+        Region(0, 1, ('test', 'ga')),
+        Region(1, 2, ('test', 'noga')),
+        Region(2, 3, ('test',)),
+        Region(3, 4, ('test',)),
+    )
+
+
+def test_while_no_nl(compiler_state):
+    compiler, state = compiler_state({
+        'scopeName': 'test',
+        'patterns': [{
+            'begin': '> ',
+            'while': '> ',
+            'contentName': 'while',
+            'patterns': [
+                {'match': r'\Ga', 'name': 'ga'},
+                {'match': 'a', 'name': 'noga'},
+            ],
+        }],
+    })
+
+    state, regions1 = highlight_line(compiler, state, '> aa\n', True)
+    state, regions2 = highlight_line(compiler, state, '> aa\n', False)
+    state, regions3 = highlight_line(compiler, state, 'after\n', False)
+
+    assert regions1 == (
+        Region(0, 2, ('test',)),
+        Region(2, 3, ('test', 'while', 'ga')),
+        Region(3, 4, ('test', 'while', 'noga')),
+        Region(4, 5, ('test', 'while')),
+    )
+    assert regions2 == (
+        Region(0, 2, ('test', 'while')),
+        Region(2, 3, ('test', 'while', 'ga')),
+        Region(3, 4, ('test', 'while', 'noga')),
+        Region(4, 5, ('test', 'while')),
+    )
+    assert regions3 == (
+        Region(0, 6, ('test',)),
+    )
+
+
+def test_complex_captures(compiler_state):
+    compiler, state = compiler_state({
+        'scopeName': 'test',
+        'patterns': [
+            {
+                'match': '(<).([^>]+)(>)',
+                'captures': {
+                    '1': {'name': 'lbracket'},
+                    '2': {
+                        'patterns': [
+                            {'match': 'a', 'name': 'a'},
+                            {'match': 'z', 'name': 'z'},
+                        ],
+                    },
+                    '3': {'name': 'rbracket'},
+                },
+            },
+        ],
+    })
+
+    state, regions = highlight_line(compiler, state, '<qabz>', first_line=True)
+    assert regions == (
+        Region(0, 1, ('test', 'lbracket')),
+        Region(1, 2, ('test',)),
+        Region(2, 3, ('test', 'a')),
+        Region(3, 4, ('test',)),
+        Region(4, 5, ('test', 'z')),
+        Region(5, 6, ('test', 'rbracket')),
+    )
+
+
+def test_captures_multiple_applied_to_same_capture(compiler_state):
+    compiler, state = compiler_state({
+        'scopeName': 'test',
+        'patterns': [
+            {
+                'match': '((a)) ((b) c) (d (e)) ((f) )',
+                'name': 'matched',
+                'captures': {
+                    '1': {'name': 'g1'},
+                    '2': {'name': 'g2'},
+                    '3': {'name': 'g3'},
+                    '4': {'name': 'g4'},
+                    '5': {'name': 'g5'},
+                    '6': {'name': 'g6'},
+                    '7': {
+                        'patterns': [
+                            {'match': 'f', 'name': 'g7f'},
+                            {'match': ' ', 'name': 'g7space'},
+                        ],
+                    },
+                    # this one has to backtrack some
+                    '8': {'name': 'g8'},
+                },
+            },
+        ],
+    })
+
+    state, regions = highlight_line(compiler, state, 'a b c d e f ', True)
+
+    assert regions == (
+        Region(0, 1, ('test', 'matched', 'g1', 'g2')),
+        Region(1, 2, ('test', 'matched')),
+        Region(2, 3, ('test', 'matched', 'g3', 'g4')),
+        Region(3, 5, ('test', 'matched', 'g3')),
+        Region(5, 6, ('test', 'matched')),
+        Region(6, 8, ('test', 'matched', 'g5')),
+        Region(8, 9, ('test', 'matched', 'g5', 'g6')),
+        Region(9, 10, ('test', 'matched')),
+        Region(10, 11, ('test', 'matched', 'g7f', 'g8')),
+        Region(11, 12, ('test', 'matched', 'g7space')),
+    )
+
+
+def test_captures_ignores_empty(compiler_state):
+    compiler, state = compiler_state({
+        'scopeName': 'test',
+        'patterns': [{
+            'match': '(.*) hi',
+            'captures': {'1': {'name': 'before'}},
+        }],
+    })
+
+    state, regions1 = highlight_line(compiler, state, ' hi\n', True)
+    state, regions2 = highlight_line(compiler, state, 'o hi\n', False)
+
+    assert regions1 == (
+        Region(0, 3, ('test',)),
+        Region(3, 4, ('test',)),
+    )
+    assert regions2 == (
+        Region(0, 1, ('test', 'before')),
+        Region(1, 4, ('test',)),
+        Region(4, 5, ('test',)),
+    )
+
+
+def test_captures_ignores_invalid_out_of_bounds(compiler_state):
+    compiler, state = compiler_state({
+        'scopeName': 'test',
+        'patterns': [{'match': '.', 'captures': {'1': {'name': 'oob'}}}],
+    })
+
+    state, regions = highlight_line(compiler, state, 'x', first_line=True)
+
+    assert regions == (
+        Region(0, 1, ('test',)),
+    )
+
+
+def test_captures_begin_end(compiler_state):
+    compiler, state = compiler_state({
+        'scopeName': 'test',
+        'patterns': [
+            {
+                'begin': '(""")',
+                'end': '(""")',
+                'beginCaptures': {'1': {'name': 'startquote'}},
+                'endCaptures': {'1': {'name': 'endquote'}},
+            },
+        ],
+    })
+
+    state, regions = highlight_line(compiler, state, '"""x"""', True)
+
+    assert regions == (
+        Region(0, 3, ('test', 'startquote')),
+        Region(3, 4, ('test',)),
+        Region(4, 7, ('test', 'endquote')),
+    )
+
+
+def test_captures_while_captures(compiler_state):
+    compiler, state = compiler_state({
+        'scopeName': 'test',
+        'patterns': [
+            {
+                'begin': '(>) ',
+                'while': '(>) ',
+                'beginCaptures': {'1': {'name': 'bblock'}},
+                'whileCaptures': {'1': {'name': 'wblock'}},
+            },
+        ],
+    })
+
+    state, regions1 = highlight_line(compiler, state, '> x\n', True)
+    state, regions2 = highlight_line(compiler, state, '> x\n', False)
+
+    assert regions1 == (
+        Region(0, 1, ('test', 'bblock')),
+        Region(1, 2, ('test',)),
+        Region(2, 4, ('test',)),
+    )
+
+    assert regions2 == (
+        Region(0, 1, ('test', 'wblock')),
+        Region(1, 2, ('test',)),
+        Region(2, 4, ('test',)),
+    )
+
+
+def test_captures_implies_begin_end_captures(compiler_state):
+    compiler, state = compiler_state({
+        'scopeName': 'test',
+        'patterns': [
+            {
+                'begin': '(""")',
+                'end': '(""")',
+                'captures': {'1': {'name': 'quote'}},
+            },
+        ],
+    })
+
+    state, regions = highlight_line(compiler, state, '"""x"""', True)
+
+    assert regions == (
+        Region(0, 3, ('test', 'quote')),
+        Region(3, 4, ('test',)),
+        Region(4, 7, ('test', 'quote')),
+    )
+
+
+def test_captures_implies_begin_while_captures(compiler_state):
+    compiler, state = compiler_state({
+        'scopeName': 'test',
+        'patterns': [
+            {
+                'begin': '(>) ',
+                'while': '(>) ',
+                'captures': {'1': {'name': 'block'}},
+            },
+        ],
+    })
+
+    state, regions1 = highlight_line(compiler, state, '> x\n', True)
+    state, regions2 = highlight_line(compiler, state, '> x\n', False)
+
+    assert regions1 == (
+        Region(0, 1, ('test', 'block')),
+        Region(1, 2, ('test',)),
+        Region(2, 4, ('test',)),
+    )
+
+    assert regions2 == (
+        Region(0, 1, ('test', 'block')),
+        Region(1, 2, ('test',)),
+        Region(2, 4, ('test',)),
+    )
+
+
+def test_include_self(compiler_state):
+    compiler, state = compiler_state({
+        'scopeName': 'test',
+        'patterns': [
+            {
+                'begin': '<',
+                'end': '>',
+                'contentName': 'bracketed',
+                'patterns': [{'include': '$self'}],
+            },
+            {'match': '.', 'name': 'content'},
+        ],
+    })
+
+    state, regions = highlight_line(compiler, state, '<<_>>', first_line=True)
+    assert regions == (
+        Region(0, 1, ('test',)),
+        Region(1, 2, ('test', 'bracketed')),
+        Region(2, 3, ('test', 'bracketed', 'bracketed', 'content')),
+        Region(3, 4, ('test', 'bracketed', 'bracketed')),
+        Region(4, 5, ('test', 'bracketed')),
+    )
+
+
+def test_include_repository_rule(compiler_state):
+    compiler, state = compiler_state({
+        'scopeName': 'test',
+        'patterns': [{'include': '#impl'}],
+        'repository': {
+            'impl': {
+                'patterns': [
+                    {'match': 'a', 'name': 'a'},
+                    {'match': '.', 'name': 'other'},
+                ],
+            },
+        },
+    })
+
+    state, regions = highlight_line(compiler, state, 'az', first_line=True)
+
+    assert regions == (
+        Region(0, 1, ('test', 'a')),
+        Region(1, 2, ('test', 'other')),
+    )
+
+
+def test_include_with_nested_repositories(compiler_state):
+    compiler, state = compiler_state({
+        'scopeName': 'test',
+        'patterns': [{
+            'begin': '<', 'end': '>', 'name': 'b',
+            'patterns': [
+                {'include': '#rule1'},
+                {'include': '#rule2'},
+                {'include': '#rule3'},
+            ],
+            'repository': {
+                'rule2': {'match': '2', 'name': 'inner2'},
+                'rule3': {'match': '3', 'name': 'inner3'},
+            },
+        }],
+        'repository': {
+            'rule1': {'match': '1', 'name': 'root1'},
+            'rule2': {'match': '2', 'name': 'root2'},
+        },
+    })
+
+    state, regions = highlight_line(compiler, state, '<123>', first_line=True)
+
+    assert regions == (
+        Region(0, 1, ('test', 'b')),
+        Region(1, 2, ('test', 'b', 'root1')),
+        Region(2, 3, ('test', 'b', 'inner2')),
+        Region(3, 4, ('test', 'b', 'inner3')),
+        Region(4, 5, ('test', 'b')),
+    )
+
+
+def test_include_other_grammar(compiler_state):
+    compiler, state = compiler_state(
+        {
+            'scopeName': 'test',
+            'patterns': [
+                {
+                    'begin': '<',
+                    'end': '>',
+                    'name': 'angle',
+                    'patterns': [{'include': 'other.grammar'}],
+                },
+                {
+                    'begin': '`',
+                    'end': '`',
+                    'name': 'tick',
+                    'patterns': [{'include': 'other.grammar#backtick'}],
+                },
+            ],
+        },
+        {
+            'scopeName': 'other.grammar',
+            'patterns': [
+                {'match': 'a', 'name': 'roota'},
+                {'match': '.', 'name': 'rootother'},
+            ],
+            'repository': {
+                'backtick': {
+                    'patterns': [
+                        {'match': 'a', 'name': 'ticka'},
+                        {'match': '.', 'name': 'tickother'},
+                    ],
+                },
+            },
+        },
+    )
+
+    state, regions1 = highlight_line(compiler, state, '<az>\n', True)
+    state, regions2 = highlight_line(compiler, state, '`az`\n', False)
+
+    assert regions1 == (
+        Region(0, 1, ('test', 'angle')),
+        Region(1, 2, ('test', 'angle', 'roota')),
+        Region(2, 3, ('test', 'angle', 'rootother')),
+        Region(3, 4, ('test', 'angle')),
+        Region(4, 5, ('test',)),
+    )
+
+    assert regions2 == (
+        Region(0, 1, ('test', 'tick')),
+        Region(1, 2, ('test', 'tick', 'ticka')),
+        Region(2, 3, ('test', 'tick', 'tickother')),
+        Region(3, 4, ('test', 'tick')),
+        Region(4, 5, ('test',)),
+    )
+
+
+def test_include_base(compiler_state):
+    compiler, state = compiler_state(
+        {
+            'scopeName': 'test',
+            'patterns': [
+                {
+                    'begin': '<',
+                    'end': '>',
+                    'name': 'bracket',
+                    # $base from root grammar includes itself
+                    'patterns': [{'include': '$base'}],
+                },
+                {'include': 'other.grammar'},
+                {'match': 'z', 'name': 'testz'},
+            ],
+        },
+        {
+            'scopeName': 'other.grammar',
+            'patterns': [
+                {
+                    'begin': '`',
+                    'end': '`',
+                    'name': 'tick',
+                    # $base from included grammar includes the root
+                    'patterns': [{'include': '$base'}],
+                },
+            ],
+        },
+    )
+
+    state, regions1 = highlight_line(compiler, state, '<z>\n', True)
+    state, regions2 = highlight_line(compiler, state, '`z`\n', False)
+
+    assert regions1 == (
+        Region(0, 1, ('test', 'bracket')),
+        Region(1, 2, ('test', 'bracket', 'testz')),
+        Region(2, 3, ('test', 'bracket')),
+        Region(3, 4, ('test',)),
+    )
+
+    assert regions2 == (
+        Region(0, 1, ('test', 'tick')),
+        Region(1, 2, ('test', 'tick', 'testz')),
+        Region(2, 3, ('test', 'tick')),
+        Region(3, 4, ('test',)),
+    )
+
+
+def test_rule_with_begin_and_no_end(compiler_state):
+    compiler, state = compiler_state({
+        'scopeName': 'test',
+        'patterns': [
+            {
+                'begin': '!', 'end': '!', 'name': 'bang',
+                'patterns': [{'begin': '--', 'name': 'invalid'}],
+            },
+        ],
+    })
+
+    state, regions = highlight_line(compiler, state, '!x! !--!', True)
+
+    assert regions == (
+        Region(0, 1, ('test', 'bang')),
+        Region(1, 2, ('test', 'bang')),
+        Region(2, 3, ('test', 'bang')),
+        Region(3, 4, ('test',)),
+        Region(4, 5, ('test', 'bang')),
+        Region(5, 7, ('test', 'bang', 'invalid')),
+        Region(7, 8, ('test', 'bang', 'invalid')),
+    )
+
+
+def test_begin_end_substitute_special_chars(compiler_state):
+    compiler, state = compiler_state({
+        'scopeName': 'test',
+        'patterns': [{'begin': r'(\*)', 'end': r'\1', 'name': 'italic'}],
+    })
+
+    state, regions = highlight_line(compiler, state, '*italic*', True)
+
+    assert regions == (
+        Region(0, 1, ('test', 'italic')),
+        Region(1, 7, ('test', 'italic')),
+        Region(7, 8, ('test', 'italic')),
+    )
+
+
+def test_backslash_z(compiler_state):
+    # similar to text.git-commit grammar, \z matches nothing!
+    compiler, state = compiler_state({
+        'scopeName': 'test',
+        'patterns': [
+            {'begin': '#', 'end': r'\z', 'name': 'comment'},
+            {'name': 'other', 'match': '.'},
+        ],
+    })
+
+    state, regions1 = highlight_line(compiler, state, '# comment', True)
+    state, regions2 = highlight_line(compiler, state, 'other?', False)
+
+    assert regions1 == (
+        Region(0, 1, ('test', 'comment')),
+        Region(1, 9, ('test', 'comment')),
+    )
+
+    assert regions2 == (
+        Region(0, 6, ('test', 'comment')),
+    )

tests/hl/syntax_test.py

@@ -0,0 +1,169 @@
+import contextlib
+import curses
+from unittest import mock
+
+import pytest
+
+from babi.buf import Buf
+from babi.color_manager import ColorManager
+from babi.hl.interface import HL
+from babi.hl.syntax import Syntax
+from babi.theme import Color
+from babi.theme import Theme
+
+
+class FakeCurses:
+    def __init__(self, *, n_colors, can_change_color):
+        self._n_colors = n_colors
+        self._can_change_color = can_change_color
+        self.colors = {}
+        self.pairs = {}
+
+    def _curses__can_change_color(self):
+        return self._can_change_color
+
+    def _curses__init_color(self, n, r, g, b):
+        self.colors[n] = (r, g, b)
+
+    def _curses__init_pair(self, n, fg, bg):
+        self.pairs[n] = (fg, bg)
+
+    def _curses__color_pair(self, n):
+        assert n == 0 or n in self.pairs
+        return n << 8
+
+    @classmethod
+    @contextlib.contextmanager
+    def patch(cls, **kwargs):
+        fake = cls(**kwargs)
+        with mock.patch.object(curses, 'COLORS', fake._n_colors, create=True):
+            with mock.patch.multiple(
+                    curses,
+                    can_change_color=fake._curses__can_change_color,
+                    color_pair=fake._curses__color_pair,
+                    init_color=fake._curses__init_color,
+                    init_pair=fake._curses__init_pair,
+            ):
+                yield fake
+
+
+class FakeScreen:
+    def __init__(self):
+        self.attr = 0
+
+    def bkgd(self, c, attr):
+        assert c == ' '
+        self.attr = attr
+
+
+@pytest.fixture
+def stdscr():
+    return FakeScreen()
+
+
+THEME = Theme.from_dct({
+    'colors': {'foreground': '#cccccc', 'background': '#333333'},
+    'tokenColors': [
+        {'scope': 'string', 'settings': {'foreground': '#009900'}},
+        {'scope': 'keyword', 'settings': {'background': '#000000'}},
+        {'scope': 'keyword', 'settings': {'fontStyle': 'bold'}},
+    ],
+})
+
+
+@pytest.fixture
+def syntax(make_grammars):
+    return Syntax(make_grammars(), THEME, ColorManager.make())
+
+
+def test_init_screen_low_color(stdscr, syntax):
+    with FakeCurses.patch(n_colors=16, can_change_color=False) as fake_curses:
+        syntax._init_screen(stdscr)
+    assert syntax.color_manager.colors == {
+        Color.parse('#cccccc'): -1,
+        Color.parse('#333333'): -1,
+        Color.parse('#000000'): -1,
+        Color.parse('#009900'): -1,
+    }
+    assert syntax.color_manager.raw_pairs == {(-1, -1): 1}
+    assert fake_curses.colors == {}
+    assert fake_curses.pairs == {1: (-1, -1)}
+    assert stdscr.attr == 1 << 8
+
+
+def test_init_screen_256_color(stdscr, syntax):
+    with FakeCurses.patch(n_colors=256, can_change_color=False) as fake_curses:
+        syntax._init_screen(stdscr)
+    assert syntax.color_manager.colors == {
+        Color.parse('#cccccc'): 252,
+        Color.parse('#333333'): 236,
+        Color.parse('#000000'): 16,
+        Color.parse('#009900'): 28,
+    }
+    assert syntax.color_manager.raw_pairs == {(252, 236): 1}
+    assert fake_curses.colors == {}
+    assert fake_curses.pairs == {1: (252, 236)}
+    assert stdscr.attr == 1 << 8
+
+
+def test_init_screen_true_color(stdscr, syntax):
+    with FakeCurses.patch(n_colors=256, can_change_color=True) as fake_curses:
+        syntax._init_screen(stdscr)
+    # weird colors happened with low color numbers so it counts down from max
+    assert syntax.color_manager.colors == {
+        Color.parse('#000000'): 255,
+        Color.parse('#009900'): 254,
+        Color.parse('#333333'): 253,
+        Color.parse('#cccccc'): 252,
+    }
+    assert syntax.color_manager.raw_pairs == {(252, 253): 1}
+    assert fake_curses.colors == {
+        255: (0, 0, 0),
+        254: (0, 600, 0),
+        253: (200, 200, 200),
+        252: (800, 800, 800),
+    }
+    assert fake_curses.pairs == {1: (252, 253)}
+    assert stdscr.attr == 1 << 8
+
+
+def test_lazily_instantiated_pairs(stdscr, syntax):
+    # pairs are assigned lazily to avoid hard upper limit (256) on pairs
+    with FakeCurses.patch(n_colors=256, can_change_color=False) as fake_curses:
+        syntax._init_screen(stdscr)
+
+        assert len(syntax.color_manager.raw_pairs) == 1
+        assert len(fake_curses.pairs) == 1
+
+        style = THEME.select(('string.python',))
+        attr = syntax.blank_file_highlighter().attr(style)
+        assert attr == 2 << 8
+
+        assert len(syntax.color_manager.raw_pairs) == 2
+        assert len(fake_curses.pairs) == 2
+
+
+def test_style_attributes_applied(stdscr, syntax):
+    with FakeCurses.patch(n_colors=256, can_change_color=False):
+        syntax._init_screen(stdscr)
+
+        style = THEME.select(('keyword.python',))
+        attr = syntax.blank_file_highlighter().attr(style)
+        assert attr == 2 << 8 | curses.A_BOLD
+
+
+def test_syntax_highlight_cache_first_line(stdscr, make_grammars):
+    with FakeCurses.patch(n_colors=256, can_change_color=False):
+        grammars = make_grammars({
+            'scopeName': 'source.demo',
+            'fileTypes': ['demo'],
+            'patterns': [{'match': r'\Aint', 'name': 'keyword'}],
+        })
+        syntax = Syntax(grammars, THEME, ColorManager.make())
+        syntax._init_screen(stdscr)
+        file_hl = syntax.file_highlighter('foo.demo', '')
+        file_hl.highlight_until(Buf(['int', 'int']), 2)
+        assert file_hl.regions == [
+            (HL(0, 3, curses.A_BOLD | 2 << 8),),
+            (),
+        ]

tests/reg_test.py

@@ -0,0 +1,74 @@
+import onigurumacffi
+import pytest
+
+from babi.reg import _Reg
+from babi.reg import _RegSet
+
+
+def test_reg_first_line():
+    reg = _Reg(r'\Ahello')
+    assert reg.match('hello', 0, first_line=True, boundary=True)
+    assert reg.search('hello', 0, first_line=True, boundary=True)
+    assert not reg.match('hello', 0, first_line=False, boundary=True)
+    assert not reg.search('hello', 0, first_line=False, boundary=True)
+
+
+def test_reg_boundary():
+    reg = _Reg(r'\Ghello')
+    assert reg.search('ohello', 1, first_line=True, boundary=True)
+    assert reg.match('ohello', 1, first_line=True, boundary=True)
+    assert not reg.search('ohello', 1, first_line=True, boundary=False)
+    assert not reg.match('ohello', 1, first_line=True, boundary=False)
+
+
+def test_reg_neither():
+    reg = _Reg(r'(\A|\G)hello')
+    assert not reg.search('hello', 0, first_line=False, boundary=False)
+    assert not reg.search('ohello', 1, first_line=False, boundary=False)
+
+
+def test_reg_other_escapes_left_untouched():
+    reg = _Reg(r'(^|\A|\G)\w\s\w')
+    assert reg.match('a b', 0, first_line=False, boundary=False)
+
+
+def test_reg_not_out_of_bounds_at_end():
+    # the only way this is triggerable is with an illegal regex, we'd rather
+    # produce an error about the regex being wrong than an IndexError
+    reg = _Reg('\\A\\')
+    with pytest.raises(onigurumacffi.OnigError) as excinfo:
+        reg.search('\\', 0, first_line=False, boundary=False)
+    msg, = excinfo.value.args
+    assert msg == 'end pattern at escape'
+
+
+def test_reg_repr():
+    assert repr(_Reg(r'\A123')) == r"_Reg('\\A123')"
+
+
+def test_regset_first_line():
+    regset = _RegSet(r'\Ahello', 'hello')
+    idx, _ = regset.search('hello', 0, first_line=True, boundary=True)
+    assert idx == 0
+    idx, _ = regset.search('hello', 0, first_line=False, boundary=True)
+    assert idx == 1
+
+
+def test_regset_boundary():
+    regset = _RegSet(r'\Ghello', 'hello')
+    idx, _ = regset.search('ohello', 1, first_line=True, boundary=True)
+    assert idx == 0
+    idx, _ = regset.search('ohello', 1, first_line=True, boundary=False)
+    assert idx == 1
+
+
+def test_regset_neither():
+    regset = _RegSet(r'\Ahello', r'\Ghello', 'hello')
+    idx, _ = regset.search('hello', 0, first_line=False, boundary=False)
+    assert idx == 2
+    idx, _ = regset.search('ohello', 1, first_line=False, boundary=False)
+    assert idx == 2
+
+
+def test_regset_repr():
+    assert repr(_RegSet('ohai', r'\Aworld')) == r"_RegSet('ohai', '\\Aworld')"

tests/textmate_demo_test.py

@@ -0,0 +1,84 @@
+import json
+
+import pytest
+
+from babi.textmate_demo import main
+
+THEME = {
+    'colors': {'foreground': '#ffffff', 'background': '#000000'},
+    'tokenColors': [
+        {'scope': 'bold', 'settings': {'fontStyle': 'bold'}},
+        {'scope': 'italic', 'settings': {'fontStyle': 'italic'}},
+        {'scope': 'underline', 'settings': {'fontStyle': 'underline'}},
+        {'scope': 'comment', 'settings': {'foreground': '#1e77d3'}},
+    ],
+}
+
+GRAMMAR = {
+    'scopeName': 'source.demo',
+    'fileTypes': ['demo'],
+    'patterns': [
+        {'match': r'\*[^*]*\*', 'name': 'bold'},
+        {'match': '/[^/]*/', 'name': 'italic'},
+        {'match': '_[^_]*_', 'name': 'underline'},
+        {'match': '#.*', 'name': 'comment'},
+    ],
+}
+
+
+@pytest.fixture
+def theme_grammars(tmpdir):
+    theme = tmpdir.join('config/theme.json').ensure()
+    theme.write(json.dumps(THEME))
+    grammars = tmpdir.join('grammar_v1').ensure_dir()
+    grammars.join('source.demo.json').write(json.dumps(GRAMMAR))
+    return theme, grammars
+
+
+def test_basic(theme_grammars, tmpdir, capsys):
+    theme, grammars = theme_grammars
+
+    f = tmpdir.join('f.demo')
+    f.write('*bold*/italic/_underline_# comment\n')
+
+    assert not main((
+        '--theme', str(theme), '--grammar-dir', str(grammars),
+        str(f),
+    ))
+
+    out, _ = capsys.readouterr()
+
+    assert out == (
+        '\x1b[48;2;0;0;0m\n'
+        '\x1b[38;2;255;255;255m\x1b[48;2;0;0;0m\x1b[1m'
+        '*bold*'
+        '\x1b[39m\x1b[49m\x1b[22m'
+        '\x1b[38;2;255;255;255m\x1b[48;2;0;0;0m\x1b[3m'
+        '/italic/'
+        '\x1b[39m\x1b[49m\x1b[23m'
+        '\x1b[38;2;255;255;255m\x1b[48;2;0;0;0m\x1b[4m'
+        '_underline_'
+        '\x1b[39m\x1b[49m\x1b[24m'
+        '\x1b[38;2;30;119;211m\x1b[48;2;0;0;0m'
+        '# comment'
+        '\x1b[39m\x1b[49m\x1b'
+        '[38;2;255;255;255m\x1b[48;2;0;0;0m\n\x1b[39m\x1b[49m'
+        '\x1b[m'
+    )
+
+
+def test_basic_with_blank_theme(theme_grammars, tmpdir, capsys):
+    theme, grammars = theme_grammars
+    theme.write('{}')
+
+    f = tmpdir.join('f.demo')
+    f.write('*bold*/italic/_underline_# comment\n')
+
+    assert not main((
+        '--theme', str(theme), '--grammar-dir', str(grammars),
+        str(f),
+    ))
+
+    out, _ = capsys.readouterr()
+
+    assert out == '*bold*/italic/_underline_# comment\n\x1b[m'

tests/theme_test.py

@@ -0,0 +1,110 @@
+import pytest
+
+from babi.color import Color
+from babi.theme import Theme
+
+THEME = Theme.from_dct({
+    'colors': {'foreground': '#100000', 'background': '#aaaaaa'},
+    'tokenColors': [
+        {'scope': 'foo.bar', 'settings': {'foreground': '#200000'}},
+        {'scope': 'foo', 'settings': {'foreground': '#300000'}},
+        {'scope': 'parent foo.bar', 'settings': {'foreground': '#400000'}},
+    ],
+})
+
+
+def unhex(color):
+    return f'#{hex(color.r << 16 | color.g << 8 | color.b)[2:]}'
+
+
+@pytest.mark.parametrize(
+    ('scope', 'expected'),
+    (
+        pytest.param(('',), '#100000', id='trivial'),
+        pytest.param(('unknown',), '#100000', id='unknown'),
+        pytest.param(('foo.bar',), '#200000', id='exact match'),
+        pytest.param(('foo.baz',), '#300000', id='prefix match'),
+        pytest.param(('src.diff', 'foo.bar'), '#200000', id='nested scope'),
+        pytest.param(
+            ('foo.bar', 'unrelated'), '#200000',
+            id='nested scope not last one',
+        ),
+    ),
+)
+def test_select(scope, expected):
+    ret = THEME.select(scope)
+    assert unhex(ret.fg) == expected
+
+
+def test_theme_default_settings_from_no_scope():
+    theme = Theme.from_dct({
+        'tokenColors': [
+            {'settings': {'foreground': '#cccccc', 'background': '#333333'}},
+        ],
+    })
+    assert theme.default.fg == Color.parse('#cccccc')
+    assert theme.default.bg == Color.parse('#333333')
+
+
+def test_theme_default_settings_from_empty_string_scope():
+    theme = Theme.from_dct({
+        'tokenColors': [
+            {
+                'scope': '',
+                'settings': {'foreground': '#cccccc', 'background': '#333333'},
+            },
+        ],
+    })
+    assert theme.default.fg == Color.parse('#cccccc')
+    assert theme.default.bg == Color.parse('#333333')
+
+
+def test_theme_scope_split_by_commas():
+    theme = Theme.from_dct({
+        'colors': {'foreground': '#cccccc', 'background': '#333333'},
+        'tokenColors': [
+            {'scope': 'a, b, c', 'settings': {'fontStyle': 'italic'}},
+        ],
+    })
+    assert theme.select(('d',)).i is False
+    assert theme.select(('a',)).i is True
+    assert theme.select(('b',)).i is True
+    assert theme.select(('c',)).i is True
+
+
+def test_theme_scope_comma_at_beginning_and_end():
+    theme = Theme.from_dct({
+        'colors': {'foreground': '#cccccc', 'background': '#333333'},
+        'tokenColors': [
+            {'scope': '\n,a,b,\n', 'settings': {'fontStyle': 'italic'}},
+        ],
+    })
+    assert theme.select(('d',)).i is False
+    assert theme.select(('a',)).i is True
+    assert theme.select(('b',)).i is True
+
+
+def test_theme_scope_internal_newline_commas():
+    # this is arguably malformed, but `cobalt2` in the wild has this issue
+    theme = Theme.from_dct({
+        'colors': {'foreground': '#cccccc', 'background': '#333333'},
+        'tokenColors': [
+            {'scope': '\n,a,\n,b,\n', 'settings': {'fontStyle': 'italic'}},
+        ],
+    })
+    assert theme.select(('d',)).i is False
+    assert theme.select(('a',)).i is True
+    assert theme.select(('b',)).i is True
+
+
+def test_theme_scope_as_A_list():
+    theme = Theme.from_dct({
+        'colors': {'foreground': '#cccccc', 'background': '#333333'},
+        'tokenColors': [
+            {'scope': ['a', 'b', 'c'], 'settings': {'fontStyle': 'underline'}},
+        ],
+    })
+    assert theme.select(('d',)).u is False
+    assert theme.select(('a',)).u is True
+    assert theme.select(('b',)).u is True
+    assert theme.select(('c',)).u is True

tests/user_data_test.py

@@ -0,0 +1,20 @@
+import os
+from unittest import mock
+
+from babi.user_data import xdg_data
+
+
+def test_when_xdg_data_home_is_set():
+    with mock.patch.dict(os.environ, {'XDG_DATA_HOME': '/foo'}):
+        ret = xdg_data('history', 'command')
+    assert ret == '/foo/babi/history/command'
+
+
+def test_when_xdg_data_home_is_not_set():
+    def fake_expanduser(s):
+        return s.replace('~', '/home/username')
+
+    with mock.patch.object(os.path, 'expanduser', fake_expanduser):
+        with mock.patch.dict(os.environ, clear=True):
+            ret = xdg_data('history')
+    assert ret == '/home/username/.local/share/babi/history'

tox.ini

@@ -13,3 +13,6 @@ commands =
 skip_install = true
 deps = pre-commit
 commands = pre-commit run --all-files --show-diff-on-failure
+
+[pep8]
+ignore = E265,E501,W504