v0.0.13

fix highlighting hang with empty begin end rules

README.md

@@ -12,7 +12,7 @@ a text editor, eventually...
 
 ### why is it called babi?
 
-I usually use the text editor `nano`, frequently I typo this.  on a qwerty
+I used to use the text editor `nano`, frequently I typo this.  on a qwerty
 keyboard, when the right hand is shifted left by one, `nano` becomes `babi`.
 
 ### quitting babi
@@ -87,7 +87,7 @@ this opens the file, displays it, and can be edited and can save! unknown keys
 are displayed as errors in the status bar.  babi will scroll if the cursor
 goes off screen either from resize events or from movement.  babi can edit
 multiple files.  babi has a command mode (so you can quit it like vim
-<kbd>:q</kbd>!).  babi also support syntax highlighting
+<kbd>:q</kbd>!).  babi also supports syntax highlighting
 
 ![](https://i.fluffy.cc/5WFZBJ4mWs7wtThD9strQnGlJqw4Z9KS.png)
 

azure-pipelines.yml

@@ -10,11 +10,11 @@ resources:
       type: github
       endpoint: github
       name: asottile/azure-pipeline-templates
-      ref: refs/tags/v1.0.0
+      ref: refs/tags/v2.0.0
 
 jobs:
 - template: job--pre-commit.yml@asottile
 - template: job--python-tox.yml@asottile
   parameters:
-    toxenvs: [py36, py37, py38]
+    toxenvs: [pypy3, py36, py37, py38, py39]
     os: linux

babi/buf.py

@@ -19,11 +19,11 @@ DelCallback = Callable[['Buf', int, str], None]
 InsCallback = Callable[['Buf', int], None]
 
 
-def _offsets(s: str) -> Tuple[int, ...]:
+def _offsets(s: str, tab_size: int) -> Tuple[int, ...]:
     ret = [0]
     for c in s:
         if c == '\t':
-            ret.append(ret[-1] + (4 - ret[-1] % 4))
+            ret.append(ret[-1] + (tab_size - ret[-1] % tab_size))
         else:
             ret.append(ret[-1] + wcwidth(c))
     return tuple(ret)
@@ -57,8 +57,9 @@ class DelModification(NamedTuple):
 
 
 class Buf:
-    def __init__(self, lines: List[str]) -> None:
+    def __init__(self, lines: List[str], tab_size: int = 4) -> None:
         self._lines = lines
+        self.tab_size = tab_size
         self.file_y = self.y = self._x = self._x_hint = 0
 
         self._set_callbacks: List[SetCallback] = [self._set_cb]
@@ -136,6 +137,10 @@ class Buf:
         if self[-1] != '':
             self.append('')
 
+    def set_tab_size(self, tab_size: int) -> None:
+        self.tab_size = tab_size
+        self._positions = [None]
+
     # event handling
 
     def add_set_callback(self, cb: SetCallback) -> None:
@@ -219,7 +224,8 @@ class Buf:
         self._extend_positions(idx)
         value = self._positions[idx]
         if value is None:
-            value = self._positions[idx] = _offsets(self._lines[idx])
+            value = _offsets(self._lines[idx], self.tab_size)
+            self._positions[idx] = value
         return value
 
     def line_x(self, margin: Margin) -> int:
@@ -238,7 +244,8 @@ class Buf:
 
     def rendered_line(self, idx: int, margin: Margin) -> str:
         x = self._cursor_x if idx == self.y else 0
-        return scrolled_line(self._lines[idx].expandtabs(4), x, margin.cols)
+        expanded = self._lines[idx].expandtabs(self.tab_size)
+        return scrolled_line(expanded, x, margin.cols)
 
     # movement
 

babi/file.py

@@ -244,7 +244,7 @@ class File:
                     status.update('(new file)')
             lines, self.nl, mixed, self.sha256 = get_lines(io.StringIO(''))
 
-        self.buf = Buf(lines)
+        self.buf = Buf(lines, self.buf.tab_size)
 
         if mixed:
             status.update(f'mixed newlines will be converted to {self.nl!r}')
@@ -523,16 +523,16 @@ class File:
         (s_y, _), (e_y, _) = self.selection.get()
         for l_y in range(s_y, e_y + 1):
             if self.buf[l_y]:
-                self.buf[l_y] = ' ' * 4 + self.buf[l_y]
+                self.buf[l_y] = ' ' * self.buf.tab_size + self.buf[l_y]
                 if l_y == self.buf.y:
-                    self.buf.x += 4
+                    self.buf.x += self.buf.tab_size
                 if l_y == sel_y and sel_x != 0:
-                    sel_x += 4
+                    sel_x += self.buf.tab_size
         self.selection.set(sel_y, sel_x, self.buf.y, self.buf.x)
 
     @edit_action('insert tab', final=False)
     def _tab(self, margin: Margin) -> None:
-        n = 4 - self.buf.x % 4
+        n = self.buf.tab_size - self.buf.x % self.buf.tab_size
         line = self.buf[self.buf.y]
         self.buf[self.buf.y] = line[:self.buf.x] + n * ' ' + line[self.buf.x:]
         self.buf.x += n
@@ -544,9 +544,8 @@ class File:
         else:
             self._tab(margin)
 
-    @staticmethod
-    def _dedent_line(s: str) -> int:
-        bound = min(len(s), 4)
+    def _dedent_line(self, s: str) -> int:
+        bound = min(len(s), self.buf.tab_size)
         i = 0
         while i < bound and s[i] == ' ':
             i += 1
@@ -641,9 +640,9 @@ class File:
         self.buf[self.buf.y] += self.buf.pop(self.buf.y + 1)
         self.buf.restore_eof_invariant()
 
-    def _sort(self, margin: Margin, s_y: int, e_y: int) -> None:
+    def _sort(self, margin: Margin, s_y: int, e_y: int, reverse: bool) -> None:
         # self.buf intentionally does not support slicing so we use islice
-        lines = sorted(itertools.islice(self.buf, s_y, e_y))
+        lines = sorted(itertools.islice(self.buf, s_y, e_y), reverse=reverse)
         for i, line in zip(range(s_y, e_y), lines):
             self.buf[i] = line
 
@@ -652,17 +651,17 @@ class File:
         self.buf.scroll_screen_if_needed(margin)
 
     @edit_action('sort', final=True)
-    def sort(self, margin: Margin) -> None:
-        self._sort(margin, 0, len(self.buf) - 1)
+    def sort(self, margin: Margin, reverse: bool = False) -> None:
+        self._sort(margin, 0, len(self.buf) - 1, reverse=reverse)
 
     @edit_action('sort selection', final=True)
     @clear_selection
-    def sort_selection(self, margin: Margin) -> None:
+    def sort_selection(self, margin: Margin, reverse: bool = False) -> None:
         (s_y, _), (e_y, _) = self.selection.get()
         e_y = min(e_y + 1, len(self.buf) - 1)
         if self.buf[e_y - 1] == '':
             e_y -= 1
-        self._sort(margin, s_y, e_y)
+        self._sort(margin, s_y, e_y, reverse=reverse)
 
     DISPATCH = {
         # movement

babi/highlight.py

@@ -273,6 +273,7 @@ class CompiledRegsetRule(CompiledRule, Protocol):
 class Entry(NamedTuple):
     scope: Tuple[str, ...]
     rule: CompiledRule
+    start: Tuple[str, int]
     reg: _Reg = ERR_REG
     boundary: bool = False
 
@@ -284,7 +285,7 @@ def _inner_capture_parse(
         scope: Scope,
         rule: CompiledRule,
 ) -> Regions:
-    state = State.root(Entry(scope + rule.name, rule))
+    state = State.root(Entry(scope + rule.name, rule, (s, 0)))
     _, regions = highlight_line(compiler, state, s, first_line=False)
     return tuple(
         r._replace(start=r.start + start, end=r.end + start) for r in regions
@@ -440,7 +441,8 @@ class EndRule(NamedTuple):
 
         boundary = match.end() == len(match.string)
         reg = make_reg(expand_escaped(match, self.end))
-        state = state.push(Entry(next_scope, self, reg, boundary))
+        start = (match.string, match.start())
+        state = state.push(Entry(next_scope, self, start, reg, boundary))
         regions = _captures(compiler, scope, match, self.begin_captures)
         return state, True, regions
 
@@ -455,7 +457,16 @@ class EndRule(NamedTuple):
         if m.start() > pos:
             ret.append(Region(pos, m.start(), state.cur.scope))
         ret.extend(_captures(compiler, state.cur.scope, m, self.end_captures))
-        return state.pop(), m.end(), False, tuple(ret)
+        # this is probably a bug in the grammar, but it pushed and popped at
+        # the same position.
+        # we'll advance the highlighter by one position to get past the loop
+        # this appears to be what vs code does as well
+        if state.entries[-1].start == (m.string, m.end()):
+            ret.append(Region(m.end(), m.end() + 1, state.cur.scope))
+            end = m.end() + 1
+        else:
+            end = m.end()
+        return state.pop(), end, False, tuple(ret)
 
     def search(
             self,
@@ -501,7 +512,9 @@ class WhileRule(NamedTuple):
 
         boundary = match.end() == len(match.string)
         reg = make_reg(expand_escaped(match, self.while_))
-        state = state.push_while(self, Entry(next_scope, self, reg, boundary))
+        start = (match.string, match.start())
+        entry = Entry(next_scope, self, start, reg, boundary)
+        state = state.push_while(self, entry)
         regions = _captures(compiler, scope, match, self.begin_captures)
         return state, True, regions
 
@@ -541,7 +554,7 @@ class Compiler:
         self._rule_to_grammar: Dict[_Rule, Grammar] = {}
         self._c_rules: Dict[_Rule, CompiledRule] = {}
         root = self._compile_root(grammar)
-        self.root_state = State.root(Entry(root.name, root))
+        self.root_state = State.root(Entry(root.name, root, ('', 0)))
 
     def _visit_rule(self, grammar: Grammar, rule: _Rule) -> _Rule:
         self._rule_to_grammar[rule] = grammar

babi/main.py

@@ -2,6 +2,7 @@ import argparse
 import curses
 import os
 import re
+import signal
 import sys
 from typing import List
 from typing import Optional
@@ -138,6 +139,11 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
     else:
         stdin = ''
 
+    # ignore backgrounding signals, we'll handle those in curses
+    # fixes a problem with ^Z on termination which would break the terminal
+    if sys.platform != 'win32':  # pragma: win32 no cover  # pragma: no branch
+        signal.signal(signal.SIGTSTP, signal.SIG_IGN)
+
     with perf_log(args.perf_log) as perf, make_stdscr() as stdscr:
         if args.key_debug:
             return _key_debug(stdscr, perf)

babi/reg.py

@@ -6,80 +6,36 @@ from typing import Tuple
 
 import onigurumacffi
 
-from babi.cached_property import cached_property
-
 _BACKREF_RE = re.compile(r'((?<!\\)(?:\\\\)*)\\([0-9]+)')
 
 
-def _replace_esc(s: str, chars: str) -> str:
-    """replace the given escape sequences of `chars` with \\uffff"""
-    for c in chars:
-        if f'\\{c}' in s:
-            break
-    else:
-        return s
-
-    b = []
-    i = 0
-    length = len(s)
-    while i < length:
-        try:
-            sbi = s.index('\\', i)
-        except ValueError:
-            b.append(s[i:])
-            break
-        if sbi > i:
-            b.append(s[i:sbi])
-        b.append('\\')
-        i = sbi + 1
-        if i < length:
-            if s[i] in chars:
-                b.append('\uffff')
-            else:
-                b.append(s[i])
-        i += 1
-    return ''.join(b)
+_FLAGS = {
+    # (first_line, boundary)
+    (False, False): (
+        onigurumacffi.OnigSearchOption.NOT_END_STRING |
+        onigurumacffi.OnigSearchOption.NOT_BEGIN_STRING |
+        onigurumacffi.OnigSearchOption.NOT_BEGIN_POSITION
+    ),
+    (False, True): (
+        onigurumacffi.OnigSearchOption.NOT_END_STRING |
+        onigurumacffi.OnigSearchOption.NOT_BEGIN_STRING
+    ),
+    (True, False): (
+        onigurumacffi.OnigSearchOption.NOT_END_STRING |
+        onigurumacffi.OnigSearchOption.NOT_BEGIN_POSITION
+    ),
+    (True, True): onigurumacffi.OnigSearchOption.NOT_END_STRING,
+}
 
 
 class _Reg:
     def __init__(self, s: str) -> None:
         self._pattern = s
+        self._reg = onigurumacffi.compile(self._pattern)
 
     def __repr__(self) -> str:
         return f'{type(self).__name__}({self._pattern!r})'
 
-    @cached_property
-    def _reg(self) -> onigurumacffi._Pattern:
-        return onigurumacffi.compile(_replace_esc(self._pattern, 'z'))
-
-    @cached_property
-    def _reg_no_A(self) -> onigurumacffi._Pattern:
-        return onigurumacffi.compile(_replace_esc(self._pattern, 'Az'))
-
-    @cached_property
-    def _reg_no_G(self) -> onigurumacffi._Pattern:
-        return onigurumacffi.compile(_replace_esc(self._pattern, 'Gz'))
-
-    @cached_property
-    def _reg_no_A_no_G(self) -> onigurumacffi._Pattern:
-        return onigurumacffi.compile(_replace_esc(self._pattern, 'AGz'))
-
-    def _get_reg(
-            self,
-            first_line: bool,
-            boundary: bool,
-    ) -> onigurumacffi._Pattern:
-        if boundary:
-            if first_line:
-                return self._reg
-            else:
-                return self._reg_no_A
-        else:
-            if first_line:
-                return self._reg_no_G
-            else:
-                return self._reg_no_A_no_G
-
     def search(
             self,
             line: str,
@@ -87,7 +43,7 @@ class _Reg:
             first_line: bool,
             boundary: bool,
     ) -> Optional[Match[str]]:
-        return self._get_reg(first_line, boundary).search(line, pos)
+        return self._reg.search(line, pos, flags=_FLAGS[first_line, boundary])
 
     def match(
             self,
@@ -96,36 +52,18 @@ class _Reg:
             first_line: bool,
             boundary: bool,
     ) -> Optional[Match[str]]:
-        return self._get_reg(first_line, boundary).match(line, pos)
+        return self._reg.match(line, pos, flags=_FLAGS[first_line, boundary])
 
 
 class _RegSet:
     def __init__(self, *s: str) -> None:
         self._patterns = s
+        self._set = onigurumacffi.compile_regset(*self._patterns)
 
     def __repr__(self) -> str:
         args = ', '.join(repr(s) for s in self._patterns)
         return f'{type(self).__name__}({args})'
 
-    @cached_property
-    def _set(self) -> onigurumacffi._RegSet:
-        return onigurumacffi.compile_regset(*self._patterns)
-
-    @cached_property
-    def _set_no_A(self) -> onigurumacffi._RegSet:
-        patterns = (_replace_esc(p, 'A') for p in self._patterns)
-        return onigurumacffi.compile_regset(*patterns)
-
-    @cached_property
-    def _set_no_G(self) -> onigurumacffi._RegSet:
-        patterns = (_replace_esc(p, 'G') for p in self._patterns)
-        return onigurumacffi.compile_regset(*patterns)
-
-    @cached_property
-    def _set_no_A_no_G(self) -> onigurumacffi._RegSet:
-        patterns = (_replace_esc(p, 'AG') for p in self._patterns)
-        return onigurumacffi.compile_regset(*patterns)
-
     def search(
             self,
             line: str,
@@ -133,16 +71,7 @@ class _RegSet:
             first_line: bool,
             boundary: bool,
     ) -> Tuple[int, Optional[Match[str]]]:
-        if boundary:
-            if first_line:
-                return self._set.search(line, pos)
-            else:
-                return self._set_no_A.search(line, pos)
-        else:
-            if first_line:
-                return self._set_no_G.search(line, pos)
-            else:
-                return self._set_no_A_no_G.search(line, pos)
+        return self._set.search(line, pos, flags=_FLAGS[first_line, boundary])
 
 
 def expand_escaped(match: Match[str], s: str) -> str:
@@ -151,4 +80,4 @@ def expand_escaped(match: Match[str], s: str) -> str:
 
 make_reg = functools.lru_cache(maxsize=None)(_Reg)
 make_regset = functools.lru_cache(maxsize=None)(_RegSet)
-ERR_REG = make_reg(')this pattern always triggers an error when used(')
+ERR_REG = make_reg('$ ^')

babi/screen.py

@@ -226,7 +226,10 @@ class Screen:
         if self._buffered_input is not None:
             wch, self._buffered_input = self._buffered_input, None
         else:
-            wch = self.stdscr.get_wch()
+            try:
+                wch = self.stdscr.get_wch()
+            except curses.error:  # pragma: no cover (macos bug?)
+                wch = self.stdscr.get_wch()
         if isinstance(wch, str) and wch == '\x1b':
             wch = self._get_sequence(wch)
             if len(wch) == 2:
@@ -418,7 +421,9 @@ class Screen:
 
     def command(self) -> Optional[EditResult]:
         response = self.prompt('', history='command')
-        if response == ':q':
+        if response is PromptResult.CANCELLED:
+            pass
+        elif response == ':q':
             return self.quit_save_modified()
         elif response == ':q!':
             return EditResult.EXIT
@@ -433,7 +438,26 @@ class Screen:
             else:
                 self.file.sort(self.margin)
             self.status.update('sorted!')
-        elif response is not PromptResult.CANCELLED:
+        elif response == ':sort!':
+            if self.file.selection.start:
+                self.file.sort_selection(self.margin, reverse=True)
+            else:
+                self.file.sort(self.margin, reverse=True)
+            self.status.update('sorted!')
+        elif response.startswith((':tabstop ', ':tabsize ')):
+            _, _, tab_size = response.partition(' ')
+            try:
+                parsed_tab_size = int(tab_size)
+            except ValueError:
+                self.status.update(f'invalid size: {tab_size}')
+            else:
+                if parsed_tab_size <= 0:
+                    self.status.update(f'invalid size: {parsed_tab_size}')
+                else:
+                    for file in self.files:
+                        file.buf.set_tab_size(parsed_tab_size)
+                    self.status.update('updated!')
+        else:
             self.status.update(f'invalid command: {response}')
         return None
 

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = babi
-version = 0.0.9
+version = 0.0.13
 description = a text editor
 long_description = file: README.md
 long_description_content_type = text/markdown
@@ -24,7 +24,7 @@ packages = find:
 install_requires =
     babi-grammars
     identify
-    onigurumacffi>=0.0.10
+    onigurumacffi>=0.0.18
     importlib_metadata>=1;python_version<"3.8"
     windows-curses;sys_platform=="win32"
 python_requires = >=3.6.1

tests/features/conftest.py

@@ -391,6 +391,7 @@ class DeferredRunner:
 
     _curses_cbreak = _curses_endwin = _curses_noecho = _curses__noop
     _curses_nonl = _curses_raw = _curses_use_default_colors = _curses__noop
+    _curses_set_escdelay = _curses__noop
 
     _curses_error = curses.error  # so we don't mock the exception
 

tests/features/sort_test.py

@@ -21,6 +21,16 @@ def test_sort_entire_file(run, unsorted):
     assert unsorted.read() == 'a\nb\nc\nd\n'
 
 
+def test_reverse_sort_entire_file(run, unsorted):
+    with run(str(unsorted)) as h, and_exit(h):
+        trigger_command_mode(h)
+        h.press_and_enter(':sort!')
+        h.await_text('sorted!')
+        h.await_cursor_position(x=0, y=1)
+        h.press('^S')
+    assert unsorted.read() == 'd\nc\nb\na\n'
+
+
 def test_sort_selection(run, unsorted):
     with run(str(unsorted)) as h, and_exit(h):
         h.press('S-Down')
@@ -32,6 +42,18 @@ def test_sort_selection(run, unsorted):
     assert unsorted.read() == 'b\nd\nc\na\n'
 
 
+def test_reverse_sort_selection(run, unsorted):
+    with run(str(unsorted)) as h, and_exit(h):
+        h.press('Down')
+        h.press('S-Down')
+        trigger_command_mode(h)
+        h.press_and_enter(':sort!')
+        h.await_text('sorted!')
+        h.await_cursor_position(x=0, y=2)
+        h.press('^S')
+    assert unsorted.read() == 'd\nc\nb\na\n'
+
+
 def test_sort_selection_does_not_include_eof(run, unsorted):
     with run(str(unsorted)) as h, and_exit(h):
         for _ in range(5):

tests/features/tabsize_test.py

@@ -0,0 +1,30 @@
+import pytest
+
+from testing.runner import and_exit
+from testing.runner import trigger_command_mode
+
+
+@pytest.mark.parametrize('setting', ('tabsize', 'tabstop'))
+def test_set_tabstop(run, setting):
+    with run() as h, and_exit(h):
+        h.press('a')
+        h.press('Left')
+        trigger_command_mode(h)
+        h.press_and_enter(f':{setting} 2')
+        h.await_text('updated!')
+        h.press('Tab')
+        h.await_text('\n  a')
+        h.await_cursor_position(x=2, y=1)
+
+
+@pytest.mark.parametrize('tabsize', ('-1', '0', 'wat'))
+def test_set_invalid_tabstop(run, tabsize):
+    with run() as h, and_exit(h):
+        h.press('a')
+        h.press('Left')
+        trigger_command_mode(h)
+        h.press_and_enter(f':tabstop {tabsize}')
+        h.await_text(f'invalid size: {tabsize}')
+        h.press('Tab')
+        h.await_text('    a')
+        h.await_cursor_position(x=4, y=1)

tests/highlight_test.py

@@ -637,3 +637,25 @@ def test_backslash_z(compiler_state):
     assert regions2 == (
         Region(0, 6, ('test', 'comment')),
     )
+
+
+def test_buggy_begin_end_grammar(compiler_state):
+    # before this would result in an infinite loop of start / end
+    compiler, state = compiler_state({
+        'scopeName': 'test',
+        'patterns': [
+            {
+                'begin': '(?=</style)',
+                'end': '(?=</style)',
+                'name': 'css',
+            },
+        ],
+    })
+
+    state, regions = highlight_line(compiler, state, 'test </style', True)
+
+    assert regions == (
+        Region(0, 5, ('test',)),
+        Region(5, 6, ('test', 'css')),
+        Region(6, 12, ('test',)),
+    )

tests/reg_test.py

@@ -35,9 +35,8 @@ def test_reg_other_escapes_left_untouched():
 def test_reg_not_out_of_bounds_at_end():
     # the only way this is triggerable is with an illegal regex, we'd rather
     # produce an error about the regex being wrong than an IndexError
-    reg = _Reg('\\A\\')
     with pytest.raises(onigurumacffi.OnigError) as excinfo:
-        reg.search('\\', 0, first_line=False, boundary=False)
+        _Reg('\\A\\')
     msg, = excinfo.value.args
     assert msg == 'end pattern at escape'