From 166e2630c0db5764d067dbb69b6852a382fff4a2 Mon Sep 17 00:00:00 2001 From: Ed Page Date: Sat, 6 Jul 2019 21:54:43 -0600 Subject: [PATCH] fix(parse): Don't assume boundary characters are one byte This was inspired by heck. They have an invariant to ensure this isn't a problem (only accept `_` as boundary) while on the other hand we accept a lot of things as boundaries. --- src/tokens.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/tokens.rs b/src/tokens.rs index 5d7503e..7841f26 100644 --- a/src/tokens.rs +++ b/src/tokens.rs @@ -176,13 +176,12 @@ fn split_ident(ident: &str, offset: usize) -> impl Iterator> { while let Some((i, c)) = char_indices.next() { let cur_mode = WordMode::classify(c); if cur_mode == WordMode::Boundary { - if start == i { - start += 1; - } + assert!(start_mode == WordMode::Boundary); continue; } if start_mode == WordMode::Boundary { start_mode = cur_mode; + start = i; } if let Some(&(next_i, next)) = char_indices.peek() {