//! Lexer for el-ui component syntax. //! //! Produces a flat `Vec` from source text. //! The lexer is context-sensitive: it switches between "code mode" //! and "template mode" when it encounters the `template` keyword and `{` / `}`. use crate::error::{CompileError, CompileResult}; #[derive(Debug, Clone, PartialEq)] pub enum Token { // Keywords Component, Props, State, Fn, Template, If, Else, Return, // Identifiers and literals Ident(String), StringLit(String), IntLit(i64), FloatLit(f64), BoolLit(bool), // Punctuation LBrace, // { RBrace, // } LParen, // ( RParen, // ) LAngle, // < RAngle, // > LBracket, // [ RBracket, // ] Colon, // : Semicolon,// ; Comma, // , Dot, // . Eq, // = EqEq, // == Bang, // ! BangEq, // != Plus, // + Minus, // - Star, // * Slash, // / Arrow, // -> FatArrow, // => Ampersand,// & Pipe, // | AmpAmp, // && PipePipe, // || Question, // ? Hash, // # At, // @ // Template-specific SlashIdent(String), // /if /each /activate ColonIdent(String), // :else HashIdent(String), // #if #each #activate OnColon(String), // on:click on:input etc. SelfClose, // /> CloseTag(String), // // Raw text in templates RawText(String), Eof, } pub fn tokenize(source: &str) -> CompileResult> { let mut lexer = Lexer::new(source); lexer.run() } struct Lexer<'a> { src: &'a [u8], pos: usize, } impl<'a> Lexer<'a> { fn new(source: &'a str) -> Self { Self { src: source.as_bytes(), pos: 0 } } fn peek(&self) -> Option { self.src.get(self.pos).copied() } fn peek2(&self) -> Option { self.src.get(self.pos + 1).copied() } fn advance(&mut self) -> Option { let ch = self.src.get(self.pos).copied(); if ch.is_some() { self.pos += 1; } ch } fn skip_whitespace_and_comments(&mut self) { loop { // Skip whitespace while matches!(self.peek(), Some(b' ' | b'\t' | b'\n' | b'\r')) { self.advance(); } // Skip // line comments if self.peek() == Some(b'/') && self.peek2() == Some(b'/') { while self.peek().is_some() && self.peek() != Some(b'\n') { self.advance(); } continue; } break; } } fn read_ident(&mut self) -> String { let start = self.pos; while matches!(self.peek(), Some(b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'_')) { self.advance(); } String::from_utf8_lossy(&self.src[start..self.pos]).into_owned() } fn read_string(&mut self) -> CompileResult { // Opening quote already consumed let mut s = String::new(); loop { match self.advance() { None => return Err(CompileError::Lex { pos: self.pos, msg: "unterminated string".into() }), Some(b'"') => break, Some(b'\\') => { match self.advance() { Some(b'n') => s.push('\n'), Some(b't') => s.push('\t'), Some(b'r') => s.push('\r'), Some(b'"') => s.push('"'), Some(b'\\') => s.push('\\'), Some(b'0') => s.push('\0'), Some(c) => s.push(c as char), None => return Err(CompileError::Lex { pos: self.pos, msg: "unterminated escape".into() }), } } Some(c) => s.push(c as char), } } Ok(s) } fn read_number(&mut self, first: u8) -> Token { let mut s = String::new(); s.push(first as char); while matches!(self.peek(), Some(b'0'..=b'9' | b'_')) { let c = self.advance().unwrap(); if c != b'_' { s.push(c as char); } } if self.peek() == Some(b'.') && matches!(self.peek2(), Some(b'0'..=b'9')) { s.push('.'); self.advance(); while matches!(self.peek(), Some(b'0'..=b'9')) { s.push(self.advance().unwrap() as char); } Token::FloatLit(s.parse().unwrap_or(0.0)) } else { Token::IntLit(s.parse().unwrap_or(0)) } } /// Read a template block — everything between the outer `{` and matching `}` /// of `template { ... }`. Returns the raw text inside. fn read_template_inner(&mut self) -> CompileResult> { // We are positioned right after `template` keyword and the `{` that opened it. // We tokenize the template body using a template-aware mini-lexer. let mut toks: Vec = Vec::new(); let mut depth = 1i32; // we've consumed the opening { let mut text_buf = String::new(); macro_rules! flush_text { () => { if !text_buf.is_empty() { let t = text_buf.trim().to_owned(); if !t.is_empty() { toks.push(Token::RawText(t)); } text_buf.clear(); } }; } loop { match self.peek() { None => return Err(CompileError::Lex { pos: self.pos, msg: "unterminated template block".into() }), Some(b'{') => { self.advance(); // Could be interpolation {expr}, or {#if}, {/if}, {:else} // Peek at what follows // Skip whitespace inside while matches!(self.peek(), Some(b' ' | b'\t')) { self.advance(); } if self.peek() == Some(b'#') { // Block tag: {#if ...} {#each ...} {#activate ...} self.advance(); // consume # let kw = self.read_ident(); flush_text!(); toks.push(Token::HashIdent(kw)); // Read the rest up to } let mut inner = String::new(); let mut brace_d = 1i32; loop { match self.peek() { None => break, Some(b'{') => { brace_d += 1; inner.push('{'); self.advance(); } Some(b'}') => { brace_d -= 1; self.advance(); if brace_d == 0 { break; } inner.push('}'); } Some(c) => { inner.push(c as char); self.advance(); } } } toks.push(Token::RawText(inner.trim().to_owned())); } else if self.peek() == Some(b'/') { // Close tag: {/if} {/each} {/activate} self.advance(); // consume / let kw = self.read_ident(); flush_text!(); toks.push(Token::SlashIdent(kw)); while self.peek() == Some(b'}') { self.advance(); break; } } else if self.peek() == Some(b':') { // {:else} self.advance(); // consume : let kw = self.read_ident(); flush_text!(); toks.push(Token::ColonIdent(kw)); while self.peek() == Some(b'}') { self.advance(); break; } } else { // Regular interpolation or outer brace tracking // Check if this closes the template if depth == 1 && self.peek() == Some(b'}') { // Empty brace—skip self.advance(); let _ = depth - 1; // depth tracked by outer loop break; } // Read the expression until matching } let mut expr = String::new(); let mut brace_d = 1i32; loop { match self.peek() { None => break, Some(b'{') => { brace_d += 1; expr.push('{'); self.advance(); } Some(b'}') => { brace_d -= 1; self.advance(); if brace_d == 0 { break; } expr.push('}'); } Some(c) => { expr.push(c as char); self.advance(); } } } let expr = expr.trim().to_owned(); if !expr.is_empty() { flush_text!(); toks.push(Token::LBrace); toks.push(Token::RawText(expr)); toks.push(Token::RBrace); } } } Some(b'}') => { depth -= 1; self.advance(); if depth == 0 { flush_text!(); break; } text_buf.push('}'); } Some(b'<') => { // HTML element or close tag self.advance(); if self.peek() == Some(b'/') { // Close tag self.advance(); let tag = self.read_tag_name(); while self.peek() != Some(b'>') && self.peek().is_some() { self.advance(); } self.advance(); // consume > flush_text!(); toks.push(Token::CloseTag(tag)); } else { // Open tag let tag = self.read_tag_name(); flush_text!(); toks.push(Token::LAngle); toks.push(Token::Ident(tag)); // Read attributes self.read_attrs_into(&mut toks)?; } } Some(b'\n' | b'\r') => { self.advance(); text_buf.push(' '); } Some(c) => { text_buf.push(c as char); self.advance(); } } } Ok(toks) } fn read_tag_name(&mut self) -> String { while matches!(self.peek(), Some(b' ' | b'\t' | b'\n')) { self.advance(); } let start = self.pos; while matches!(self.peek(), Some(b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'-' | b'_')) { self.advance(); } String::from_utf8_lossy(&self.src[start..self.pos]).into_owned() } /// Read attributes until `>` or `/>`. fn read_attrs_into(&mut self, toks: &mut Vec) -> CompileResult<()> { loop { // Skip whitespace while matches!(self.peek(), Some(b' ' | b'\t' | b'\n' | b'\r')) { self.advance(); } match self.peek() { None => break, Some(b'/') if self.peek2() == Some(b'>') => { self.advance(); self.advance(); toks.push(Token::SelfClose); break; } Some(b'>') => { self.advance(); toks.push(Token::RAngle); break; } Some(b'o') if self.src.get(self.pos..self.pos+3) == Some(b"on:") => { // on:event={handler} self.pos += 3; // skip "on:" let event = self.read_tag_name(); // skip whitespace and = while matches!(self.peek(), Some(b' ' | b'=')) { self.advance(); } // read {expr} let expr = if self.peek() == Some(b'{') { self.advance(); self.read_until_brace_close()? } else { self.read_quoted_string()? }; toks.push(Token::OnColon(event)); toks.push(Token::RawText(expr)); } _ => { // Regular attribute: name="val" or name={expr} let name = self.read_attr_name(); if name.is_empty() { break; } // Skip whitespace and = while matches!(self.peek(), Some(b' ' | b'\t')) { self.advance(); } if self.peek() != Some(b'=') { // Boolean attribute with no value toks.push(Token::Ident(name)); continue; } self.advance(); // consume = while matches!(self.peek(), Some(b' ' | b'\t')) { self.advance(); } let value = if self.peek() == Some(b'"') { self.advance(); // consume " let s = self.read_string()?; toks.push(Token::Ident(name.clone())); toks.push(Token::Eq); toks.push(Token::StringLit(s)); continue; } else if self.peek() == Some(b'{') { self.advance(); self.read_until_brace_close()? } else { self.read_attr_name() }; toks.push(Token::Ident(name)); toks.push(Token::Eq); toks.push(Token::RawText(value)); } } } Ok(()) } fn read_attr_name(&mut self) -> String { let start = self.pos; while matches!(self.peek(), Some(b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'-' | b'_' | b':')) { self.advance(); } String::from_utf8_lossy(&self.src[start..self.pos]).into_owned() } fn read_quoted_string(&mut self) -> CompileResult { if self.peek() == Some(b'"') { self.advance(); self.read_string() } else { Ok(self.read_attr_name()) } } fn read_until_brace_close(&mut self) -> CompileResult { let mut s = String::new(); let mut depth = 1i32; loop { match self.peek() { None => return Err(CompileError::Lex { pos: self.pos, msg: "unterminated {".into() }), Some(b'{') => { depth += 1; s.push('{'); self.advance(); } Some(b'}') => { depth -= 1; self.advance(); if depth == 0 { break; } s.push('}'); } Some(c) => { s.push(c as char); self.advance(); } } } Ok(s) } fn run(&mut self) -> CompileResult> { let mut tokens: Vec = Vec::new(); loop { self.skip_whitespace_and_comments(); if self.peek().is_none() { tokens.push(Token::Eof); break; } let ch = self.advance().unwrap(); match ch { b'a'..=b'z' | b'A'..=b'Z' | b'_' => { let mut ident = String::new(); ident.push(ch as char); while matches!(self.peek(), Some(b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'_')) { ident.push(self.advance().unwrap() as char); } let tok = match ident.as_str() { "component" => Token::Component, "props" => Token::Props, "state" => Token::State, "fn" => Token::Fn, "template" => Token::Template, "if" => Token::If, "else" => Token::Else, "return" => Token::Return, "true" => Token::BoolLit(true), "false" => Token::BoolLit(false), other => Token::Ident(other.to_owned()), }; // Special handling: after `template`, read the block specially if tok == Token::Template { tokens.push(tok); self.skip_whitespace_and_comments(); if self.peek() == Some(b'{') { self.advance(); // consume { tokens.push(Token::LBrace); let inner = self.read_template_inner()?; tokens.extend(inner); tokens.push(Token::RBrace); } } else { tokens.push(tok); } } b'"' => { let s = self.read_string()?; tokens.push(Token::StringLit(s)); } b'0'..=b'9' => { let tok = self.read_number(ch); tokens.push(tok); } b'{' => tokens.push(Token::LBrace), b'}' => tokens.push(Token::RBrace), b'(' => tokens.push(Token::LParen), b')' => tokens.push(Token::RParen), b'[' => tokens.push(Token::LBracket), b']' => tokens.push(Token::RBracket), b':' => tokens.push(Token::Colon), b';' => tokens.push(Token::Semicolon), b',' => tokens.push(Token::Comma), b'.' => tokens.push(Token::Dot), b'=' => { if self.peek() == Some(b'=') { self.advance(); tokens.push(Token::EqEq); } else if self.peek() == Some(b'>') { self.advance(); tokens.push(Token::FatArrow); } else { tokens.push(Token::Eq); } } b'!' => { if self.peek() == Some(b'=') { self.advance(); tokens.push(Token::BangEq); } else { tokens.push(Token::Bang); } } b'+' => tokens.push(Token::Plus), b'-' => { if self.peek() == Some(b'>') { self.advance(); tokens.push(Token::Arrow); } else { tokens.push(Token::Minus); } } b'*' => tokens.push(Token::Star), b'/' => { if self.peek() == Some(b'/') { // Line comment (shouldn't reach here after skip, but guard) while self.peek().is_some() && self.peek() != Some(b'\n') { self.advance(); } } else { tokens.push(Token::Slash); } } b'&' => { if self.peek() == Some(b'&') { self.advance(); tokens.push(Token::AmpAmp); } else { tokens.push(Token::Ampersand); } } b'|' => { if self.peek() == Some(b'|') { self.advance(); tokens.push(Token::PipePipe); } else { tokens.push(Token::Pipe); } } b'?' => tokens.push(Token::Question), b'#' => tokens.push(Token::Hash), b'@' => tokens.push(Token::At), b'<' => tokens.push(Token::LAngle), b'>' => tokens.push(Token::RAngle), _ => { // Ignore unknown characters (whitespace already skipped) } } } Ok(tokens) } }