feat: engram-lang — new programming language, quantum-sealed prod target, spreading activation types

crates/el-lexer/src/token.rs

@@ -0,0 +1,201 @@
+//! Token definitions and span types.
+
+/// A span in the source file — byte offsets plus human-readable location.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct Span {
+    /// Byte offset of the first character of this token.
+    pub start: usize,
+    /// Byte offset one past the last character of this token.
+    pub end: usize,
+    /// 1-based line number.
+    pub line: u32,
+    /// 1-based column number (byte column within the line).
+    pub col: u32,
+}
+
+impl Span {
+    pub fn new(start: usize, end: usize, line: u32, col: u32) -> Self {
+        Self { start, end, line, col }
+    }
+
+    /// A zero-width span at the given position (used for EOF).
+    pub fn point(pos: usize, line: u32, col: u32) -> Self {
+        Self { start: pos, end: pos, line, col }
+    }
+}
+
+impl std::fmt::Display for Span {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}:{}", self.line, self.col)
+    }
+}
+
+/// A value annotated with its source location.
+#[derive(Debug, Clone, PartialEq)]
+pub struct Spanned<T> {
+    pub node: T,
+    pub span: Span,
+}
+
+impl<T> Spanned<T> {
+    pub fn new(node: T, span: Span) -> Self {
+        Self { node, span }
+    }
+}
+
+/// All tokens the Engram language lexer can produce.
+#[derive(Debug, Clone, PartialEq)]
+pub enum Token {
+    // ── Keywords ──────────────────────────────────────────────────────────────
+    /// `let`
+    Let,
+    /// `fn`
+    Fn,
+    /// `type`
+    Type,
+    /// `enum`
+    Enum,
+    /// `match`
+    Match,
+    /// `return`
+    Return,
+    /// `activate` — the spreading-activation query construct
+    Activate,
+    /// `where` — used in `activate T where "query"`
+    Where,
+    /// `sealed` — quantum-sealed block
+    Sealed,
+    /// `if`
+    If,
+    /// `else`
+    Else,
+    /// `for`
+    For,
+    /// `in`
+    In,
+    /// `true` / `false`
+    BoolLiteral(bool),
+
+    // ── Literals ──────────────────────────────────────────────────────────────
+    IntLiteral(i64),
+    FloatLiteral(f64),
+    /// String literal with escape sequences already resolved.
+    StringLiteral(String),
+
+    // ── Identifiers ───────────────────────────────────────────────────────────
+    Ident(String),
+
+    // ── Operators ─────────────────────────────────────────────────────────────
+    /// `+`
+    Plus,
+    /// `-`
+    Minus,
+    /// `*`
+    Star,
+    /// `/`
+    Slash,
+    /// `=`
+    Eq,
+    /// `==`
+    EqEq,
+    /// `!=`
+    NotEq,
+    /// `<`
+    Lt,
+    /// `>`
+    Gt,
+    /// `<=`
+    LtEq,
+    /// `>=`
+    GtEq,
+    /// `&&`
+    And,
+    /// `||`
+    Or,
+    /// `!`
+    Not,
+    /// `->` (function return type arrow)
+    Arrow,
+    /// `=>`  (match arm)
+    FatArrow,
+
+    // ── Delimiters ────────────────────────────────────────────────────────────
+    /// `(`
+    LParen,
+    /// `)`
+    RParen,
+    /// `{`
+    LBrace,
+    /// `}`
+    RBrace,
+    /// `[`
+    LBracket,
+    /// `]`
+    RBracket,
+    /// `,`
+    Comma,
+    /// `:`
+    Colon,
+    /// `::`
+    ColonColon,
+    /// `.`
+    Dot,
+    /// `;`
+    Semicolon,
+
+    // ── Special ───────────────────────────────────────────────────────────────
+    Eof,
+}
+
+impl std::fmt::Display for Token {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Token::Let => write!(f, "let"),
+            Token::Fn => write!(f, "fn"),
+            Token::Type => write!(f, "type"),
+            Token::Enum => write!(f, "enum"),
+            Token::Match => write!(f, "match"),
+            Token::Return => write!(f, "return"),
+            Token::Activate => write!(f, "activate"),
+            Token::Where => write!(f, "where"),
+            Token::Sealed => write!(f, "sealed"),
+            Token::If => write!(f, "if"),
+            Token::Else => write!(f, "else"),
+            Token::For => write!(f, "for"),
+            Token::In => write!(f, "in"),
+            Token::BoolLiteral(b) => write!(f, "{b}"),
+            Token::IntLiteral(n) => write!(f, "{n}"),
+            Token::FloatLiteral(n) => write!(f, "{n}"),
+            Token::StringLiteral(s) => write!(f, "\"{s}\""),
+            Token::Ident(s) => write!(f, "{s}"),
+            Token::Plus => write!(f, "+"),
+            Token::Minus => write!(f, "-"),
+            Token::Star => write!(f, "*"),
+            Token::Slash => write!(f, "/"),
+            Token::Eq => write!(f, "="),
+            Token::EqEq => write!(f, "=="),
+            Token::NotEq => write!(f, "!="),
+            Token::Lt => write!(f, "<"),
+            Token::Gt => write!(f, ">"),
+            Token::LtEq => write!(f, "<="),
+            Token::GtEq => write!(f, ">="),
+            Token::And => write!(f, "&&"),
+            Token::Or => write!(f, "||"),
+            Token::Not => write!(f, "!"),
+            Token::Arrow => write!(f, "->"),
+            Token::FatArrow => write!(f, "=>"),
+            Token::LParen => write!(f, "("),
+            Token::RParen => write!(f, ")"),
+            Token::LBrace => write!(f, "{{"),
+            Token::RBrace => write!(f, "}}"),
+            Token::LBracket => write!(f, "["),
+            Token::RBracket => write!(f, "]"),
+            Token::Comma => write!(f, ","),
+            Token::Colon => write!(f, ":"),
+            Token::ColonColon => write!(f, "::"),
+            Token::Dot => write!(f, "."),
+            Token::Semicolon => write!(f, ";"),
+            Token::Eof => write!(f, "<eof>"),
+        }
+    }
+}