lexer: strip JS/CSS comments from code-bearing string literals at compile time

scan_string() is the right gate for this: every El source that embeds JS
or CSS does so as a quoted string literal, and the lexer is the single
chokepoint every backend reads. Strip there and the // line comments
and /* */ block comments never reach the parser, codegen, or the served
HTML.

looks_like_code is intentionally narrow:
  - contains "<script" or "<style" (the embedded-asset case), or
  - contains "function" AND ";" (a JS body without an opening tag)
Plain prose with stray // sequences passes through verbatim.

strip_code_comments tracks JS string state (single, double, backtick)
and never strips inside one. Backslash escapes inside JS strings consume
the next char verbatim. URL guard: when the char before / is ':', emit
the / literally and advance one — preserves https:// inside string
literals. Block-comment scan walks until the matching '*/' pair.

elc-cli.el is now a one-line `import "el-compiler/src/compiler.el"`
shim. Top-level `let _argv = args()` was clashing with C int main()'s
`char** _argv` parameter once compiler.el's fn main() body got folded
into C main. compiler.el owns the CLI entry point now.

Self-host fixed point reached: gen2 == gen3 byte-identical.
Tagged dist/platform/elc.20260502-1104-self-host alongside dist/platform/elc.
This commit is contained in:
Will Anderson
2026-05-02 11:14:18 -05:00
parent f5c457a823
commit 990ce72539
4 changed files with 217 additions and 8 deletions
BIN
View File
Binary file not shown.
BIN
View File
Binary file not shown.
+211 -1
View File
@@ -195,6 +195,208 @@ fn scan_ident(chars: [String], start: Int, total: Int) -> Map<String, Any> {
{ "text": text, "pos": i }
}
// Code-bearing string detection + comment strip
// Inline JS/CSS literals embedded in El source (e.g. <script></script> blobs
// or stylesheet payloads inside string literals) carry their own line and
// block comments. Those comments leak into the served HTML and reveal build
// notes the visitor should never see. We strip them at the lexer so every
// downstream consumer (codegen-c, codegen-js, parser) gets the cleaned form.
//
// looks_like_code heuristic gate so we only strip strings that actually
// embed JS or CSS. Plain prose, hex blobs, JSON, etc. pass through verbatim.
fn substr_at(chars: [String], start: Int, total: Int, needle: String) -> Bool {
let nchars: [String] = native_string_chars(needle)
let nlen: Int = native_list_len(nchars)
if start + nlen > total { return false }
let i = 0
let matched = true
while i < nlen {
let a: String = native_list_get(chars, start + i)
let b: String = native_list_get(nchars, i)
if a == b { let i = i + 1 } else { let matched = false; let i = nlen }
}
matched
}
fn str_has(s: String, needle: String) -> Bool {
let chars: [String] = native_string_chars(s)
let total: Int = native_list_len(chars)
let i = 0
let found = false
while i < total {
if substr_at(chars, i, total, needle) {
let found = true
let i = total
} else {
let i = i + 1
}
}
found
}
fn looks_like_code(s: String) -> Bool {
if str_has(s, "<script") { return true }
if str_has(s, "<style") { return true }
if str_has(s, "function") {
if str_has(s, ";") { return true }
}
false
}
// strip_code_comments character-by-character walk. Tracks JS string state
// (single, double, backtick) and never strips inside one. Backslash escapes
// inside JS strings consume the next char verbatim. URLs like https:// are
// preserved by checking the previous char before treating // as a line
// comment opener: if the char immediately before '/' is ':', emit the '/'
// literally and advance one position.
fn strip_code_comments(s: String) -> String {
let chars: [String] = native_string_chars(s)
let total: Int = native_list_len(chars)
let out = ""
let i = 0
let in_squote = false
let in_dquote = false
let in_btick = false
let prev = ""
while i < total {
let ch: String = native_list_get(chars, i)
let in_js_string = false
if in_squote { let in_js_string = true }
if in_dquote { let in_js_string = true }
if in_btick { let in_js_string = true }
if in_js_string {
// Backslash escape: consume next char verbatim regardless of which.
if ch == "\\" {
let out = out + ch
let next_i = i + 1
if next_i < total {
let nc: String = native_list_get(chars, next_i)
let out = out + nc
let prev = nc
let i = next_i + 1
} else {
let prev = ch
let i = next_i
}
} else {
if in_squote {
if ch == "'" { let in_squote = false }
} else {
if in_dquote {
if ch == "\"" { let in_dquote = false }
} else {
if in_btick {
if ch == "`" { let in_btick = false }
}
}
}
let out = out + ch
let prev = ch
let i = i + 1
}
} else {
// Not in a JS string. Check for comment openers.
let next_i = i + 1
let next_ch = ""
if next_i < total {
let next_ch: String = native_list_get(chars, next_i)
}
if ch == "/" {
if next_ch == "/" {
// URL guard: prev char ':' means this is "://", not a comment.
if prev == ":" {
let out = out + ch
let prev = ch
let i = i + 1
} else {
// Skip until newline (newline itself is preserved so
// surrounding line counts/structure stay sane).
let i = i + 2
let scanning = true
while scanning {
if i >= total {
let scanning = false
} else {
let lc: String = native_list_get(chars, i)
if lc == "\n" {
let scanning = false
} else {
let i = i + 1
}
}
}
let prev = ""
}
} else {
if next_ch == "*" {
// Skip until matching "*/".
let i = i + 2
let scanning2 = true
while scanning2 {
if i >= total {
let scanning2 = false
} else {
let bc: String = native_list_get(chars, i)
if bc == "*" {
let after = i + 1
if after < total {
let nc2: String = native_list_get(chars, after)
if nc2 == "/" {
let i = after + 1
let scanning2 = false
} else {
let i = i + 1
}
} else {
let i = i + 1
}
} else {
let i = i + 1
}
}
}
let prev = ""
} else {
let out = out + ch
let prev = ch
let i = i + 1
}
}
} else {
// Open a JS string?
if ch == "'" {
let in_squote = true
let out = out + ch
let prev = ch
let i = i + 1
} else {
if ch == "\"" {
let in_dquote = true
let out = out + ch
let prev = ch
let i = i + 1
} else {
if ch == "`" {
let in_btick = true
let out = out + ch
let prev = ch
let i = i + 1
} else {
let out = out + ch
let prev = ch
let i = i + 1
}
}
}
}
}
}
out
}
// scan_string scan a quoted string literal, handling \" escapes.
// Starts AFTER the opening quote. Returns { "text": content, "pos": i_after_close }
fn scan_string(chars: [String], start: Int, total: Int) -> Map<String, Any> {
@@ -305,7 +507,15 @@ fn lex(source: String) -> [Map<String, Any>] {
let result = scan_string(chars, i + 1, total)
let str_text: String = result["text"]
let new_pos: Int = result["pos"]
let tokens = native_list_append(tokens, make_tok("Str", str_text))
// Compile-time scrub: strings that embed JS or CSS get
// their // line comments and /* block comments stripped
// before the token reaches the parser. Plain prose passes
// through untouched.
let clean_text = str_text
if looks_like_code(str_text) {
let clean_text = strip_code_comments(str_text)
}
let tokens = native_list_append(tokens, make_tok("Str", clean_text))
let i = new_pos
} else {
// Number literal
+6 -7
View File
@@ -1,8 +1,7 @@
// elc-cli.el entry point for the self-hosted el compiler.
//
// All logic lives in el-compiler/src/compiler.el (which defines fn main()).
// Importing it pulls in compiler.el + parser + lexer + codegen transitively.
// The native compiler resolves imports textually and folds fn main() into
// C's int main(), so this file does not declare any top-level work itself.
import "el-compiler/src/compiler.el"
// compile() streams C source to stdout via println.
// We read source from args()[0] and compile it.
let _argv: [String] = args()
let src_path: String = native_list_get(_argv, 0)
let source: String = fs_read(src_path)
compile(source)