lexer: strip JS/CSS comments from code-bearing string literals at compile time
scan_string() is the right gate for this: every El source that embeds JS or CSS does so as a quoted string literal, and the lexer is the single chokepoint every backend reads. Strip there and the // line comments and /* */ block comments never reach the parser, codegen, or the served HTML. looks_like_code is intentionally narrow: - contains "<script" or "<style" (the embedded-asset case), or - contains "function" AND ";" (a JS body without an opening tag) Plain prose with stray // sequences passes through verbatim. strip_code_comments tracks JS string state (single, double, backtick) and never strips inside one. Backslash escapes inside JS strings consume the next char verbatim. URL guard: when the char before / is ':', emit the / literally and advance one — preserves https:// inside string literals. Block-comment scan walks until the matching '*/' pair. elc-cli.el is now a one-line `import "el-compiler/src/compiler.el"` shim. Top-level `let _argv = args()` was clashing with C int main()'s `char** _argv` parameter once compiler.el's fn main() body got folded into C main. compiler.el owns the CLI entry point now. Self-host fixed point reached: gen2 == gen3 byte-identical. Tagged dist/platform/elc.20260502-1104-self-host alongside dist/platform/elc.
This commit is contained in:
Vendored
BIN
Binary file not shown.
BIN
Binary file not shown.
+211
-1
@@ -195,6 +195,208 @@ fn scan_ident(chars: [String], start: Int, total: Int) -> Map<String, Any> {
|
||||
{ "text": text, "pos": i }
|
||||
}
|
||||
|
||||
// ── Code-bearing string detection + comment strip ────────────────────────────
|
||||
// Inline JS/CSS literals embedded in El source (e.g. <script>…</script> blobs
|
||||
// or stylesheet payloads inside string literals) carry their own line and
|
||||
// block comments. Those comments leak into the served HTML and reveal build
|
||||
// notes the visitor should never see. We strip them at the lexer so every
|
||||
// downstream consumer (codegen-c, codegen-js, parser) gets the cleaned form.
|
||||
//
|
||||
// looks_like_code — heuristic gate so we only strip strings that actually
|
||||
// embed JS or CSS. Plain prose, hex blobs, JSON, etc. pass through verbatim.
|
||||
|
||||
fn substr_at(chars: [String], start: Int, total: Int, needle: String) -> Bool {
|
||||
let nchars: [String] = native_string_chars(needle)
|
||||
let nlen: Int = native_list_len(nchars)
|
||||
if start + nlen > total { return false }
|
||||
let i = 0
|
||||
let matched = true
|
||||
while i < nlen {
|
||||
let a: String = native_list_get(chars, start + i)
|
||||
let b: String = native_list_get(nchars, i)
|
||||
if a == b { let i = i + 1 } else { let matched = false; let i = nlen }
|
||||
}
|
||||
matched
|
||||
}
|
||||
|
||||
fn str_has(s: String, needle: String) -> Bool {
|
||||
let chars: [String] = native_string_chars(s)
|
||||
let total: Int = native_list_len(chars)
|
||||
let i = 0
|
||||
let found = false
|
||||
while i < total {
|
||||
if substr_at(chars, i, total, needle) {
|
||||
let found = true
|
||||
let i = total
|
||||
} else {
|
||||
let i = i + 1
|
||||
}
|
||||
}
|
||||
found
|
||||
}
|
||||
|
||||
fn looks_like_code(s: String) -> Bool {
|
||||
if str_has(s, "<script") { return true }
|
||||
if str_has(s, "<style") { return true }
|
||||
if str_has(s, "function") {
|
||||
if str_has(s, ";") { return true }
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
// strip_code_comments — character-by-character walk. Tracks JS string state
|
||||
// (single, double, backtick) and never strips inside one. Backslash escapes
|
||||
// inside JS strings consume the next char verbatim. URLs like https:// are
|
||||
// preserved by checking the previous char before treating // as a line
|
||||
// comment opener: if the char immediately before '/' is ':', emit the '/'
|
||||
// literally and advance one position.
|
||||
fn strip_code_comments(s: String) -> String {
|
||||
let chars: [String] = native_string_chars(s)
|
||||
let total: Int = native_list_len(chars)
|
||||
let out = ""
|
||||
let i = 0
|
||||
let in_squote = false
|
||||
let in_dquote = false
|
||||
let in_btick = false
|
||||
let prev = ""
|
||||
while i < total {
|
||||
let ch: String = native_list_get(chars, i)
|
||||
let in_js_string = false
|
||||
if in_squote { let in_js_string = true }
|
||||
if in_dquote { let in_js_string = true }
|
||||
if in_btick { let in_js_string = true }
|
||||
|
||||
if in_js_string {
|
||||
// Backslash escape: consume next char verbatim regardless of which.
|
||||
if ch == "\\" {
|
||||
let out = out + ch
|
||||
let next_i = i + 1
|
||||
if next_i < total {
|
||||
let nc: String = native_list_get(chars, next_i)
|
||||
let out = out + nc
|
||||
let prev = nc
|
||||
let i = next_i + 1
|
||||
} else {
|
||||
let prev = ch
|
||||
let i = next_i
|
||||
}
|
||||
} else {
|
||||
if in_squote {
|
||||
if ch == "'" { let in_squote = false }
|
||||
} else {
|
||||
if in_dquote {
|
||||
if ch == "\"" { let in_dquote = false }
|
||||
} else {
|
||||
if in_btick {
|
||||
if ch == "`" { let in_btick = false }
|
||||
}
|
||||
}
|
||||
}
|
||||
let out = out + ch
|
||||
let prev = ch
|
||||
let i = i + 1
|
||||
}
|
||||
} else {
|
||||
// Not in a JS string. Check for comment openers.
|
||||
let next_i = i + 1
|
||||
let next_ch = ""
|
||||
if next_i < total {
|
||||
let next_ch: String = native_list_get(chars, next_i)
|
||||
}
|
||||
|
||||
if ch == "/" {
|
||||
if next_ch == "/" {
|
||||
// URL guard: prev char ':' means this is "://", not a comment.
|
||||
if prev == ":" {
|
||||
let out = out + ch
|
||||
let prev = ch
|
||||
let i = i + 1
|
||||
} else {
|
||||
// Skip until newline (newline itself is preserved so
|
||||
// surrounding line counts/structure stay sane).
|
||||
let i = i + 2
|
||||
let scanning = true
|
||||
while scanning {
|
||||
if i >= total {
|
||||
let scanning = false
|
||||
} else {
|
||||
let lc: String = native_list_get(chars, i)
|
||||
if lc == "\n" {
|
||||
let scanning = false
|
||||
} else {
|
||||
let i = i + 1
|
||||
}
|
||||
}
|
||||
}
|
||||
let prev = ""
|
||||
}
|
||||
} else {
|
||||
if next_ch == "*" {
|
||||
// Skip until matching "*/".
|
||||
let i = i + 2
|
||||
let scanning2 = true
|
||||
while scanning2 {
|
||||
if i >= total {
|
||||
let scanning2 = false
|
||||
} else {
|
||||
let bc: String = native_list_get(chars, i)
|
||||
if bc == "*" {
|
||||
let after = i + 1
|
||||
if after < total {
|
||||
let nc2: String = native_list_get(chars, after)
|
||||
if nc2 == "/" {
|
||||
let i = after + 1
|
||||
let scanning2 = false
|
||||
} else {
|
||||
let i = i + 1
|
||||
}
|
||||
} else {
|
||||
let i = i + 1
|
||||
}
|
||||
} else {
|
||||
let i = i + 1
|
||||
}
|
||||
}
|
||||
}
|
||||
let prev = ""
|
||||
} else {
|
||||
let out = out + ch
|
||||
let prev = ch
|
||||
let i = i + 1
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Open a JS string?
|
||||
if ch == "'" {
|
||||
let in_squote = true
|
||||
let out = out + ch
|
||||
let prev = ch
|
||||
let i = i + 1
|
||||
} else {
|
||||
if ch == "\"" {
|
||||
let in_dquote = true
|
||||
let out = out + ch
|
||||
let prev = ch
|
||||
let i = i + 1
|
||||
} else {
|
||||
if ch == "`" {
|
||||
let in_btick = true
|
||||
let out = out + ch
|
||||
let prev = ch
|
||||
let i = i + 1
|
||||
} else {
|
||||
let out = out + ch
|
||||
let prev = ch
|
||||
let i = i + 1
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
// scan_string — scan a quoted string literal, handling \" escapes.
|
||||
// Starts AFTER the opening quote. Returns { "text": content, "pos": i_after_close }
|
||||
fn scan_string(chars: [String], start: Int, total: Int) -> Map<String, Any> {
|
||||
@@ -305,7 +507,15 @@ fn lex(source: String) -> [Map<String, Any>] {
|
||||
let result = scan_string(chars, i + 1, total)
|
||||
let str_text: String = result["text"]
|
||||
let new_pos: Int = result["pos"]
|
||||
let tokens = native_list_append(tokens, make_tok("Str", str_text))
|
||||
// Compile-time scrub: strings that embed JS or CSS get
|
||||
// their // line comments and /* block comments stripped
|
||||
// before the token reaches the parser. Plain prose passes
|
||||
// through untouched.
|
||||
let clean_text = str_text
|
||||
if looks_like_code(str_text) {
|
||||
let clean_text = strip_code_comments(str_text)
|
||||
}
|
||||
let tokens = native_list_append(tokens, make_tok("Str", clean_text))
|
||||
let i = new_pos
|
||||
} else {
|
||||
// Number literal
|
||||
|
||||
+6
-7
@@ -1,8 +1,7 @@
|
||||
// elc-cli.el — entry point for the self-hosted el compiler.
|
||||
//
|
||||
// All logic lives in el-compiler/src/compiler.el (which defines fn main()).
|
||||
// Importing it pulls in compiler.el + parser + lexer + codegen transitively.
|
||||
// The native compiler resolves imports textually and folds fn main() into
|
||||
// C's int main(), so this file does not declare any top-level work itself.
|
||||
import "el-compiler/src/compiler.el"
|
||||
|
||||
// compile() streams C source to stdout via println.
|
||||
// We read source from args()[0] and compile it.
|
||||
let _argv: [String] = args()
|
||||
let src_path: String = native_list_get(_argv, 0)
|
||||
let source: String = fs_read(src_path)
|
||||
compile(source)
|
||||
|
||||
Reference in New Issue
Block a user