slow.monster

STUDIO HAZE · topaz

Topaz source

Open by house policy. Topaz is our own language.

Topaz 5.21 · 50 files

Compiler sources from the Topaz 5.21.0 release. Compatibility code for earlier language modes is retained.

Publication edits are limited to explanatory comments. Executable code is unchanged.
Rights and source notices · Source manifest

compiler/crates/topaz_self_frontend/topaz/literal.tpz

Topaz · 14052 B · Published file

1// Source literal grammar and parser-close cooking. Coordinates are UTF-8 bytes.
2import types { SyntaxNode, PreviewAstAttribute }
4function byteAt(bytes: Bytes, at: int) -> int { bytes.get(at) ?? -1 }
5function slice(bytes: Bytes, lo: int, hi: int) -> string {
6 match bytes.slice(lo, hi).decodeUtf8() { case Ok(text) => text; case Err(_) => "" }
7}
8function digit(value: int) -> int {
9 if value >= 48 && value <= 57 { value - 48 }
10 else if value >= 65 && value <= 70 { value - 65 + 10 }
11 else if value >= 97 && value <= 102 { value - 97 + 10 }
12 else { -1 }
14export function validDigitRun(bytes: Bytes, lo: int, hi: int, radix: int) -> bool {
15 if lo == hi { return false }
16 let mut at = lo
17 while at < hi {
18 let b = byteAt(bytes, at)
19 let d = digit(b)
20 if d < 0 || d >= radix {
21 if b != 95 || at == lo || at + 1 == hi { return false }
22 let before = digit(byteAt(bytes, at - 1))
23 let after = digit(byteAt(bytes, at + 1))
24 if before < 0 || before >= radix || after < 0 || after >= radix { return false }
25 }
26 at += 1
27 }
28 true
30export function integerRadix(spelling: string) -> int {
31 if spelling.startsWith("0x") || spelling.startsWith("0X") { 16 }
32 else if spelling.startsWith("0b") || spelling.startsWith("0B") { 2 }
33 else { 10 }
35export function validIntegerLiteral(spelling: string) -> bool {
36 let bytes = spelling.encodeUtf8()
37 let radix = integerRadix(spelling)
38 validDigitRun(bytes, if radix == 10 { 0 } else { 2 }, bytes.length(), radix)
40export function validDecimalLiteral(spelling: string) -> bool {
41 let parts = spelling.split(".")
42 if parts.length == 0 || parts.length > 2 { return false }
43 for part in parts {
44 let bytes = part.encodeUtf8()
45 if !validDigitRun(bytes, 0, bytes.length(), 10) { return false }
46 }
47 true
49export function integerMagnitude(spelling: string) -> Option<BigInt> {
50 if !validIntegerLiteral(spelling) { return None }
51 let bytes = spelling.encodeUtf8()
52 let radix = integerRadix(spelling)
53 let mut at = if radix == 10 { 0 } else { 2 }
54 let mut value = match BigInt.parse("0", 10) { case Some(value) => value; case None => return None }
55 let base = match BigInt.parse("{radix}", 10) { case Some(value) => value; case None => return None }
56 while at < bytes.length() {
57 if byteAt(bytes, at) != 95 {
58 let d = digit(byteAt(bytes, at))
59 let next = match BigInt.parse("{d}", 10) { case Some(value) => value; case None => return None }
60 value = value * base + next
61 }
62 at += 1
63 }
64 Some(value)
66export function decodeIntegerLiteral(spelling: string, negative: bool) -> Option<int> {
67 let value = match integerMagnitude(spelling) { case Some(value) => value; case None => return None }
68 if negative { (-value).toInt() } else { value.toInt() }
70export function decodeDurationMilliseconds(spelling: string) -> Option<string> {
71 let bytes = spelling.encodeUtf8()
72 let unitSize = if spelling.endsWith("ms") { 2 } else { 1 }
73 if bytes.length() <= unitSize { return None }
74 let unit = slice(bytes, bytes.length() - unitSize, bytes.length())
75 let magnitude = slice(bytes, 0, bytes.length() - unitSize)
76 if unit != "ms" && unit != "s" && unit != "m" { return None }
77 if integerRadix(magnitude) != 10 || !validIntegerLiteral(magnitude) { return None }
78 let value = match integerMagnitude(magnitude) { case Some(value) => value; case None => return None }
79 let factor = match BigInt.parse(if unit == "ms" { "1" } else if unit == "s" { "1000" } else { "60000" }, 10) { case Some(value) => value; case None => return None }
80 let maximum = match BigInt.parse("18446744073709551615", 10) { case Some(value) => value; case None => return None }
81 let milliseconds = value * factor
82 if milliseconds > maximum { None } else { Some(milliseconds.toString(10)) }
84export function decodeDecimalFloatLiteral(spelling: string) -> Option<float> {
85 if !validDecimalLiteral(spelling) || !spelling.contains(".") { return None }
86 let canonical = spelling.replace("_", "")
87 match canonical.parseFloat() { case Ok(value) => Some(value); case Err(_) => Some(1.0 / 0.0) }
89function hexWord(value: int) -> string {
90 let integer = match BigInt.parse("{value}", 10) { case Some(value) => value; case None => return "" }
91 let mut text = integer.toString(16)
92 while text.encodeUtf8().length() < 8 { text = "0{text}" }
93 text
95// Resolve a positive source float to IEEE bits before IR/backend boundaries.
96export function floatLiteralBits(spelling: string) -> string {
97 match decodeDecimalFloatLiteral(spelling) { case Some(value) => floatValueBits(value); case None => "" }
99function floatValueBits(input: float) -> string {
100 if input.isNaN() { return "7ff8000000000000" }
101 let negative = input < 0.0 || 1.0 / input == -1.0 / 0.0
102 let sign = if negative { 2147483648 } else { 0 }
103 let mut value = if negative { -input } else { input }
104 if value == 1.0 / 0.0 { return "{hexWord(sign + 2146435072)}00000000" }
105 if value == 0.0 { return "{hexWord(sign)}00000000" }
106 let mut exponent = 1023
107 while value >= 2.0 { value /= 2.0; exponent += 1 }
108 while value < 1.0 && exponent > 1 { value *= 2.0; exponent -= 1 }
109 let significand = if value < 1.0 { value * 4503599627370496.0 }
110 else { (value - 1.0) * 4503599627370496.0 }
111 if value < 1.0 { exponent = 0 }
112 // Binary64 scaling leaves an exact integer in [0, 2^52); trunc is in range.
113 let mantissa = significand.trunc()
114 let high = sign + exponent * 1048576 + mantissa / 4294967296
115 let low = mantissa % 4294967296
116 "{hexWord(high)}{hexWord(low)}"
118function escapeValue(bytes: Bytes, at: int, width: int) -> string {
119 match byteAt(bytes, at) {
120 case 110 => "\n"; case 114 => "\r"; case 116 => "\t"
121 case _ => slice(bytes, at, at + width)
122 }
124// Historical IR is decoded at its input boundary, before any backend consumes it.
125export function normalizeLegacyIrDetail(kind: string, detail: string, languageMode: string) -> Result<string, string> {
126 if kind == "expression/float" {
127 if detail == "" { return Err("invalid legacy float literal") }
128 for scalar in detail.scalars() {
129 if !"0123456789._eE+-".contains(scalar) { return Err("invalid legacy float literal") }
130 }
131 let normalized = detail.replace("_", "")
132 match normalized.parseFloat() {
133 case Ok(value) => return Ok(floatValueBits(value))
134 case Err(_) => ()
135 }
136 // Source literal overflow was IEEE infinity in the legacy compiler too.
137 if languageMode == "topaz-5.21" && detail.contains(".") && validDecimalLiteral(detail) && validDecimalLiteral(normalized) { return Ok("7ff0000000000000") }
138 return Err("invalid legacy float literal")
139 }
140 if kind == "expression/duration" {
141 let bytes = detail.encodeUtf8()
142 let end = bytes.length() - (if detail.endsWith("ms") { 2 } else { 1 })
143 if end <= 0 { return Err("invalid legacy duration literal") }
144 let mut at = 0
145 while at < end {
146 let b = byteAt(bytes, at)
147 if b < 48 || b > 57 { return Err("invalid legacy duration literal") }
148 at += 1
149 }
150 return match decodeDurationMilliseconds(detail) { case Some(value) => Ok(value); case None => Err("invalid legacy duration literal") }
151 }
152 if kind != "expression/string-text" { return Ok(detail) }
153 let bytes = detail.encodeUtf8()
154 let mut at = 0
155 let mut out: Array<string> = []
156 while at < bytes.length() {
157 let b = byteAt(bytes, at)
158 if b == 92 {
159 at += 1
160 let escaped = byteAt(bytes, at)
161 if escaped != 110 && escaped != 114 && escaped != 116 && escaped != 92 && escaped != 34 && escaped != 123 && escaped != 125 { return Err("invalid legacy string escape") }
162 out.push(escapeValue(bytes, at, 1))
163 at += 1
164 } else {
165 let width = if b < 128 { 1 } else if b < 224 { 2 } else if b < 240 { 3 } else { 4 }
166 out.push(slice(bytes, at, at + width))
167 at += width
168 }
169 }
170 Ok(out.join(""))
172export record IntegerLiteralInput { spelling: string, negative: bool }
173export function literalAttribute(node: SyntaxNode, name: string) -> string {
174 for attr in node.attributes {
175 if attr.name == name {
176 return attr.value
177 }
178 }
179 ""
181function unparen(node: SyntaxNode) -> SyntaxNode {
182 let mut current = node
183 while current.kind == "expression/parenthesized" && current.children.length == 1 { current = current.children[0] }
184 current
186export function signedIntegerInput(node: SyntaxNode, source: Bytes) -> Option<IntegerLiteralInput> {
187 let mut value = unparen(node)
188 let mut negative = false
189 if value.kind == "expression/unary" {
190 let op = literalAttribute(value, "operator")
191 if op != "minus" && op != "plus" { return None }
192 negative = op == "minus"
193 if value.children.length != 1 { return None }
194 value = unparen(value.children[0])
195 }
196 if value.kind != "expression/integer" { return None }
197 Some(IntegerLiteralInput { spelling: slice(source, value.lo, value.hi), negative: negative })
200export record LiteralRange { lo: int, hi: int }
201export record StringLayout { removed: Array<LiteralRange>, offending: Array<LiteralRange> }
202export function stringLayout(bytes: Bytes, openingEnd: int, closing: int, lines: Array<int>) -> StringLayout {
203 let mut removed: Array<LiteralRange> = []
204 let mut offending: Array<LiteralRange> = []
205 let openingBreak = if byteAt(bytes, openingEnd) == 13 && byteAt(bytes, openingEnd + 1) == 10 { 2 }
206 else if byteAt(bytes, openingEnd) == 10 { 1 } else { 0 }
207 if openingBreak > 0 { removed.push(LiteralRange { lo: openingEnd, hi: openingEnd + openingBreak }) }
208 let mut closeLine = closing
209 while closeLine > 0 && byteAt(bytes, closeLine - 1) != 10 { closeLine -= 1 }
210 let indent = closing - closeLine
211 let mut at = closeLine
212 while at < closing {
213 let b = byteAt(bytes, at)
214 if b != 32 && b != 9 { return StringLayout { removed: removed, offending: offending } }
215 at += 1
216 }
217 for line in lines {
218 if line >= closeLine || line < openingEnd + openingBreak { continue }
219 let mut end = line
220 while end < closing && byteAt(bytes, end) != 10 { end += 1 }
221 if end > line && byteAt(bytes, end - 1) == 13 { end -= 1 }
222 let mut blank = true
223 at = line
224 while at < end {
225 let b = byteAt(bytes, at)
226 if b != 32 && b != 9 { blank = false }
227 at += 1
228 }
229 if blank { continue }
230 let mut matchesIndent = end - line >= indent
231 at = 0
232 while matchesIndent && at < indent {
233 if byteAt(bytes, line + at) != byteAt(bytes, closeLine + at) { matchesIndent = false }
234 at += 1
235 }
236 if matchesIndent {
237 if indent > 0 { removed.push(LiteralRange { lo: line, hi: line + indent }) }
238 } else { offending.push(LiteralRange { lo: line, hi: end }) }
239 }
240 // Keep removed ranges in source order for the cooking cursor.
241 if indent > 0 { removed.push(LiteralRange { lo: closeLine, hi: closing }) }
242 StringLayout { removed: removed, offending: offending }
244function textAttribute(name: string, value: string) -> PreviewAstAttribute {
245 PreviewAstAttribute { name: name, kind: "string", value: value }
247export function cookStringParts(bytes: Bytes, openingEnd: int, closing: int, multiline: bool, parts: Array<SyntaxNode>) -> Array<SyntaxNode> {
248 let mut lines: Array<int> = []
249 if multiline {
250 if byteAt(bytes, openingEnd) != 10 && !(byteAt(bytes, openingEnd) == 13 && byteAt(bytes, openingEnd + 1) == 10) { lines.push(openingEnd) }
251 for part in parts {
252 if part.kind != "string-part/text" { continue }
253 let mut at = part.lo
254 while at < part.hi {
255 if byteAt(bytes, at) == 10 { lines.push(at + 1) }
256 at += 1
257 }
258 }
259 ()
260 }
261 let layout = if multiline { stringLayout(bytes, openingEnd, closing, lines) } else { StringLayout { removed: [], offending: [] } }
262 let mut result: Array<SyntaxNode> = []
263 let mut removedIndex = 0
264 for part in parts {
265 if part.kind != "string-part/text" { result.push(part); continue }
266 let mut out: Array<string> = []
267 let mut runs: Array<string> = []
268 let mut offset = 0
269 let mut at = part.lo
270 while at < part.hi {
271 let start = at
272 while removedIndex < layout.removed.length && layout.removed[removedIndex].hi <= at { removedIndex += 1 }
273 let nextRemoved = if removedIndex < layout.removed.length { layout.removed[removedIndex].lo } else { part.hi }
274 let removed = if nextRemoved <= at && removedIndex < layout.removed.length { layout.removed[removedIndex].hi.atMost(part.hi) } else { at }
275 if removed > at {
276 runs.push("{offset}:{offset}:{at}:{removed}")
277 at = removed
278 continue
279 }
280 let b = byteAt(bytes, at)
281 let mut value = ""
282 if multiline && b == 13 && byteAt(bytes, at + 1) == 10 {
283 value = "\n"; at += 2
284 } else if b == 92 {
285 at += 1
286 if at < part.hi {
287 let escaped = byteAt(bytes, at)
288 let width = if escaped < 128 { 1 } else if escaped < 224 { 2 } else if escaped < 240 { 3 } else { 4 }
289 value = escapeValue(bytes, at, width)
290 at += width
291 }
292 } else {
293 // Copy unchanged UTF-8 once per run. Preserve every scalar's map entry:
294 // consumers also observe zero-width removals and escape boundaries.
295 while at < part.hi && at < nextRemoved {
296 let plain = byteAt(bytes, at)
297 if plain == 92 || (multiline && plain == 13 && byteAt(bytes, at + 1) == 10) { break }
298 let width = if plain < 128 { 1 } else if plain < 224 { 2 } else if plain < 240 { 3 } else { 4 }
299 let end = offset + width
300 runs.push("{offset}:{end}:{at}:{at + width}")
301 offset = end
302 at += width
303 }
304 out.push(slice(bytes, start, at))
305 continue
306 }
307 out.push(value)
308 let end = offset + value.encodeUtf8().length()
309 runs.push("{offset}:{end}:{start}:{at}")
310 offset = end
311 }
312 result.push(SyntaxNode {
313 kind: part.kind, lo: part.lo, hi: part.hi, field: part.field, index: part.index,
314 attributes: [textAttribute("cooked", out.join("")), textAttribute("sourceMap", runs.join(";"))], children: part.children,
315 })
316 }
317 result

SHA256 e9887081f1a848d6e10434dab87c9efc283ee17302b8dd27764f13a27d2def77