STUDIO HAZE · topaz
Topaz source
Open by house policy. Topaz is our own language.
Topaz 5.21 · 50 files
Compiler sources from the Topaz 5.21.0 release. Compatibility code for earlier language modes is retained.
Publication edits are limited to explanatory comments. Executable code is unchanged.
Rights and source notices · Source manifest
compiler/crates/topaz_self_frontend/topaz/literal.tpz
Topaz · 14052 B · Published file
1
// Source literal grammar and parser-close cooking. Coordinates are UTF-8 bytes.2
import types { SyntaxNode, PreviewAstAttribute }4
function byteAt(bytes: Bytes, at: int) -> int { bytes.get(at) ?? -1 }5
function slice(bytes: Bytes, lo: int, hi: int) -> string {6
match bytes.slice(lo, hi).decodeUtf8() { case Ok(text) => text; case Err(_) => "" }7
}8
function digit(value: int) -> int {9
if value >= 48 && value <= 57 { value - 48 }10
else if value >= 65 && value <= 70 { value - 65 + 10 }11
else if value >= 97 && value <= 102 { value - 97 + 10 }12
else { -1 }13
}14
export function validDigitRun(bytes: Bytes, lo: int, hi: int, radix: int) -> bool {15
if lo == hi { return false }16
let mut at = lo17
while at < hi {18
let b = byteAt(bytes, at)19
let d = digit(b)20
if d < 0 || d >= radix {21
if b != 95 || at == lo || at + 1 == hi { return false }22
let before = digit(byteAt(bytes, at - 1))23
let after = digit(byteAt(bytes, at + 1))24
if before < 0 || before >= radix || after < 0 || after >= radix { return false }25
}26
at += 127
}28
true29
}30
export function integerRadix(spelling: string) -> int {31
if spelling.startsWith("0x") || spelling.startsWith("0X") { 16 }32
else if spelling.startsWith("0b") || spelling.startsWith("0B") { 2 }33
else { 10 }34
}35
export function validIntegerLiteral(spelling: string) -> bool {36
let bytes = spelling.encodeUtf8()37
let radix = integerRadix(spelling)38
validDigitRun(bytes, if radix == 10 { 0 } else { 2 }, bytes.length(), radix)39
}40
export function validDecimalLiteral(spelling: string) -> bool {41
let parts = spelling.split(".")42
if parts.length == 0 || parts.length > 2 { return false }43
for part in parts {44
let bytes = part.encodeUtf8()45
if !validDigitRun(bytes, 0, bytes.length(), 10) { return false }46
}47
true48
}49
export function integerMagnitude(spelling: string) -> Option<BigInt> {50
if !validIntegerLiteral(spelling) { return None }51
let bytes = spelling.encodeUtf8()52
let radix = integerRadix(spelling)53
let mut at = if radix == 10 { 0 } else { 2 }54
let mut value = match BigInt.parse("0", 10) { case Some(value) => value; case None => return None }55
let base = match BigInt.parse("{radix}", 10) { case Some(value) => value; case None => return None }56
while at < bytes.length() {57
if byteAt(bytes, at) != 95 {58
let d = digit(byteAt(bytes, at))59
let next = match BigInt.parse("{d}", 10) { case Some(value) => value; case None => return None }60
value = value * base + next61
}62
at += 163
}64
Some(value)65
}66
export function decodeIntegerLiteral(spelling: string, negative: bool) -> Option<int> {67
let value = match integerMagnitude(spelling) { case Some(value) => value; case None => return None }68
if negative { (-value).toInt() } else { value.toInt() }69
}70
export function decodeDurationMilliseconds(spelling: string) -> Option<string> {71
let bytes = spelling.encodeUtf8()72
let unitSize = if spelling.endsWith("ms") { 2 } else { 1 }73
if bytes.length() <= unitSize { return None }74
let unit = slice(bytes, bytes.length() - unitSize, bytes.length())75
let magnitude = slice(bytes, 0, bytes.length() - unitSize)76
if unit != "ms" && unit != "s" && unit != "m" { return None }77
if integerRadix(magnitude) != 10 || !validIntegerLiteral(magnitude) { return None }78
let value = match integerMagnitude(magnitude) { case Some(value) => value; case None => return None }79
let factor = match BigInt.parse(if unit == "ms" { "1" } else if unit == "s" { "1000" } else { "60000" }, 10) { case Some(value) => value; case None => return None }80
let maximum = match BigInt.parse("18446744073709551615", 10) { case Some(value) => value; case None => return None }81
let milliseconds = value * factor82
if milliseconds > maximum { None } else { Some(milliseconds.toString(10)) }83
}84
export function decodeDecimalFloatLiteral(spelling: string) -> Option<float> {85
if !validDecimalLiteral(spelling) || !spelling.contains(".") { return None }86
let canonical = spelling.replace("_", "")87
match canonical.parseFloat() { case Ok(value) => Some(value); case Err(_) => Some(1.0 / 0.0) }88
}89
function hexWord(value: int) -> string {90
let integer = match BigInt.parse("{value}", 10) { case Some(value) => value; case None => return "" }91
let mut text = integer.toString(16)92
while text.encodeUtf8().length() < 8 { text = "0{text}" }93
text94
}95
// Resolve a positive source float to IEEE bits before IR/backend boundaries.96
export function floatLiteralBits(spelling: string) -> string {97
match decodeDecimalFloatLiteral(spelling) { case Some(value) => floatValueBits(value); case None => "" }98
}99
function floatValueBits(input: float) -> string {100
if input.isNaN() { return "7ff8000000000000" }101
let negative = input < 0.0 || 1.0 / input == -1.0 / 0.0102
let sign = if negative { 2147483648 } else { 0 }103
let mut value = if negative { -input } else { input }104
if value == 1.0 / 0.0 { return "{hexWord(sign + 2146435072)}00000000" }105
if value == 0.0 { return "{hexWord(sign)}00000000" }106
let mut exponent = 1023107
while value >= 2.0 { value /= 2.0; exponent += 1 }108
while value < 1.0 && exponent > 1 { value *= 2.0; exponent -= 1 }109
let significand = if value < 1.0 { value * 4503599627370496.0 }110
else { (value - 1.0) * 4503599627370496.0 }111
if value < 1.0 { exponent = 0 }112
// Binary64 scaling leaves an exact integer in [0, 2^52); trunc is in range.113
let mantissa = significand.trunc()114
let high = sign + exponent * 1048576 + mantissa / 4294967296115
let low = mantissa % 4294967296116
"{hexWord(high)}{hexWord(low)}"117
}118
function escapeValue(bytes: Bytes, at: int, width: int) -> string {119
match byteAt(bytes, at) {120
case 110 => "\n"; case 114 => "\r"; case 116 => "\t"121
case _ => slice(bytes, at, at + width)122
}123
}124
// Historical IR is decoded at its input boundary, before any backend consumes it.125
export function normalizeLegacyIrDetail(kind: string, detail: string, languageMode: string) -> Result<string, string> {126
if kind == "expression/float" {127
if detail == "" { return Err("invalid legacy float literal") }128
for scalar in detail.scalars() {129
if !"0123456789._eE+-".contains(scalar) { return Err("invalid legacy float literal") }130
}131
let normalized = detail.replace("_", "")132
match normalized.parseFloat() {133
case Ok(value) => return Ok(floatValueBits(value))134
case Err(_) => ()135
}136
// Source literal overflow was IEEE infinity in the legacy compiler too.137
if languageMode == "topaz-5.21" && detail.contains(".") && validDecimalLiteral(detail) && validDecimalLiteral(normalized) { return Ok("7ff0000000000000") }138
return Err("invalid legacy float literal")139
}140
if kind == "expression/duration" {141
let bytes = detail.encodeUtf8()142
let end = bytes.length() - (if detail.endsWith("ms") { 2 } else { 1 })143
if end <= 0 { return Err("invalid legacy duration literal") }144
let mut at = 0145
while at < end {146
let b = byteAt(bytes, at)147
if b < 48 || b > 57 { return Err("invalid legacy duration literal") }148
at += 1149
}150
return match decodeDurationMilliseconds(detail) { case Some(value) => Ok(value); case None => Err("invalid legacy duration literal") }151
}152
if kind != "expression/string-text" { return Ok(detail) }153
let bytes = detail.encodeUtf8()154
let mut at = 0155
let mut out: Array<string> = []156
while at < bytes.length() {157
let b = byteAt(bytes, at)158
if b == 92 {159
at += 1160
let escaped = byteAt(bytes, at)161
if escaped != 110 && escaped != 114 && escaped != 116 && escaped != 92 && escaped != 34 && escaped != 123 && escaped != 125 { return Err("invalid legacy string escape") }162
out.push(escapeValue(bytes, at, 1))163
at += 1164
} else {165
let width = if b < 128 { 1 } else if b < 224 { 2 } else if b < 240 { 3 } else { 4 }166
out.push(slice(bytes, at, at + width))167
at += width168
}169
}170
Ok(out.join(""))171
}172
export record IntegerLiteralInput { spelling: string, negative: bool }173
export function literalAttribute(node: SyntaxNode, name: string) -> string {174
for attr in node.attributes {175
if attr.name == name {176
return attr.value177
}178
}179
""180
}181
function unparen(node: SyntaxNode) -> SyntaxNode {182
let mut current = node183
while current.kind == "expression/parenthesized" && current.children.length == 1 { current = current.children[0] }184
current185
}186
export function signedIntegerInput(node: SyntaxNode, source: Bytes) -> Option<IntegerLiteralInput> {187
let mut value = unparen(node)188
let mut negative = false189
if value.kind == "expression/unary" {190
let op = literalAttribute(value, "operator")191
if op != "minus" && op != "plus" { return None }192
negative = op == "minus"193
if value.children.length != 1 { return None }194
value = unparen(value.children[0])195
}196
if value.kind != "expression/integer" { return None }197
Some(IntegerLiteralInput { spelling: slice(source, value.lo, value.hi), negative: negative })198
}200
export record LiteralRange { lo: int, hi: int }201
export record StringLayout { removed: Array<LiteralRange>, offending: Array<LiteralRange> }202
export function stringLayout(bytes: Bytes, openingEnd: int, closing: int, lines: Array<int>) -> StringLayout {203
let mut removed: Array<LiteralRange> = []204
let mut offending: Array<LiteralRange> = []205
let openingBreak = if byteAt(bytes, openingEnd) == 13 && byteAt(bytes, openingEnd + 1) == 10 { 2 }206
else if byteAt(bytes, openingEnd) == 10 { 1 } else { 0 }207
if openingBreak > 0 { removed.push(LiteralRange { lo: openingEnd, hi: openingEnd + openingBreak }) }208
let mut closeLine = closing209
while closeLine > 0 && byteAt(bytes, closeLine - 1) != 10 { closeLine -= 1 }210
let indent = closing - closeLine211
let mut at = closeLine212
while at < closing {213
let b = byteAt(bytes, at)214
if b != 32 && b != 9 { return StringLayout { removed: removed, offending: offending } }215
at += 1216
}217
for line in lines {218
if line >= closeLine || line < openingEnd + openingBreak { continue }219
let mut end = line220
while end < closing && byteAt(bytes, end) != 10 { end += 1 }221
if end > line && byteAt(bytes, end - 1) == 13 { end -= 1 }222
let mut blank = true223
at = line224
while at < end {225
let b = byteAt(bytes, at)226
if b != 32 && b != 9 { blank = false }227
at += 1228
}229
if blank { continue }230
let mut matchesIndent = end - line >= indent231
at = 0232
while matchesIndent && at < indent {233
if byteAt(bytes, line + at) != byteAt(bytes, closeLine + at) { matchesIndent = false }234
at += 1235
}236
if matchesIndent {237
if indent > 0 { removed.push(LiteralRange { lo: line, hi: line + indent }) }238
} else { offending.push(LiteralRange { lo: line, hi: end }) }239
}240
// Keep removed ranges in source order for the cooking cursor.241
if indent > 0 { removed.push(LiteralRange { lo: closeLine, hi: closing }) }242
StringLayout { removed: removed, offending: offending }243
}244
function textAttribute(name: string, value: string) -> PreviewAstAttribute {245
PreviewAstAttribute { name: name, kind: "string", value: value }246
}247
export function cookStringParts(bytes: Bytes, openingEnd: int, closing: int, multiline: bool, parts: Array<SyntaxNode>) -> Array<SyntaxNode> {248
let mut lines: Array<int> = []249
if multiline {250
if byteAt(bytes, openingEnd) != 10 && !(byteAt(bytes, openingEnd) == 13 && byteAt(bytes, openingEnd + 1) == 10) { lines.push(openingEnd) }251
for part in parts {252
if part.kind != "string-part/text" { continue }253
let mut at = part.lo254
while at < part.hi {255
if byteAt(bytes, at) == 10 { lines.push(at + 1) }256
at += 1257
}258
}259
()260
}261
let layout = if multiline { stringLayout(bytes, openingEnd, closing, lines) } else { StringLayout { removed: [], offending: [] } }262
let mut result: Array<SyntaxNode> = []263
let mut removedIndex = 0264
for part in parts {265
if part.kind != "string-part/text" { result.push(part); continue }266
let mut out: Array<string> = []267
let mut runs: Array<string> = []268
let mut offset = 0269
let mut at = part.lo270
while at < part.hi {271
let start = at272
while removedIndex < layout.removed.length && layout.removed[removedIndex].hi <= at { removedIndex += 1 }273
let nextRemoved = if removedIndex < layout.removed.length { layout.removed[removedIndex].lo } else { part.hi }274
let removed = if nextRemoved <= at && removedIndex < layout.removed.length { layout.removed[removedIndex].hi.atMost(part.hi) } else { at }275
if removed > at {276
runs.push("{offset}:{offset}:{at}:{removed}")277
at = removed278
continue279
}280
let b = byteAt(bytes, at)281
let mut value = ""282
if multiline && b == 13 && byteAt(bytes, at + 1) == 10 {283
value = "\n"; at += 2284
} else if b == 92 {285
at += 1286
if at < part.hi {287
let escaped = byteAt(bytes, at)288
let width = if escaped < 128 { 1 } else if escaped < 224 { 2 } else if escaped < 240 { 3 } else { 4 }289
value = escapeValue(bytes, at, width)290
at += width291
}292
} else {293
// Copy unchanged UTF-8 once per run. Preserve every scalar's map entry:294
// consumers also observe zero-width removals and escape boundaries.295
while at < part.hi && at < nextRemoved {296
let plain = byteAt(bytes, at)297
if plain == 92 || (multiline && plain == 13 && byteAt(bytes, at + 1) == 10) { break }298
let width = if plain < 128 { 1 } else if plain < 224 { 2 } else if plain < 240 { 3 } else { 4 }299
let end = offset + width300
runs.push("{offset}:{end}:{at}:{at + width}")301
offset = end302
at += width303
}304
out.push(slice(bytes, start, at))305
continue306
}307
out.push(value)308
let end = offset + value.encodeUtf8().length()309
runs.push("{offset}:{end}:{start}:{at}")310
offset = end311
}312
result.push(SyntaxNode {313
kind: part.kind, lo: part.lo, hi: part.hi, field: part.field, index: part.index,314
attributes: [textAttribute("cooked", out.join("")), textAttribute("sourceMap", runs.join(";"))], children: part.children,315
})316
}317
result318
}SHA256 e9887081f1a848d6e10434dab87c9efc283ee17302b8dd27764f13a27d2def77