slow.monster

STUDIO HAZE · topaz

Topaz source

Open by house policy. Topaz is our own language.

Topaz 5.20 · 56 files

compiler/crates/topaz_self_frontend/topaz/raw.tpz

Topaz · 21428 B · Original file

1import types { PreviewDiagnostic, PreviewToken, RawResult }
2import unicode {
3 identifierContinueEnd,
4 identifierEnd,
5 identifierStartWidth,
6 utf8Width,
7}
9function token(kind: string, lo: int, hi: int) -> PreviewToken {
10 PreviewToken {
11 kind: kind,
12 lo: lo,
13 hi: hi,
14 stream: "raw",
15 synthetic: false,
16 }
19function syntheticToken(kind: string, at: int) -> PreviewToken {
20 PreviewToken {
21 kind: kind,
22 lo: at,
23 hi: at,
24 stream: "raw",
25 synthetic: true,
26 }
29function diagnostic(code: string, message: string, lo: int, hi: int) -> PreviewDiagnostic {
30 PreviewDiagnostic { code: code, message: message, lo: lo, hi: hi, notes: [] }
33function byteAt(bytes: Bytes, index: int) -> int {
34 match bytes.get(index) {
35 case Some(value) => value
36 case None => -1
37 }
40function isAsciiSpace(value: int) -> bool {
41 value == 32 || value == 9
44function lineCommentEnd(bytes: Bytes, start: int) -> int {
45 let mut index = start + 2
46 while index < bytes.length() && byteAt(bytes, index) != 10 {
47 index += 1
48 }
49 index
52function blockCommentEnd(bytes: Bytes, start: int) -> int {
53 let mut index = start + 2
54 while index < bytes.length() {
55 if byteAt(bytes, index) == 42 && byteAt(bytes, index + 1) == 47 {
56 return index + 2
57 }
58 index += 1
59 }
60 index
63function isAsciiDigit(value: int) -> bool {
64 value >= 48 && value <= 57
67export function keywordKind(text: string) -> string {
68 match text {
69 case "let" => "keyword/let"
70 case "mut" => "keyword/mut"
71 case "const" => "keyword/const"
72 case "type" => "keyword/type"
73 case "function" => "keyword/function"
74 case "return" => "keyword/return"
75 case "if" => "keyword/if"
76 case "else" => "keyword/else"
77 case "match" => "keyword/match"
78 case "case" => "keyword/case"
79 case "for" => "keyword/for"
80 case "in" => "keyword/in"
81 case "by" => "keyword/by"
82 case "while" => "keyword/while"
83 case "break" => "keyword/break"
84 case "continue" => "keyword/continue"
85 case "defer" => "keyword/defer"
86 case "concurrent" => "keyword/concurrent"
87 case "true" => "keyword/true"
88 case "false" => "keyword/false"
89 case "null" => "keyword/null"
90 case "_" => "underscore"
91 case _ => "identifier"
92 }
95function decode(bytes: Bytes, lo: int, hi: int) -> string {
96 match bytes.slice(lo, hi).decodeUtf8() {
97 case Ok(value) => value
98 case Err(_) => ""
99 }
102function identifierHeadKind(bytes: Bytes, start: int, end: int) -> string {
103 let kind = keywordKind(decode(bytes, start, end))
104 if kind == "identifier" && byteAt(bytes, end) == 34 {
105 "tagged-string"
106 } else {
107 kind
108 }
111function punctuation(bytes: Bytes, index: int) -> Option<PreviewToken> {
112 let a = byteAt(bytes, index)
113 let b = byteAt(bytes, index + 1)
114 let c = byteAt(bytes, index + 2)
115 match [a, b, c] {
116 case [46, 46, 46] => Some(token("ellipsis", index, index + 3))
117 case [46, 46, 60] => Some(token("dot-dot-lt", index, index + 3))
118 case [63, 63, 61] => Some(token("question-question-eq", index, index + 3))
119 case [42, 42, _] => Some(token("star-star", index, index + 2))
120 case [46, 46, _] => Some(token("dot-dot", index, index + 2))
121 case [63, 46, _] => Some(token("question-dot", index, index + 2))
122 case [63, 63, _] => Some(token("question-question", index, index + 2))
123 case [60, 61, _] => Some(token("le", index, index + 2))
124 case [62, 61, _] => Some(token("ge", index, index + 2))
125 case [62, 62, _] => Some(token("gt-gt", index, index + 2))
126 case [61, 61, _] => Some(token("eq-eq", index, index + 2))
127 case [33, 61, _] => Some(token("not-eq", index, index + 2))
128 case [43, 61, _] => Some(token("plus-eq", index, index + 2))
129 case [45, 61, _] => Some(token("minus-eq", index, index + 2))
130 case [42, 61, _] => Some(token("star-eq", index, index + 2))
131 case [47, 61, _] => Some(token("slash-eq", index, index + 2))
132 case [37, 61, _] => Some(token("percent-eq", index, index + 2))
133 case [38, 38, _] => Some(token("and-and", index, index + 2))
134 case [124, 124, _] => Some(token("or-or", index, index + 2))
135 case [124, 62, _] => Some(token("pipe-gt", index, index + 2))
136 case [61, 62, _] => Some(token("fat-arrow", index, index + 2))
137 case [45, 62, _] => Some(token("thin-arrow", index, index + 2))
138 case [40, _, _] => Some(token("left-paren", index, index + 1))
139 case [41, _, _] => Some(token("right-paren", index, index + 1))
140 case [91, _, _] => Some(token("left-bracket", index, index + 1))
141 case [93, _, _] => Some(token("right-bracket", index, index + 1))
142 case [123, _, _] => Some(token("left-brace", index, index + 1))
143 case [125, _, _] => Some(token("right-brace", index, index + 1))
144 case [43, _, _] => Some(token("plus", index, index + 1))
145 case [45, _, _] => Some(token("minus", index, index + 1))
146 case [42, _, _] => Some(token("star", index, index + 1))
147 case [47, _, _] => Some(token("slash", index, index + 1))
148 case [37, _, _] => Some(token("percent", index, index + 1))
149 case [46, _, _] => Some(token("dot", index, index + 1))
150 case [63, _, _] => Some(token("question", index, index + 1))
151 case [60, _, _] => Some(token("lt", index, index + 1))
152 case [62, _, _] => Some(token("gt", index, index + 1))
153 case [61, _, _] => Some(token("eq", index, index + 1))
154 case [33, _, _] => Some(token("bang", index, index + 1))
155 case [126, _, _] => Some(token("tilde", index, index + 1))
156 case [124, _, _] => Some(token("pipe", index, index + 1))
157 case [44, _, _] => Some(token("comma", index, index + 1))
158 case [58, _, _] => Some(token("colon", index, index + 1))
159 case [59, _, _] => Some(token("semicolon", index, index + 1))
160 case _ => None
161 }
164function isValidEscape(value: int) -> bool {
165 value == 110 ||
166 value == 116 ||
167 value == 114 ||
168 value == 92 ||
169 value == 34 ||
170 value == 123 ||
171 value == 125
174function startsTripleQuote(bytes: Bytes, index: int) -> bool {
175 byteAt(bytes, index) == 34 &&
176 byteAt(bytes, index + 1) == 34 &&
177 byteAt(bytes, index + 2) == 34
180function multilineIndentDiagnostics(
181 bytes: Bytes,
182 lineStarts: Array<int>,
183 delimiter: int,
184) -> Array<PreviewDiagnostic> {
185 let mut diagnostics: Array<PreviewDiagnostic> = []
186 let mut closeLine = delimiter
187 while closeLine > 0 && byteAt(bytes, closeLine - 1) != 10 {
188 closeLine -= 1
189 }
190 let indentLength = delimiter - closeLine
191 if indentLength == 0 {
192 return diagnostics
193 }
194 let mut offset = 0
195 while offset < indentLength {
196 let value = byteAt(bytes, closeLine + offset)
197 if value != 32 && value != 9 {
198 return diagnostics
199 }
200 offset += 1
201 }
202 for line in lineStarts {
203 if line >= closeLine {
204 break
205 }
206 let mut end = line
207 while end < bytes.length() && byteAt(bytes, end) != 10 {
208 end += 1
209 }
210 let textEnd = if end > line && byteAt(bytes, end - 1) == 13 { end - 1 } else { end }
211 let mut blank = true
212 let mut cursor = line
213 while cursor < textEnd {
214 let value = byteAt(bytes, cursor)
215 if value != 32 && value != 9 {
216 blank = false
217 break
218 }
219 cursor += 1
220 }
221 if blank {
222 continue
223 }
224 let mut matchesIndent = textEnd - line >= indentLength
225 offset = 0
226 while matchesIndent && offset < indentLength {
227 if byteAt(bytes, line + offset) != byteAt(bytes, closeLine + offset) {
228 matchesIndent = false
229 }
230 offset += 1
231 }
232 if !matchesIndent {
233 diagnostics.push(diagnostic(
234 "TPZ0006",
235 "line does not begin with the closing indent of its multiline string",
236 line,
237 textEnd,
238 ))
239 }
240 }
241 diagnostics
244record LexemeScan {
245 tokens: Array<PreviewToken>,
246 diagnostics: Array<PreviewDiagnostic>,
247 next: int,
248 propagatedLineBreak: bool,
251record StringInterpolationOutcome {
252 next: int,
253 lineBreak: bool,
256record StringScanState {
257 bytes: Bytes,
258 tokens: Array<PreviewToken>,
259 diagnostics: Array<PreviewDiagnostic>,
260 lineStarts: Array<int>,
261 quote: int,
262 multiline: bool,
263 lineBreakForbidden: bool,
266function stringDelimiterEnd(start: int, multiline: bool) -> int {
267 start + if multiline { 3 } else { 1 }
270function appendStringText(state: StringScanState, start: int, end: int) -> () {
271 if end > start {
272 let mut tokens = state.tokens
273 tokens.push(token("string-text", start, end))
274 }
275 ()
278function finishString(state: StringScanState, textStart: int, delimiter: int) -> LexemeScan {
279 appendStringText(state, textStart, delimiter)
280 let end = stringDelimiterEnd(delimiter, state.multiline)
281 let mut tokens = state.tokens
282 tokens.push(token("string-end", delimiter, end))
283 if state.multiline {
284 let mut diagnostics = state.diagnostics
285 for found in multilineIndentDiagnostics(state.bytes, state.lineStarts, delimiter) {
286 diagnostics.push(found)
287 }
288 }
289 LexemeScan {
290 tokens: state.tokens,
291 diagnostics: state.diagnostics,
292 next: end,
293 propagatedLineBreak: false,
294 }
297function scanStringEscape(state: StringScanState, start: int) -> int {
298 let mut index = start + 1
299 let escaped = byteAt(state.bytes, index)
300 if isValidEscape(escaped) {
301 return index + 1
302 }
303 let mut diagnostics = state.diagnostics
304 if escaped == 10 || escaped == 13 || escaped == -1 {
305 diagnostics.push(diagnostic(
306 "TPZ0004",
307 "invalid escape: `\\` before the end of the line",
308 start,
309 index,
310 ))
311 return index
312 }
313 let escapedWidth = utf8Width(escaped)
314 let consumed = if escapedWidth > 0 { escapedWidth } else { 1 }
315 let escapedText = decode(state.bytes, index, index + consumed)
316 index += consumed
317 diagnostics.push(diagnostic(
318 "TPZ0004",
319 "invalid escape `\\{escapedText}` in a string literal",
320 start,
321 index,
322 ))
323 index
326function scanStringInterpolation(
327 state: StringScanState,
328 textStart: int,
329 start: int,
330) -> StringInterpolationOutcome {
331 appendStringText(state, textStart, start)
332 let mut tokens = state.tokens
333 tokens.push(token("interpolation-start", start, start + 1))
334 let interpolation = scanCode(
335 state.bytes,
336 start + 1,
337 true,
338 state.lineBreakForbidden || !state.multiline,
339 )
340 let mut diagnostics = state.diagnostics
341 for found in interpolation.tokens {
342 tokens.push(found)
343 }
344 for found in interpolation.diagnostics {
345 diagnostics.push(found)
346 }
347 StringInterpolationOutcome {
348 next: interpolation.next,
349 lineBreak: interpolation.propagatedLineBreak,
350 }
353function interruptedString(
354 state: StringScanState,
355 textStart: int,
356 end: int,
357 message: string,
358) -> LexemeScan {
359 appendStringText(state, textStart, end)
360 let mut diagnostics = state.diagnostics
361 diagnostics.push(diagnostic(
362 "TPZ0003",
363 message,
364 state.quote,
365 stringDelimiterEnd(state.quote, state.multiline),
366 ))
367 let mut tokens = state.tokens
368 tokens.push(syntheticToken("string-end", end))
369 LexemeScan {
370 tokens: state.tokens,
371 diagnostics: state.diagnostics,
372 next: end,
373 propagatedLineBreak: false,
374 }
377function propagateStringLineBreak(
378 state: StringScanState,
379 textStart: int,
380 end: int,
381) -> LexemeScan {
382 appendStringText(state, textStart, end)
383 let mut tokens = state.tokens
384 tokens.push(syntheticToken("string-end", end))
385 LexemeScan {
386 tokens: state.tokens,
387 diagnostics: state.diagnostics,
388 next: end,
389 propagatedLineBreak: true,
390 }
393function scanString(
394 bytes: Bytes,
395 tokenStart: int,
396 quote: int,
397 tagged: bool,
398 lineBreakForbidden: bool,
399) -> LexemeScan {
400 let multiline = startsTripleQuote(bytes, quote)
401 let mut index = stringDelimiterEnd(quote, multiline)
402 let mut textStart = index
403 let state = StringScanState {
404 bytes: bytes,
405 tokens: [token(
406 "string-start/tagged={tagged}/multiline={multiline}",
407 tokenStart,
408 index,
409 )],
410 diagnostics: [],
411 lineStarts: [],
412 quote: quote,
413 multiline: multiline,
414 lineBreakForbidden: lineBreakForbidden,
415 }
416 if multiline && index < bytes.length() && byteAt(bytes, index) != 10 &&
417 !(byteAt(bytes, index) == 13 && byteAt(bytes, index + 1) == 10) {
418 let mut lineStarts = state.lineStarts
419 lineStarts.push(index)
420 }
421 while index < bytes.length() {
422 if (multiline && startsTripleQuote(bytes, index)) ||
423 (!multiline && byteAt(bytes, index) == 34) {
424 return finishString(state, textStart, index)
425 }
426 let value = byteAt(bytes, index)
427 if value == 92 {
428 index = scanStringEscape(state, index)
429 continue
430 }
431 if value == 123 {
432 let interpolation = scanStringInterpolation(state, textStart, index)
433 if interpolation.lineBreak {
434 if state.multiline && state.lineBreakForbidden {
435 return propagateStringLineBreak(state, interpolation.next, interpolation.next)
436 }
437 return interruptedString(
438 state,
439 interpolation.next,
440 interpolation.next,
441 "string interpolation in a single-line string cannot contain a line break",
442 )
443 }
444 index = interpolation.next
445 textStart = index
446 continue
447 }
448 if multiline && state.lineBreakForbidden &&
449 (value == 10 || (value == 13 && byteAt(bytes, index + 1) == 10)) {
450 return propagateStringLineBreak(state, textStart, index)
451 }
452 if !multiline && (value == 10 ||
453 (value == 13 && byteAt(bytes, index + 1) == 10)) {
454 return interruptedString(
455 state,
456 textStart,
457 index,
458 "single-line string literal is not closed before the line break",
459 )
460 }
461 if !multiline && value == 125 {
462 let mut diagnostics = state.diagnostics
463 diagnostics.push(diagnostic(
464 "TPZ0005",
465 "unescaped `\}` in a string literal; write `\\\}`",
466 index,
467 index + 1,
468 ))
469 }
470 if multiline && value == 10 {
471 let mut lineStarts = state.lineStarts
472 lineStarts.push(index + 1)
473 }
474 index += 1
475 }
476 interruptedString(state, textStart, index, "unterminated string literal")
479function appendScan(
480 targetTokens: Array<PreviewToken>,
481 targetDiagnostics: Array<PreviewDiagnostic>,
482 foundTokens: Array<PreviewToken>,
483 foundDiagnostics: Array<PreviewDiagnostic>,
484) -> () {
485 let mut tokens = targetTokens
486 for found in foundTokens {
487 tokens.push(found)
488 }
489 let mut diagnostics = targetDiagnostics
490 for found in foundDiagnostics {
491 diagnostics.push(found)
492 }
493 ()
496function scanComment(bytes: Bytes, start: int) -> LexemeScan {
497 let mut index = start + 2
498 let mut diagnostics: Array<PreviewDiagnostic> = []
499 if byteAt(bytes, start + 1) == 47 {
500 index = lineCommentEnd(bytes, start)
501 } else {
502 index = blockCommentEnd(bytes, start)
503 let closed = index >= start + 4 &&
504 byteAt(bytes, index - 2) == 42 && byteAt(bytes, index - 1) == 47
505 if !closed {
506 diagnostics.push(diagnostic("TPZ0002", "unterminated block comment", start, index))
507 }
508 }
509 LexemeScan {
510 tokens: [],
511 diagnostics: diagnostics,
512 next: index,
513 propagatedLineBreak: false,
514 }
517function scanNumber(bytes: Bytes, start: int) -> LexemeScan {
518 let mut index = start + 1
519 while index < bytes.length() && isAsciiDigit(byteAt(bytes, index)) {
520 index += 1
521 }
522 let mut kind = "integer"
523 if byteAt(bytes, index) == 46 && isAsciiDigit(byteAt(bytes, index + 1)) {
524 kind = "float"
525 index += 1
526 while index < bytes.length() && isAsciiDigit(byteAt(bytes, index)) {
527 index += 1
528 }
529 } else {
530 let unitStart = index
531 let unitEnd = identifierContinueEnd(bytes, index)
532 let unit = decode(bytes, unitStart, unitEnd)
533 if unit == "ms" || unit == "s" || unit == "m" {
534 kind = "duration/{unit}"
535 index = unitEnd
536 }
537 }
538 LexemeScan {
539 tokens: [token(kind, start, index)],
540 diagnostics: [],
541 next: index,
542 propagatedLineBreak: false,
543 }
546function scanLabel(bytes: Bytes, start: int) -> LexemeScan {
547 let identifierStart = start + 1
548 let index = identifierEnd(bytes, identifierStart)
549 if index == identifierStart {
550 return LexemeScan {
551 tokens: [],
552 diagnostics: [diagnostic(
553 "TPZ0001",
554 "a `'` begins a loop label `'name`; a bare `'` is not valid",
555 start,
556 identifierStart,
557 )],
558 next: identifierStart,
559 propagatedLineBreak: false,
560 }
561 }
562 LexemeScan {
563 tokens: [token("label", start, index)],
564 diagnostics: [],
565 next: index,
566 propagatedLineBreak: false,
567 }
570function scanIdentifier(bytes: Bytes, start: int, lineBreakForbidden: bool) -> LexemeScan {
571 let index = identifierEnd(bytes, start)
572 let kind = identifierHeadKind(bytes, start, index)
573 if kind == "tagged-string" {
574 return scanString(bytes, start, index, true, lineBreakForbidden)
575 }
576 LexemeScan {
577 tokens: [token(kind, start, index)],
578 diagnostics: [],
579 next: index,
580 propagatedLineBreak: false,
581 }
584function scanUnknown(bytes: Bytes, start: int) -> LexemeScan {
585 let scalarWidth = utf8Width(byteAt(bytes, start))
586 let consumed = if scalarWidth > 0 { scalarWidth } else { 1 }
587 let found = decode(bytes, start, start + consumed)
588 LexemeScan {
589 tokens: [],
590 diagnostics: [diagnostic(
591 "TPZ0001",
592 "unknown character `{found}`",
593 start,
594 start + consumed,
595 )],
596 next: start + consumed,
597 propagatedLineBreak: false,
598 }
601function scanCode(
602 bytes: Bytes,
603 startIndex: int,
604 interpolation: bool,
605 lineBreakForbidden: bool,
606) -> LexemeScan {
607 let mut tokens: Array<PreviewToken> = []
608 let mut diagnostics: Array<PreviewDiagnostic> = []
609 let mut index = startIndex
610 let mut braceDepth = 0
612 while index < bytes.length() {
613 let value = byteAt(bytes, index)
614 if interpolation && value == 125 {
615 if braceDepth == 0 {
616 tokens.push(token("interpolation-end", index, index + 1))
617 return LexemeScan {
618 tokens: tokens,
619 diagnostics: diagnostics,
620 next: index + 1,
621 propagatedLineBreak: false,
622 }
623 }
624 tokens.push(token("right-brace", index, index + 1))
625 braceDepth -= 1
626 index += 1
627 continue
628 }
629 if interpolation && lineBreakForbidden &&
630 (value == 10 || (value == 13 && byteAt(bytes, index + 1) == 10)) {
631 tokens.push(syntheticToken("interpolation-end", index))
632 return LexemeScan {
633 tokens: tokens,
634 diagnostics: diagnostics,
635 next: index,
636 propagatedLineBreak: true,
637 }
638 }
639 if isAsciiSpace(value) {
640 index += 1
641 continue
642 }
643 if value == 10 {
644 let lo = if index > 0 && byteAt(bytes, index - 1) == 13 { index - 1 } else { index }
645 tokens.push(token("newline", lo, index + 1))
646 index += 1
647 continue
648 }
649 if value == 13 && byteAt(bytes, index + 1) == 10 {
650 tokens.push(token("newline", index, index + 2))
651 index += 2
652 continue
653 }
654 if value == 13 {
655 index += 1
656 continue
657 }
658 if value == 47 &&
659 (byteAt(bytes, index + 1) == 47 || byteAt(bytes, index + 1) == 42) {
660 let found = scanComment(bytes, index)
661 appendScan(tokens, diagnostics, found.tokens, found.diagnostics)
662 index = found.next
663 continue
664 }
665 if value == 34 {
666 let result = scanString(bytes, index, index, false, lineBreakForbidden)
667 appendScan(tokens, diagnostics, result.tokens, result.diagnostics)
668 index = result.next
669 if result.propagatedLineBreak {
670 tokens.push(syntheticToken("interpolation-end", index))
671 return LexemeScan {
672 tokens: tokens,
673 diagnostics: diagnostics,
674 next: index,
675 propagatedLineBreak: true,
676 }
677 }
678 continue
679 }
680 if isAsciiDigit(value) {
681 let found = scanNumber(bytes, index)
682 appendScan(tokens, diagnostics, found.tokens, found.diagnostics)
683 index = found.next
684 continue
685 }
686 if value == 39 {
687 let found = scanLabel(bytes, index)
688 appendScan(tokens, diagnostics, found.tokens, found.diagnostics)
689 index = found.next
690 continue
691 }
692 let startWidth = identifierStartWidth(bytes, index)
693 if startWidth > 0 {
694 let found = scanIdentifier(bytes, index, lineBreakForbidden)
695 appendScan(tokens, diagnostics, found.tokens, found.diagnostics)
696 index = found.next
697 if found.propagatedLineBreak {
698 tokens.push(syntheticToken("interpolation-end", index))
699 return LexemeScan {
700 tokens: tokens,
701 diagnostics: diagnostics,
702 next: index,
703 propagatedLineBreak: true,
704 }
705 }
706 continue
707 }
708 match punctuation(bytes, index) {
709 case Some(found) => {
710 tokens.push(found)
711 if interpolation && found.kind == "left-brace" {
712 braceDepth += 1
713 }
714 index = found.hi
715 }
716 case None => {
717 let found = scanUnknown(bytes, index)
718 appendScan(tokens, diagnostics, found.tokens, found.diagnostics)
719 index = found.next
720 }
721 }
722 }
723 if interpolation {
724 tokens.push(syntheticToken("interpolation-end", index))
725 } else {
726 tokens.push(token("eof", bytes.length(), bytes.length()))
727 }
728 LexemeScan {
729 tokens: tokens,
730 diagnostics: diagnostics,
731 next: index,
732 propagatedLineBreak: false,
733 }
736export function lex(source: string) -> RawResult {
737 let bytes = Bytes.encodeUtf8(source)
738 let result = scanCode(bytes, 0, false, false)
739 RawResult { tokens: result.tokens, diagnostics: result.diagnostics }

SHA256 03f62f0b96a06b4fde80b2420fb87e0bac37f3e6c31e022d15e47f321254254b