STUDIO HAZE · topaz

Topaz source

Open by house policy. Topaz is our own language.

Topaz 5.20 · 56 files

compiler/crates/topaz_self_frontend/topaz/src/main.tpz

Topaz · 16632 B · Original file

1import checker { check }

2import emitter { emitRust, emitStep }

3import layout { normalize }

4import lowering { lowerStep, lowerValue }

5import parser { parse }

6import profile { checkProfile }

7import raw { lex }

8import resolver { resolvePreview, resolveWithFacts }

9import resolver.model_facts { ResolverHostFactCatalog, resolverHostFacts }

10import stage1_types { LoweringOutput, STAGE1_EXCHANGE_SCHEMA, Stage1Provenance, compilerProvenance }

11import types {

12 CheckResult,

13 LowerBridgeRequest,

14 encodeLowerBridgeRequest,

15 LayoutResult,

16 ParseResult,

17 PreviewAstNode,

18 PreviewCallArgument,

19 PreviewCallEvaluation,

20 PreviewCheckLabel,

21 PreviewDiagnostic,

22 PreviewDirectoryEntry,

23 PreviewGeneratedStdModule,

24 PreviewHostFact,

25 PreviewHostQuery,

26 PreviewImportEdge,

27 PreviewModuleResult,

28 PreviewResolvedDeclaration,

29 PreviewResolvedDiagnostic,

30 PreviewResolvedExport,

31 PreviewResolvedReference,

32 PreviewResolvedScope,

33 PreviewToken,

34 PreviewTypeAtom,

35 PreviewTypedCall,

36 PreviewTypedCapture,

37 PreviewTypedNode,

38 PreviewCheckDiagnostic,

39 RawResult,

40 ResolveResult,

41}

43const EXCHANGE_SCHEMA = "topaz.compiler.frontend-preview-exchange/v1"

45export record RequestPackageFacts derives JSON {

46 buildRole: string,

47 externModules: Array<string>,

48 externReplayModules: Array<string>,

49 externReplayErrors: Array<string>,

50 generatedStdModules: Array<PreviewGeneratedStdModule>,

51}

53record PreviewRequest derives JSON {

54 schema: string,

55 terminal: string,

56 entry: string,

57 root: string,

58 source: string,

59 sourceId: string,

60 facts: Array<PreviewHostFact>,

61 package: RequestPackageFacts,

62 maxAstNodes: int,

63 maxAstDepth: int,

64}

66export record PreviewResponse derives JSON {

67 schema: string,

68 status: string,

69 sourceId: string,

70 raw: Array<PreviewToken>,

71 layout: Array<PreviewToken>,

72 ast: Array<PreviewAstNode>,

73 diagnostics: Array<PreviewDiagnostic>,

74 queries: Array<PreviewHostQuery>,

75 modules: Array<PreviewModuleResult>,

76 edges: Array<PreviewImportEdge>,

77 scopes: Array<PreviewResolvedScope>,

78 declarations: Array<PreviewResolvedDeclaration>,

79 references: Array<PreviewResolvedReference>,

80 exports: Array<PreviewResolvedExport>,

81 resolverDiagnostics: Array<PreviewResolvedDiagnostic>,

82 typedNodes: Array<PreviewTypedNode>,

83 typedCalls: Array<PreviewTypedCall>,

84 typedCaptures: Array<PreviewTypedCapture>,

85 checkerDiagnostics: Array<PreviewCheckDiagnostic>,

86}

88export record Stage1Request derives JSON {

89 schema: string,

90 producer: string,

91 compilerSourceSetId: string,

92 terminal: string,

93 entry: string,

94 root: string,

95 source: string,

96 sourceId: string,

97 languageMode: string,

98 profile: string,

99 facts: Array<PreviewHostFact>,

100 package: RequestPackageFacts,

101 maxSourceFacts: int,

102 maxTotalSourceBytes: int,

103 maxAstNodes: int,

104 maxAstDepth: int,

105 maxRawTokens: int,

106 maxLayoutTokens: int,

107 maxHirNodes: int,

108 maxLoweredNodes: int,

109 maxDiagnostics: int,

110 maxGeneratedRustBytes: int,

111 maxFactRounds: int,

112}

113

114record SyntaxPreview {

115 ast: Array<PreviewAstNode>,

116 diagnostics: Array<PreviewDiagnostic>,

117}

118

119record CompilerTypedResponse derives JSON {

120 schema: string,

121 status: string,

122 frontEnd: string,

123 provenance: Stage1Provenance,

124}

125

126function emptyCheckResult() -> CheckResult {

127 CheckResult { nodes: [], calls: [], captures: [], diagnostics: [] }

128}

129

130function resolveRequest(request: PreviewRequest) -> Result<ResolveResult, string> {

131 resolvePreview(

132 request.entry,

133 request.root,

134 request.facts,

135 request.package.externModules,

136 request.package.generatedStdModules,

137 request.maxAstNodes,

138 request.maxAstDepth,

139 )

140}

141

142function checkResolvedRequest(

143 request: PreviewRequest,

144 resolved: ResolveResult,

145) -> Result<CheckResult, string> {

146 if request.terminal == "typed" &&

147 resolved.queries.length == 0 && resolved.diagnostics.length == 0 {

148 check(

149 resolved,

150 request.package.externReplayModules,

151 request.package.externReplayErrors,

152 request.maxAstNodes,

153 request.maxAstDepth,

154 )

155 } else {

156 Ok(emptyCheckResult())

157 }

158}

159

160function resolvedStatus(resolved: ResolveResult, checked: CheckResult) -> string {

161 if resolved.queries.length > 0 {

162 "need-facts"

163 } else if resolved.diagnostics.length > 0 || checked.diagnostics.length > 0 {

164 "rejected"

165 } else {

166 "completed"

167 }

168}

169

170function resolvedResponse(

171 request: PreviewRequest,

172 resolved: ResolveResult,

173 checked: CheckResult,

174) -> PreviewResponse {

175 PreviewResponse {

176 schema: EXCHANGE_SCHEMA,

177 status: resolvedStatus(resolved, checked),

178 sourceId: request.sourceId,

179 raw: [],

180 layout: [],

181 ast: [],

182 diagnostics: [],

183 queries: resolved.queries,

184 modules: resolved.modules,

185 edges: resolved.edges,

186 scopes: resolved.scopes,

187 declarations: resolved.declarations,

188 references: resolved.references,

189 exports: resolved.exports,

190 resolverDiagnostics: resolved.diagnostics,

191 typedNodes: checked.nodes,

192 typedCalls: checked.calls,

193 typedCaptures: checked.captures,

194 checkerDiagnostics: checked.diagnostics,

195 }

196}

197

198function processResolvedRequest(request: PreviewRequest) -> Result<PreviewResponse, string> {

199 let resolved = resolveRequest(request)?

200 let checked = checkResolvedRequest(request, resolved)?

201 Ok(resolvedResponse(request, resolved, checked))

202}

203

204function parseSyntax(

205 request: PreviewRequest,

206 raw: RawResult,

207 layout: LayoutResult,

208) -> Result<SyntaxPreview, string> {

209 let mut diagnostics = raw.diagnostics

210 for found in layout.diagnostics {

211 diagnostics.push(found)

212 }

213 let mut ast: Array<PreviewAstNode> = []

214 if diagnostics.length == 0 {

215 let parsed = parse(

216 layout.tokens,

217 request.source,

218 request.maxAstNodes,

219 request.maxAstDepth,

220 )?

221 ast = parsed.nodes

222 for found in parsed.diagnostics {

223 diagnostics.push(found)

224 }

225 }

226 Ok(SyntaxPreview { ast: ast, diagnostics: diagnostics })

227}

228

229function syntaxResponse(

230 request: PreviewRequest,

231 raw: RawResult,

232 layout: LayoutResult,

233 syntax: SyntaxPreview,

234) -> PreviewResponse {

235 PreviewResponse {

236 schema: EXCHANGE_SCHEMA,

237 status: if syntax.diagnostics.length == 0 { "completed" } else { "rejected" },

238 sourceId: request.sourceId,

239 raw: raw.tokens,

240 layout: layout.tokens,

241 ast: syntax.ast,

242 diagnostics: syntax.diagnostics,

243 queries: [],

244 modules: [],

245 edges: [],

246 scopes: [],

247 declarations: [],

248 references: [],

249 exports: [],

250 resolverDiagnostics: [],

251 typedNodes: [],

252 typedCalls: [],

253 typedCaptures: [],

254 checkerDiagnostics: [],

255 }

256}

257

258function processSyntaxRequest(request: PreviewRequest) -> Result<PreviewResponse, string> {

259 let raw = lex(request.source)

260 let layout = normalize(raw.tokens, request.source)

261 let syntax = parseSyntax(request, raw, layout)?

262 Ok(syntaxResponse(request, raw, layout, syntax))

263}

264

265function process(request: PreviewRequest) -> Result<PreviewResponse, string> {

266 if request.schema != EXCHANGE_SCHEMA {

267 return Err("unsupported front-end preview exchange schema")

268 }

269 if request.terminal == "resolved" || request.terminal == "typed" {

270 processResolvedRequest(request)

271 } else if request.terminal == "ast" {

272 processSyntaxRequest(request)

273 } else {

274 Err("unsupported front-end preview terminal `{request.terminal}`")

275 }

276}

277

278export function frontEndStep(request: Bytes) -> Result<Bytes, string> {

279 let text = request.decodeUtf8()?

280 let decoded = JSON.parseAs<PreviewRequest>(text)?

281 let response = process(decoded)?

282 let encoded = JSON.stringify(response)?

283 Ok(Bytes.encodeUtf8(encoded))

284}

285

286function supportsSelfHostedLanguage(mode: string) -> bool {

287 mode == "topaz-5.16" ||

288 mode == "topaz-5.17" ||

289 mode == "topaz-5.18" ||

290 mode == "topaz-5.19" ||

291 mode == "topaz-5.20"

292}

293

294function validateStage1Request(request: Stage1Request) -> Result<int, string> {

295 if request.schema != STAGE1_EXCHANGE_SCHEMA {

296 return Err("unsupported Stage 1 exchange schema")

297 }

298 let producerStage = if request.producer == "topaz-stage1" {

299 1

300 } else if request.producer == "topaz-stage2" {

301 2

302 } else {

303 return Err("unsupported compiler producer")

304 }

305 if request.terminal != "typed" && request.terminal != "lowered" && request.terminal != "rust-source" {

306 return Err("Stage 1 supports typed, lowered and rust-source terminals")

307 }

308 if !supportsSelfHostedLanguage(request.languageMode) {

309 return Err("unsupported self-hosted language mode `{request.languageMode}`")

310 }

311 if request.maxFactRounds < 1 {

312 return Err("Stage 1 fact-round budget must be positive")

313 }

314 Ok(producerStage)

315}

316

317function stage1PreviewRequest(request: Stage1Request) -> PreviewRequest {

318 PreviewRequest {

319 schema: EXCHANGE_SCHEMA,

320 terminal: "typed",

321 entry: request.entry,

322 root: request.root,

323 source: "",

324 sourceId: "",

325 facts: request.facts,

326 package: request.package,

327 maxAstNodes: request.maxAstNodes,

328 maxAstDepth: request.maxAstDepth,

329 }

330}

331

332function applyCompilerProfile(

333 profile: string,

334 base: PreviewResponse,

335) -> Result<PreviewResponse, string> {

336 let mut checkerDiagnostics = base.checkerDiagnostics

337 if base.queries.length == 0 {

338 for diagnostic in checkProfile(

339 profile,

340 base.modules,

341 base.references,

342 base.typedNodes,

343 base.typedCalls,

344 base.typedCaptures,

345 )? {

346 checkerDiagnostics.push(diagnostic)

347 }

348 }

349 Ok(PreviewResponse {

350 ...base,

351 status: if base.status == "completed" && checkerDiagnostics.length > 0 {

352 "rejected"

353 } else {

354 base.status

355 },

356 checkerDiagnostics: checkerDiagnostics,

357 })

358}

359

360// Lowering follows imported names to their declarations, which live in module

361// scope; local declarations stay out of the bridge.

362function moduleScopeDeclarations(

363 values: Array<PreviewResolvedDeclaration>,

364) -> Array<PreviewResolvedDeclaration> {

365 let mut declarations: Array<PreviewResolvedDeclaration> = []

366 for value in values {

367 if value.scopeOrdinal == 0 {

368 declarations.push(value)

369 }

370 }

371 declarations

372}

373

374function lowerBridgeRequest(

375 request: Stage1Request,

376 producerStage: int,

377 frontEnd: PreviewResponse,

378) -> Result<Bytes, string> {

379 encodeLowerBridgeRequest(LowerBridgeRequest {

380 ...lowerBridgeValue(request, producerStage, frontEnd),

381 frontEnd: JSON.stringify(frontEnd)?,

382 })

383}

384

385function lowerBridgeValue(

386 request: Stage1Request,

387 producerStage: int,

388 frontEnd: PreviewResponse,

389) -> LowerBridgeRequest {

390 LowerBridgeRequest {

391 producer: request.producer,

392 producerStage: producerStage,

393 frontEnd: "",

394 frontEndStatus: frontEnd.status,

395 modules: frontEnd.modules,

396 references: frontEnd.references,

397 typedNodes: frontEnd.typedNodes,

398 typedCalls: frontEnd.typedCalls,

399 typedCaptures: frontEnd.typedCaptures,

400 edges: frontEnd.edges,

401 moduleDeclarations: moduleScopeDeclarations(frontEnd.declarations),

402 languageMode: request.languageMode,

403 sourceSetId: request.compilerSourceSetId,

404 maxLoweredNodes: request.maxLoweredNodes,

405 }

406}

407

408function finishCompilerRequest(

409 request: Stage1Request,

410 lowered: Bytes,

411) -> Result<Bytes, string> {

412 if request.terminal == "rust-source" {

413 emitStep(lowered, request.maxGeneratedRustBytes)

414 } else {

415 Ok(lowered)

416 }

417}

418

419export function compilerStep(input: Bytes) -> Result<Bytes, string> {

420 let text = input.decodeUtf8()?

421 let decoded = JSON.parseAs<Stage1Request>(text)?

422 let producerStage = validateStage1Request(decoded)?

423 let baseFrontEnd = process(stage1PreviewRequest(decoded))?

424 let frontEnd = applyCompilerProfile(decoded.profile, baseFrontEnd)?

425 if decoded.terminal == "typed" {

426 let response = CompilerTypedResponse {

427 schema: STAGE1_EXCHANGE_SCHEMA,

428 status: frontEnd.status,

429 frontEnd: JSON.stringify(frontEnd)?,

430 provenance: compilerProvenance(decoded.compilerSourceSetId, decoded.producer, producerStage),

431 }

432 return Ok(Bytes.encodeUtf8(JSON.stringify(response)?))

433 }

434 let bridge = lowerBridgeRequest(decoded, producerStage, frontEnd)?

435 let lowered = lowerStep(bridge)?

436 finishCompilerRequest(decoded, lowered)

437}

438

439// Results are values. Whole frontend observations and lowered tables are

440// returned only when requested; a typed completion never denotes Rust output.

441export record CompilerResult {

442 status: string,

443 kind: string,

444 diagnostics: Array<PreviewResolvedDiagnostic>,

445 checkerDiagnostics: Array<PreviewCheckDiagnostic>,

446 exports: Array<PreviewResolvedExport>,

447 provenance: Stage1Provenance,

448 generatedRust: Option<string>,

449 lowered: Option<LoweringOutput>,

450 observation: Option<PreviewResponse>,

451}

452

453function resourceLimit(label: string, observed: int, limit: int) -> Result<(), string> {

454 if observed > limit {

455 Err("front-end resolver preview {label} resource limit: observed {observed}, limit {limit}")

456 } else {

457 Ok(())

458 }

459}

460

461function accountSourceFact(

462 request: Stage1Request,

463 totals: Array<int>,

464 fact: PreviewHostFact,

465) -> Result<(), string> {

466 let mut counts = totals

467 if fact.kind == "read-source" {

468 counts[0] += 1

469 if fact.status == "present" { counts[1] += fact.source.byteLength() }

470 resourceLimit("source-fact", counts[0], request.maxSourceFacts)?

471 resourceLimit("total-source-byte", counts[1], request.maxTotalSourceBytes)?

472 }

473 Ok(())

474}

475

476function resolverBudgets(request: Stage1Request, resolved: ResolveResult) -> Result<(), string> {

477 let mut raw = 0

478 let mut layout = 0

479 let mut ast = 0

480 for module in resolved.modules {

481 raw += module.raw.length

482 layout += module.layout.length

483 ast += module.ast.length

484 }

485 resourceLimit("raw-token", raw, request.maxRawTokens)?

486 resourceLimit("layout-token", layout, request.maxLayoutTokens)?

487 resourceLimit("ast-node", ast, request.maxAstNodes)?

488 resourceLimit("resolved-fact", resolved.scopes.length + resolved.declarations.length +

489 resolved.references.length + resolved.exports.length, request.maxHirNodes)?

490 resourceLimit("diagnostic", resolved.diagnostics.length, request.maxDiagnostics)

491}

492

493// The host owns I/O and package facts. K owns this invocation's fact cache,

494// parsing, graph discovery, checking and requested backend work. No state is

495// retained by a subsequent call and no compilerStep replay occurs here.

496export function compile(

497 request: Stage1Request,

498 provide: (PreviewHostQuery) -> Result<PreviewHostFact, string>,

499 observe: bool,

500 includeExports: bool,

501) -> Result<CompilerResult, string> {

502 let producerStage = validateStage1Request(request)?

503 let mut facts: Array<PreviewHostFact> = []

504 let counts: Array<int> = [0, 0]

505 for fact in request.facts {

506 accountSourceFact(request, counts, fact)?

507 facts.push(fact)

508 }

509 let catalog = resolverHostFacts(facts)

510 let hostFacts = ResolverHostFactCatalog {

511 ...catalog,

512 provider: Some((query: PreviewHostQuery) => {

513 let fact = provide(query)?

514 accountSourceFact(request, counts, fact)?

515 Ok(fact)

516 }),

517 }

518 let resolved = resolveWithFacts(request.entry, request.root, hostFacts,

519 request.package.externModules, request.package.generatedStdModules,

520 request.maxAstNodes, request.maxAstDepth)?

521 resolverBudgets(request, resolved)?

522 let preview = stage1PreviewRequest(request)

523 let checked = checkResolvedRequest(preview, resolved)?

524 let frontEnd = applyCompilerProfile(request.profile, resolvedResponse(preview, resolved, checked))?

525 resourceLimit("diagnostic", frontEnd.resolverDiagnostics.length +

526 frontEnd.checkerDiagnostics.length, request.maxDiagnostics)?

527 let base = CompilerResult {

528 status: frontEnd.status,

529 kind: "typed",

530 diagnostics: frontEnd.resolverDiagnostics,

531 checkerDiagnostics: frontEnd.checkerDiagnostics,

532 exports: if includeExports { frontEnd.exports } else { [] },

533 provenance: compilerProvenance(request.compilerSourceSetId, request.producer, producerStage),

534 generatedRust: None,

535 lowered: None,

536 observation: if observe { Some(frontEnd) } else { None },

537 }

538 if frontEnd.status != "completed" || request.terminal == "typed" {

539 return Ok(base)

540 }

541 let lowered = lowerValue(lowerBridgeValue(request, producerStage, frontEnd))?

542 if request.terminal == "lowered" {

543 return Ok(CompilerResult {

544 ...base,

545 kind: "lowered",

546 status: if lowered.unsupported.length > 0 { "unsupported" } else { "completed" },

547 lowered: Some(lowered),

548 })

549 }

550 let rust = emitRust(lowered)?

551 if rust.byteLength() > request.maxGeneratedRustBytes {

552 return Err("Stage 1 generated Rust exceeds {request.maxGeneratedRustBytes} bytes")

553 }

554 Ok(CompilerResult { ...base, kind: "rust-source", generatedRust: Some(rust) })

555}

556

SHA256 2aba863dd6653221f007066facdea8e6ebaff94998d20fe5d2b427f36461d804