slow.monster

STUDIO HAZE · topaz

Topaz source

Open by house policy. Topaz is our own language.

Topaz 5.20 · 56 files

compiler/crates/topaz_self_frontend/topaz/unicode.tpz

Topaz · 9698 B · Original file

1import unicode_tables {
2 CANONICAL_COMBINING_CLASS_KEYS,
3 CANONICAL_COMBINING_CLASS_VALUES,
4 CANONICAL_DECOMPOSITION_KEYS,
5 CANONICAL_DECOMPOSITION_OFFSETS,
6 CANONICAL_DECOMPOSITION_VALUES,
7 CASE_FOLD_KEYS,
8 CASE_FOLD_OFFSETS,
9 CASE_FOLD_VALUES,
10 CodePointRange,
11 EMOJI_RANGES,
12 LETTER_RANGES,
13 NUMBER_RANGES,
16// Unicode Standard Annex #15 constants for algorithmic Hangul decomposition.
17const HANGUL_SYLLABLE_BASE = 44032
18const HANGUL_SYLLABLE_LIMIT = 55204
19const HANGUL_LEADING_BASE = 4352
20const HANGUL_VOWEL_BASE = 4449
21const HANGUL_TRAILING_BASE = 4519
22const HANGUL_SYLLABLES_PER_LEADING = 588
23const HANGUL_TRAILING_COUNT = 28
24// Suggestions skip short names and admit edits up to one third of the longer spelling.
25const MIN_SUGGESTION_SCALARS = 4
26const MAX_SUGGESTION_SCALARS = 64
27const SUGGESTION_DISTANCE_DIVISOR = 3
28const MAX_SUGGESTION_BYTES = 256
29// This sentinel exceeds every admitted suggestion length.
30const NO_SUGGESTION_DISTANCE = 1000000
32function contains(ranges: Array<CodePointRange>, codePoint: int) -> bool {
33 let mut lo = 0
34 let mut hi = ranges.length
35 while lo < hi {
36 let middle = lo + (hi - lo) / 2
37 let range = ranges[middle]
38 if codePoint < range.first {
39 hi = middle
40 } else if codePoint > range.last {
41 lo = middle + 1
42 } else {
43 return true
44 }
45 }
46 false
49function sequenceIndex(keys: Array<int>, codePoint: int) -> int {
50 let mut lo = 0
51 let mut hi = keys.length
52 while lo < hi {
53 let middle = lo + (hi - lo) / 2
54 let candidate = keys[middle]
55 if codePoint < candidate {
56 hi = middle
57 } else if codePoint > candidate {
58 lo = middle + 1
59 } else {
60 return middle
61 }
62 }
63 -1
66function combiningClass(codePoint: int) -> int {
67 let index = sequenceIndex(CANONICAL_COMBINING_CLASS_KEYS, codePoint)
68 if index >= 0 {
69 return CANONICAL_COMBINING_CLASS_VALUES[index]
70 }
71 0
74function decompose(codePoint: int, output: Array<int>) -> () {
75 let mut target = output
76 if codePoint >= HANGUL_SYLLABLE_BASE && codePoint < HANGUL_SYLLABLE_LIMIT {
77 let syllable = codePoint - HANGUL_SYLLABLE_BASE
78 target.push(HANGUL_LEADING_BASE + syllable / HANGUL_SYLLABLES_PER_LEADING)
79 target.push(HANGUL_VOWEL_BASE +
80 (syllable % HANGUL_SYLLABLES_PER_LEADING) / HANGUL_TRAILING_COUNT)
81 let trailing = syllable % HANGUL_TRAILING_COUNT
82 if trailing != 0 {
83 target.push(HANGUL_TRAILING_BASE + trailing)
84 }
85 return ()
86 }
87 let index = sequenceIndex(CANONICAL_DECOMPOSITION_KEYS, codePoint)
88 if index < 0 {
89 target.push(codePoint)
90 return ()
91 }
92 let mut offset = CANONICAL_DECOMPOSITION_OFFSETS[index]
93 let limit = CANONICAL_DECOMPOSITION_OFFSETS[index + 1]
94 while offset < limit {
95 target.push(CANONICAL_DECOMPOSITION_VALUES[offset])
96 offset += 1
97 }
98 ()
101function codePointText(codePoint: int) -> string {
102 match fromCodePoint(codePoint) {
103 case Some(value) => value
104 case None => ""
105 }
108export function nfd(value: string) -> string {
109 let mut codePoints: Array<int> = []
110 for scalar in value.scalars() {
111 match scalar.codePointAt(0) {
112 case Some(codePoint) => decompose(codePoint, codePoints)
113 case None => ()
114 }
115 }
116 let mut index = 1
117 while index < codePoints.length {
118 let left = codePoints[index - 1]
119 let right = codePoints[index]
120 let leftClass = combiningClass(left)
121 let rightClass = combiningClass(right)
122 if rightClass != 0 && leftClass > rightClass {
123 codePoints[index - 1] = right
124 codePoints[index] = left
125 if index > 1 {
126 index -= 1
127 } else {
128 index += 1
129 }
130 } else {
131 index += 1
132 }
133 }
134 let mut output: Array<string> = []
135 for codePoint in codePoints {
136 output.push(codePointText(codePoint))
137 }
138 output.join("")
141export function caseFold(value: string) -> string {
142 let mut output: Array<string> = []
143 for scalar in value.scalars() {
144 match scalar.codePointAt(0) {
145 case Some(codePoint) => {
146 let index = sequenceIndex(CASE_FOLD_KEYS, codePoint)
147 if index < 0 {
148 output.push(scalar)
149 } else {
150 let mut offset = CASE_FOLD_OFFSETS[index]
151 let limit = CASE_FOLD_OFFSETS[index + 1]
152 while offset < limit {
153 output.push(codePointText(CASE_FOLD_VALUES[offset]))
154 offset += 1
155 }
156 }
157 }
158 case None => ()
159 }
160 }
161 output.join("")
164function isIdentifierStartCodePoint(codePoint: int) -> bool {
165 codePoint == 95 ||
166 contains(LETTER_RANGES, codePoint) ||
167 contains(EMOJI_RANGES, codePoint)
170function isIdentifierContinueCodePoint(codePoint: int) -> bool {
171 isIdentifierStartCodePoint(codePoint) || contains(NUMBER_RANGES, codePoint)
174function byteAt(bytes: Bytes, index: int) -> int {
175 match bytes.get(index) {
176 case Some(value) => value
177 case None => -1
178 }
181export function utf8Width(first: int) -> int {
182 if first >= 0 && first < 128 { return 1 }
183 if first >= 194 && first <= 223 { return 2 }
184 if first >= 224 && first <= 239 { return 3 }
185 if first >= 240 && first <= 244 { return 4 }
186 0
189function codePointAt(bytes: Bytes, index: int, width: int) -> Option<int> {
190 match bytes.slice(index, index + width).decodeUtf8() {
191 case Ok(text) => text.codePointAt(0)
192 case Err(_) => None
193 }
196function identifierWidth(bytes: Bytes, index: int, allowNumber: bool) -> int {
197 let first = byteAt(bytes, index)
198 if first >= 0 && first < 128 {
199 let isStart = first == 95 ||
200 (first >= 65 && first <= 90) ||
201 (first >= 97 && first <= 122)
202 return if isStart || allowNumber && first >= 48 && first <= 57 { 1 } else { 0 }
203 }
204 let width = utf8Width(first)
205 if width == 0 { return 0 }
206 match codePointAt(bytes, index, width) {
207 case Some(codePoint) => {
208 let accepted = if allowNumber {
209 isIdentifierContinueCodePoint(codePoint)
210 } else {
211 isIdentifierStartCodePoint(codePoint)
212 }
213 if accepted { width } else { 0 }
214 }
215 case None => 0
216 }
219export function identifierStartWidth(bytes: Bytes, index: int) -> int {
220 identifierWidth(bytes, index, false)
223function identifierContinueWidth(bytes: Bytes, index: int) -> int {
224 identifierWidth(bytes, index, true)
227export function identifierContinueEnd(bytes: Bytes, start: int) -> int {
228 let mut index = start
229 let mut width = identifierContinueWidth(bytes, index)
230 while width > 0 {
231 index += width
232 width = identifierContinueWidth(bytes, index)
233 }
234 index
237export function identifierEnd(bytes: Bytes, start: int) -> int {
238 let width = identifierStartWidth(bytes, start)
239 if width == 0 { return start }
240 identifierContinueEnd(bytes, start + width)
243function minimum3(left: int, middle: int, right: int) -> int {
244 let mut best = left
245 if middle < best {
246 best = middle
247 }
248 if right < best {
249 best = right
250 }
251 best
254function osaDistance(leftScalars: Array<string>, rightScalars: Array<string>) -> int {
255 let mut previousPrevious: Array<int> = []
256 let mut previous: Array<int> = []
257 let mut current: Array<int> = []
258 let mut column = 0
259 while column <= rightScalars.length {
260 previousPrevious.push(0)
261 previous.push(column)
262 current.push(0)
263 column += 1
264 }
265 let mut row = 1
266 while row <= leftScalars.length {
267 current[0] = row
268 column = 1
269 while column <= rightScalars.length {
270 let substitution = previous[column - 1] +
271 if leftScalars[row - 1] == rightScalars[column - 1] {
272 0
273 } else {
274 1
275 }
276 let deletion = previous[column] + 1
277 let insertion = current[column - 1] + 1
278 let mut best = minimum3(substitution, deletion, insertion)
279 if row > 1 && column > 1 &&
280 leftScalars[row - 1] == rightScalars[column - 2] &&
281 leftScalars[row - 2] == rightScalars[column - 1] {
282 let transposition = previousPrevious[column - 2] + 1
283 if transposition < best {
284 best = transposition
285 }
286 }
287 current[column] = best
288 column += 1
289 }
290 let recycled = previousPrevious
291 previousPrevious = previous
292 previous = current
293 current = recycled
294 row += 1
295 }
296 previous[rightScalars.length]
299export function didYouMean(target: string, candidates: Array<string>) -> string {
300 if target.byteLength() > MAX_SUGGESTION_BYTES {
301 return ""
302 }
303 let targetScalars = target.scalars()
304 let targetLength = targetScalars.length
305 if targetLength > MAX_SUGGESTION_SCALARS {
306 return ""
307 }
308 let mut best = ""
309 let mut bestDistance = NO_SUGGESTION_DISTANCE
310 for candidate in candidates {
311 if candidate != target && candidate.byteLength() <= MAX_SUGGESTION_BYTES {
312 let candidateScalars = candidate.scalars()
313 let candidateLength = candidateScalars.length
314 if candidateLength >= MIN_SUGGESTION_SCALARS &&
315 candidateLength <= MAX_SUGGESTION_SCALARS {
316 let longest = if targetLength > candidateLength { targetLength } else { candidateLength }
317 let mut threshold = longest / SUGGESTION_DISTANCE_DIVISOR
318 if threshold < 1 {
319 threshold = 1
320 }
321 let lengthDifference = if targetLength > candidateLength {
322 targetLength - candidateLength
323 } else {
324 candidateLength - targetLength
325 }
326 if lengthDifference <= threshold {
327 let distance = osaDistance(targetScalars, candidateScalars)
328 if distance <= threshold &&
329 (best == "" || distance < bestDistance ||
330 distance == bestDistance && candidate < best) {
331 best = candidate
332 bestDistance = distance
333 }
334 }
335 }
336 }
337 }
338 if best == "" { "" } else { "; did you mean `{best}`?" }

SHA256 a71b3b9c72d4aaf02b86a81be553a3421af4bee9960a363cfbc515201f7afeac