Coverage for src/wiktextract/extractor/en/pronunciation.py: 84%

918 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 07:03 +0000

1import hashlib 

2import re 

3import urllib 

4from copy import deepcopy 

5from dataclasses import dataclass 

6from typing import Iterator, Literal, NamedTuple 

7 

8from wikitextprocessor import ( 

9 HTMLNode, 

10 LevelNode, 

11 NodeKind, 

12 TemplateNode, 

13 WikiNode, 

14) 

15 

16from ...clean import clean_value 

17from ...datautils import data_append, data_extend, split_at_comma_semi 

18from ...page import LEVEL_KINDS, clean_node, is_panel_template 

19from ...tags import valid_tags 

20from ...wxr_context import WiktextractContext 

21from ..share import create_audio_url_dict 

22from .form_descriptions import ( 

23 classify_desc, 

24 decode_tags, 

25 parse_pronunciation_tags, 

26) 

27from .parts_of_speech import part_of_speech_map 

28from .type_utils import Hyphenation, SoundData, TemplateArgs, WordData 

29 

30PronunciationPoses = tuple[str, ...] 

31 

32# Prefixes, tags, and regexp for finding romanizations from the pronuncation 

33# section 

34pron_romanizations = { 

35 " Revised Romanization ": "romanization revised", 

36 " Revised Romanization (translit.) ": ( 

37 "romanization revised transliteration" 

38 ), 

39 " McCune-Reischauer ": "McCune-Reischauer romanization", 

40 " McCune–Reischauer ": "McCune-Reischauer romanization", 

41 " Yale Romanization ": "Yale romanization", 

42} 

43pron_romanization_re = re.compile( 

44 "(?m)^(" 

45 + "|".join( 

46 re.escape(x) 

47 for x in sorted(pron_romanizations.keys(), key=len, reverse=True) 

48 ) 

49 + ")([^\n]+)" 

50) 

51 

52IPA_EXTRACT = r"^(\((.+)\) )?(IPA⁽ᵏᵉʸ⁾|enPR): ((.+?)( \(([^(]+)\))?\s*)$" 

53IPA_EXTRACT_RE = re.compile(IPA_EXTRACT) 

54 

55IPA_MATCHES_RE = re.compile( 

56 r"(?m)/[^][\n/,]+?/" r"|" r"\[[^]\n0-9,/][^],/]*?\]" 

57) 

58 

59 

60class PronunciationPosMatch(NamedTuple): 

61 pos_values: PronunciationPoses 

62 residual: str 

63 

64 

65class PronunciationPosPrefix(NamedTuple): 

66 pos_values: PronunciationPoses 

67 text: str 

68 is_persistent: bool 

69 

70 

71class FlattenedListNode(NamedTuple): 

72 node: WikiNode | str 

73 list_depth: int 

74 

75 

76PRON_POS_TEMPLATE_NAMES = { 

77 "q", 

78 "qualifier", 

79 "qual", 

80 "i", 

81 "sense", 

82 "a", 

83 "accent", 

84 "lb", 

85 "lbl", 

86 "label", 

87} 

88 

89PRON_POS_BY_LABEL = { 

90 label: pos_data["pos"] for label, pos_data in part_of_speech_map.items() 

91} 

92 

93PRON_POS_LABEL_RE = re.compile( 

94 r"^(?:(?P<residual>.+?)\s+)?(?P<label>" 

95 + "|".join( 

96 re.escape(label) 

97 for label in sorted(PRON_POS_BY_LABEL, key=len, reverse=True) 

98 ) 

99 + r")$" 

100) 

101 

102 

103def normalize_pronunciation_pos_label(label: str) -> str: 

104 label = label.strip().lower() 

105 label = re.sub(r"\s+", " ", label) 

106 # Drop explanatory suffixes such as "noun (barren areas)" before 

107 # matching the label against part-of-speech names. 

108 label = re.sub(r"\s*\([^)]*\)\s*$", "", label).strip() 

109 label = label.strip(" \t\n\r():") 

110 label = re.sub(r"\s+senses?$", "", label).strip() 

111 return label 

112 

113 

114def split_pronunciation_pos_text(text: str) -> PronunciationPosMatch: 

115 pos_values: list[str] = [] 

116 residual: list[str] = [] 

117 for part in split_pronunciation_pos_parts(text): 

118 pos, residual_part = pronunciation_pos_from_part(part) 

119 if pos: 

120 # POS-bearing qualifier text may also contain normal pronunciation 

121 # tags before the POS label, e.g. "attributive adjective". 

122 if pos not in pos_values: 122 ↛ 124line 122 didn't jump to line 124 because the condition on line 122 was always true

123 pos_values.append(pos) 

124 if residual_part: 

125 residual.append(residual_part) 

126 elif residual_part: 126 ↛ 117line 126 didn't jump to line 117 because the condition on line 126 was always true

127 residual.append(residual_part) 

128 if not pos_values: 

129 # If nothing in the text was a POS label, preserve the original text 

130 # for normal pronunciation tag/note parsing. 

131 return PronunciationPosMatch((), text.strip()) 

132 return PronunciationPosMatch(tuple(pos_values), ", ".join(residual)) 

133 

134 

135def pronunciation_pos_from_part(part: str) -> tuple[str | None, str]: 

136 normalized = normalize_pronunciation_pos_label(part) 

137 if normalized in PRON_POS_BY_LABEL: 

138 return PRON_POS_BY_LABEL[normalized], "" 

139 # Match residual tag text followed by a POS label: 

140 # "attributive adjective" -> ("adj", "attributive") 

141 # "attributive proper noun" -> ("name", "attributive") 

142 # The label alternation is sorted longest-first so multi-word POS labels 

143 # such as "proper noun" win over their suffixes. 

144 match = PRON_POS_LABEL_RE.match(normalized) 

145 if match: 

146 label = match.group("label") 

147 residual = (match.group("residual") or "").rstrip(" ,;:") 

148 if not residual or classify_desc(residual) == "tags": 

149 return PRON_POS_BY_LABEL[label], residual 

150 return None, part 

151 

152 

153def split_pronunciation_pos_parts(text: str) -> list[str]: 

154 parts: list[str] = [] 

155 for comma_part in re.split(r"[,;]", text): 

156 comma_part = comma_part.strip() 

157 if not comma_part: 

158 continue 

159 # Commas and semicolons reliably separate qualifier chunks. Only split 

160 # "and"/"or" when at least one side is a POS label, so prose notes 

161 # stay intact. 

162 conjunction_parts = re.split(r"\s+(?:and|or)\s+", comma_part) 

163 if len(conjunction_parts) > 1 and any( 

164 pronunciation_pos_from_part(part)[0] 

165 for part in conjunction_parts 

166 ): 

167 parts.extend(conjunction_parts) 

168 else: 

169 parts.append(comma_part) 

170 return parts 

171 

172 

173def set_sound_pos( 

174 sound: SoundData, pos_values: PronunciationPoses | None 

175) -> PronunciationPoses | None: 

176 if pos_values: 

177 sound["pos"] = pos_values # type: ignore[typeddict-unknown-key] 

178 return pos_values 

179 if "pos" in sound: 

180 return sound["pos"] # type: ignore[typeddict-item] 

181 return None 

182 

183 

184def common_sound_pos( 

185 pos_candidates: set[PronunciationPoses], 

186) -> PronunciationPoses | None: 

187 if len(pos_candidates) != 1: 

188 return None 

189 return next(iter(pos_candidates)) 

190 

191 

192def merge_pronunciation_tag_data( 

193 sound: SoundData, tag_data: SoundData 

194) -> None: 

195 for value in tag_data.get("tags", []): 

196 if value not in sound.get("tags", []): 196 ↛ 195line 196 didn't jump to line 195 because the condition on line 196 was always true

197 data_append(sound, "tags", value) 

198 for value in tag_data.get("topics", []): 198 ↛ 199line 198 didn't jump to line 199 because the loop on line 198 never started

199 if value not in sound.get("topics", []): 

200 data_append(sound, "topics", value) 

201 if note := tag_data.get("note"): 

202 existing_note = sound.get("note") 

203 if not existing_note: 

204 sound["note"] = note 

205 elif note not in [n.strip() for n in existing_note.split(";")]: 205 ↛ exitline 205 didn't return from function 'merge_pronunciation_tag_data' because the condition on line 205 was always true

206 sound["note"] = f"{existing_note}; {note}" 

207 

208 

209def inherit_pronunciation_tag_data( 

210 sound: SoundData, parent_tag_data: SoundData 

211) -> None: 

212 """Add the tags and topics of a parent list item, such as 

213 "* {{a|en|GA}}", to a nested pronunciation, and put its note before 

214 the nested pronunciation's own.""" 

215 if not parent_tag_data: 

216 return 

217 tag_data: SoundData = {} 

218 merge_pronunciation_tag_data(tag_data, parent_tag_data) 

219 merge_pronunciation_tag_data(tag_data, sound) 

220 if "tags" in tag_data: 220 ↛ 222line 220 didn't jump to line 222 because the condition on line 220 was always true

221 tag_data["tags"] = sorted(tag_data["tags"]) 

222 if "topics" in tag_data: 222 ↛ 223line 222 didn't jump to line 223 because the condition on line 222 was never true

223 tag_data["topics"] = sorted(tag_data["topics"]) 

224 sound.update(tag_data) 

225 

226 

227def is_toggled_expansion(text: str) -> bool: 

228 """A big, toggleable section, whose collapsed "header" may be a duplicate 

229 of its first line.""" 

230 return text.startswith("<") and "vsToggleElement" in text 

231 

232 

233def ipa_qualifiers(text: str) -> list[tuple[re.Match[str], str]]: 

234 """Each IPA match in the text, with the parenthesized qualifier that follows 

235 it: "/a/ [b] (Spain), /c/ (Peru (north))" gives "Spain" for the first two 

236 and "Peru (north)" for the last. Language modules such as {{es-pr}} expand 

237 to plain lines like that, so these never reach extract_pron_template(). 

238 Parentheses inside a transcription, as in /ˈhæp(ə)n/, are no qualifier, nor 

239 is a group holding a transcription itself, as in "/a/ (also /b/)". Anything 

240 else between a transcription and the group, such as the comma of 

241 "/a/, (old) /b/" or "/a/, /b/ (Spain)", means the group is not about it.""" 

242 ipa_matches = list(IPA_MATCHES_RE.finditer(text)) 

243 masked = text 

244 for m in ipa_matches: 

245 masked = masked[: m.start()] + " " * len(m.group()) + masked[m.end() :] 

246 groups: list[tuple[int, int]] = [] 

247 depth = group_start = 0 

248 for i, char in enumerate(masked): 

249 if char == "(": 

250 if depth == 0: 

251 group_start = i 

252 depth += 1 

253 elif char == ")" and depth > 0: 

254 depth -= 1 

255 if depth == 0: 

256 groups.append((group_start, i)) 

257 qualifiers = [] 

258 for m in ipa_matches: 

259 qualifier = "" 

260 for start, end in groups: 

261 if m.start() < start: 

262 if not masked[m.end() : start].strip() and not any( 

263 start < o.start() < end for o in ipa_matches 

264 ): 

265 qualifier = text[start + 1 : end].strip() 

266 break 

267 if m.start() < end: 

268 break 

269 qualifiers.append((m, qualifier)) 

270 return qualifiers 

271 

272 

273def parse_pronunciation_tags_with_pos( 

274 wxr: WiktextractContext, text: str, sound: SoundData 

275) -> PronunciationPoses: 

276 match = split_pronunciation_pos_text(text) 

277 set_sound_pos(sound, match.pos_values) 

278 if match.residual: 

279 tag_data: SoundData = {} 

280 parse_pronunciation_tags(wxr, match.residual, tag_data) 

281 merge_pronunciation_tag_data(sound, tag_data) 

282 return match.pos_values 

283 

284 

285def extract_pos_prefix(text: str) -> PronunciationPosPrefix | None: 

286 stripped = text.strip() 

287 if not (stripped.startswith("(") and stripped.endswith(")")): 

288 bare_match = split_pronunciation_pos_text(text) 

289 if bare_match.pos_values and not bare_match.residual: 

290 return PronunciationPosPrefix(bare_match.pos_values, "", True) 

291 

292 colon_match = re.match(r"\s*([^:()]+?)\s*:\s*(.*)$", text) 

293 if colon_match: 

294 match = split_pronunciation_pos_text(colon_match.group(1)) 

295 if match.pos_values and not match.residual: 295 ↛ 296line 295 didn't jump to line 296 because the condition on line 295 was never true

296 return PronunciationPosPrefix( 

297 match.pos_values, colon_match.group(2).strip(), True 

298 ) 

299 

300 paren_match = re.match(r"\s*\(([^()]*)\)\s*(.*)$", text) 

301 if paren_match: 

302 match = split_pronunciation_pos_text(paren_match.group(1)) 

303 if match.pos_values and not match.residual: 

304 return PronunciationPosPrefix( 

305 match.pos_values, paren_match.group(2).strip(), False 

306 ) 

307 

308 return None 

309 

310 

311def extract_pronunciation_pos_template( 

312 wxr: WiktextractContext, 

313 name: str, 

314 ht: TemplateArgs, 

315 lang_code: str, 

316) -> PronunciationPosMatch: 

317 if name in {"a", "accent", "lb", "lbl", "label"}: 

318 pos_args = [ 

319 value 

320 for key, value in ht.items() 

321 if isinstance(key, int) and key >= 2 

322 ] 

323 if not pos_args and ht.get(1) != lang_code: 

324 pos_args = [ht.get(1, "")] 

325 else: 

326 pos_args = [ 

327 value 

328 for key, value in ht.items() 

329 if isinstance(key, int) and key >= 1 

330 ] 

331 

332 pos_values: list[str] = [] 

333 residual: list[str] = [] 

334 for arg in pos_args: 

335 text = clean_node(wxr, None, [arg]) 

336 match = split_pronunciation_pos_text(text) 

337 for pos in match.pos_values: 

338 if pos not in pos_values: 338 ↛ 337line 338 didn't jump to line 337 because the condition on line 338 was always true

339 pos_values.append(pos) 

340 if match.residual: 

341 residual.append(match.residual) 

342 return PronunciationPosMatch(tuple(pos_values), ", ".join(residual)) 

343 

344 

345def extract_pron_template( 

346 wxr: WiktextractContext, tname: str, targs: TemplateArgs, expanded: str 

347) -> tuple[SoundData, list[SoundData]] | None: 

348 """In post_template_fn, this is used to handle all enPR and IPA templates 

349 so that we can leave breadcrumbs in the text that can later be handled 

350 there. We return a `base_data` so that if there are two 

351 or more templates on the same line, like this: 

352 (Tags for the whole line, really) enPR: foo, IPA(keys): /foo/ 

353 then we can apply base_data fields to other templates, too, if needed. 

354 """ 

355 cleaned = clean_value(wxr, expanded) 

356 # print(f"extract_pron_template input: {tname=} {expanded=}-> {cleaned=}") 

357 m = IPA_EXTRACT_RE.match(cleaned) 

358 if not m: 

359 wxr.wtp.error( 

360 f"Text cannot match IPA_EXTRACT_RE regex: " 

361 f"{cleaned=}, {tname=}, {targs=}", 

362 sortid="en/pronunciation/54", 

363 ) 

364 return None 

365 # for i, group in enumerate(m.groups()): 

366 # print(i + 1, repr(group)) 

367 main_qual = m.group(2) or "" 

368 if "qq" in targs: 

369 # If the template has been given a qualifier that applies to 

370 # every entry, but which also happens to appear at the end 

371 # which can be confused with the post-qualifier of a single 

372 # entry in the style of "... /ipa3/ (foo) (bar)", where foo 

373 # might not be present so the bar looks like it only might 

374 # apply to `/ipa3/` 

375 pron_body = m.group(5) 

376 post_qual = m.group(7) 

377 else: 

378 pron_body = m.group(4) 

379 post_qual = "" 

380 

381 if not pron_body: 381 ↛ 382line 381 didn't jump to line 382 because the condition on line 381 was never true

382 wxr.wtp.error( 

383 f"Regex failed to find 'body' from {cleaned=}", 

384 sortid="en/pronunciation/81", 

385 ) 

386 return None 

387 

388 base_data: SoundData = {} 

389 if main_qual: 

390 parse_pronunciation_tags_with_pos(wxr, main_qual, base_data) 

391 if post_qual: 

392 parse_pronunciation_tags_with_pos(wxr, post_qual, base_data) 

393 # This base_data is used as the base copy for all entries from this 

394 # template, but it is also returned so that its contents may be applied 

395 # to other templates on the same line. 

396 # print(f"{base_data=}") 

397 

398 sound_datas: list[SoundData] = [] 

399 

400 parts: list[list[str]] = [[]] 

401 inside = 0 

402 current: list[str] = [] 

403 for i, p in enumerate(re.split(r"(\s*,|;|\(|\)\s*)", pron_body)): 

404 # Split the line on commas and semicolons outside of parens. This 

405 # gives us lines with "(main-qualifier) /phon/ (post-qualifier, maybe)" 

406 # print(f" {i=}, {p=}") 

407 comp = p.strip() 

408 if not p: 

409 continue 

410 if comp == "(": 

411 if not inside and i > 0: 411 ↛ 414line 411 didn't jump to line 414 because the condition on line 411 was always true

412 if stripped := "".join(current).strip(): 

413 parts[-1].append("".join(current).strip()) # type:ignore[arg-type] 

414 current = [p] 

415 inside += 1 

416 continue 

417 if comp == ")": 

418 inside -= 1 

419 if not inside: 419 ↛ 424line 419 didn't jump to line 424 because the condition on line 419 was always true

420 if stripped := "".join(current).strip(): 420 ↛ 424line 420 didn't jump to line 424 because the condition on line 420 was always true

421 current.append(p) 

422 parts[-1].append("".join(current).strip()) # type:ignore[arg-type] 

423 current = [] 

424 continue 

425 if not inside and comp in (",", ";"): 

426 if stripped := "".join(current).strip(): 

427 parts[-1].append(stripped) # type:ignore[arg-type] 

428 current = [] 

429 parts.append([]) 

430 continue 

431 current.append(p) 

432 if current: 

433 parts[-1].append("".join(current).strip()) 

434 

435 # print(f">>>>>> {parts=}") 

436 new_parts: list[list[str]] = [] 

437 for entry in parts: 

438 if not entry: 438 ↛ 439line 438 didn't jump to line 439 because the condition on line 438 was never true

439 continue 

440 new_entry: list[str] = [] 

441 i1: int = entry[0].startswith("(") and entry[0].endswith(")") 

442 if i1: 

443 new_entry.append(entry[0][1:-1].strip()) 

444 else: 

445 new_entry.append("") 

446 i2: int = ( 

447 entry[-1].startswith("(") 

448 and entry[-1].endswith(")") 

449 and len(entry) > 1 

450 ) 

451 if i2 == 0: 

452 i2 = len(entry) 

453 else: 

454 i2 = -1 

455 new_entry.append("".join(entry[i1:i2]).strip()) 

456 if not new_entry[-1]: 456 ↛ 457line 456 didn't jump to line 457 because the condition on line 456 was never true

457 wxr.wtp.error( 

458 f"Missing IPA/enPRO sound data between qualifiers?{entry=}", 

459 sortid="en/pronunciation/153", 

460 ) 

461 if i2 == -1: 

462 new_entry.append(entry[-1][1:-1].strip()) 

463 else: 

464 new_entry.append("") 

465 new_parts.append(new_entry) 

466 

467 # print(f">>>>> {new_parts=}") 

468 

469 for part in new_parts: 

470 sd = deepcopy(base_data) 

471 if part[0]: 

472 parse_pronunciation_tags_with_pos(wxr, part[0], sd) 

473 if part[2]: 

474 parse_pronunciation_tags_with_pos(wxr, part[2], sd) 

475 if tname == "enPR": 

476 sd["enpr"] = part[1] 

477 else: 

478 sd["ipa"] = part[1] 

479 sound_datas.append(sd) 

480 

481 # print(f"BASE_DATA: {base_data}") 

482 # print(f"SOUND_DATAS: {sound_datas=}") 

483 

484 return base_data, sound_datas 

485 

486 

487def parse_pronunciation( 

488 wxr: WiktextractContext, 

489 level_node: LevelNode, 

490 data: WordData, 

491 etym_data: WordData, 

492 have_etym: bool, 

493 base_data: WordData, 

494 lang_code: str, 

495) -> None: 

496 """Parses the pronunciation section from a language section on a 

497 page.""" 

498 if level_node.kind in LEVEL_KINDS: 498 ↛ 511line 498 didn't jump to line 511 because the condition on line 498 was always true

499 contents: list[str | WikiNode | TemplateNode] = [] 

500 for node in level_node.children: 

501 if isinstance(node, TemplateNode): 

502 if node.template_name == "th-pron": 

503 extract_th_pron_template(wxr, data, node) 

504 elif node.template_name == "zh-pron": 

505 extract_zh_pron_template(wxr, data, node) 

506 else: 

507 contents.append(node) 

508 else: 

509 contents.append(node) 

510 else: 

511 contents = [level_node] 

512 # Remove subsections, such as Usage notes. They may contain IPAchar 

513 # templates in running text, and we do not want to extract IPAs from 

514 # those. 

515 # Filter out only LEVEL_KINDS; 'or' is doing heavy lifting here 

516 # Slip through not-WikiNodes, then slip through WikiNodes that 

517 # are not LEVEL_KINDS. 

518 contents = [ 

519 x 

520 for x in contents 

521 if not isinstance(x, WikiNode) or x.kind not in LEVEL_KINDS 

522 ] 

523 # Set when a template's expansion may generate duplicate sound data 

524 may_be_duplicates = False 

525 if not any( 

526 isinstance(x, WikiNode) and x.kind == NodeKind.LIST for x in contents 

527 ): 

528 # expand all templates 

529 new_contents: list[str | WikiNode | TemplateNode] = [] 

530 for lst in contents: 

531 if isinstance(lst, TemplateNode): 

532 temp = wxr.wtp.node_to_wikitext(lst) 

533 temp = wxr.wtp.expand(temp) 

534 if is_toggled_expansion(temp): 

535 may_be_duplicates = True 

536 temp_parsed = wxr.wtp.parse(temp) 

537 new_contents.extend(temp_parsed.children) 

538 else: 

539 new_contents.append(lst) 

540 contents = new_contents 

541 

542 if have_etym and data is base_data: 542 ↛ 543line 542 didn't jump to line 543 because the condition on line 542 was never true

543 data = etym_data 

544 pron_templates: list[tuple[SoundData, list[SoundData]]] = [] 

545 pron_pos_markers: list[PronunciationPoses] = [] 

546 hyphenations: list[Hyphenation] = [] 

547 audios: list[SoundData] = [] 

548 have_panel_templates = False 

549 

550 def parse_pronunciation_template_fn( 

551 name: str, ht: TemplateArgs 

552 ) -> str | None: 

553 """Handle pronunciation and hyphenation templates""" 

554 # _template_fn handles templates *before* they are expanded; 

555 # this allows for special handling before all the work needed 

556 # for expansion is done. 

557 nonlocal have_panel_templates 

558 if is_panel_template(wxr, name): 

559 have_panel_templates = True 

560 return "" 

561 if name == "audio": 

562 filename = ht.get(2) or "" 

563 audio: SoundData = {"audio": filename.strip()} 

564 dialect = ht.get("a", "") 

565 if "aa" in ht: 565 ↛ 566line 565 didn't jump to line 566 because the condition on line 565 was never true

566 dialect += ", " + ht.get("aa", "") 

567 if dialect: 

568 dialect = dialect.replace("<", "").replace(">", "") 

569 dialect = clean_node(wxr, None, [dialect]) 

570 for part in split_at_comma_semi(dialect): 

571 if "(" not in part: 

572 parse_pronunciation_tags(wxr, part, audio) 

573 else: 

574 for ppart in re.split(r"[][()]", part): 

575 parse_pronunciation_tags(wxr, ppart, audio) 

576 desc = ht.get(3) or "" 

577 desc = clean_node(wxr, None, [desc]) 

578 if desc: 578 ↛ 579line 578 didn't jump to line 579 because the condition on line 578 was never true

579 audio["text"] = desc 

580 m = re.search(r"\((([^()]|\([^()]*\))*)\)", desc) 

581 skip = False 

582 if m: 582 ↛ 583line 582 didn't jump to line 583 because the condition on line 582 was never true

583 par = m.group(1) 

584 cls = classify_desc(par) 

585 if cls == "tags": 

586 parse_pronunciation_tags(wxr, par, audio) 

587 else: 

588 skip = True 

589 if skip: 589 ↛ 590line 589 didn't jump to line 590 because the condition on line 589 was never true

590 return "" 

591 audios.append(audio) 

592 return "__AUDIO_IGNORE_THIS__" + str(len(audios) - 1) + "__" 

593 if name == "audio-IPA": 593 ↛ 594line 593 didn't jump to line 594 because the condition on line 593 was never true

594 filename = ht.get(2) or "" 

595 ipa = ht.get(3) or "" 

596 dial = ht.get("dial") 

597 audio = {"audio": filename.strip()} 

598 if dial: 

599 dial = clean_node(wxr, None, [dial]) 

600 audio["text"] = dial 

601 if ipa: 

602 audio["audio-ipa"] = ipa 

603 audios.append(audio) 

604 # The problem with these IPAs is that they often just describe 

605 # what's in the sound file, rather than giving the pronunciation 

606 # of the word alone. It is common for audio files to contain 

607 # multiple pronunciations or articles in the same file, and then 

608 # this IPA often describes what is in the file. 

609 return "__AUDIO_IGNORE_THIS__" + str(len(audios) - 1) + "__" 

610 if name == "audio-pron": 

611 filename = ht.get(2) or "" 

612 ipa = ht.get("ipa") or "" 

613 dial = ht.get("dial") 

614 country = ht.get("country") 

615 audio = {"audio": filename.strip()} 

616 if dial: 616 ↛ 620line 616 didn't jump to line 620 because the condition on line 616 was always true

617 dial = clean_node(wxr, None, [dial]) 

618 audio["text"] = dial 

619 parse_pronunciation_tags(wxr, dial, audio) 

620 if country: 620 ↛ 622line 620 didn't jump to line 622 because the condition on line 620 was always true

621 parse_pronunciation_tags(wxr, country, audio) 

622 if ipa: 622 ↛ 624line 622 didn't jump to line 624 because the condition on line 622 was always true

623 audio["audio-ipa"] = ipa 

624 audios.append(audio) 

625 # XXX do we really want to extract pronunciations from these? 

626 # Or are they spurious / just describing what is in the 

627 # audio file? 

628 # if ipa: 

629 # pron = {"ipa": ipa} 

630 # if dial: 

631 # parse_pronunciation_tags(wxr, dial, pron) 

632 # if country: 

633 # parse_pronunciation_tags(wxr, country, pron) 

634 # data_append(data, "sounds", pron) 

635 return "__AUDIO_IGNORE_THIS__" + str(len(audios) - 1) + "__" 

636 if name in ("hyph", "hyphenation"): 

637 # {{hyph|en|re|late|caption="Hyphenation UK:"}} 

638 # {{hyphenation|it|quiè|to||qui|è|to||quié|to||qui|é|to}} 

639 # and also nocaption=1 

640 caption = clean_node(wxr, None, ht.get("caption", "")) 

641 tagsets, _ = decode_tags(caption) 

642 # flatten the tagsets into one; it would be really weird to have 

643 # several tagsets for a hyphenation caption 

644 tags = sorted(set(tag for tagset in tagsets for tag in tagset)) 

645 # We'll just ignore any errors from tags, it's not very important 

646 # for hyphenation 

647 tags = [tag for tag in tags if not tag.startswith("error")] 

648 hyph_sequences: list[list[str]] = [[]] 

649 for text in [ 

650 t for (k, t) in ht.items() if (isinstance(k, int) and k >= 2) 

651 ]: 

652 if not text: 

653 hyph_sequences.append([]) 

654 else: 

655 hyph_sequences[-1].append(clean_node(wxr, None, text)) 

656 for seq in hyph_sequences: 

657 hyphenations.append(Hyphenation(parts=seq, tags=tags)) 

658 return "" 

659 return None 

660 

661 def parse_pron_post_template_fn( 

662 name: str, ht: TemplateArgs, text: str 

663 ) -> str | None: 

664 # _post_template_fn handles templates *after* the work to expand 

665 # them has been done; this is exactly the same as _template_fn, 

666 # except with the additional expanded text as an input, and 

667 # possible side-effects from the expansion and recursion (like 

668 # calling other subtemplates that are handled in _template_fn. 

669 nonlocal may_be_duplicates 

670 if is_panel_template(wxr, name): 670 ↛ 671line 670 didn't jump to line 671 because the condition on line 670 was never true

671 return "" 

672 if name in PRON_POS_TEMPLATE_NAMES: 

673 pos_match = extract_pronunciation_pos_template( 

674 wxr, name, ht, lang_code 

675 ) 

676 if pos_match.pos_values: 

677 pron_pos_markers.append(pos_match.pos_values) 

678 marker = ( 

679 f"__PRON_POS_MARKER_{len(pron_pos_markers) - 1}__" 

680 ) 

681 if pos_match.residual: 681 ↛ 682line 681 didn't jump to line 682 because the condition on line 681 was never true

682 return f"{marker} ({pos_match.residual})" 

683 return marker 

684 if name in { 

685 *PRON_POS_TEMPLATE_NAMES, 

686 "l", 

687 "link", 

688 }: 

689 # Kludge: when these templates expand to /.../ or [...], 

690 # replace the expansion by something safe. This is used 

691 # to filter spurious IPA-looking expansions that aren't really 

692 # IPAs. We probably don't care about these templates in the 

693 # contexts where they expand to something containing these. 

694 v = re.sub(r'href="[^"]*"', "", text) # Ignore URLs 

695 v = re.sub(r'src="[^"]*"', "", v) 

696 v = clean_value(wxr, v) 

697 if re.search(r"/[^/,]+?/|\[[^]0-9,/][^],/]*?\]", v): 

698 # Note: replacing by empty results in Lua errors that we 

699 # would rather not have. For example, voi/Middle Vietnamese 

700 # uses {{a|{{l{{vi|...}}}}, and the {{a|...}} will fail 

701 # if {{l|...}} returns empty. 

702 return "stripped-by-parse_pron_post_template_fn" 

703 if name in ("IPA", "enPR"): 

704 # Extract the data from IPA and enPR templates (same underlying 

705 # template) and replace them in-text with magical cookie that 

706 # can be later used to refer to the data's index inside 

707 # pron_templates. 

708 if pron_t := extract_pron_template(wxr, name, ht, text): 

709 pron_templates.append(pron_t) 

710 return f"__PRON_TEMPLATE_{len(pron_templates) - 1}__" 

711 # Catch templates that generate duplicate sound data entries 

712 # here; if the text produces a big, toggleable section, the 

713 # "header" for that section might be duplicated. Add more conditions 

714 # if necessary. 

715 if is_toggled_expansion(text): 715 ↛ 716line 715 didn't jump to line 716 because the condition on line 715 was never true

716 may_be_duplicates = True 

717 return text 

718 

719 def flattened_tree( 

720 lines: list[WikiNode | str], 

721 ) -> Iterator[FlattenedListNode]: 

722 assert isinstance(lines, list) 

723 for line in lines: 

724 yield from flattened_tree1(line, 0) 

725 

726 def flattened_tree1( 

727 node: WikiNode | str, list_depth: int 

728 ) -> Iterator[FlattenedListNode]: 

729 assert isinstance(node, (WikiNode, str)) 

730 if isinstance(node, str): 

731 yield FlattenedListNode(node, list_depth) 

732 return 

733 elif node.kind == NodeKind.LIST: 

734 for item in node.children: 

735 yield from flattened_tree1(item, list_depth) 

736 elif node.kind == NodeKind.LIST_ITEM: 

737 item_depth = ( 

738 len(node.sarg) if isinstance(node.sarg, str) else list_depth 

739 ) 

740 new_children = [] 

741 # A list item can have several sublists, e.g. "**" lines 

742 # followed by a "*:" line. 

743 sublists = [] 

744 for child in node.children: 

745 if isinstance(child, WikiNode) and child.kind == NodeKind.LIST: 

746 sublists.append(child) 

747 else: 

748 new_children.append(child) 

749 node.children = new_children 

750 node.sarg = "*" 

751 yield FlattenedListNode(node, item_depth) 

752 for sublist in sublists: 

753 yield from flattened_tree1(sublist, item_depth) 

754 else: 

755 yield FlattenedListNode(node, list_depth) 

756 

757 # XXX Do not use flattened_tree more than once here, for example for 

758 # debug printing... The underlying data is changed, and the separated 

759 # sublists disappear. 

760 

761 # Kludge for templates that generate several lines, but haven't 

762 # been caught by earlier kludges... 

763 def split_cleaned_node_on_newlines( 

764 contents: list[WikiNode | str], 

765 ) -> Iterator[tuple[str, int]]: 

766 for flattened in flattened_tree(contents): 

767 ipa_text = clean_node( 

768 wxr, 

769 data, 

770 flattened.node, 

771 template_fn=parse_pronunciation_template_fn, 

772 post_template_fn=parse_pron_post_template_fn, 

773 ) 

774 for line in ipa_text.splitlines(): 

775 yield line, flattened.list_depth 

776 

777 # have_pronunciations = False 

778 active_pos: PronunciationPoses | None = None 

779 # POS values from parent pronunciation lines by original list depth. 

780 # Audio-only child lines can inherit from a parent pronunciation line, 

781 # but same-depth audio lines must not inherit from a preceding IPA. 

782 pronunciation_pos_stack: list[tuple[int, PronunciationPoses]] = [] 

783 

784 def parent_pronunciation_pos( 

785 list_depth: int, 

786 ) -> PronunciationPoses | None: 

787 if not pronunciation_pos_stack: 

788 return None 

789 parent_depth, parent_pos = pronunciation_pos_stack[-1] 

790 return parent_pos if parent_depth < list_depth else None 

791 

792 # Tags from label-only lines by original list depth, e.g. 

793 # "* {{a|en|GA}}" over "** {{IPA|en|...}}". Nested pronunciations 

794 # start from their parent's tags and add their own. 

795 pronunciation_tags_stack: list[tuple[int, SoundData]] = [] 

796 

797 def parent_pronunciation_tags(list_depth: int) -> SoundData: 

798 if not pronunciation_tags_stack: 

799 return {} 

800 parent_depth, parent_tags = pronunciation_tags_stack[-1] 

801 return parent_tags if parent_depth < list_depth else {} 

802 

803 for line, list_depth in split_cleaned_node_on_newlines(contents): 

804 prefix: str | None = None 

805 earlier_base_data: SoundData | None = None 

806 line_pos: PronunciationPoses | None = None 

807 current_group_sounds: list[SoundData] = [] 

808 # POS values seen on sounds extracted from this physical line. A 

809 # single candidate can seed adjacent audio-only child lines; multiple 

810 # POS-marked sounds on one line are too ambiguous for inheritance. 

811 line_sound_pos_candidates: set[PronunciationPoses] = set() 

812 line_has_sound = False 

813 if not line: 813 ↛ 814line 813 didn't jump to line 814 because the condition on line 813 was never true

814 continue 

815 while ( 

816 pronunciation_pos_stack 

817 and pronunciation_pos_stack[-1][0] >= list_depth 

818 ): 

819 pronunciation_pos_stack.pop() 

820 while ( 

821 pronunciation_tags_stack 

822 and pronunciation_tags_stack[-1][0] >= list_depth 

823 ): 

824 pronunciation_tags_stack.pop() 

825 parent_tags = parent_pronunciation_tags(list_depth) 

826 

827 split_templates = re.split(r"__PRON_TEMPLATE_(\d+)__", line) 

828 for i, text in enumerate(split_templates): 

829 if not text: 

830 continue 

831 # clean up starts at the start of the line 

832 text = re.sub(r"^\**\s*", "", text).strip() 

833 if i == 0: 

834 # At the start of a line, check for stuff like "Noun:" 

835 # or "(verb)" for POS labels that apply to this line or 

836 # structurally nested pronunciation lines. 

837 # These labels feed the inheritance state that later sets the 

838 # temporary sound["pos"] field used to route pronunciation 

839 # data into matching POS sections. 

840 if pos_prefix := extract_pos_prefix(text): 

841 text = pos_prefix.text 

842 line_pos = pos_prefix.pos_values 

843 if pos_prefix.is_persistent: 

844 active_pos = pos_prefix.pos_values 

845 if not text: 

846 continue 

847 

848 m = re.search(r"__PRON_POS_MARKER_(\d+)__", text) 

849 while m: 

850 if current_group_sounds and re.search( 

851 r"[,;]", text[: m.start()] 

852 ): 

853 current_group_sounds = [] 

854 pos_values = pron_pos_markers[int(m.group(1))] 

855 if current_group_sounds: 

856 for sound in current_group_sounds: 

857 set_sound_pos(sound, pos_values) 

858 line_sound_pos_candidates.add(pos_values) 

859 line_pos = pos_values 

860 text = text[: m.start()] + text[m.end() :] 

861 m = re.search(r"__PRON_POS_MARKER_(\d+)__", text) 

862 text = text.strip() 

863 if not text: 

864 continue 

865 # POS inheritance for normal pronunciation data: 

866 # 1. line_pos: explicit POS marker on this line, e.g. 

867 # "* {{q|noun}} {{IPA|...}}". 

868 # 2. parent_pronunciation_pos: structurally inherited from a 

869 # parent list item, e.g. "* {{q|noun}}" then "** {{IPA|...}}". 

870 # 3. active_pos: support for "* Noun:" followed by 

871 # "* {{IPA|...}}"; broad, so it stays after structural data. 

872 inherited_pos = ( 

873 line_pos or parent_pronunciation_pos(list_depth) or active_pos 

874 ) 

875 

876 if i % 2 == 1: 

877 # re.split (with capture groups) splits the lines so that 

878 # every even entry is a captured splitter; odd lines are either 

879 # empty strings or stuff around the splitters. 

880 base_pron_data, first_prons = pron_templates[int(text)] 

881 if base_pron_data: 

882 earlier_base_data = base_pron_data 

883 # print(f"Set {earlier_base_data=}") 

884 elif earlier_base_data is not None: 

885 # merge data from an earlier iteration of this loop 

886 for pr in first_prons: 

887 if "note" in pr and "note" in earlier_base_data: 887 ↛ 888line 887 didn't jump to line 888 because the condition on line 887 was never true

888 pr["note"] += ";" + earlier_base_data.get( 

889 "note", "" 

890 ) 

891 elif "note" in earlier_base_data: 891 ↛ 892line 891 didn't jump to line 892 because the condition on line 891 was never true

892 pr["note"] = earlier_base_data["note"] 

893 if "topics" in earlier_base_data: 893 ↛ 894line 893 didn't jump to line 894 because the condition on line 893 was never true

894 data_extend( 

895 pr, "topics", earlier_base_data["topics"] 

896 ) 

897 if "tags" in pr and "tags" in earlier_base_data: 897 ↛ 898line 897 didn't jump to line 898 because the condition on line 897 was never true

898 pr["tags"].extend(earlier_base_data["tags"]) 

899 elif "tags" in earlier_base_data: 899 ↛ 886line 899 didn't jump to line 886 because the condition on line 899 was always true

900 pr["tags"] = sorted(set(earlier_base_data["tags"])) 

901 for pr in first_prons: 

902 inherit_pronunciation_tag_data(pr, parent_tags) 

903 if sound_pos := set_sound_pos( 

904 pr, 

905 None if "pos" in pr else inherited_pos, 

906 ): 

907 line_sound_pos_candidates.add(sound_pos) 

908 if pr not in data.get("sounds", ()): 908 ↛ 910line 908 didn't jump to line 910 because the condition on line 908 was always true

909 data_append(data, "sounds", pr) 

910 current_group_sounds.append(pr) 

911 line_has_sound = True 

912 # This bit is handled 

913 continue 

914 

915 if "IPA" in text: 

916 field: Literal[ 

917 "audio", 

918 "audio-ipa", 

919 "enpr", 

920 "form", 

921 "hangeul", 

922 "homophone", 

923 "ipa", 

924 "mp3_url", 

925 "note", 

926 "ogg_url", 

927 "other", 

928 "rhymes", 

929 "tags", 

930 "text", 

931 "topics", 

932 "zh-pron", 

933 ] = "ipa" 

934 else: 

935 # This is used for Rhymes, Homophones, etc 

936 field = "other" 

937 

938 # Check if it contains Japanese "Tokyo" pronunciation with 

939 # special syntax 

940 m = re.search(r"(?m)\(Tokyo\) +([^ ]+) +\[", text) 

941 if m: 941 ↛ 942line 941 didn't jump to line 942 because the condition on line 941 was never true

942 pron: SoundData = {field: m.group(1)} # type: ignore[misc] 

943 if sound_pos := set_sound_pos(pron, inherited_pos): 

944 line_sound_pos_candidates.add(sound_pos) 

945 data_append(data, "sounds", pron) 

946 current_group_sounds.append(pron) 

947 line_has_sound = True 

948 # have_pronunciations = True 

949 continue 

950 

951 # Check if it contains Rhymes 

952 m = re.match(r"\s*Rhymes?: (.*)", text) 

953 if m: 

954 for ending in split_at_comma_semi(m.group(1)): 

955 ending = ending.strip() 

956 if ending: 956 ↛ 954line 956 didn't jump to line 954 because the condition on line 956 was always true

957 pron = {"rhymes": ending} 

958 if sound_pos := set_sound_pos(pron, inherited_pos): 

959 line_sound_pos_candidates.add(sound_pos) 

960 data_append(data, "sounds", pron) 

961 current_group_sounds.append(pron) 

962 line_has_sound = True 

963 # have_pronunciations = True 

964 continue 

965 

966 # Check if it contains homophones 

967 m = re.search(r"(?m)\bHomophones?: (.*)", text) 

968 if m: 

969 for w in split_at_comma_semi(m.group(1)): 

970 w = w.strip() 

971 if w: 971 ↛ 969line 971 didn't jump to line 969 because the condition on line 971 was always true

972 pron = {"homophone": w} 

973 if sound_pos := set_sound_pos(pron, inherited_pos): 

974 line_sound_pos_candidates.add(sound_pos) 

975 data_append(data, "sounds", pron) 

976 current_group_sounds.append(pron) 

977 line_has_sound = True 

978 # have_pronunciations = True 

979 continue 

980 

981 # Check if it contains Phonetic hangeul 

982 m = re.search(r"(?m)\bPhonetic hange?ul: \[([^]]+)\]", text) 

983 if m: 983 ↛ 984line 983 didn't jump to line 984 because the condition on line 983 was never true

984 seen = set() 

985 for w in m.group(1).split("/"): 

986 w = w.strip() 

987 if w and w not in seen: 

988 seen.add(w) 

989 pron = {"hangeul": w} 

990 if sound_pos := set_sound_pos(pron, inherited_pos): 

991 line_sound_pos_candidates.add(sound_pos) 

992 data_append(data, "sounds", pron) 

993 current_group_sounds.append(pron) 

994 line_has_sound = True 

995 # have_pronunciations = True 

996 

997 # This regex-based hyphenation detection left as backup 

998 m = re.search(r"\b(Syllabification|Hyphenation): *([^\n.]*)", text) 

999 if m: 

1000 data_append(data, "hyphenation", m.group(2)) 

1001 commaseparated = m.group(2).split(",") 

1002 if len(commaseparated) > 1: 1002 ↛ 1013line 1002 didn't jump to line 1013 because the condition on line 1002 was always true

1003 for h in commaseparated: 

1004 # That second characters looks like a dash but it's 

1005 # actually unicode decimal code 8231, hyphenation dash 

1006 # Add more delimiters here if needed. 

1007 parts = re.split(r"-|‧", h.strip()) 

1008 data_append( 

1009 data, "hyphenations", Hyphenation(parts=parts) 

1010 ) 

1011 ... 

1012 else: 

1013 data_append( 

1014 data, 

1015 "hyphenations", 

1016 Hyphenation(parts=m.group(2).split(sep="-")), 

1017 ) 

1018 # have_pronunciations = True 

1019 

1020 # See if it contains a word prefix restricting which forms the 

1021 # pronunciation applies to (see amica/Latin) and/or parenthesized 

1022 # tags. 

1023 m = re.match( 

1024 r"^[*#\s]*(([-\w]+):\s+)?\((([^()]|\([^()]*\))*?)\)", text 

1025 ) 

1026 if m: 

1027 prefix = m.group(2) or "" 

1028 tagstext = m.group(3) 

1029 text = text[m.end() :] 

1030 else: 

1031 m = re.match(r"^[*#\s]*([-\w]+):\s+", text) 

1032 if m: 

1033 prefix = m.group(1) 

1034 tagstext = "" 

1035 text = text[m.end() :] 

1036 else: 

1037 # Spanish has tags before pronunciations, eg. aceite/Spanish 

1038 m = re.match(r".*:\s+\(([^)]*)\)\s+(.*)", text) 

1039 if m: 1039 ↛ 1040line 1039 didn't jump to line 1040 because the condition on line 1039 was never true

1040 tagstext = m.group(1) 

1041 text = m.group(2) 

1042 else: 

1043 # No prefix. In this case, we inherit prefix 

1044 # from previous entry. This particularly 

1045 # applies for nested Audio files. 

1046 tagstext = "" 

1047 if tagstext: 

1048 earlier_base_data = {} 

1049 parse_pronunciation_tags_with_pos( 

1050 wxr, tagstext, earlier_base_data 

1051 ) 

1052 

1053 # Find romanizations from the pronunciation section (routinely 

1054 # produced for Korean by {{ko-IPA}}) 

1055 for m in re.finditer(pron_romanization_re, text): 1055 ↛ 1056line 1055 didn't jump to line 1056 because the loop on line 1055 never started

1056 prefix = m.group(1) 

1057 w = m.group(2).strip() 

1058 tag = pron_romanizations[prefix] 

1059 form = {"form": w, "tags": tag.split()} 

1060 data_append(data, "forms", form) 

1061 

1062 # Find IPA pronunciations 

1063 for m, qualifier in ipa_qualifiers(text): 

1064 v = m.group(0) 

1065 # The regexp above can match file links. Skip them. 

1066 if v.startswith("[[File:"): 1066 ↛ 1067line 1066 didn't jump to line 1067 because the condition on line 1066 was never true

1067 continue 

1068 if v == "/wiki.local/": 1068 ↛ 1069line 1068 didn't jump to line 1069 because the condition on line 1068 was never true

1069 continue 

1070 if field == "ipa" and "__AUDIO_IGNORE_THIS__" in text: 1070 ↛ 1071line 1070 didn't jump to line 1071 because the condition on line 1070 was never true

1071 m = re.search(r"__AUDIO_IGNORE_THIS__(\d+)__", text) 

1072 assert m 

1073 idx = int(m.group(1)) 

1074 if idx >= len(audios): 

1075 continue 

1076 if not audios[idx].get("audio-ipa"): 

1077 audios[idx]["audio-ipa"] = v 

1078 if prefix: 

1079 audios[idx]["form"] = prefix 

1080 else: 

1081 if earlier_base_data: 

1082 pron = deepcopy(earlier_base_data) 

1083 pron[field] = v 

1084 else: 

1085 pron = {field: v} # type: ignore[misc] 

1086 if prefix: 

1087 pron["form"] = prefix 

1088 if qualifier: 

1089 parse_pronunciation_tags_with_pos(wxr, qualifier, pron) 

1090 inherit_pronunciation_tag_data(pron, parent_tags) 

1091 if sound_pos := set_sound_pos( 

1092 pron, 

1093 None if "pos" in pron else inherited_pos, 

1094 ): 

1095 line_sound_pos_candidates.add(sound_pos) 

1096 if may_be_duplicates is True: 

1097 ok = True 

1098 for comp_sound in data.get("sounds", []): 

1099 # Python has dict comparison since 3.8 

1100 if pron == comp_sound: 1100 ↛ 1101line 1100 didn't jump to line 1101 because the condition on line 1100 was never true

1101 ok = False 

1102 break 

1103 # The collapsed header of a toggle repeats the 

1104 # line without its qualifier: keep the labelled. 

1105 if qualifier and all( 

1106 pron.get(k) == val # type: ignore[misc] 

1107 for k, val in comp_sound.items() 

1108 ): 

1109 comp_sound.update(pron) 

1110 pron = comp_sound 

1111 ok = False 

1112 break 

1113 if ok: 

1114 data_append(data, "sounds", pron) 

1115 else: 

1116 data_append(data, "sounds", pron) 

1117 current_group_sounds.append(pron) 

1118 line_has_sound = True 

1119 # have_pronunciations = True 

1120 if current_group_sounds and re.search(r"[,;]", text): 

1121 current_group_sounds = [] 

1122 # XXX what about {{hyphenation|...}}, {{hyph|...}} 

1123 # and those used to be stored under "hyphenation" 

1124 

1125 # Add data that was collected in template_fn 

1126 # POS inheritance for audio has one extra source: 

1127 # common_sound_pos(line_sound_pos_candidates), from pronunciations 

1128 # extracted earlier on the same physical line, e.g. 

1129 # "* {{IPA|en|/foo/|a=verb}} {{audio|en|foo.wav}}". 

1130 # Explicit line_pos still wins, then same-line sound agreement, then 

1131 # parent-list structure, then active_pos. 

1132 audio_inherited_pos = ( 

1133 line_pos 

1134 or common_sound_pos(line_sound_pos_candidates) 

1135 or parent_pronunciation_pos(list_depth) 

1136 or active_pos 

1137 ) 

1138 for audio in audios: 

1139 if "audio" in audio: 1139 ↛ 1196line 1139 didn't jump to line 1196 because the condition on line 1139 was always true

1140 # Compute audio file URLs 

1141 fn = audio["audio"] 

1142 # Strip certain characters, e.g., left-to-right mark 

1143 fn = re.sub(r"[\u200f\u200e]", "", fn) 

1144 fn = fn.strip() 

1145 fn = urllib.parse.unquote(fn) 

1146 # First character is usually uppercased 

1147 if re.match(r"^[a-z][a-z]+", fn): 

1148 fn = fn[0].upper() + fn[1:] 

1149 if fn in wxr.config.redirects: 1149 ↛ 1150line 1149 didn't jump to line 1150 because the condition on line 1149 was never true

1150 fn = wxr.config.redirects[fn] 

1151 # File extension is lowercased 

1152 # XXX some words seem to need this, some don't seem to 

1153 # have this??? what is the exact rule? 

1154 # fn = re.sub(r"\.[^.]*$", lambda m: m.group(0).lower(), fn) 

1155 # Spaces are converted to underscores 

1156 fn = re.sub(r"\s+", "_", fn) 

1157 # Compute hash digest part 

1158 h = hashlib.md5() 

1159 hname = fn.encode("utf-8") 

1160 h.update(hname) 

1161 digest = h.hexdigest() 

1162 # Quote filename for URL 

1163 qfn = urllib.parse.quote(fn) 

1164 # For safety when writing files 

1165 qfn = qfn.replace("/", "__slash__") 

1166 if re.search(r"(?i)\.(ogg|oga)$", fn): 

1167 ogg = ( 

1168 "https://upload.wikimedia.org/wikipedia/" 

1169 "commons/{}/{}/{}".format(digest[:1], digest[:2], qfn) 

1170 ) 

1171 else: 

1172 ogg = ( 

1173 "https://upload.wikimedia.org/wikipedia/" 

1174 "commons/transcoded/" 

1175 "{}/{}/{}/{}.ogg".format( 

1176 digest[:1], digest[:2], qfn, qfn 

1177 ) 

1178 ) 

1179 if re.search(r"(?i)\.(mp3)$", fn): 1179 ↛ 1180line 1179 didn't jump to line 1180 because the condition on line 1179 was never true

1180 mp3 = ( 

1181 "https://upload.wikimedia.org/wikipedia/" 

1182 "commons/{}/{}/{}".format(digest[:1], digest[:2], qfn) 

1183 ) 

1184 else: 

1185 mp3 = ( 

1186 "https://upload.wikimedia.org/wikipedia/" 

1187 "commons/transcoded/" 

1188 "{}/{}/{}/{}.mp3".format( 

1189 digest[:1], digest[:2], qfn, qfn 

1190 ) 

1191 ) 

1192 audio["ogg_url"] = ogg 

1193 audio["mp3_url"] = mp3 

1194 if "pos" not in audio: 1194 ↛ 1196line 1194 didn't jump to line 1196 because the condition on line 1194 was always true

1195 set_sound_pos(audio, audio_inherited_pos) 

1196 if audio not in data.get("sounds", ()): 

1197 data_append(data, "sounds", audio) 

1198 line_has_sound = True 

1199 

1200 # if audios: 

1201 # have_pronunciations = True 

1202 audios = [] 

1203 

1204 data_extend(data, "hyphenations", hyphenations) 

1205 hyphenations = [] 

1206 

1207 if line_pos and not line_has_sound: 

1208 active_pos = line_pos 

1209 pronunciation_pos_stack.append((list_depth, line_pos)) 

1210 elif line_pronunciation_pos := common_sound_pos( 

1211 line_sound_pos_candidates 

1212 ): 

1213 pronunciation_pos_stack.append((list_depth, line_pronunciation_pos)) 

1214 

1215 if not line_has_sound and earlier_base_data: 

1216 line_tags: SoundData = {} 

1217 merge_pronunciation_tag_data(line_tags, earlier_base_data) 

1218 if line_tags: 1218 ↛ 803line 1218 didn't jump to line 803 because the condition on line 1218 was always true

1219 inherit_pronunciation_tag_data(line_tags, parent_tags) 

1220 pronunciation_tags_stack.append((list_depth, line_tags)) 

1221 

1222 ## I have commented out the otherwise unused have_pronunciation 

1223 ## toggles; uncomment them to use this debug print 

1224 # if not have_pronunciations and not have_panel_templates: 

1225 # wxr.wtp.debug("no pronunciations found from pronunciation section", 

1226 # sortid="pronunciations/533") 

1227 

1228 

1229def extract_th_pron_template( 

1230 wxr: WiktextractContext, word_entry: WordData, t_node: TemplateNode 

1231): 

1232 # https://en.wiktionary.org/wiki/Template:th-pron 

1233 @dataclass 

1234 class TableHeader: 

1235 raw_tags: list[str] 

1236 rowspan: int 

1237 

1238 expanded_node = wxr.wtp.parse( 

1239 wxr.wtp.node_to_wikitext(t_node), expand_all=True 

1240 ) 

1241 sounds = [] 

1242 for table_tag in expanded_node.find_html("table"): 

1243 row_headers = [] 

1244 for tr_tag in table_tag.find_html("tr"): 

1245 field = "other" 

1246 new_headers = [] 

1247 for header in row_headers: 

1248 if header.rowspan > 1: 

1249 header.rowspan -= 1 

1250 new_headers.append(header) 

1251 row_headers = new_headers 

1252 for th_tag in tr_tag.find_html("th"): 

1253 header_str = clean_node(wxr, None, th_tag) 

1254 if header_str.startswith("(standard) IPA"): 

1255 field = "ipa" 

1256 elif header_str.startswith("Homophones"): 1256 ↛ 1257line 1256 didn't jump to line 1257 because the condition on line 1256 was never true

1257 field = "homophone" 

1258 elif header_str == "Audio": 

1259 field = "audio" 

1260 elif header_str != "": 1260 ↛ 1252line 1260 didn't jump to line 1252 because the condition on line 1260 was always true

1261 rowspan = 1 

1262 rowspan_str = th_tag.attrs.get("rowspan", "1") 

1263 if re.fullmatch(r"\d+", rowspan_str): 1263 ↛ 1265line 1263 didn't jump to line 1265 because the condition on line 1263 was always true

1264 rowspan = int(rowspan_str) 

1265 header = TableHeader([], rowspan) 

1266 for line in header_str.splitlines(): 

1267 for raw_tag in line.strip("{}\n ").split(";"): 

1268 raw_tag = raw_tag.strip() 

1269 if raw_tag != "": 1269 ↛ 1267line 1269 didn't jump to line 1267 because the condition on line 1269 was always true

1270 header.raw_tags.append(raw_tag) 

1271 row_headers.append(header) 

1272 

1273 for td_tag in tr_tag.find_html("td"): 

1274 if field == "audio": 

1275 for link_node in td_tag.find_child(NodeKind.LINK): 

1276 filename = clean_node(wxr, None, link_node.largs[0]) 

1277 if filename != "": 1277 ↛ 1275line 1277 didn't jump to line 1275 because the condition on line 1277 was always true

1278 sound = create_audio_url_dict(filename) 

1279 sounds.append(sound) 

1280 elif field == "homophone": 1280 ↛ 1281line 1280 didn't jump to line 1281 because the condition on line 1280 was never true

1281 for span_tag in td_tag.find_html_recursively( 

1282 "span", attr_name="lang", attr_value="th" 

1283 ): 

1284 word = clean_node(wxr, None, span_tag) 

1285 if word != "": 

1286 sounds.append({"homophone": word}) 

1287 else: 

1288 raw_tags = [] 

1289 for html_node in td_tag.find_child_recursively( 

1290 NodeKind.HTML 

1291 ): 

1292 if html_node.tag == "small": 

1293 node_str = clean_node(wxr, None, html_node) 

1294 if node_str.startswith("[") and node_str.endswith( 

1295 "]" 

1296 ): 

1297 for raw_tag in node_str.strip("[]").split(","): 

1298 raw_tag = raw_tag.strip() 

1299 if raw_tag != "": 1299 ↛ 1297line 1299 didn't jump to line 1297 because the condition on line 1299 was always true

1300 raw_tags.append(raw_tag) 

1301 elif len(sounds) > 0: 1301 ↛ 1289line 1301 didn't jump to line 1289 because the condition on line 1301 was always true

1302 sounds[-1]["roman"] = node_str 

1303 elif html_node.tag == "span": 

1304 node_str = clean_node(wxr, None, html_node) 

1305 span_lang = html_node.attrs.get("lang", "") 

1306 span_class = html_node.attrs.get("class", "") 

1307 if node_str != "" and ( 

1308 span_lang == "th" or span_class in ["IPA", "tr"] 

1309 ): 

1310 sound = {} 

1311 for raw_tag in raw_tags: 

1312 if raw_tag in valid_tags: 1312 ↛ 1315line 1312 didn't jump to line 1315 because the condition on line 1312 was always true

1313 data_append(sound, "tags", raw_tag) 

1314 else: 

1315 data_append(sound, "raw_tags", raw_tag) 

1316 for header in row_headers: 

1317 for raw_tag in header.raw_tags: 

1318 if raw_tag.lower() in valid_tags: 

1319 data_append( 

1320 sound, "tags", raw_tag.lower() 

1321 ) 

1322 else: 

1323 data_append( 

1324 sound, "raw_tags", raw_tag 

1325 ) 

1326 if "romanization" in sound.get("tags", []): 

1327 field = "roman" 

1328 sound[field] = node_str 

1329 sounds.append(sound) 

1330 

1331 clean_node(wxr, word_entry, expanded_node) 

1332 data_extend(word_entry, "sounds", sounds) 

1333 

1334 

1335def extract_zh_pron_template( 

1336 wxr: WiktextractContext, word_entry: WordData, t_node: TemplateNode 

1337): 

1338 # https://en.wiktionary.org/wiki/Template:zh-pron 

1339 expanded_node = wxr.wtp.parse( 

1340 wxr.wtp.node_to_wikitext(t_node), expand_all=True 

1341 ) 

1342 seen_lists = set() 

1343 sounds = [] 

1344 for list_node in expanded_node.find_child_recursively(NodeKind.LIST): 

1345 if list_node not in seen_lists: 

1346 for list_item in list_node.find_child(NodeKind.LIST_ITEM): 

1347 sounds.extend( 

1348 extract_zh_pron_list_item(wxr, list_item, [], seen_lists) 

1349 ) 

1350 clean_node(wxr, word_entry, expanded_node) 

1351 data_extend(word_entry, "sounds", sounds) 

1352 

1353 

1354def extract_zh_pron_list_item( 

1355 wxr: WiktextractContext, 

1356 list_item: WikiNode, 

1357 raw_tags: list[str], 

1358 seen_lists: set[WikiNode], 

1359) -> list[SoundData]: 

1360 current_tags = raw_tags[:] 

1361 sounds = [] 

1362 is_first_small_tag = True 

1363 for node in list_item.children: 

1364 if isinstance(node, WikiNode) and node.kind == NodeKind.LINK: 

1365 link_str = clean_node(wxr, None, node.largs) 

1366 node_str = clean_node(wxr, None, node) 

1367 if link_str.startswith("File:"): 1367 ↛ 1368line 1367 didn't jump to line 1368 because the condition on line 1367 was never true

1368 sound = create_audio_url_dict(link_str.removeprefix("File:")) 

1369 sound["raw_tags"] = current_tags[:] 

1370 translate_zh_pron_raw_tags(sound) 

1371 sounds.append(sound) 

1372 elif node_str != "": 1372 ↛ 1363line 1372 didn't jump to line 1363 because the condition on line 1372 was always true

1373 current_tags.append(node_str) 

1374 elif isinstance(node, HTMLNode): 

1375 if node.tag == "small": 

1376 if is_first_small_tag: 1376 ↛ 1387line 1376 didn't jump to line 1387 because the condition on line 1376 was always true

1377 raw_tag_text = clean_node( 

1378 wxr, 

1379 None, 

1380 [ 

1381 n 

1382 for n in node.children 

1383 if not (isinstance(n, HTMLNode) and n.tag == "sup") 

1384 ], 

1385 ) 

1386 current_tags.extend(split_zh_pron_raw_tag(raw_tag_text)) 

1387 elif len(sounds) > 0: 

1388 data_extend( 

1389 sounds[-1], 

1390 "raw_tags", 

1391 split_zh_pron_raw_tag(clean_node(wxr, None, node)), 

1392 ) 

1393 translate_zh_pron_raw_tags(sounds[-1]) 

1394 is_first_small_tag = False 

1395 elif node.tag == "span": 

1396 sounds.extend(extract_zh_pron_span(wxr, node, current_tags)) 

1397 elif ( 1397 ↛ 1402line 1397 didn't jump to line 1402 because the condition on line 1397 was never true

1398 node.tag == "table" 

1399 and len(current_tags) > 0 

1400 and current_tags[-1] == "Homophones" 

1401 ): 

1402 sounds.extend( 

1403 extract_zh_pron_homophone_table(wxr, node, current_tags) 

1404 ) 

1405 elif isinstance(node, WikiNode) and node.kind == NodeKind.LIST: 

1406 seen_lists.add(node) 

1407 for child_list_item in node.find_child(NodeKind.LIST_ITEM): 

1408 sounds.extend( 

1409 extract_zh_pron_list_item( 

1410 wxr, child_list_item, current_tags, seen_lists 

1411 ) 

1412 ) 

1413 

1414 return sounds 

1415 

1416 

1417def extract_zh_pron_homophone_table( 

1418 wxr: WiktextractContext, table: HTMLNode, raw_tags: list[str] 

1419) -> list[SoundData]: 

1420 sounds = [] 

1421 for td_tag in table.find_html_recursively("td"): 

1422 for span_tag in td_tag.find_html("span"): 

1423 span_class = span_tag.attrs.get("class", "") 

1424 span_lang = span_tag.attrs.get("lang", "") 

1425 span_str = clean_node(wxr, None, span_tag) 

1426 if ( 

1427 span_str not in ["", "/"] 

1428 and span_lang != "" 

1429 and span_class in ["Hant", "Hans", "Hani"] 

1430 ): 

1431 sound = {"homophone": span_str, "raw_tags": raw_tags[:]} 

1432 if span_class == "Hant": 

1433 data_append(sound, "tags", "Traditional-Chinese") 

1434 elif span_class == "Hans": 

1435 data_append(sound, "tags", "Simplified-Chinese") 

1436 translate_zh_pron_raw_tags(sound) 

1437 sounds.append(sound) 

1438 

1439 return sounds 

1440 

1441 

1442def translate_zh_pron_raw_tags(sound: SoundData): 

1443 from .zh_pron_tags import ZH_PRON_TAGS 

1444 

1445 raw_tags = [] 

1446 for raw_tag in sound.get("raw_tags", []): 

1447 if raw_tag in ZH_PRON_TAGS: 

1448 tr_tag = ZH_PRON_TAGS[raw_tag] 

1449 if isinstance(tr_tag, str): 

1450 data_append(sound, "tags", tr_tag) 

1451 elif isinstance(tr_tag, list) and tr_tag not in sound.get( 1451 ↛ 1446line 1451 didn't jump to line 1446 because the condition on line 1451 was always true

1452 "tags", [] 

1453 ): 

1454 data_extend(sound, "tags", tr_tag) 

1455 elif raw_tag in valid_tags: 

1456 if raw_tag not in sound.get("tags", []): 1456 ↛ 1446line 1456 didn't jump to line 1446 because the condition on line 1456 was always true

1457 data_append(sound, "tags", raw_tag) 

1458 elif raw_tag not in raw_tags: 1458 ↛ 1446line 1458 didn't jump to line 1446 because the condition on line 1458 was always true

1459 raw_tags.append(raw_tag) 

1460 

1461 if len(raw_tags) > 0: 

1462 sound["raw_tags"] = raw_tags 

1463 elif "raw_tags" in sound: 1463 ↛ exitline 1463 didn't return from function 'translate_zh_pron_raw_tags' because the condition on line 1463 was always true

1464 del sound["raw_tags"] 

1465 

1466 

1467def split_zh_pron_raw_tag(raw_tag_text: str) -> list[str]: 

1468 raw_tags = [] 

1469 if "(" not in raw_tag_text: 

1470 for raw_tag in re.split(r",|:|;| and ", raw_tag_text): 

1471 raw_tag = raw_tag.strip().removeprefix("incl. ").strip() 

1472 if raw_tag != "": 

1473 raw_tags.append(raw_tag) 

1474 else: 

1475 processed_offsets = [] 

1476 for match in re.finditer(r"\([^()]+\)", raw_tag_text): 

1477 processed_offsets.append((match.start(), match.end())) 

1478 raw_tags.extend( 

1479 split_zh_pron_raw_tag( 

1480 raw_tag_text[match.start() + 1 : match.end() - 1] 

1481 ) 

1482 ) 

1483 not_processed = "" 

1484 last_end = 0 

1485 for start, end in processed_offsets: 

1486 not_processed += raw_tag_text[last_end:start] 

1487 last_end = end 

1488 not_processed += raw_tag_text[last_end:] 

1489 if not_processed != raw_tag_text: 1489 ↛ 1492line 1489 didn't jump to line 1492 because the condition on line 1489 was always true

1490 raw_tags = split_zh_pron_raw_tag(not_processed) + raw_tags 

1491 else: 

1492 raw_tags.append(not_processed) 

1493 

1494 return raw_tags 

1495 

1496 

1497def extract_zh_pron_span( 

1498 wxr: WiktextractContext, span_tag: HTMLNode, raw_tags: list[str] 

1499) -> list[SoundData]: 

1500 sounds = [] 

1501 small_tags = [] 

1502 pron_nodes = [] 

1503 roman = "" 

1504 phonetic_pron = "" 

1505 for index, node in enumerate(span_tag.children): 

1506 if isinstance(node, HTMLNode) and node.tag == "small": 1506 ↛ 1507line 1506 didn't jump to line 1507 because the condition on line 1506 was never true

1507 small_tags = split_zh_pron_raw_tag(clean_node(wxr, None, node)) 

1508 elif ( 1508 ↛ 1513line 1508 didn't jump to line 1513 because the condition on line 1508 was never true

1509 isinstance(node, HTMLNode) 

1510 and node.tag == "span" 

1511 and "-Latn" in node.attrs.get("lang", "") 

1512 ): 

1513 roman = clean_node(wxr, None, node).strip("() ") 

1514 elif isinstance(node, str) and node.strip() == "[Phonetic:": 1514 ↛ 1515line 1514 didn't jump to line 1515 because the condition on line 1514 was never true

1515 phonetic_pron = clean_node( 

1516 wxr, None, span_tag.children[index + 1 :] 

1517 ).strip("] ") 

1518 break 

1519 else: 

1520 pron_nodes.append(node) 

1521 for zh_pron in split_zh_pron(clean_node(wxr, None, pron_nodes)): 

1522 zh_pron = zh_pron.strip("[]: ") 

1523 if len(zh_pron) > 0: 1523 ↛ 1521line 1523 didn't jump to line 1521 because the condition on line 1523 was always true

1524 if "IPA" in span_tag.attrs.get("class", ""): 1524 ↛ 1525line 1524 didn't jump to line 1525 because the condition on line 1524 was never true

1525 sound = {"ipa": zh_pron, "raw_tags": raw_tags[:]} 

1526 else: 

1527 sound = {"zh_pron": zh_pron, "raw_tags": raw_tags[:]} 

1528 if roman != "": 1528 ↛ 1529line 1528 didn't jump to line 1529 because the condition on line 1528 was never true

1529 sound["roman"] = roman 

1530 sounds.append(sound) 

1531 if len(sounds) > 0: 1531 ↛ 1533line 1531 didn't jump to line 1533 because the condition on line 1531 was always true

1532 data_extend(sounds[-1], "raw_tags", small_tags) 

1533 if phonetic_pron != "": 1533 ↛ 1534line 1533 didn't jump to line 1534 because the condition on line 1533 was never true

1534 sound = { 

1535 "zh_pron": phonetic_pron, 

1536 "raw_tags": raw_tags[:] + ["Phonetic"], 

1537 } 

1538 if roman != "": 

1539 sound["roman"] = roman 

1540 sounds.append(sound) 

1541 for sound in sounds: 

1542 translate_zh_pron_raw_tags(sound) 

1543 return sounds 

1544 

1545 

1546def split_zh_pron(zh_pron: str) -> list[str]: 

1547 # split by comma and other symbols that outside parentheses 

1548 parentheses = 0 

1549 pron_list = [] 

1550 pron = "" 

1551 for c in zh_pron: 

1552 if ( 

1553 (c in [",", ";", "→"] or (c == "/" and not zh_pron.startswith("/"))) 

1554 and parentheses == 0 

1555 and len(pron.strip()) > 0 

1556 ): 

1557 pron_list.append(pron.strip()) 

1558 pron = "" 

1559 elif c == "(": 

1560 parentheses += 1 

1561 pron += c 

1562 elif c == ")": 

1563 parentheses -= 1 

1564 pron += c 

1565 else: 

1566 pron += c 

1567 

1568 if pron.strip() != "": 1568 ↛ 1570line 1568 didn't jump to line 1570 because the condition on line 1568 was always true

1569 pron_list.append(pron) 

1570 return pron_list