Coverage for src/wiktextract/extractor/en/pronunciation.py: 84%
918 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 07:03 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 07:03 +0000
1import hashlib
2import re
3import urllib
4from copy import deepcopy
5from dataclasses import dataclass
6from typing import Iterator, Literal, NamedTuple
8from wikitextprocessor import (
9 HTMLNode,
10 LevelNode,
11 NodeKind,
12 TemplateNode,
13 WikiNode,
14)
16from ...clean import clean_value
17from ...datautils import data_append, data_extend, split_at_comma_semi
18from ...page import LEVEL_KINDS, clean_node, is_panel_template
19from ...tags import valid_tags
20from ...wxr_context import WiktextractContext
21from ..share import create_audio_url_dict
22from .form_descriptions import (
23 classify_desc,
24 decode_tags,
25 parse_pronunciation_tags,
26)
27from .parts_of_speech import part_of_speech_map
28from .type_utils import Hyphenation, SoundData, TemplateArgs, WordData
30PronunciationPoses = tuple[str, ...]
32# Prefixes, tags, and regexp for finding romanizations from the pronuncation
33# section
34pron_romanizations = {
35 " Revised Romanization ": "romanization revised",
36 " Revised Romanization (translit.) ": (
37 "romanization revised transliteration"
38 ),
39 " McCune-Reischauer ": "McCune-Reischauer romanization",
40 " McCune–Reischauer ": "McCune-Reischauer romanization",
41 " Yale Romanization ": "Yale romanization",
42}
43pron_romanization_re = re.compile(
44 "(?m)^("
45 + "|".join(
46 re.escape(x)
47 for x in sorted(pron_romanizations.keys(), key=len, reverse=True)
48 )
49 + ")([^\n]+)"
50)
52IPA_EXTRACT = r"^(\((.+)\) )?(IPA⁽ᵏᵉʸ⁾|enPR): ((.+?)( \(([^(]+)\))?\s*)$"
53IPA_EXTRACT_RE = re.compile(IPA_EXTRACT)
55IPA_MATCHES_RE = re.compile(
56 r"(?m)/[^][\n/,]+?/" r"|" r"\[[^]\n0-9,/][^],/]*?\]"
57)
60class PronunciationPosMatch(NamedTuple):
61 pos_values: PronunciationPoses
62 residual: str
65class PronunciationPosPrefix(NamedTuple):
66 pos_values: PronunciationPoses
67 text: str
68 is_persistent: bool
71class FlattenedListNode(NamedTuple):
72 node: WikiNode | str
73 list_depth: int
76PRON_POS_TEMPLATE_NAMES = {
77 "q",
78 "qualifier",
79 "qual",
80 "i",
81 "sense",
82 "a",
83 "accent",
84 "lb",
85 "lbl",
86 "label",
87}
89PRON_POS_BY_LABEL = {
90 label: pos_data["pos"] for label, pos_data in part_of_speech_map.items()
91}
93PRON_POS_LABEL_RE = re.compile(
94 r"^(?:(?P<residual>.+?)\s+)?(?P<label>"
95 + "|".join(
96 re.escape(label)
97 for label in sorted(PRON_POS_BY_LABEL, key=len, reverse=True)
98 )
99 + r")$"
100)
103def normalize_pronunciation_pos_label(label: str) -> str:
104 label = label.strip().lower()
105 label = re.sub(r"\s+", " ", label)
106 # Drop explanatory suffixes such as "noun (barren areas)" before
107 # matching the label against part-of-speech names.
108 label = re.sub(r"\s*\([^)]*\)\s*$", "", label).strip()
109 label = label.strip(" \t\n\r():")
110 label = re.sub(r"\s+senses?$", "", label).strip()
111 return label
114def split_pronunciation_pos_text(text: str) -> PronunciationPosMatch:
115 pos_values: list[str] = []
116 residual: list[str] = []
117 for part in split_pronunciation_pos_parts(text):
118 pos, residual_part = pronunciation_pos_from_part(part)
119 if pos:
120 # POS-bearing qualifier text may also contain normal pronunciation
121 # tags before the POS label, e.g. "attributive adjective".
122 if pos not in pos_values: 122 ↛ 124line 122 didn't jump to line 124 because the condition on line 122 was always true
123 pos_values.append(pos)
124 if residual_part:
125 residual.append(residual_part)
126 elif residual_part: 126 ↛ 117line 126 didn't jump to line 117 because the condition on line 126 was always true
127 residual.append(residual_part)
128 if not pos_values:
129 # If nothing in the text was a POS label, preserve the original text
130 # for normal pronunciation tag/note parsing.
131 return PronunciationPosMatch((), text.strip())
132 return PronunciationPosMatch(tuple(pos_values), ", ".join(residual))
135def pronunciation_pos_from_part(part: str) -> tuple[str | None, str]:
136 normalized = normalize_pronunciation_pos_label(part)
137 if normalized in PRON_POS_BY_LABEL:
138 return PRON_POS_BY_LABEL[normalized], ""
139 # Match residual tag text followed by a POS label:
140 # "attributive adjective" -> ("adj", "attributive")
141 # "attributive proper noun" -> ("name", "attributive")
142 # The label alternation is sorted longest-first so multi-word POS labels
143 # such as "proper noun" win over their suffixes.
144 match = PRON_POS_LABEL_RE.match(normalized)
145 if match:
146 label = match.group("label")
147 residual = (match.group("residual") or "").rstrip(" ,;:")
148 if not residual or classify_desc(residual) == "tags":
149 return PRON_POS_BY_LABEL[label], residual
150 return None, part
153def split_pronunciation_pos_parts(text: str) -> list[str]:
154 parts: list[str] = []
155 for comma_part in re.split(r"[,;]", text):
156 comma_part = comma_part.strip()
157 if not comma_part:
158 continue
159 # Commas and semicolons reliably separate qualifier chunks. Only split
160 # "and"/"or" when at least one side is a POS label, so prose notes
161 # stay intact.
162 conjunction_parts = re.split(r"\s+(?:and|or)\s+", comma_part)
163 if len(conjunction_parts) > 1 and any(
164 pronunciation_pos_from_part(part)[0]
165 for part in conjunction_parts
166 ):
167 parts.extend(conjunction_parts)
168 else:
169 parts.append(comma_part)
170 return parts
173def set_sound_pos(
174 sound: SoundData, pos_values: PronunciationPoses | None
175) -> PronunciationPoses | None:
176 if pos_values:
177 sound["pos"] = pos_values # type: ignore[typeddict-unknown-key]
178 return pos_values
179 if "pos" in sound:
180 return sound["pos"] # type: ignore[typeddict-item]
181 return None
184def common_sound_pos(
185 pos_candidates: set[PronunciationPoses],
186) -> PronunciationPoses | None:
187 if len(pos_candidates) != 1:
188 return None
189 return next(iter(pos_candidates))
192def merge_pronunciation_tag_data(
193 sound: SoundData, tag_data: SoundData
194) -> None:
195 for value in tag_data.get("tags", []):
196 if value not in sound.get("tags", []): 196 ↛ 195line 196 didn't jump to line 195 because the condition on line 196 was always true
197 data_append(sound, "tags", value)
198 for value in tag_data.get("topics", []): 198 ↛ 199line 198 didn't jump to line 199 because the loop on line 198 never started
199 if value not in sound.get("topics", []):
200 data_append(sound, "topics", value)
201 if note := tag_data.get("note"):
202 existing_note = sound.get("note")
203 if not existing_note:
204 sound["note"] = note
205 elif note not in [n.strip() for n in existing_note.split(";")]: 205 ↛ exitline 205 didn't return from function 'merge_pronunciation_tag_data' because the condition on line 205 was always true
206 sound["note"] = f"{existing_note}; {note}"
209def inherit_pronunciation_tag_data(
210 sound: SoundData, parent_tag_data: SoundData
211) -> None:
212 """Add the tags and topics of a parent list item, such as
213 "* {{a|en|GA}}", to a nested pronunciation, and put its note before
214 the nested pronunciation's own."""
215 if not parent_tag_data:
216 return
217 tag_data: SoundData = {}
218 merge_pronunciation_tag_data(tag_data, parent_tag_data)
219 merge_pronunciation_tag_data(tag_data, sound)
220 if "tags" in tag_data: 220 ↛ 222line 220 didn't jump to line 222 because the condition on line 220 was always true
221 tag_data["tags"] = sorted(tag_data["tags"])
222 if "topics" in tag_data: 222 ↛ 223line 222 didn't jump to line 223 because the condition on line 222 was never true
223 tag_data["topics"] = sorted(tag_data["topics"])
224 sound.update(tag_data)
227def is_toggled_expansion(text: str) -> bool:
228 """A big, toggleable section, whose collapsed "header" may be a duplicate
229 of its first line."""
230 return text.startswith("<") and "vsToggleElement" in text
233def ipa_qualifiers(text: str) -> list[tuple[re.Match[str], str]]:
234 """Each IPA match in the text, with the parenthesized qualifier that follows
235 it: "/a/ [b] (Spain), /c/ (Peru (north))" gives "Spain" for the first two
236 and "Peru (north)" for the last. Language modules such as {{es-pr}} expand
237 to plain lines like that, so these never reach extract_pron_template().
238 Parentheses inside a transcription, as in /ˈhæp(ə)n/, are no qualifier, nor
239 is a group holding a transcription itself, as in "/a/ (also /b/)". Anything
240 else between a transcription and the group, such as the comma of
241 "/a/, (old) /b/" or "/a/, /b/ (Spain)", means the group is not about it."""
242 ipa_matches = list(IPA_MATCHES_RE.finditer(text))
243 masked = text
244 for m in ipa_matches:
245 masked = masked[: m.start()] + " " * len(m.group()) + masked[m.end() :]
246 groups: list[tuple[int, int]] = []
247 depth = group_start = 0
248 for i, char in enumerate(masked):
249 if char == "(":
250 if depth == 0:
251 group_start = i
252 depth += 1
253 elif char == ")" and depth > 0:
254 depth -= 1
255 if depth == 0:
256 groups.append((group_start, i))
257 qualifiers = []
258 for m in ipa_matches:
259 qualifier = ""
260 for start, end in groups:
261 if m.start() < start:
262 if not masked[m.end() : start].strip() and not any(
263 start < o.start() < end for o in ipa_matches
264 ):
265 qualifier = text[start + 1 : end].strip()
266 break
267 if m.start() < end:
268 break
269 qualifiers.append((m, qualifier))
270 return qualifiers
273def parse_pronunciation_tags_with_pos(
274 wxr: WiktextractContext, text: str, sound: SoundData
275) -> PronunciationPoses:
276 match = split_pronunciation_pos_text(text)
277 set_sound_pos(sound, match.pos_values)
278 if match.residual:
279 tag_data: SoundData = {}
280 parse_pronunciation_tags(wxr, match.residual, tag_data)
281 merge_pronunciation_tag_data(sound, tag_data)
282 return match.pos_values
285def extract_pos_prefix(text: str) -> PronunciationPosPrefix | None:
286 stripped = text.strip()
287 if not (stripped.startswith("(") and stripped.endswith(")")):
288 bare_match = split_pronunciation_pos_text(text)
289 if bare_match.pos_values and not bare_match.residual:
290 return PronunciationPosPrefix(bare_match.pos_values, "", True)
292 colon_match = re.match(r"\s*([^:()]+?)\s*:\s*(.*)$", text)
293 if colon_match:
294 match = split_pronunciation_pos_text(colon_match.group(1))
295 if match.pos_values and not match.residual: 295 ↛ 296line 295 didn't jump to line 296 because the condition on line 295 was never true
296 return PronunciationPosPrefix(
297 match.pos_values, colon_match.group(2).strip(), True
298 )
300 paren_match = re.match(r"\s*\(([^()]*)\)\s*(.*)$", text)
301 if paren_match:
302 match = split_pronunciation_pos_text(paren_match.group(1))
303 if match.pos_values and not match.residual:
304 return PronunciationPosPrefix(
305 match.pos_values, paren_match.group(2).strip(), False
306 )
308 return None
311def extract_pronunciation_pos_template(
312 wxr: WiktextractContext,
313 name: str,
314 ht: TemplateArgs,
315 lang_code: str,
316) -> PronunciationPosMatch:
317 if name in {"a", "accent", "lb", "lbl", "label"}:
318 pos_args = [
319 value
320 for key, value in ht.items()
321 if isinstance(key, int) and key >= 2
322 ]
323 if not pos_args and ht.get(1) != lang_code:
324 pos_args = [ht.get(1, "")]
325 else:
326 pos_args = [
327 value
328 for key, value in ht.items()
329 if isinstance(key, int) and key >= 1
330 ]
332 pos_values: list[str] = []
333 residual: list[str] = []
334 for arg in pos_args:
335 text = clean_node(wxr, None, [arg])
336 match = split_pronunciation_pos_text(text)
337 for pos in match.pos_values:
338 if pos not in pos_values: 338 ↛ 337line 338 didn't jump to line 337 because the condition on line 338 was always true
339 pos_values.append(pos)
340 if match.residual:
341 residual.append(match.residual)
342 return PronunciationPosMatch(tuple(pos_values), ", ".join(residual))
345def extract_pron_template(
346 wxr: WiktextractContext, tname: str, targs: TemplateArgs, expanded: str
347) -> tuple[SoundData, list[SoundData]] | None:
348 """In post_template_fn, this is used to handle all enPR and IPA templates
349 so that we can leave breadcrumbs in the text that can later be handled
350 there. We return a `base_data` so that if there are two
351 or more templates on the same line, like this:
352 (Tags for the whole line, really) enPR: foo, IPA(keys): /foo/
353 then we can apply base_data fields to other templates, too, if needed.
354 """
355 cleaned = clean_value(wxr, expanded)
356 # print(f"extract_pron_template input: {tname=} {expanded=}-> {cleaned=}")
357 m = IPA_EXTRACT_RE.match(cleaned)
358 if not m:
359 wxr.wtp.error(
360 f"Text cannot match IPA_EXTRACT_RE regex: "
361 f"{cleaned=}, {tname=}, {targs=}",
362 sortid="en/pronunciation/54",
363 )
364 return None
365 # for i, group in enumerate(m.groups()):
366 # print(i + 1, repr(group))
367 main_qual = m.group(2) or ""
368 if "qq" in targs:
369 # If the template has been given a qualifier that applies to
370 # every entry, but which also happens to appear at the end
371 # which can be confused with the post-qualifier of a single
372 # entry in the style of "... /ipa3/ (foo) (bar)", where foo
373 # might not be present so the bar looks like it only might
374 # apply to `/ipa3/`
375 pron_body = m.group(5)
376 post_qual = m.group(7)
377 else:
378 pron_body = m.group(4)
379 post_qual = ""
381 if not pron_body: 381 ↛ 382line 381 didn't jump to line 382 because the condition on line 381 was never true
382 wxr.wtp.error(
383 f"Regex failed to find 'body' from {cleaned=}",
384 sortid="en/pronunciation/81",
385 )
386 return None
388 base_data: SoundData = {}
389 if main_qual:
390 parse_pronunciation_tags_with_pos(wxr, main_qual, base_data)
391 if post_qual:
392 parse_pronunciation_tags_with_pos(wxr, post_qual, base_data)
393 # This base_data is used as the base copy for all entries from this
394 # template, but it is also returned so that its contents may be applied
395 # to other templates on the same line.
396 # print(f"{base_data=}")
398 sound_datas: list[SoundData] = []
400 parts: list[list[str]] = [[]]
401 inside = 0
402 current: list[str] = []
403 for i, p in enumerate(re.split(r"(\s*,|;|\(|\)\s*)", pron_body)):
404 # Split the line on commas and semicolons outside of parens. This
405 # gives us lines with "(main-qualifier) /phon/ (post-qualifier, maybe)"
406 # print(f" {i=}, {p=}")
407 comp = p.strip()
408 if not p:
409 continue
410 if comp == "(":
411 if not inside and i > 0: 411 ↛ 414line 411 didn't jump to line 414 because the condition on line 411 was always true
412 if stripped := "".join(current).strip():
413 parts[-1].append("".join(current).strip()) # type:ignore[arg-type]
414 current = [p]
415 inside += 1
416 continue
417 if comp == ")":
418 inside -= 1
419 if not inside: 419 ↛ 424line 419 didn't jump to line 424 because the condition on line 419 was always true
420 if stripped := "".join(current).strip(): 420 ↛ 424line 420 didn't jump to line 424 because the condition on line 420 was always true
421 current.append(p)
422 parts[-1].append("".join(current).strip()) # type:ignore[arg-type]
423 current = []
424 continue
425 if not inside and comp in (",", ";"):
426 if stripped := "".join(current).strip():
427 parts[-1].append(stripped) # type:ignore[arg-type]
428 current = []
429 parts.append([])
430 continue
431 current.append(p)
432 if current:
433 parts[-1].append("".join(current).strip())
435 # print(f">>>>>> {parts=}")
436 new_parts: list[list[str]] = []
437 for entry in parts:
438 if not entry: 438 ↛ 439line 438 didn't jump to line 439 because the condition on line 438 was never true
439 continue
440 new_entry: list[str] = []
441 i1: int = entry[0].startswith("(") and entry[0].endswith(")")
442 if i1:
443 new_entry.append(entry[0][1:-1].strip())
444 else:
445 new_entry.append("")
446 i2: int = (
447 entry[-1].startswith("(")
448 and entry[-1].endswith(")")
449 and len(entry) > 1
450 )
451 if i2 == 0:
452 i2 = len(entry)
453 else:
454 i2 = -1
455 new_entry.append("".join(entry[i1:i2]).strip())
456 if not new_entry[-1]: 456 ↛ 457line 456 didn't jump to line 457 because the condition on line 456 was never true
457 wxr.wtp.error(
458 f"Missing IPA/enPRO sound data between qualifiers?{entry=}",
459 sortid="en/pronunciation/153",
460 )
461 if i2 == -1:
462 new_entry.append(entry[-1][1:-1].strip())
463 else:
464 new_entry.append("")
465 new_parts.append(new_entry)
467 # print(f">>>>> {new_parts=}")
469 for part in new_parts:
470 sd = deepcopy(base_data)
471 if part[0]:
472 parse_pronunciation_tags_with_pos(wxr, part[0], sd)
473 if part[2]:
474 parse_pronunciation_tags_with_pos(wxr, part[2], sd)
475 if tname == "enPR":
476 sd["enpr"] = part[1]
477 else:
478 sd["ipa"] = part[1]
479 sound_datas.append(sd)
481 # print(f"BASE_DATA: {base_data}")
482 # print(f"SOUND_DATAS: {sound_datas=}")
484 return base_data, sound_datas
487def parse_pronunciation(
488 wxr: WiktextractContext,
489 level_node: LevelNode,
490 data: WordData,
491 etym_data: WordData,
492 have_etym: bool,
493 base_data: WordData,
494 lang_code: str,
495) -> None:
496 """Parses the pronunciation section from a language section on a
497 page."""
498 if level_node.kind in LEVEL_KINDS: 498 ↛ 511line 498 didn't jump to line 511 because the condition on line 498 was always true
499 contents: list[str | WikiNode | TemplateNode] = []
500 for node in level_node.children:
501 if isinstance(node, TemplateNode):
502 if node.template_name == "th-pron":
503 extract_th_pron_template(wxr, data, node)
504 elif node.template_name == "zh-pron":
505 extract_zh_pron_template(wxr, data, node)
506 else:
507 contents.append(node)
508 else:
509 contents.append(node)
510 else:
511 contents = [level_node]
512 # Remove subsections, such as Usage notes. They may contain IPAchar
513 # templates in running text, and we do not want to extract IPAs from
514 # those.
515 # Filter out only LEVEL_KINDS; 'or' is doing heavy lifting here
516 # Slip through not-WikiNodes, then slip through WikiNodes that
517 # are not LEVEL_KINDS.
518 contents = [
519 x
520 for x in contents
521 if not isinstance(x, WikiNode) or x.kind not in LEVEL_KINDS
522 ]
523 # Set when a template's expansion may generate duplicate sound data
524 may_be_duplicates = False
525 if not any(
526 isinstance(x, WikiNode) and x.kind == NodeKind.LIST for x in contents
527 ):
528 # expand all templates
529 new_contents: list[str | WikiNode | TemplateNode] = []
530 for lst in contents:
531 if isinstance(lst, TemplateNode):
532 temp = wxr.wtp.node_to_wikitext(lst)
533 temp = wxr.wtp.expand(temp)
534 if is_toggled_expansion(temp):
535 may_be_duplicates = True
536 temp_parsed = wxr.wtp.parse(temp)
537 new_contents.extend(temp_parsed.children)
538 else:
539 new_contents.append(lst)
540 contents = new_contents
542 if have_etym and data is base_data: 542 ↛ 543line 542 didn't jump to line 543 because the condition on line 542 was never true
543 data = etym_data
544 pron_templates: list[tuple[SoundData, list[SoundData]]] = []
545 pron_pos_markers: list[PronunciationPoses] = []
546 hyphenations: list[Hyphenation] = []
547 audios: list[SoundData] = []
548 have_panel_templates = False
550 def parse_pronunciation_template_fn(
551 name: str, ht: TemplateArgs
552 ) -> str | None:
553 """Handle pronunciation and hyphenation templates"""
554 # _template_fn handles templates *before* they are expanded;
555 # this allows for special handling before all the work needed
556 # for expansion is done.
557 nonlocal have_panel_templates
558 if is_panel_template(wxr, name):
559 have_panel_templates = True
560 return ""
561 if name == "audio":
562 filename = ht.get(2) or ""
563 audio: SoundData = {"audio": filename.strip()}
564 dialect = ht.get("a", "")
565 if "aa" in ht: 565 ↛ 566line 565 didn't jump to line 566 because the condition on line 565 was never true
566 dialect += ", " + ht.get("aa", "")
567 if dialect:
568 dialect = dialect.replace("<", "").replace(">", "")
569 dialect = clean_node(wxr, None, [dialect])
570 for part in split_at_comma_semi(dialect):
571 if "(" not in part:
572 parse_pronunciation_tags(wxr, part, audio)
573 else:
574 for ppart in re.split(r"[][()]", part):
575 parse_pronunciation_tags(wxr, ppart, audio)
576 desc = ht.get(3) or ""
577 desc = clean_node(wxr, None, [desc])
578 if desc: 578 ↛ 579line 578 didn't jump to line 579 because the condition on line 578 was never true
579 audio["text"] = desc
580 m = re.search(r"\((([^()]|\([^()]*\))*)\)", desc)
581 skip = False
582 if m: 582 ↛ 583line 582 didn't jump to line 583 because the condition on line 582 was never true
583 par = m.group(1)
584 cls = classify_desc(par)
585 if cls == "tags":
586 parse_pronunciation_tags(wxr, par, audio)
587 else:
588 skip = True
589 if skip: 589 ↛ 590line 589 didn't jump to line 590 because the condition on line 589 was never true
590 return ""
591 audios.append(audio)
592 return "__AUDIO_IGNORE_THIS__" + str(len(audios) - 1) + "__"
593 if name == "audio-IPA": 593 ↛ 594line 593 didn't jump to line 594 because the condition on line 593 was never true
594 filename = ht.get(2) or ""
595 ipa = ht.get(3) or ""
596 dial = ht.get("dial")
597 audio = {"audio": filename.strip()}
598 if dial:
599 dial = clean_node(wxr, None, [dial])
600 audio["text"] = dial
601 if ipa:
602 audio["audio-ipa"] = ipa
603 audios.append(audio)
604 # The problem with these IPAs is that they often just describe
605 # what's in the sound file, rather than giving the pronunciation
606 # of the word alone. It is common for audio files to contain
607 # multiple pronunciations or articles in the same file, and then
608 # this IPA often describes what is in the file.
609 return "__AUDIO_IGNORE_THIS__" + str(len(audios) - 1) + "__"
610 if name == "audio-pron":
611 filename = ht.get(2) or ""
612 ipa = ht.get("ipa") or ""
613 dial = ht.get("dial")
614 country = ht.get("country")
615 audio = {"audio": filename.strip()}
616 if dial: 616 ↛ 620line 616 didn't jump to line 620 because the condition on line 616 was always true
617 dial = clean_node(wxr, None, [dial])
618 audio["text"] = dial
619 parse_pronunciation_tags(wxr, dial, audio)
620 if country: 620 ↛ 622line 620 didn't jump to line 622 because the condition on line 620 was always true
621 parse_pronunciation_tags(wxr, country, audio)
622 if ipa: 622 ↛ 624line 622 didn't jump to line 624 because the condition on line 622 was always true
623 audio["audio-ipa"] = ipa
624 audios.append(audio)
625 # XXX do we really want to extract pronunciations from these?
626 # Or are they spurious / just describing what is in the
627 # audio file?
628 # if ipa:
629 # pron = {"ipa": ipa}
630 # if dial:
631 # parse_pronunciation_tags(wxr, dial, pron)
632 # if country:
633 # parse_pronunciation_tags(wxr, country, pron)
634 # data_append(data, "sounds", pron)
635 return "__AUDIO_IGNORE_THIS__" + str(len(audios) - 1) + "__"
636 if name in ("hyph", "hyphenation"):
637 # {{hyph|en|re|late|caption="Hyphenation UK:"}}
638 # {{hyphenation|it|quiè|to||qui|è|to||quié|to||qui|é|to}}
639 # and also nocaption=1
640 caption = clean_node(wxr, None, ht.get("caption", ""))
641 tagsets, _ = decode_tags(caption)
642 # flatten the tagsets into one; it would be really weird to have
643 # several tagsets for a hyphenation caption
644 tags = sorted(set(tag for tagset in tagsets for tag in tagset))
645 # We'll just ignore any errors from tags, it's not very important
646 # for hyphenation
647 tags = [tag for tag in tags if not tag.startswith("error")]
648 hyph_sequences: list[list[str]] = [[]]
649 for text in [
650 t for (k, t) in ht.items() if (isinstance(k, int) and k >= 2)
651 ]:
652 if not text:
653 hyph_sequences.append([])
654 else:
655 hyph_sequences[-1].append(clean_node(wxr, None, text))
656 for seq in hyph_sequences:
657 hyphenations.append(Hyphenation(parts=seq, tags=tags))
658 return ""
659 return None
661 def parse_pron_post_template_fn(
662 name: str, ht: TemplateArgs, text: str
663 ) -> str | None:
664 # _post_template_fn handles templates *after* the work to expand
665 # them has been done; this is exactly the same as _template_fn,
666 # except with the additional expanded text as an input, and
667 # possible side-effects from the expansion and recursion (like
668 # calling other subtemplates that are handled in _template_fn.
669 nonlocal may_be_duplicates
670 if is_panel_template(wxr, name): 670 ↛ 671line 670 didn't jump to line 671 because the condition on line 670 was never true
671 return ""
672 if name in PRON_POS_TEMPLATE_NAMES:
673 pos_match = extract_pronunciation_pos_template(
674 wxr, name, ht, lang_code
675 )
676 if pos_match.pos_values:
677 pron_pos_markers.append(pos_match.pos_values)
678 marker = (
679 f"__PRON_POS_MARKER_{len(pron_pos_markers) - 1}__"
680 )
681 if pos_match.residual: 681 ↛ 682line 681 didn't jump to line 682 because the condition on line 681 was never true
682 return f"{marker} ({pos_match.residual})"
683 return marker
684 if name in {
685 *PRON_POS_TEMPLATE_NAMES,
686 "l",
687 "link",
688 }:
689 # Kludge: when these templates expand to /.../ or [...],
690 # replace the expansion by something safe. This is used
691 # to filter spurious IPA-looking expansions that aren't really
692 # IPAs. We probably don't care about these templates in the
693 # contexts where they expand to something containing these.
694 v = re.sub(r'href="[^"]*"', "", text) # Ignore URLs
695 v = re.sub(r'src="[^"]*"', "", v)
696 v = clean_value(wxr, v)
697 if re.search(r"/[^/,]+?/|\[[^]0-9,/][^],/]*?\]", v):
698 # Note: replacing by empty results in Lua errors that we
699 # would rather not have. For example, voi/Middle Vietnamese
700 # uses {{a|{{l{{vi|...}}}}, and the {{a|...}} will fail
701 # if {{l|...}} returns empty.
702 return "stripped-by-parse_pron_post_template_fn"
703 if name in ("IPA", "enPR"):
704 # Extract the data from IPA and enPR templates (same underlying
705 # template) and replace them in-text with magical cookie that
706 # can be later used to refer to the data's index inside
707 # pron_templates.
708 if pron_t := extract_pron_template(wxr, name, ht, text):
709 pron_templates.append(pron_t)
710 return f"__PRON_TEMPLATE_{len(pron_templates) - 1}__"
711 # Catch templates that generate duplicate sound data entries
712 # here; if the text produces a big, toggleable section, the
713 # "header" for that section might be duplicated. Add more conditions
714 # if necessary.
715 if is_toggled_expansion(text): 715 ↛ 716line 715 didn't jump to line 716 because the condition on line 715 was never true
716 may_be_duplicates = True
717 return text
719 def flattened_tree(
720 lines: list[WikiNode | str],
721 ) -> Iterator[FlattenedListNode]:
722 assert isinstance(lines, list)
723 for line in lines:
724 yield from flattened_tree1(line, 0)
726 def flattened_tree1(
727 node: WikiNode | str, list_depth: int
728 ) -> Iterator[FlattenedListNode]:
729 assert isinstance(node, (WikiNode, str))
730 if isinstance(node, str):
731 yield FlattenedListNode(node, list_depth)
732 return
733 elif node.kind == NodeKind.LIST:
734 for item in node.children:
735 yield from flattened_tree1(item, list_depth)
736 elif node.kind == NodeKind.LIST_ITEM:
737 item_depth = (
738 len(node.sarg) if isinstance(node.sarg, str) else list_depth
739 )
740 new_children = []
741 # A list item can have several sublists, e.g. "**" lines
742 # followed by a "*:" line.
743 sublists = []
744 for child in node.children:
745 if isinstance(child, WikiNode) and child.kind == NodeKind.LIST:
746 sublists.append(child)
747 else:
748 new_children.append(child)
749 node.children = new_children
750 node.sarg = "*"
751 yield FlattenedListNode(node, item_depth)
752 for sublist in sublists:
753 yield from flattened_tree1(sublist, item_depth)
754 else:
755 yield FlattenedListNode(node, list_depth)
757 # XXX Do not use flattened_tree more than once here, for example for
758 # debug printing... The underlying data is changed, and the separated
759 # sublists disappear.
761 # Kludge for templates that generate several lines, but haven't
762 # been caught by earlier kludges...
763 def split_cleaned_node_on_newlines(
764 contents: list[WikiNode | str],
765 ) -> Iterator[tuple[str, int]]:
766 for flattened in flattened_tree(contents):
767 ipa_text = clean_node(
768 wxr,
769 data,
770 flattened.node,
771 template_fn=parse_pronunciation_template_fn,
772 post_template_fn=parse_pron_post_template_fn,
773 )
774 for line in ipa_text.splitlines():
775 yield line, flattened.list_depth
777 # have_pronunciations = False
778 active_pos: PronunciationPoses | None = None
779 # POS values from parent pronunciation lines by original list depth.
780 # Audio-only child lines can inherit from a parent pronunciation line,
781 # but same-depth audio lines must not inherit from a preceding IPA.
782 pronunciation_pos_stack: list[tuple[int, PronunciationPoses]] = []
784 def parent_pronunciation_pos(
785 list_depth: int,
786 ) -> PronunciationPoses | None:
787 if not pronunciation_pos_stack:
788 return None
789 parent_depth, parent_pos = pronunciation_pos_stack[-1]
790 return parent_pos if parent_depth < list_depth else None
792 # Tags from label-only lines by original list depth, e.g.
793 # "* {{a|en|GA}}" over "** {{IPA|en|...}}". Nested pronunciations
794 # start from their parent's tags and add their own.
795 pronunciation_tags_stack: list[tuple[int, SoundData]] = []
797 def parent_pronunciation_tags(list_depth: int) -> SoundData:
798 if not pronunciation_tags_stack:
799 return {}
800 parent_depth, parent_tags = pronunciation_tags_stack[-1]
801 return parent_tags if parent_depth < list_depth else {}
803 for line, list_depth in split_cleaned_node_on_newlines(contents):
804 prefix: str | None = None
805 earlier_base_data: SoundData | None = None
806 line_pos: PronunciationPoses | None = None
807 current_group_sounds: list[SoundData] = []
808 # POS values seen on sounds extracted from this physical line. A
809 # single candidate can seed adjacent audio-only child lines; multiple
810 # POS-marked sounds on one line are too ambiguous for inheritance.
811 line_sound_pos_candidates: set[PronunciationPoses] = set()
812 line_has_sound = False
813 if not line: 813 ↛ 814line 813 didn't jump to line 814 because the condition on line 813 was never true
814 continue
815 while (
816 pronunciation_pos_stack
817 and pronunciation_pos_stack[-1][0] >= list_depth
818 ):
819 pronunciation_pos_stack.pop()
820 while (
821 pronunciation_tags_stack
822 and pronunciation_tags_stack[-1][0] >= list_depth
823 ):
824 pronunciation_tags_stack.pop()
825 parent_tags = parent_pronunciation_tags(list_depth)
827 split_templates = re.split(r"__PRON_TEMPLATE_(\d+)__", line)
828 for i, text in enumerate(split_templates):
829 if not text:
830 continue
831 # clean up starts at the start of the line
832 text = re.sub(r"^\**\s*", "", text).strip()
833 if i == 0:
834 # At the start of a line, check for stuff like "Noun:"
835 # or "(verb)" for POS labels that apply to this line or
836 # structurally nested pronunciation lines.
837 # These labels feed the inheritance state that later sets the
838 # temporary sound["pos"] field used to route pronunciation
839 # data into matching POS sections.
840 if pos_prefix := extract_pos_prefix(text):
841 text = pos_prefix.text
842 line_pos = pos_prefix.pos_values
843 if pos_prefix.is_persistent:
844 active_pos = pos_prefix.pos_values
845 if not text:
846 continue
848 m = re.search(r"__PRON_POS_MARKER_(\d+)__", text)
849 while m:
850 if current_group_sounds and re.search(
851 r"[,;]", text[: m.start()]
852 ):
853 current_group_sounds = []
854 pos_values = pron_pos_markers[int(m.group(1))]
855 if current_group_sounds:
856 for sound in current_group_sounds:
857 set_sound_pos(sound, pos_values)
858 line_sound_pos_candidates.add(pos_values)
859 line_pos = pos_values
860 text = text[: m.start()] + text[m.end() :]
861 m = re.search(r"__PRON_POS_MARKER_(\d+)__", text)
862 text = text.strip()
863 if not text:
864 continue
865 # POS inheritance for normal pronunciation data:
866 # 1. line_pos: explicit POS marker on this line, e.g.
867 # "* {{q|noun}} {{IPA|...}}".
868 # 2. parent_pronunciation_pos: structurally inherited from a
869 # parent list item, e.g. "* {{q|noun}}" then "** {{IPA|...}}".
870 # 3. active_pos: support for "* Noun:" followed by
871 # "* {{IPA|...}}"; broad, so it stays after structural data.
872 inherited_pos = (
873 line_pos or parent_pronunciation_pos(list_depth) or active_pos
874 )
876 if i % 2 == 1:
877 # re.split (with capture groups) splits the lines so that
878 # every even entry is a captured splitter; odd lines are either
879 # empty strings or stuff around the splitters.
880 base_pron_data, first_prons = pron_templates[int(text)]
881 if base_pron_data:
882 earlier_base_data = base_pron_data
883 # print(f"Set {earlier_base_data=}")
884 elif earlier_base_data is not None:
885 # merge data from an earlier iteration of this loop
886 for pr in first_prons:
887 if "note" in pr and "note" in earlier_base_data: 887 ↛ 888line 887 didn't jump to line 888 because the condition on line 887 was never true
888 pr["note"] += ";" + earlier_base_data.get(
889 "note", ""
890 )
891 elif "note" in earlier_base_data: 891 ↛ 892line 891 didn't jump to line 892 because the condition on line 891 was never true
892 pr["note"] = earlier_base_data["note"]
893 if "topics" in earlier_base_data: 893 ↛ 894line 893 didn't jump to line 894 because the condition on line 893 was never true
894 data_extend(
895 pr, "topics", earlier_base_data["topics"]
896 )
897 if "tags" in pr and "tags" in earlier_base_data: 897 ↛ 898line 897 didn't jump to line 898 because the condition on line 897 was never true
898 pr["tags"].extend(earlier_base_data["tags"])
899 elif "tags" in earlier_base_data: 899 ↛ 886line 899 didn't jump to line 886 because the condition on line 899 was always true
900 pr["tags"] = sorted(set(earlier_base_data["tags"]))
901 for pr in first_prons:
902 inherit_pronunciation_tag_data(pr, parent_tags)
903 if sound_pos := set_sound_pos(
904 pr,
905 None if "pos" in pr else inherited_pos,
906 ):
907 line_sound_pos_candidates.add(sound_pos)
908 if pr not in data.get("sounds", ()): 908 ↛ 910line 908 didn't jump to line 910 because the condition on line 908 was always true
909 data_append(data, "sounds", pr)
910 current_group_sounds.append(pr)
911 line_has_sound = True
912 # This bit is handled
913 continue
915 if "IPA" in text:
916 field: Literal[
917 "audio",
918 "audio-ipa",
919 "enpr",
920 "form",
921 "hangeul",
922 "homophone",
923 "ipa",
924 "mp3_url",
925 "note",
926 "ogg_url",
927 "other",
928 "rhymes",
929 "tags",
930 "text",
931 "topics",
932 "zh-pron",
933 ] = "ipa"
934 else:
935 # This is used for Rhymes, Homophones, etc
936 field = "other"
938 # Check if it contains Japanese "Tokyo" pronunciation with
939 # special syntax
940 m = re.search(r"(?m)\(Tokyo\) +([^ ]+) +\[", text)
941 if m: 941 ↛ 942line 941 didn't jump to line 942 because the condition on line 941 was never true
942 pron: SoundData = {field: m.group(1)} # type: ignore[misc]
943 if sound_pos := set_sound_pos(pron, inherited_pos):
944 line_sound_pos_candidates.add(sound_pos)
945 data_append(data, "sounds", pron)
946 current_group_sounds.append(pron)
947 line_has_sound = True
948 # have_pronunciations = True
949 continue
951 # Check if it contains Rhymes
952 m = re.match(r"\s*Rhymes?: (.*)", text)
953 if m:
954 for ending in split_at_comma_semi(m.group(1)):
955 ending = ending.strip()
956 if ending: 956 ↛ 954line 956 didn't jump to line 954 because the condition on line 956 was always true
957 pron = {"rhymes": ending}
958 if sound_pos := set_sound_pos(pron, inherited_pos):
959 line_sound_pos_candidates.add(sound_pos)
960 data_append(data, "sounds", pron)
961 current_group_sounds.append(pron)
962 line_has_sound = True
963 # have_pronunciations = True
964 continue
966 # Check if it contains homophones
967 m = re.search(r"(?m)\bHomophones?: (.*)", text)
968 if m:
969 for w in split_at_comma_semi(m.group(1)):
970 w = w.strip()
971 if w: 971 ↛ 969line 971 didn't jump to line 969 because the condition on line 971 was always true
972 pron = {"homophone": w}
973 if sound_pos := set_sound_pos(pron, inherited_pos):
974 line_sound_pos_candidates.add(sound_pos)
975 data_append(data, "sounds", pron)
976 current_group_sounds.append(pron)
977 line_has_sound = True
978 # have_pronunciations = True
979 continue
981 # Check if it contains Phonetic hangeul
982 m = re.search(r"(?m)\bPhonetic hange?ul: \[([^]]+)\]", text)
983 if m: 983 ↛ 984line 983 didn't jump to line 984 because the condition on line 983 was never true
984 seen = set()
985 for w in m.group(1).split("/"):
986 w = w.strip()
987 if w and w not in seen:
988 seen.add(w)
989 pron = {"hangeul": w}
990 if sound_pos := set_sound_pos(pron, inherited_pos):
991 line_sound_pos_candidates.add(sound_pos)
992 data_append(data, "sounds", pron)
993 current_group_sounds.append(pron)
994 line_has_sound = True
995 # have_pronunciations = True
997 # This regex-based hyphenation detection left as backup
998 m = re.search(r"\b(Syllabification|Hyphenation): *([^\n.]*)", text)
999 if m:
1000 data_append(data, "hyphenation", m.group(2))
1001 commaseparated = m.group(2).split(",")
1002 if len(commaseparated) > 1: 1002 ↛ 1013line 1002 didn't jump to line 1013 because the condition on line 1002 was always true
1003 for h in commaseparated:
1004 # That second characters looks like a dash but it's
1005 # actually unicode decimal code 8231, hyphenation dash
1006 # Add more delimiters here if needed.
1007 parts = re.split(r"-|‧", h.strip())
1008 data_append(
1009 data, "hyphenations", Hyphenation(parts=parts)
1010 )
1011 ...
1012 else:
1013 data_append(
1014 data,
1015 "hyphenations",
1016 Hyphenation(parts=m.group(2).split(sep="-")),
1017 )
1018 # have_pronunciations = True
1020 # See if it contains a word prefix restricting which forms the
1021 # pronunciation applies to (see amica/Latin) and/or parenthesized
1022 # tags.
1023 m = re.match(
1024 r"^[*#\s]*(([-\w]+):\s+)?\((([^()]|\([^()]*\))*?)\)", text
1025 )
1026 if m:
1027 prefix = m.group(2) or ""
1028 tagstext = m.group(3)
1029 text = text[m.end() :]
1030 else:
1031 m = re.match(r"^[*#\s]*([-\w]+):\s+", text)
1032 if m:
1033 prefix = m.group(1)
1034 tagstext = ""
1035 text = text[m.end() :]
1036 else:
1037 # Spanish has tags before pronunciations, eg. aceite/Spanish
1038 m = re.match(r".*:\s+\(([^)]*)\)\s+(.*)", text)
1039 if m: 1039 ↛ 1040line 1039 didn't jump to line 1040 because the condition on line 1039 was never true
1040 tagstext = m.group(1)
1041 text = m.group(2)
1042 else:
1043 # No prefix. In this case, we inherit prefix
1044 # from previous entry. This particularly
1045 # applies for nested Audio files.
1046 tagstext = ""
1047 if tagstext:
1048 earlier_base_data = {}
1049 parse_pronunciation_tags_with_pos(
1050 wxr, tagstext, earlier_base_data
1051 )
1053 # Find romanizations from the pronunciation section (routinely
1054 # produced for Korean by {{ko-IPA}})
1055 for m in re.finditer(pron_romanization_re, text): 1055 ↛ 1056line 1055 didn't jump to line 1056 because the loop on line 1055 never started
1056 prefix = m.group(1)
1057 w = m.group(2).strip()
1058 tag = pron_romanizations[prefix]
1059 form = {"form": w, "tags": tag.split()}
1060 data_append(data, "forms", form)
1062 # Find IPA pronunciations
1063 for m, qualifier in ipa_qualifiers(text):
1064 v = m.group(0)
1065 # The regexp above can match file links. Skip them.
1066 if v.startswith("[[File:"): 1066 ↛ 1067line 1066 didn't jump to line 1067 because the condition on line 1066 was never true
1067 continue
1068 if v == "/wiki.local/": 1068 ↛ 1069line 1068 didn't jump to line 1069 because the condition on line 1068 was never true
1069 continue
1070 if field == "ipa" and "__AUDIO_IGNORE_THIS__" in text: 1070 ↛ 1071line 1070 didn't jump to line 1071 because the condition on line 1070 was never true
1071 m = re.search(r"__AUDIO_IGNORE_THIS__(\d+)__", text)
1072 assert m
1073 idx = int(m.group(1))
1074 if idx >= len(audios):
1075 continue
1076 if not audios[idx].get("audio-ipa"):
1077 audios[idx]["audio-ipa"] = v
1078 if prefix:
1079 audios[idx]["form"] = prefix
1080 else:
1081 if earlier_base_data:
1082 pron = deepcopy(earlier_base_data)
1083 pron[field] = v
1084 else:
1085 pron = {field: v} # type: ignore[misc]
1086 if prefix:
1087 pron["form"] = prefix
1088 if qualifier:
1089 parse_pronunciation_tags_with_pos(wxr, qualifier, pron)
1090 inherit_pronunciation_tag_data(pron, parent_tags)
1091 if sound_pos := set_sound_pos(
1092 pron,
1093 None if "pos" in pron else inherited_pos,
1094 ):
1095 line_sound_pos_candidates.add(sound_pos)
1096 if may_be_duplicates is True:
1097 ok = True
1098 for comp_sound in data.get("sounds", []):
1099 # Python has dict comparison since 3.8
1100 if pron == comp_sound: 1100 ↛ 1101line 1100 didn't jump to line 1101 because the condition on line 1100 was never true
1101 ok = False
1102 break
1103 # The collapsed header of a toggle repeats the
1104 # line without its qualifier: keep the labelled.
1105 if qualifier and all(
1106 pron.get(k) == val # type: ignore[misc]
1107 for k, val in comp_sound.items()
1108 ):
1109 comp_sound.update(pron)
1110 pron = comp_sound
1111 ok = False
1112 break
1113 if ok:
1114 data_append(data, "sounds", pron)
1115 else:
1116 data_append(data, "sounds", pron)
1117 current_group_sounds.append(pron)
1118 line_has_sound = True
1119 # have_pronunciations = True
1120 if current_group_sounds and re.search(r"[,;]", text):
1121 current_group_sounds = []
1122 # XXX what about {{hyphenation|...}}, {{hyph|...}}
1123 # and those used to be stored under "hyphenation"
1125 # Add data that was collected in template_fn
1126 # POS inheritance for audio has one extra source:
1127 # common_sound_pos(line_sound_pos_candidates), from pronunciations
1128 # extracted earlier on the same physical line, e.g.
1129 # "* {{IPA|en|/foo/|a=verb}} {{audio|en|foo.wav}}".
1130 # Explicit line_pos still wins, then same-line sound agreement, then
1131 # parent-list structure, then active_pos.
1132 audio_inherited_pos = (
1133 line_pos
1134 or common_sound_pos(line_sound_pos_candidates)
1135 or parent_pronunciation_pos(list_depth)
1136 or active_pos
1137 )
1138 for audio in audios:
1139 if "audio" in audio: 1139 ↛ 1196line 1139 didn't jump to line 1196 because the condition on line 1139 was always true
1140 # Compute audio file URLs
1141 fn = audio["audio"]
1142 # Strip certain characters, e.g., left-to-right mark
1143 fn = re.sub(r"[\u200f\u200e]", "", fn)
1144 fn = fn.strip()
1145 fn = urllib.parse.unquote(fn)
1146 # First character is usually uppercased
1147 if re.match(r"^[a-z][a-z]+", fn):
1148 fn = fn[0].upper() + fn[1:]
1149 if fn in wxr.config.redirects: 1149 ↛ 1150line 1149 didn't jump to line 1150 because the condition on line 1149 was never true
1150 fn = wxr.config.redirects[fn]
1151 # File extension is lowercased
1152 # XXX some words seem to need this, some don't seem to
1153 # have this??? what is the exact rule?
1154 # fn = re.sub(r"\.[^.]*$", lambda m: m.group(0).lower(), fn)
1155 # Spaces are converted to underscores
1156 fn = re.sub(r"\s+", "_", fn)
1157 # Compute hash digest part
1158 h = hashlib.md5()
1159 hname = fn.encode("utf-8")
1160 h.update(hname)
1161 digest = h.hexdigest()
1162 # Quote filename for URL
1163 qfn = urllib.parse.quote(fn)
1164 # For safety when writing files
1165 qfn = qfn.replace("/", "__slash__")
1166 if re.search(r"(?i)\.(ogg|oga)$", fn):
1167 ogg = (
1168 "https://upload.wikimedia.org/wikipedia/"
1169 "commons/{}/{}/{}".format(digest[:1], digest[:2], qfn)
1170 )
1171 else:
1172 ogg = (
1173 "https://upload.wikimedia.org/wikipedia/"
1174 "commons/transcoded/"
1175 "{}/{}/{}/{}.ogg".format(
1176 digest[:1], digest[:2], qfn, qfn
1177 )
1178 )
1179 if re.search(r"(?i)\.(mp3)$", fn): 1179 ↛ 1180line 1179 didn't jump to line 1180 because the condition on line 1179 was never true
1180 mp3 = (
1181 "https://upload.wikimedia.org/wikipedia/"
1182 "commons/{}/{}/{}".format(digest[:1], digest[:2], qfn)
1183 )
1184 else:
1185 mp3 = (
1186 "https://upload.wikimedia.org/wikipedia/"
1187 "commons/transcoded/"
1188 "{}/{}/{}/{}.mp3".format(
1189 digest[:1], digest[:2], qfn, qfn
1190 )
1191 )
1192 audio["ogg_url"] = ogg
1193 audio["mp3_url"] = mp3
1194 if "pos" not in audio: 1194 ↛ 1196line 1194 didn't jump to line 1196 because the condition on line 1194 was always true
1195 set_sound_pos(audio, audio_inherited_pos)
1196 if audio not in data.get("sounds", ()):
1197 data_append(data, "sounds", audio)
1198 line_has_sound = True
1200 # if audios:
1201 # have_pronunciations = True
1202 audios = []
1204 data_extend(data, "hyphenations", hyphenations)
1205 hyphenations = []
1207 if line_pos and not line_has_sound:
1208 active_pos = line_pos
1209 pronunciation_pos_stack.append((list_depth, line_pos))
1210 elif line_pronunciation_pos := common_sound_pos(
1211 line_sound_pos_candidates
1212 ):
1213 pronunciation_pos_stack.append((list_depth, line_pronunciation_pos))
1215 if not line_has_sound and earlier_base_data:
1216 line_tags: SoundData = {}
1217 merge_pronunciation_tag_data(line_tags, earlier_base_data)
1218 if line_tags: 1218 ↛ 803line 1218 didn't jump to line 803 because the condition on line 1218 was always true
1219 inherit_pronunciation_tag_data(line_tags, parent_tags)
1220 pronunciation_tags_stack.append((list_depth, line_tags))
1222 ## I have commented out the otherwise unused have_pronunciation
1223 ## toggles; uncomment them to use this debug print
1224 # if not have_pronunciations and not have_panel_templates:
1225 # wxr.wtp.debug("no pronunciations found from pronunciation section",
1226 # sortid="pronunciations/533")
1229def extract_th_pron_template(
1230 wxr: WiktextractContext, word_entry: WordData, t_node: TemplateNode
1231):
1232 # https://en.wiktionary.org/wiki/Template:th-pron
1233 @dataclass
1234 class TableHeader:
1235 raw_tags: list[str]
1236 rowspan: int
1238 expanded_node = wxr.wtp.parse(
1239 wxr.wtp.node_to_wikitext(t_node), expand_all=True
1240 )
1241 sounds = []
1242 for table_tag in expanded_node.find_html("table"):
1243 row_headers = []
1244 for tr_tag in table_tag.find_html("tr"):
1245 field = "other"
1246 new_headers = []
1247 for header in row_headers:
1248 if header.rowspan > 1:
1249 header.rowspan -= 1
1250 new_headers.append(header)
1251 row_headers = new_headers
1252 for th_tag in tr_tag.find_html("th"):
1253 header_str = clean_node(wxr, None, th_tag)
1254 if header_str.startswith("(standard) IPA"):
1255 field = "ipa"
1256 elif header_str.startswith("Homophones"): 1256 ↛ 1257line 1256 didn't jump to line 1257 because the condition on line 1256 was never true
1257 field = "homophone"
1258 elif header_str == "Audio":
1259 field = "audio"
1260 elif header_str != "": 1260 ↛ 1252line 1260 didn't jump to line 1252 because the condition on line 1260 was always true
1261 rowspan = 1
1262 rowspan_str = th_tag.attrs.get("rowspan", "1")
1263 if re.fullmatch(r"\d+", rowspan_str): 1263 ↛ 1265line 1263 didn't jump to line 1265 because the condition on line 1263 was always true
1264 rowspan = int(rowspan_str)
1265 header = TableHeader([], rowspan)
1266 for line in header_str.splitlines():
1267 for raw_tag in line.strip("{}\n ").split(";"):
1268 raw_tag = raw_tag.strip()
1269 if raw_tag != "": 1269 ↛ 1267line 1269 didn't jump to line 1267 because the condition on line 1269 was always true
1270 header.raw_tags.append(raw_tag)
1271 row_headers.append(header)
1273 for td_tag in tr_tag.find_html("td"):
1274 if field == "audio":
1275 for link_node in td_tag.find_child(NodeKind.LINK):
1276 filename = clean_node(wxr, None, link_node.largs[0])
1277 if filename != "": 1277 ↛ 1275line 1277 didn't jump to line 1275 because the condition on line 1277 was always true
1278 sound = create_audio_url_dict(filename)
1279 sounds.append(sound)
1280 elif field == "homophone": 1280 ↛ 1281line 1280 didn't jump to line 1281 because the condition on line 1280 was never true
1281 for span_tag in td_tag.find_html_recursively(
1282 "span", attr_name="lang", attr_value="th"
1283 ):
1284 word = clean_node(wxr, None, span_tag)
1285 if word != "":
1286 sounds.append({"homophone": word})
1287 else:
1288 raw_tags = []
1289 for html_node in td_tag.find_child_recursively(
1290 NodeKind.HTML
1291 ):
1292 if html_node.tag == "small":
1293 node_str = clean_node(wxr, None, html_node)
1294 if node_str.startswith("[") and node_str.endswith(
1295 "]"
1296 ):
1297 for raw_tag in node_str.strip("[]").split(","):
1298 raw_tag = raw_tag.strip()
1299 if raw_tag != "": 1299 ↛ 1297line 1299 didn't jump to line 1297 because the condition on line 1299 was always true
1300 raw_tags.append(raw_tag)
1301 elif len(sounds) > 0: 1301 ↛ 1289line 1301 didn't jump to line 1289 because the condition on line 1301 was always true
1302 sounds[-1]["roman"] = node_str
1303 elif html_node.tag == "span":
1304 node_str = clean_node(wxr, None, html_node)
1305 span_lang = html_node.attrs.get("lang", "")
1306 span_class = html_node.attrs.get("class", "")
1307 if node_str != "" and (
1308 span_lang == "th" or span_class in ["IPA", "tr"]
1309 ):
1310 sound = {}
1311 for raw_tag in raw_tags:
1312 if raw_tag in valid_tags: 1312 ↛ 1315line 1312 didn't jump to line 1315 because the condition on line 1312 was always true
1313 data_append(sound, "tags", raw_tag)
1314 else:
1315 data_append(sound, "raw_tags", raw_tag)
1316 for header in row_headers:
1317 for raw_tag in header.raw_tags:
1318 if raw_tag.lower() in valid_tags:
1319 data_append(
1320 sound, "tags", raw_tag.lower()
1321 )
1322 else:
1323 data_append(
1324 sound, "raw_tags", raw_tag
1325 )
1326 if "romanization" in sound.get("tags", []):
1327 field = "roman"
1328 sound[field] = node_str
1329 sounds.append(sound)
1331 clean_node(wxr, word_entry, expanded_node)
1332 data_extend(word_entry, "sounds", sounds)
1335def extract_zh_pron_template(
1336 wxr: WiktextractContext, word_entry: WordData, t_node: TemplateNode
1337):
1338 # https://en.wiktionary.org/wiki/Template:zh-pron
1339 expanded_node = wxr.wtp.parse(
1340 wxr.wtp.node_to_wikitext(t_node), expand_all=True
1341 )
1342 seen_lists = set()
1343 sounds = []
1344 for list_node in expanded_node.find_child_recursively(NodeKind.LIST):
1345 if list_node not in seen_lists:
1346 for list_item in list_node.find_child(NodeKind.LIST_ITEM):
1347 sounds.extend(
1348 extract_zh_pron_list_item(wxr, list_item, [], seen_lists)
1349 )
1350 clean_node(wxr, word_entry, expanded_node)
1351 data_extend(word_entry, "sounds", sounds)
1354def extract_zh_pron_list_item(
1355 wxr: WiktextractContext,
1356 list_item: WikiNode,
1357 raw_tags: list[str],
1358 seen_lists: set[WikiNode],
1359) -> list[SoundData]:
1360 current_tags = raw_tags[:]
1361 sounds = []
1362 is_first_small_tag = True
1363 for node in list_item.children:
1364 if isinstance(node, WikiNode) and node.kind == NodeKind.LINK:
1365 link_str = clean_node(wxr, None, node.largs)
1366 node_str = clean_node(wxr, None, node)
1367 if link_str.startswith("File:"): 1367 ↛ 1368line 1367 didn't jump to line 1368 because the condition on line 1367 was never true
1368 sound = create_audio_url_dict(link_str.removeprefix("File:"))
1369 sound["raw_tags"] = current_tags[:]
1370 translate_zh_pron_raw_tags(sound)
1371 sounds.append(sound)
1372 elif node_str != "": 1372 ↛ 1363line 1372 didn't jump to line 1363 because the condition on line 1372 was always true
1373 current_tags.append(node_str)
1374 elif isinstance(node, HTMLNode):
1375 if node.tag == "small":
1376 if is_first_small_tag: 1376 ↛ 1387line 1376 didn't jump to line 1387 because the condition on line 1376 was always true
1377 raw_tag_text = clean_node(
1378 wxr,
1379 None,
1380 [
1381 n
1382 for n in node.children
1383 if not (isinstance(n, HTMLNode) and n.tag == "sup")
1384 ],
1385 )
1386 current_tags.extend(split_zh_pron_raw_tag(raw_tag_text))
1387 elif len(sounds) > 0:
1388 data_extend(
1389 sounds[-1],
1390 "raw_tags",
1391 split_zh_pron_raw_tag(clean_node(wxr, None, node)),
1392 )
1393 translate_zh_pron_raw_tags(sounds[-1])
1394 is_first_small_tag = False
1395 elif node.tag == "span":
1396 sounds.extend(extract_zh_pron_span(wxr, node, current_tags))
1397 elif ( 1397 ↛ 1402line 1397 didn't jump to line 1402 because the condition on line 1397 was never true
1398 node.tag == "table"
1399 and len(current_tags) > 0
1400 and current_tags[-1] == "Homophones"
1401 ):
1402 sounds.extend(
1403 extract_zh_pron_homophone_table(wxr, node, current_tags)
1404 )
1405 elif isinstance(node, WikiNode) and node.kind == NodeKind.LIST:
1406 seen_lists.add(node)
1407 for child_list_item in node.find_child(NodeKind.LIST_ITEM):
1408 sounds.extend(
1409 extract_zh_pron_list_item(
1410 wxr, child_list_item, current_tags, seen_lists
1411 )
1412 )
1414 return sounds
1417def extract_zh_pron_homophone_table(
1418 wxr: WiktextractContext, table: HTMLNode, raw_tags: list[str]
1419) -> list[SoundData]:
1420 sounds = []
1421 for td_tag in table.find_html_recursively("td"):
1422 for span_tag in td_tag.find_html("span"):
1423 span_class = span_tag.attrs.get("class", "")
1424 span_lang = span_tag.attrs.get("lang", "")
1425 span_str = clean_node(wxr, None, span_tag)
1426 if (
1427 span_str not in ["", "/"]
1428 and span_lang != ""
1429 and span_class in ["Hant", "Hans", "Hani"]
1430 ):
1431 sound = {"homophone": span_str, "raw_tags": raw_tags[:]}
1432 if span_class == "Hant":
1433 data_append(sound, "tags", "Traditional-Chinese")
1434 elif span_class == "Hans":
1435 data_append(sound, "tags", "Simplified-Chinese")
1436 translate_zh_pron_raw_tags(sound)
1437 sounds.append(sound)
1439 return sounds
1442def translate_zh_pron_raw_tags(sound: SoundData):
1443 from .zh_pron_tags import ZH_PRON_TAGS
1445 raw_tags = []
1446 for raw_tag in sound.get("raw_tags", []):
1447 if raw_tag in ZH_PRON_TAGS:
1448 tr_tag = ZH_PRON_TAGS[raw_tag]
1449 if isinstance(tr_tag, str):
1450 data_append(sound, "tags", tr_tag)
1451 elif isinstance(tr_tag, list) and tr_tag not in sound.get( 1451 ↛ 1446line 1451 didn't jump to line 1446 because the condition on line 1451 was always true
1452 "tags", []
1453 ):
1454 data_extend(sound, "tags", tr_tag)
1455 elif raw_tag in valid_tags:
1456 if raw_tag not in sound.get("tags", []): 1456 ↛ 1446line 1456 didn't jump to line 1446 because the condition on line 1456 was always true
1457 data_append(sound, "tags", raw_tag)
1458 elif raw_tag not in raw_tags: 1458 ↛ 1446line 1458 didn't jump to line 1446 because the condition on line 1458 was always true
1459 raw_tags.append(raw_tag)
1461 if len(raw_tags) > 0:
1462 sound["raw_tags"] = raw_tags
1463 elif "raw_tags" in sound: 1463 ↛ exitline 1463 didn't return from function 'translate_zh_pron_raw_tags' because the condition on line 1463 was always true
1464 del sound["raw_tags"]
1467def split_zh_pron_raw_tag(raw_tag_text: str) -> list[str]:
1468 raw_tags = []
1469 if "(" not in raw_tag_text:
1470 for raw_tag in re.split(r",|:|;| and ", raw_tag_text):
1471 raw_tag = raw_tag.strip().removeprefix("incl. ").strip()
1472 if raw_tag != "":
1473 raw_tags.append(raw_tag)
1474 else:
1475 processed_offsets = []
1476 for match in re.finditer(r"\([^()]+\)", raw_tag_text):
1477 processed_offsets.append((match.start(), match.end()))
1478 raw_tags.extend(
1479 split_zh_pron_raw_tag(
1480 raw_tag_text[match.start() + 1 : match.end() - 1]
1481 )
1482 )
1483 not_processed = ""
1484 last_end = 0
1485 for start, end in processed_offsets:
1486 not_processed += raw_tag_text[last_end:start]
1487 last_end = end
1488 not_processed += raw_tag_text[last_end:]
1489 if not_processed != raw_tag_text: 1489 ↛ 1492line 1489 didn't jump to line 1492 because the condition on line 1489 was always true
1490 raw_tags = split_zh_pron_raw_tag(not_processed) + raw_tags
1491 else:
1492 raw_tags.append(not_processed)
1494 return raw_tags
1497def extract_zh_pron_span(
1498 wxr: WiktextractContext, span_tag: HTMLNode, raw_tags: list[str]
1499) -> list[SoundData]:
1500 sounds = []
1501 small_tags = []
1502 pron_nodes = []
1503 roman = ""
1504 phonetic_pron = ""
1505 for index, node in enumerate(span_tag.children):
1506 if isinstance(node, HTMLNode) and node.tag == "small": 1506 ↛ 1507line 1506 didn't jump to line 1507 because the condition on line 1506 was never true
1507 small_tags = split_zh_pron_raw_tag(clean_node(wxr, None, node))
1508 elif ( 1508 ↛ 1513line 1508 didn't jump to line 1513 because the condition on line 1508 was never true
1509 isinstance(node, HTMLNode)
1510 and node.tag == "span"
1511 and "-Latn" in node.attrs.get("lang", "")
1512 ):
1513 roman = clean_node(wxr, None, node).strip("() ")
1514 elif isinstance(node, str) and node.strip() == "[Phonetic:": 1514 ↛ 1515line 1514 didn't jump to line 1515 because the condition on line 1514 was never true
1515 phonetic_pron = clean_node(
1516 wxr, None, span_tag.children[index + 1 :]
1517 ).strip("] ")
1518 break
1519 else:
1520 pron_nodes.append(node)
1521 for zh_pron in split_zh_pron(clean_node(wxr, None, pron_nodes)):
1522 zh_pron = zh_pron.strip("[]: ")
1523 if len(zh_pron) > 0: 1523 ↛ 1521line 1523 didn't jump to line 1521 because the condition on line 1523 was always true
1524 if "IPA" in span_tag.attrs.get("class", ""): 1524 ↛ 1525line 1524 didn't jump to line 1525 because the condition on line 1524 was never true
1525 sound = {"ipa": zh_pron, "raw_tags": raw_tags[:]}
1526 else:
1527 sound = {"zh_pron": zh_pron, "raw_tags": raw_tags[:]}
1528 if roman != "": 1528 ↛ 1529line 1528 didn't jump to line 1529 because the condition on line 1528 was never true
1529 sound["roman"] = roman
1530 sounds.append(sound)
1531 if len(sounds) > 0: 1531 ↛ 1533line 1531 didn't jump to line 1533 because the condition on line 1531 was always true
1532 data_extend(sounds[-1], "raw_tags", small_tags)
1533 if phonetic_pron != "": 1533 ↛ 1534line 1533 didn't jump to line 1534 because the condition on line 1533 was never true
1534 sound = {
1535 "zh_pron": phonetic_pron,
1536 "raw_tags": raw_tags[:] + ["Phonetic"],
1537 }
1538 if roman != "":
1539 sound["roman"] = roman
1540 sounds.append(sound)
1541 for sound in sounds:
1542 translate_zh_pron_raw_tags(sound)
1543 return sounds
1546def split_zh_pron(zh_pron: str) -> list[str]:
1547 # split by comma and other symbols that outside parentheses
1548 parentheses = 0
1549 pron_list = []
1550 pron = ""
1551 for c in zh_pron:
1552 if (
1553 (c in [",", ";", "→"] or (c == "/" and not zh_pron.startswith("/")))
1554 and parentheses == 0
1555 and len(pron.strip()) > 0
1556 ):
1557 pron_list.append(pron.strip())
1558 pron = ""
1559 elif c == "(":
1560 parentheses += 1
1561 pron += c
1562 elif c == ")":
1563 parentheses -= 1
1564 pron += c
1565 else:
1566 pron += c
1568 if pron.strip() != "": 1568 ↛ 1570line 1568 didn't jump to line 1570 because the condition on line 1568 was always true
1569 pron_list.append(pron)
1570 return pron_list