Coverage for src/wiktextract/extractor/en/page.py: 79%
1845 statements
« prev ^ index » next coverage.py v7.17.0, created at 2026-10-26 05:53 +0000
1# Code for parsing information from a single Wiktionary page.
2#
3# Copyright (c) 2018-2022 Tatu Ylonen. See file LICENSE and https://ylonen.org
5import copy
6import html
7import re
8from collections import defaultdict
9from functools import partial
10from typing import (
11 TYPE_CHECKING,
12 Any,
13 Iterable,
14 Literal,
15 Optional,
16 Set,
17 Union,
18 cast,
19)
21from mediawiki_langcodes import get_all_names, name_to_code
22from wikitextprocessor.core import TemplateArgs, TemplateFnCallable
23from wikitextprocessor.parser import (
24 LEVEL_KIND_FLAGS,
25 GeneralNode,
26 HTMLNode,
27 LevelNode,
28 NodeKind,
29 TemplateNode,
30 WikiNode,
31)
33from ...clean import clean_template_args, clean_value
34from ...datautils import (
35 data_append,
36 data_extend,
37 ns_title_prefix_tuple,
38)
39from ...page import (
40 LEVEL_KINDS,
41 clean_node,
42 is_panel_template,
43 recursively_extract,
44)
45from ...tags import valid_tags
46from ...wxr_context import WiktextractContext
47from ...wxr_logging import logger
48from ..ruby import extract_ruby, parse_ruby
49from ..share import strip_nodes
50from .descendant import (
51 ETYMOLOGY_TEMPLATES_IN_HEADS,
52 etymology_template_append,
53 extract_descendant_section,
54)
55from .example import extract_example_list_item, extract_template_zh_x
56from .form_descriptions import (
57 classify_desc,
58 decode_tags,
59 distw,
60 parse_alt_or_inflection_of,
61 parse_sense_qualifier,
62 parse_word_head,
63)
64from .inflection import TableContext, parse_inflection_section
65from .info_templates import (
66 INFO_TEMPLATE_FUNCS,
67 parse_info_template_arguments,
68 parse_info_template_node,
69)
70from .linkages import (
71 extract_alt_form_section,
72 parse_linkage,
73)
74from .parts_of_speech import PARTS_OF_SPEECH
75from .section_titles import (
76 COMPOUNDS_TITLE,
77 DESCENDANTS_TITLE,
78 ETYMOLOGY_TITLES,
79 IGNORED_TITLES,
80 INFLECTION_TITLES,
81 LINKAGE_TITLES,
82 POS_TITLES,
83 PRONUNCIATION_TITLE,
84 PROTO_ROOT_DERIVED_TITLES,
85 TRANSLATIONS_TITLE,
86)
87from .translations import parse_translation_item_text
88from .type_utils import (
89 AttestationData,
90 ExampleData,
91 FormData,
92 LinkageData,
93 ReferenceData,
94 SenseData,
95 SoundData,
96 TemplateData,
97 WordData,
98)
99from .unsupported_titles import unsupported_title_map
101# When determining whether a string is 'english', classify_desc
102# might return 'taxonomic' which is English text 99% of the time.
103ENGLISH_TEXTS = ("english", "taxonomic")
105# Matches head tag
106HEAD_TAG_RE = re.compile(
107 r"^(head|Han char|arabic-noun|arabic-noun-form|"
108 r"hangul-symbol|syllable-hangul)$|"
109 + r"^(latin|"
110 + "|".join(lang_code for lang_code, *_ in get_all_names("en"))
111 + r")-("
112 + "|".join(
113 [
114 "abbr",
115 "adj",
116 "adjective",
117 "adjective form",
118 "adjective-form",
119 "adv",
120 "adverb",
121 "affix",
122 "animal command",
123 "art",
124 "article",
125 "aux",
126 "bound pronoun",
127 "bound-pronoun",
128 "Buyla",
129 "card num",
130 "card-num",
131 "cardinal",
132 "chunom",
133 "classifier",
134 "clitic",
135 "cls",
136 "cmene",
137 "cmavo",
138 "colloq-verb",
139 "colverbform",
140 "combining form",
141 "combining-form",
142 "comparative",
143 "con",
144 "concord",
145 "conj",
146 "conjunction",
147 "conjug",
148 "cont",
149 "contr",
150 "converb",
151 "daybox",
152 "decl",
153 "decl noun",
154 "def",
155 "dem",
156 "det",
157 "determ",
158 "Deva",
159 "ending",
160 "entry",
161 "form",
162 "fuhivla",
163 "gerund",
164 "gismu",
165 "hanja",
166 "hantu",
167 "hanzi",
168 "head",
169 "ideophone",
170 "idiom",
171 "inf",
172 "indef",
173 "infixed pronoun",
174 "infixed-pronoun",
175 "infl",
176 "inflection",
177 "initialism",
178 "int",
179 "interfix",
180 "intj",
181 "interj",
182 "interjection",
183 "jyut",
184 "latin",
185 "letter",
186 "locative",
187 "lujvo",
188 "monthbox",
189 "mutverb",
190 "name",
191 "nisba",
192 "nom",
193 "noun",
194 "noun form",
195 "noun-form",
196 "noun plural",
197 "noun-plural",
198 "nounprefix",
199 "num",
200 "number",
201 "numeral",
202 "ord",
203 "ordinal",
204 "par",
205 "part",
206 "part form",
207 "part-form",
208 "participle",
209 "particle",
210 "past",
211 "past neg",
212 "past-neg",
213 "past participle",
214 "past-participle",
215 "pastpart",
216 "perfect participle",
217 "perfect-participle",
218 "personal pronoun",
219 "personal-pronoun",
220 "pref",
221 "prefix",
222 "phrase",
223 "pinyin",
224 "plural noun",
225 "plural-noun",
226 "pos",
227 "poss-noun",
228 "post",
229 "postp",
230 "postposition",
231 "PP",
232 "pp",
233 "ppron",
234 "pred",
235 "predicative",
236 "prep",
237 "prep phrase",
238 "prep-phrase",
239 "preposition",
240 "present participle",
241 "present-participle",
242 "pron",
243 "prondem",
244 "pronindef",
245 "pronoun",
246 "prop",
247 "propn",
248 "proper noun",
249 "proper-noun",
250 "proper noun form",
251 "proper-noun form",
252 "proper noun-form",
253 "proper-noun-form",
254 "prov",
255 "proverb",
256 "prpn",
257 "prpr",
258 "punctuation mark",
259 "punctuation-mark",
260 "regnoun",
261 "rel",
262 "rom",
263 "romanji",
264 "root",
265 "sign",
266 "suff",
267 "suffix",
268 "syllable",
269 "symbol",
270 "verb",
271 "verb form",
272 "verb-form",
273 "verbal noun",
274 "verbal-noun",
275 "verbnec",
276 "vform",
277 ]
278 )
279 + r")(-|/|\+|$)"
280)
282# Head-templates causing problems (like newlines) that can be squashed into
283# an empty string in the template handler while saving their template
284# data for later.
285WORD_LEVEL_HEAD_TEMPLATES = {"term-label", "tlb"}
288PROBLEMATIC_TEMPLATES_CLUMP = (
289 WORD_LEVEL_HEAD_TEMPLATES | ETYMOLOGY_TEMPLATES_IN_HEADS
290)
292FLOATING_TABLE_TEMPLATES: set[str] = {
293 # az-suffix-form creates a style=floatright div that is otherwise
294 # deleted; if it is not pre-expanded, we can intercept the template
295 # so we add this set into do_not_pre_expand, and intercept the
296 # templates in parse_part_of_speech
297 "az-suffix-forms",
298 "az-inf-p",
299 "kk-suffix-forms",
300 "ky-suffix-forms",
301 "tr-inf-p",
302 "tr-suffix-forms",
303 "tt-suffix-forms",
304 "uz-suffix-forms",
305}
306# These two should contain template names that should always be
307# pre-expanded when *first* processing the tree, or not pre-expanded
308# so that the template are left in place with their identifying
309# name intact for later filtering.
311DO_NOT_PRE_EXPAND_TEMPLATES: set[str] = set()
312DO_NOT_PRE_EXPAND_TEMPLATES.update(FLOATING_TABLE_TEMPLATES)
314# Additional templates to be expanded in the pre-expand phase
315ADDITIONAL_EXPAND_TEMPLATES: set[str] = {
316 "multitrans",
317 "multitrans-nowiki",
318 "trans-top",
319 "trans-top-also",
320 "trans-bottom",
321 "checktrans-top",
322 "checktrans-bottom",
323 "col",
324 "col1",
325 "col2",
326 "col3",
327 "col4",
328 "col5",
329 "col1-u",
330 "col2-u",
331 "col3-u",
332 "col4-u",
333 "col5-u",
334 "check deprecated lang param usage",
335 "deprecated code",
336 "ru-verb-alt-ё",
337 "ru-noun-alt-ё",
338 "ru-adj-alt-ё",
339 "ru-proper noun-alt-ё",
340 "ru-pos-alt-ё",
341 "ru-alt-ё",
342 "inflection of",
343 "no deprecated lang param usage",
344 "transclude", # these produce sense entries (or other lists)
345 "tcl",
346}
348# Inverse linkage for those that have them
349linkage_inverses: dict[str, str] = {
350 # XXX this is not currently used, move to post-processing
351 "synonyms": "synonyms",
352 "hypernyms": "hyponyms",
353 "hyponyms": "hypernyms",
354 "holonyms": "meronyms",
355 "meronyms": "holonyms",
356 "derived": "derived_from",
357 "coordinate_terms": "coordinate_terms",
358 "troponyms": "hypernyms",
359 "antonyms": "antonyms",
360 "instances": "instance_of",
361 "related": "related",
362}
364# Templates that are used to form panels on pages and that
365# should be ignored in various positions
366PANEL_TEMPLATES: set[str] = {
367 "Character info",
368 "CJKV",
369 "French personal pronouns",
370 "French possessive adjectives",
371 "French possessive pronouns",
372 "Han etym",
373 "Han etyl", # this redirects to Han etym and would cause Lua errors,
374 # and I don't know why, but I'm putting it here because
375 # we should be ignoring it anyhow.
376 "Japanese demonstratives",
377 "Latn-script",
378 "LDL",
379 "MW1913Abbr",
380 "Number-encoding",
381 "Nuttall",
382 "Spanish possessive adjectives",
383 "Spanish possessive pronouns",
384 "USRegionDisputed",
385 "Webster 1913",
386 "ase-rfr",
387 "attention",
388 "attn",
389 "beer",
390 "broken ref",
391 "ca-compass",
392 "character info",
393 "character info/var",
394 "checksense",
395 "compass-fi",
396 "copyvio suspected",
397 "delete",
398 "dial syn", # Currently ignore these, but could be useful in Chinese/Korean
399 "etystub",
400 "examples",
401 "hu-corr",
402 "hu-suff-pron",
403 "interwiktionary",
404 "ja-kanjitab",
405 "ja-kt",
406 "ko-hanja-search",
407 "look",
408 "maintenance box",
409 "maintenance line",
410 "mediagenic terms",
411 "merge",
412 "missing template",
413 "morse links",
414 "move",
415 "multiple images",
416 "no inline",
417 "picdic",
418 "picdicimg",
419 "picdiclabel",
420 "polyominoes",
421 "predidential nomics",
422 "punctuation", # This actually gets pre-expanded
423 "reconstructed",
424 "request box",
425 "rf-sound example",
426 "rfaccents",
427 "rfap",
428 "rfaspect",
429 "rfc",
430 "rfc-auto",
431 "rfc-header",
432 "rfc-level",
433 "rfc-pron-n",
434 "rfc-sense",
435 "rfclarify",
436 "rfd",
437 "rfd-redundant",
438 "rfd-sense",
439 "rfdate",
440 "rfdatek",
441 "rfdef",
442 "rfe",
443 "rfe/dowork",
444 "rfex",
445 "rfexp",
446 "rfform",
447 "rfgender",
448 "rfi",
449 "rfinfl",
450 "rfm",
451 "rfm-sense",
452 "rfp",
453 "rfp-old",
454 "rfquote",
455 "rfquote-sense",
456 "rfquotek",
457 "rfref",
458 "rfscript",
459 "rft2",
460 "rftaxon",
461 "rftone",
462 "rftranslit",
463 "rfv",
464 "rfv-etym",
465 "rfv-pron",
466 "rfv-quote",
467 "rfv-sense",
468 "selfref",
469 "split",
470 "stroke order", # XXX consider capturing this?
471 "stub entry",
472 "t-needed",
473 "tbot entry",
474 "tea room",
475 "tea room sense",
476 # "ttbc", - XXX needed in at least on/Preposition/Translation page
477 "unblock",
478 "unsupportedpage",
479 "video frames",
480 "was wotd",
481 "wrongtitle",
482 "zh-forms",
483 "zh-hanzi-box",
484 "no entry",
485}
487# Template name prefixes used for language-specific panel templates (i.e.,
488# templates that create side boxes or notice boxes or that should generally
489# be ignored).
490PANEL_PREFIXES: set[str] = {
491 "list:compass points/",
492 "list:Gregorian calendar months/",
493 "RQ:",
494}
496# Templates used for wikipedia links.
497wikipedia_templates: set[str] = {
498 "wikipedia",
499 "slim-wikipedia",
500 "w",
501 "W",
502 "swp",
503 "wiki",
504 "Wikipedia",
505 "wtorw",
506}
507for x in PANEL_PREFIXES & wikipedia_templates: 507 ↛ 508line 507 didn't jump to line 508 because the loop on line 507 never started
508 print(
509 "WARNING: {!r} in both panel_templates and wikipedia_templates".format(
510 x
511 )
512 )
514# Mapping from a template name (without language prefix) for the main word
515# (e.g., fi-noun, fi-adj, en-verb) to permitted parts-of-speech in which
516# it could validly occur. This is used as just a sanity check to give
517# warnings about probably incorrect coding in Wiktionary.
518template_allowed_pos_map: dict[str, list[str]] = {
519 "abbr": ["abbrev"],
520 "noun": ["noun", "abbrev", "pron", "name", "num", "adj_noun"],
521 "plural noun": ["noun", "name"],
522 "plural-noun": ["noun", "name"],
523 "proper noun": ["noun", "name"],
524 "proper-noun": ["name", "noun"],
525 "prop": ["name", "noun"],
526 "verb": ["verb", "phrase"],
527 "gerund": ["verb"],
528 "particle": ["adv", "particle"],
529 "adj": ["adj", "adj_noun"],
530 "pron": ["pron", "noun"],
531 "name": ["name", "noun"],
532 "adv": ["adv", "intj", "conj", "particle"],
533 "phrase": ["phrase", "prep_phrase"],
534 "noun phrase": ["phrase"],
535 "ordinal": ["num"],
536 "number": ["num"],
537 "pos": ["affix", "name", "num"],
538 "suffix": ["suffix", "affix"],
539 "character": ["character"],
540 "letter": ["character"],
541 "kanji": ["character"],
542 "cont": ["abbrev"],
543 "interj": ["intj"],
544 "con": ["conj"],
545 "part": ["particle"],
546 "prep": ["prep", "postp"],
547 "postp": ["postp"],
548 "misspelling": ["noun", "adj", "verb", "adv"],
549 "part-form": ["verb"],
550}
551for k, v in template_allowed_pos_map.items():
552 for x in v:
553 if x not in PARTS_OF_SPEECH: 553 ↛ 554line 553 didn't jump to line 554 because the condition on line 553 was never true
554 print(
555 "BAD PART OF SPEECH {!r} IN template_allowed_pos_map: {}={}"
556 "".format(x, k, v)
557 )
558 assert False
561# Templates ignored during etymology extraction, i.e., these will not be listed
562# in the extracted etymology templates.
563ignored_etymology_templates: list[str] = [
564 "...",
565 "IPAchar",
566 "ipachar",
567 "ISBN",
568 "isValidPageName",
569 "redlink category",
570 "deprecated code",
571 "check deprecated lang param usage",
572 "para",
573 "p",
574 "cite",
575 "Cite news",
576 "Cite newsgroup",
577 "cite paper",
578 "cite MLLM 1976",
579 "cite journal",
580 "cite news/documentation",
581 "cite paper/documentation",
582 "cite video game",
583 "cite video game/documentation",
584 "cite newsgroup",
585 "cite newsgroup/documentation",
586 "cite web/documentation",
587 "cite news",
588 "Cite book",
589 "Cite-book",
590 "cite book",
591 "cite web",
592 "cite-usenet",
593 "cite-video/documentation",
594 "Cite-journal",
595 "rfe",
596 "catlangname",
597 "cln",
598 "langname-lite",
599 "no deprecated lang param usage",
600 "mention",
601 "m",
602 "m-self",
603 "link",
604 "l",
605 "ll",
606 "l-self",
607]
608# Regexp for matching ignored etymology template names. This adds certain
609# prefixes to the names listed above.
610ignored_etymology_templates_re = re.compile(
611 r"^((cite-|R:|RQ:).*|"
612 + r"|".join(re.escape(x) for x in ignored_etymology_templates)
613 + r")$"
614)
616# Regexp for matching ignored descendants template names. Right now we just
617# copy the ignored etymology templates
618ignored_descendants_templates_re = ignored_etymology_templates_re
620# Set of template names that are used to define usage examples. If the usage
621# example contains one of these templates, then it its type is set to
622# "example"
623usex_templates: set[str] = {
624 "afex",
625 "affixusex",
626 "co", # {{collocation}} acts like a example template, specifically for
627 # pairs of combinations of words that are more common than you'd
628 # except would be randomly; hlavní#Czech
629 "coi",
630 "collocation",
631 "el-example",
632 "el-x",
633 "example",
634 "examples",
635 "he-usex",
636 "he-x",
637 "hi-usex",
638 "hi-x",
639 "ja-usex-inline",
640 "ja-usex",
641 "ja-x",
642 "jbo-example",
643 "jbo-x",
644 "km-usex",
645 "km-x",
646 "ko-usex",
647 "ko-x",
648 "lo-usex",
649 "lo-x",
650 "ne-x",
651 "ne-usex",
652 "prefixusex",
653 "ryu-usex",
654 "ryu-x",
655 "shn-usex",
656 "shn-x",
657 "suffixusex",
658 "th-usex",
659 "th-x",
660 "ur-usex",
661 "ur-x",
662 "usex",
663 "usex-suffix",
664 "ux",
665 "uxi",
666}
668stop_head_at_these_templates: set[str] = {
669 "category",
670 "cat",
671 "topics",
672 "catlangname",
673 "c",
674 "C",
675 "top",
676 "cln",
677}
679# Set of template names that are used to define quotation examples. If the
680# usage example contains one of these templates, then its type is set to
681# "quotation".
682quotation_templates: set[str] = {
683 "collapse-quote",
684 "quote-av",
685 "quote-book",
686 "quote-GYLD",
687 "quote-hansard",
688 "quotei",
689 "quote-journal",
690 "quotelite",
691 "quote-mailing list",
692 "quote-meta",
693 "quote-newsgroup",
694 "quote-song",
695 "quote-text",
696 "quote",
697 "quote-us-patent",
698 "quote-video game",
699 "quote-web",
700 "quote-wikipedia",
701 "wikiquote",
702 "Wikiquote",
703 "Q",
704}
706taxonomy_templates = {
707 # argument 1 should be the taxonomic name, frex. "Lupus lupus"
708 "taxfmt",
709 "taxlink",
710 "taxlink2",
711 "taxlinknew",
712 "taxlook",
713}
715# Template names, this was exctracted from template_linkage_mappings,
716# because the code using template_linkage_mappings was actually not used
717# (but not removed).
718template_linkages_to_ignore_in_examples: set[str] = {
719 "syn",
720 "synonyms",
721 "ant",
722 "antonyms",
723 "hyp",
724 "hyponyms",
725 "der",
726 "derived terms",
727 "coordinate terms",
728 "cot",
729 "rel",
730 "col",
731 "inline alt forms",
732 "alti",
733 "comeronyms",
734 "holonyms",
735 "holo",
736 "hypernyms",
737 "hyper",
738 "meronyms",
739 "mero",
740 "troponyms",
741 "perfectives",
742 "pf",
743 "imperfectives",
744 "impf",
745 "syndiff",
746 "synsee",
747 # not linkage nor example templates
748 "sense",
749 "s",
750 "color panel",
751 "colour panel",
752}
754# Maps template name used in a word sense to a linkage field that it adds.
755sense_linkage_templates: dict[str, str] = {
756 "syn": "synonyms",
757 "synonyms": "synonyms",
758 "synsee": "synonyms",
759 "syndiff": "synonyms",
760 "hyp": "hyponyms",
761 "hyponyms": "hyponyms",
762 "ant": "antonyms",
763 "antonyms": "antonyms",
764 "alti": "related",
765 "inline alt forms": "related",
766 "coordinate terms": "coordinate_terms",
767 "cot": "coordinate_terms",
768 "comeronyms": "related",
769 "holonyms": "holonyms",
770 "holo": "holonyms",
771 "hypernyms": "hypernyms",
772 "hyper": "hypernyms",
773 "meronyms": "meronyms",
774 "mero": "meronyms",
775 "troponyms": "troponyms",
776 "perfectives": "related",
777 "pf": "related",
778 "imperfectives": "related",
779 "impf": "related",
780 "parasynonyms": "synonyms",
781 "par": "synonyms",
782 "parasyn": "synonyms",
783 "nearsyn": "synonyms",
784 "near-syn": "synonyms",
785}
787sense_linkage_templates_tags: dict[str, list[str]] = {
788 "alti": ["alternative"],
789 "inline alt forms": ["alternative"],
790 "comeronyms": ["comeronym"],
791 "perfectives": ["perfective"],
792 "pf": ["perfective"],
793 "imperfectives": ["imperfective"],
794 "impf": ["imperfective"],
795}
798def decode_html_entities(v: Union[str, int]) -> str:
799 """Decodes HTML entities from a value, converting them to the respective
800 Unicode characters/strings."""
801 if isinstance(v, int):
802 # I changed this to return str(v) instead of v = str(v),
803 # but there might have been the intention to have more logic
804 # here. html.unescape would not do anything special with an integer,
805 # it needs html escape symbols (&xx;).
806 return str(v)
807 return html.unescape(v)
810def parse_sense_linkage(
811 wxr: WiktextractContext,
812 data: SenseData,
813 name: str,
814 ht: TemplateArgs,
815 pos: str,
816) -> None:
817 """Parses a linkage (synonym, etc) specified in a word sense."""
818 assert isinstance(wxr, WiktextractContext)
819 assert isinstance(data, dict)
820 assert isinstance(name, str)
821 assert isinstance(ht, dict)
822 field = sense_linkage_templates[name]
823 field_tags = sense_linkage_templates_tags.get(name, [])
824 for i in range(2, 20):
825 if i not in ht:
826 break
827 w = clean_node(wxr, data, ht[i])
828 if "#" in w:
829 w = w[: w.index("#")]
830 if w in ["", "<"]: # `<` used in "hypernyms" template
831 continue
832 if ( 832 ↛ 837line 832 didn't jump to line 837 because the condition on line 832 was never true
833 i > 2
834 and w in (",", "or", ";")
835 or w.startswith(("see also", "See also"))
836 ):
837 continue
838 is_thesaurus = False
839 for alias in ns_title_prefix_tuple(wxr, "Thesaurus"):
840 if w.startswith(alias):
841 is_thesaurus = True
842 w = w[len(alias) :]
843 if w != wxr.wtp.title: 843 ↛ 863line 843 didn't jump to line 863 because the condition on line 843 was always true
844 from ...thesaurus import search_thesaurus
846 lang_code = clean_node(wxr, None, ht.get(1, ""))
847 for t_data in search_thesaurus(
848 wxr.thesaurus_db_conn, # type: ignore
849 w,
850 lang_code,
851 pos,
852 "synonyms", # GH issue #1570
853 ):
854 l_data: LinkageData = {
855 "word": t_data.term,
856 "source": "Thesaurus:" + w,
857 }
858 if len(t_data.tags) > 0: 858 ↛ 859line 858 didn't jump to line 859 because the condition on line 858 was never true
859 l_data["tags"] = t_data.tags
860 if len(t_data.raw_tags) > 0: 860 ↛ 861line 860 didn't jump to line 861 because the condition on line 860 was never true
861 l_data["raw_tags"] = t_data.raw_tags
862 data_append(data, field, l_data)
863 break
864 if is_thesaurus:
865 continue
866 tags: list[str] = []
867 topics: list[str] = []
868 english: Optional[str] = None
869 # Try to find qualifiers for this synonym
870 q = ht.get("q{}".format(i - 1))
871 if q:
872 cls = classify_desc(q)
873 if cls == "tags":
874 tagsets1, topics1 = decode_tags(q)
875 for ts in tagsets1:
876 tags.extend(ts)
877 topics.extend(topics1)
878 elif cls == "english": 878 ↛ 884line 878 didn't jump to line 884 because the condition on line 878 was always true
879 if english: 879 ↛ 880line 879 didn't jump to line 880 because the condition on line 879 was never true
880 english += "; " + q
881 else:
882 english = q
883 # Try to find English translation for this synonym
884 t = ht.get("t{}".format(i - 1))
885 if t: 885 ↛ 886line 885 didn't jump to line 886 because the condition on line 885 was never true
886 if english:
887 english += "; " + t
888 else:
889 english = t
891 # See if the linkage contains a parenthesized alt
892 alt = None
893 m = re.search(r"\(([^)]+)\)$", w)
894 if m: 894 ↛ 895line 894 didn't jump to line 895 because the condition on line 894 was never true
895 w = w[: m.start()].strip()
896 alt = m.group(1)
898 dt = {"word": w}
899 if field_tags: 899 ↛ 900line 899 didn't jump to line 900 because the condition on line 899 was never true
900 data_extend(dt, "tags", field_tags)
901 if tags:
902 data_extend(dt, "tags", tags)
903 if topics: 903 ↛ 904line 903 didn't jump to line 904 because the condition on line 903 was never true
904 data_extend(dt, "topics", topics)
905 if english:
906 dt["english"] = english # DEPRECATED for "translation"
907 dt["translation"] = english
908 if alt: 908 ↛ 909line 908 didn't jump to line 909 because the condition on line 908 was never true
909 dt["alt"] = alt
910 data_append(data, field, dt)
913EXAMPLE_SPLITTERS = r"\s*[―—]+\s*"
914example_splitter_re = re.compile(EXAMPLE_SPLITTERS)
915captured_splitters_re = re.compile(r"(" + EXAMPLE_SPLITTERS + r")")
918def synch_splits_with_args(
919 line: str, targs: TemplateArgs
920) -> Optional[list[str]]:
921 """If it looks like there's something weird with how a line of example
922 text has been split, this function will do the splitting after counting
923 occurences of the splitting regex inside the two main template arguments
924 containing the string data for the original language example and the
925 English translations.
926 """
927 # Previously, we split without capturing groups, but here we want to
928 # keep the original splitting hyphen regex intact.
929 fparts = captured_splitters_re.split(line)
930 new_parts = []
931 # ["First", " – ", "second", " – ", "third..."] from OL argument
932 first = 1 + (2 * len(example_splitter_re.findall(targs.get(2, ""))))
933 new_parts.append("".join(fparts[:first]))
934 # Translation argument
935 tr_arg = targs.get(3) or targs.get("translation") or targs.get("t", "")
936 # +2 = + 1 to skip the "expected" hyphen, + 1 as the `1 +` above.
937 second = first + 2 + (2 * len(example_splitter_re.findall(tr_arg)))
938 new_parts.append("".join(fparts[first + 1 : second]))
940 if all(new_parts): # no empty strings from the above spaghetti
941 new_parts.extend(fparts[second + 1 :: 2]) # skip rest of hyphens
942 return new_parts
943 else:
944 return None
947QUALIFIERS = r"^\((([^()]|\([^()]*\))*)\):?\s*"
948QUALIFIERS_RE = re.compile(QUALIFIERS)
949# (...): ... or (...(...)...): ...
952def parse_language(
953 wxr: WiktextractContext, langnode: WikiNode, language: str, lang_code: str
954) -> list[WordData]:
955 """Iterates over the text of the page, returning words (parts-of-speech)
956 defined on the page one at a time. (Individual word senses for the
957 same part-of-speech are typically encoded in the same entry.)"""
958 # imported here to avoid circular import
959 from .pronunciation import parse_pronunciation
961 assert isinstance(wxr, WiktextractContext)
962 assert isinstance(langnode, WikiNode)
963 assert isinstance(language, str)
964 assert isinstance(lang_code, str)
965 # print("parse_language", language)
967 is_reconstruction = False
968 word: str = wxr.wtp.title # type: ignore[assignment]
969 unsupported_prefix = "Unsupported titles/"
970 if word.startswith(unsupported_prefix):
971 w = word[len(unsupported_prefix) :]
972 if w in unsupported_title_map: 972 ↛ 975line 972 didn't jump to line 975 because the condition on line 972 was always true
973 word = unsupported_title_map[w]
974 else:
975 wxr.wtp.error(
976 "Unimplemented unsupported title: {}".format(word),
977 sortid="page/870",
978 )
979 word = w
980 elif word.startswith("Reconstruction:"):
981 word = word[word.find("/") + 1 :]
982 is_reconstruction = True
983 elif word.startswith("a/languages"): 983 ↛ 985line 983 didn't jump to line 985 because the condition on line 983 was never true
984 # ATM there's only one "mammoth page" in English wiktionary, 'a'
985 word = "a"
987 base_data: WordData = {
988 "word": word,
989 "lang": language,
990 "lang_code": lang_code,
991 }
992 if is_reconstruction:
993 data_append(base_data, "tags", "reconstruction")
994 sense_data: SenseData = {}
995 pos_data: WordData = {} # For a current part-of-speech
996 level_four_data: WordData = {} # Chinese Pronunciation-sections in-between
997 etym_data: WordData = {} # For one etymology
998 sense_datas: list[SenseData] = []
999 sense_ordinal = 0 # The recursive sense parsing messes up the ordering
1000 # Never reset, do not use as data
1001 level_four_datas: list[WordData] = []
1002 etym_datas: list[WordData] = []
1003 page_datas: list[WordData] = []
1004 have_etym = False
1005 inside_level_four = False # This is for checking if the etymology section
1006 # or article has a Pronunciation section, for Chinese mostly; because
1007 # Chinese articles can have three level three sections (two etymology
1008 # sections and pronunciation sections) one after another, we need a kludge
1009 # to better keep track of whether we're in a normal "etym" or inside a
1010 # "level four" (which is what we've turned the level three Pron sections
1011 # into in the fix_subtitle_hierarchy(); all other sections are demoted by
1012 # a step.
1013 stack: list[str] = [] # names of items on the "stack"
1015 def merge_base(data: WordData, base: WordData) -> None:
1016 for k, v in base.items():
1017 # Copy the value to ensure that we don't share lists or
1018 # dicts between structures (even nested ones).
1019 v = copy.deepcopy(v)
1020 if k not in data:
1021 # The list was copied above, so this will not create shared ref
1022 data[k] = v # type: ignore[literal-required]
1023 continue
1024 if data[k] == v: # type: ignore[literal-required]
1025 continue
1026 if ( 1026 ↛ 1034line 1026 didn't jump to line 1034 because the condition on line 1026 was always true
1027 isinstance(data[k], (list, tuple)) # type: ignore[literal-required]
1028 or isinstance(
1029 v,
1030 (list, tuple), # Should this be "and"?
1031 )
1032 ):
1033 data[k] = list(data[k]) + list(v) # type: ignore
1034 elif data[k] != v: # type: ignore[literal-required]
1035 wxr.wtp.warning(
1036 "conflicting values for {} in merge_base: "
1037 "{!r} vs {!r}".format(k, data[k], v), # type: ignore[literal-required]
1038 sortid="page/904",
1039 )
1041 def complementary_pop(pron: SoundData, key: str) -> SoundData:
1042 """Remove unnecessary keys from dict values
1043 in a list comprehension..."""
1044 if key in pron:
1045 pron.pop(key) # type: ignore
1046 return pron
1048 def sound_matches_pos(sound: SoundData, pos: str) -> bool:
1049 if "pos" not in sound:
1050 return True
1051 sound_pos = sound["pos"] # type: ignore[typeddict-item]
1052 return pos in sound_pos
1054 def strip_sound_pos(sound: SoundData) -> SoundData:
1055 complementary_pop(sound, "pos")
1056 return sound
1058 # If the result has sounds, eliminate sounds that have a prefix that
1059 # does not match "word" or one of "forms"
1060 if "sounds" in data and "word" in data:
1061 accepted = [data["word"]]
1062 accepted.extend(f["form"] for f in data.get("forms", dict()))
1063 data["sounds"] = list(
1064 s
1065 for s in data["sounds"]
1066 if "form" not in s or s["form"] in accepted
1067 )
1068 # If the result has sounds, eliminate sounds that have a pos that
1069 # does not match "pos"
1070 if "sounds" in data and "pos" in data:
1071 data["sounds"] = list(
1072 strip_sound_pos(s)
1073 for s in data["sounds"]
1074 # "pos" is not a field of SoundData, correctly, so we're
1075 # removing it here. It's a kludge on a kludge on a kludge.
1076 if sound_matches_pos(s, data["pos"])
1077 )
1078 elif "sounds" in data: 1078 ↛ 1079line 1078 didn't jump to line 1079 because the condition on line 1078 was never true
1079 data["sounds"] = [strip_sound_pos(s) for s in data["sounds"]]
1081 def push_sense(sorting_ordinal: int | None = None) -> bool:
1082 """Starts collecting data for a new word sense. This returns True
1083 if a sense was added."""
1084 nonlocal sense_data
1085 if sorting_ordinal is None:
1086 sorting_ordinal = sense_ordinal
1087 tags = sense_data.get("tags", ())
1088 if (
1089 not sense_data.get("glosses")
1090 and "translation-hub" not in tags
1091 and "no-gloss" not in tags
1092 ):
1093 return False
1095 if ( 1095 ↛ 1105line 1095 didn't jump to line 1105 because the condition on line 1095 was never true
1096 (
1097 "participle" in sense_data.get("tags", ())
1098 or "infinitive" in sense_data.get("tags", ())
1099 )
1100 and "alt_of" not in sense_data
1101 and "form_of" not in sense_data
1102 and "etymology_text" in etym_data
1103 and etym_data["etymology_text"] != ""
1104 ):
1105 etym = etym_data["etymology_text"]
1106 etym = etym.split(". ")[0]
1107 ret = parse_alt_or_inflection_of(wxr, etym, set())
1108 if ret is not None:
1109 tags, lst = ret
1110 assert isinstance(lst, (list, tuple))
1111 if "form-of" in tags:
1112 data_extend(sense_data, "form_of", lst)
1113 data_extend(sense_data, "tags", tags)
1114 elif "alt-of" in tags:
1115 data_extend(sense_data, "alt_of", lst)
1116 data_extend(sense_data, "tags", tags)
1118 if not sense_data.get("glosses") and "no-gloss" not in sense_data.get( 1118 ↛ 1121line 1118 didn't jump to line 1121 because the condition on line 1118 was never true
1119 "tags", ()
1120 ):
1121 data_append(sense_data, "tags", "no-gloss")
1123 sense_data["__temp_sense_sorting_ordinal"] = sorting_ordinal # type: ignore
1124 sense_datas.append(sense_data)
1125 sense_data = {}
1126 return True
1128 def push_pos(sorting_ordinal: int | None = None) -> None:
1129 """Starts collecting data for a new part-of-speech."""
1130 nonlocal pos_data
1131 nonlocal sense_datas
1132 push_sense(sorting_ordinal)
1133 if wxr.wtp.subsection:
1134 data: WordData = {"senses": sense_datas}
1135 merge_base(data, pos_data)
1136 level_four_datas.append(data)
1137 pos_data = {}
1138 sense_datas = []
1139 wxr.wtp.start_subsection(None)
1141 def push_level_four_section(clear_sound_data: bool) -> None:
1142 """Starts collecting data for a new level four sections, which
1143 is usually virtual and empty, unless the article has Chinese
1144 'Pronunciation' sections that are etymology-section-like but
1145 under etymology, and at the same level in the source. We modify
1146 the source to demote Pronunciation sections like that to level
1147 4, and other sections one step lower."""
1148 nonlocal level_four_data
1149 nonlocal level_four_datas
1150 nonlocal etym_datas
1151 push_pos()
1152 # print(f"======\n{etym_data=}")
1153 # print(f"======\n{etym_datas=}")
1154 # print(f"======\n{level_four_data=}")
1155 # print(f"======\n{level_four_datas=}")
1156 for data in level_four_datas:
1157 merge_base(data, level_four_data)
1158 etym_datas.append(data)
1159 for data in etym_datas:
1160 merge_base(data, etym_data)
1161 page_datas.append(data)
1162 if clear_sound_data:
1163 level_four_data = {}
1164 level_four_datas = []
1165 etym_datas = []
1167 def push_etym() -> None:
1168 """Starts collecting data for a new etymology."""
1169 nonlocal etym_data
1170 nonlocal etym_datas
1171 nonlocal have_etym
1172 nonlocal inside_level_four
1173 have_etym = True
1174 push_level_four_section(False)
1175 inside_level_four = False
1176 # etymology section could under pronunciation section
1177 etym_data = (
1178 copy.deepcopy(level_four_data) if len(level_four_data) > 0 else {}
1179 )
1181 def select_data() -> WordData:
1182 """Selects where to store data (pos or etym) based on whether we
1183 are inside a pos (part-of-speech)."""
1184 # print(f"{wxr.wtp.subsection=}")
1185 # print(f"{stack=}")
1186 if wxr.wtp.subsection is not None:
1187 return pos_data
1188 if inside_level_four:
1189 return level_four_data
1190 if stack[-1] == language:
1191 return base_data
1192 return etym_data
1194 def parse_part_of_speech(posnode: WikiNode, pos: str) -> None:
1195 """Parses the subsection for a part-of-speech under a language on
1196 a page."""
1197 assert isinstance(posnode, WikiNode)
1198 assert isinstance(pos, str)
1199 # print("parse_part_of_speech", pos)
1200 pos_data["pos"] = pos
1201 pre: list[list[Union[str, WikiNode]]] = [[]] # list of lists
1202 lists: list[list[WikiNode]] = [[]] # list of lists
1203 first_para = True
1204 first_head_tmplt = True
1205 collecting_head = True
1206 start_of_paragraph = True
1208 # XXX extract templates from posnode with recursively_extract
1209 # that break stuff, like ja-kanji or az-suffix-form.
1210 # Do the extraction with a list of template names, combined from
1211 # different lists, then separate out them into different lists
1212 # that are handled at different points of the POS section.
1213 # First, extract az-suffix-form, put it in `inflection`,
1214 # and parse `inflection`'s content when appropriate later.
1215 # The contents of az-suffix-form (and ja-kanji) that generate
1216 # divs with "floatright" in their style gets deleted by
1217 # clean_value, so templates that slip through from here won't
1218 # break anything.
1219 # XXX bookmark
1220 # print("===================")
1221 # print(posnode.children)
1223 floaters, poschildren = recursively_extract(
1224 posnode.children,
1225 lambda x: (
1226 isinstance(x, WikiNode)
1227 and (
1228 (
1229 isinstance(x, TemplateNode)
1230 and x.template_name in FLOATING_TABLE_TEMPLATES
1231 )
1232 or (
1233 x.kind == NodeKind.LINK
1234 # Need to check for stringiness because some links are
1235 # broken; for example, if a template is missing an
1236 # argument, a link might look like `[[{{{1}}}...]]`
1237 and len(x.largs) > 0
1238 and len(x.largs[0]) > 0
1239 and isinstance(x.largs[0][0], str)
1240 and x.largs[0][0].lower().startswith("file:") # type:ignore[union-attr]
1241 )
1242 )
1243 ),
1244 )
1245 tempnode = WikiNode(NodeKind.LEVEL6, 0)
1246 tempnode.largs = [["Inflection"]]
1247 tempnode.children = floaters
1248 parse_inflection(tempnode, "Floating Div", pos)
1249 # print(poschildren)
1250 # XXX new above
1252 if not poschildren: 1252 ↛ 1253line 1252 didn't jump to line 1253 because the condition on line 1252 was never true
1253 if not floaters:
1254 wxr.wtp.debug(
1255 "PoS section without contents",
1256 sortid="en/page/1051/20230612",
1257 )
1258 else:
1259 wxr.wtp.debug(
1260 "PoS section without contents except for a floating table",
1261 sortid="en/page/1056/20230612",
1262 )
1263 return
1265 for node in poschildren:
1266 if isinstance(node, str):
1267 for m in re.finditer(r"\n+|[^\n]+", node):
1268 p = m.group(0)
1269 if p.startswith("\n\n") and pre:
1270 first_para = False
1271 start_of_paragraph = True
1272 break
1273 if p and collecting_head:
1274 pre[-1].append(p)
1275 continue
1276 assert isinstance(node, WikiNode)
1277 kind = node.kind
1278 if kind == NodeKind.LIST:
1279 lists[-1].append(node)
1280 collecting_head = False
1281 start_of_paragraph = True
1282 continue
1283 elif kind in LEVEL_KINDS:
1284 # Stop parsing section if encountering any kind of
1285 # level header (like ===Noun=== or ====Further Reading====).
1286 # At a quick glance, this should be the default behavior,
1287 # but if some kinds of source articles have sub-sub-sections
1288 # that should be parsed XXX it should be handled by changing
1289 # this break.
1290 break
1291 elif collecting_head and kind == NodeKind.LINK:
1292 # We might collect relevant links as they are often pictures
1293 # relating to the word
1294 if len(node.largs[0]) >= 1 and isinstance( 1294 ↛ 1309line 1294 didn't jump to line 1309 because the condition on line 1294 was always true
1295 node.largs[0][0], str
1296 ):
1297 if node.largs[0][0].startswith( 1297 ↛ 1303line 1297 didn't jump to line 1303 because the condition on line 1297 was never true
1298 ns_title_prefix_tuple(wxr, "Category")
1299 ):
1300 # [[Category:...]]
1301 # We're at the end of the file, probably, so stop
1302 # here. Otherwise the head will get garbage.
1303 break
1304 if node.largs[0][0].startswith(
1305 ns_title_prefix_tuple(wxr, "File")
1306 ):
1307 # Skips file links
1308 continue
1309 start_of_paragraph = False
1310 pre[-1].append(node)
1311 elif kind == NodeKind.HTML:
1312 if node.sarg == "br":
1313 if pre[-1]: 1313 ↛ 1265line 1313 didn't jump to line 1265 because the condition on line 1313 was always true
1314 pre.append([]) # Switch to next head
1315 lists.append([]) # Lists parallels pre
1316 collecting_head = True
1317 start_of_paragraph = True
1318 elif collecting_head and node.sarg not in ( 1318 ↛ 1324line 1318 didn't jump to line 1324 because the condition on line 1318 was never true
1319 "gallery",
1320 "ref",
1321 "cite",
1322 "caption",
1323 ):
1324 start_of_paragraph = False
1325 pre[-1].append(node)
1326 else:
1327 start_of_paragraph = False
1328 elif isinstance(node, TemplateNode):
1329 # XXX Insert code here that disambiguates between
1330 # templates that generate word heads and templates
1331 # that don't.
1332 # There's head_tag_re that seems like a regex meant
1333 # to identify head templates. Too bad it's None.
1335 # ignore {{category}}, {{cat}}... etc.
1336 if node.template_name in stop_head_at_these_templates:
1337 # we've reached a template that should be at the end,
1338 continue
1340 # skip these templates; panel_templates is already used
1341 # to skip certain templates else, but it also applies to
1342 # head parsing quite well.
1343 # node.largs[0][0] should always be str, but can't type-check
1344 # that.
1345 if is_panel_template(wxr, node.template_name):
1346 continue
1347 # skip these templates
1348 # if node.largs[0][0] in skip_these_templates_in_head:
1349 # first_head_tmplt = False # no first_head_tmplt at all
1350 # start_of_paragraph = False
1351 # continue
1353 if first_head_tmplt and pre[-1]:
1354 first_head_tmplt = False
1355 start_of_paragraph = False
1356 pre[-1].append(node)
1357 elif pre[-1] and start_of_paragraph:
1358 pre.append([]) # Switch to the next head
1359 lists.append([]) # lists parallel pre
1360 collecting_head = True
1361 start_of_paragraph = False
1362 pre[-1].append(node)
1363 else:
1364 pre[-1].append(node)
1365 elif first_para:
1366 start_of_paragraph = False
1367 if collecting_head: 1367 ↛ 1265line 1367 didn't jump to line 1265 because the condition on line 1367 was always true
1368 pre[-1].append(node)
1369 # XXX use template_fn in clean_node to check that the head macro
1370 # is compatible with the current part-of-speech and generate warning
1371 # if not. Use template_allowed_pos_map.
1373 # Clean up empty pairs, and fix messes with extra newlines that
1374 # separate templates that are followed by lists wiktextract issue #314
1376 cleaned_pre: list[list[Union[str, WikiNode]]] = []
1377 cleaned_lists: list[list[WikiNode]] = []
1378 pairless_pre_index = None
1380 for pre1, ls in zip(pre, lists):
1381 if pre1 and not ls:
1382 pairless_pre_index = len(cleaned_pre)
1383 if not pre1 and not ls: 1383 ↛ 1385line 1383 didn't jump to line 1385 because the condition on line 1383 was never true
1384 # skip [] + []
1385 continue
1386 if not ls and all(
1387 (isinstance(x, str) and not x.strip()) for x in pre1
1388 ):
1389 # skip ["\n", " "] + []
1390 continue
1391 if ls and not pre1:
1392 if pairless_pre_index is not None: 1392 ↛ 1393line 1392 didn't jump to line 1393 because the condition on line 1392 was never true
1393 cleaned_lists[pairless_pre_index] = ls
1394 pairless_pre_index = None
1395 continue
1396 cleaned_pre.append(pre1)
1397 cleaned_lists.append(ls)
1399 pre = cleaned_pre
1400 lists = cleaned_lists
1402 there_are_many_heads = len(pre) > 1
1403 header_tags: list[str] = []
1404 header_topics: list[str] = []
1405 previous_head_had_list = False
1407 if not any(g for g in lists):
1408 process_gloss_without_list(
1409 poschildren, pos, pos_data, header_tags, header_topics
1410 )
1411 else:
1412 for i, (pre1, ls) in enumerate(zip(pre, lists)):
1413 # if len(ls) == 0:
1414 # # don't have gloss list
1415 # # XXX add code here to filter out 'garbage', like text
1416 # # that isn't a head template or head.
1417 # continue
1419 if all(not sl for sl in lists[i:]):
1420 if i == 0: 1420 ↛ 1421line 1420 didn't jump to line 1421 because the condition on line 1420 was never true
1421 if isinstance(node, str):
1422 wxr.wtp.debug(
1423 "first head without list of senses,"
1424 "string: '{}[...]', {}/{}".format(
1425 node[:20], word, language
1426 ),
1427 sortid="page/1689/20221215",
1428 )
1429 if isinstance(node, WikiNode):
1430 if node.largs and node.largs[0][0] in [
1431 "Han char",
1432 ]:
1433 # just ignore these templates
1434 pass
1435 else:
1436 wxr.wtp.debug(
1437 "first head without "
1438 "list of senses, "
1439 "template node "
1440 "{}, {}/{}".format(
1441 node.largs, word, language
1442 ),
1443 sortid="page/1694/20221215",
1444 )
1445 else:
1446 wxr.wtp.debug(
1447 "first head without list of senses, "
1448 "{}/{}".format(word, language),
1449 sortid="page/1700/20221215",
1450 )
1451 # no break here so that the first head always
1452 # gets processed.
1453 else:
1454 if isinstance(node, str): 1454 ↛ 1455line 1454 didn't jump to line 1455 because the condition on line 1454 was never true
1455 wxr.wtp.debug(
1456 "later head without list of senses,"
1457 "string: '{}[...]', {}/{}".format(
1458 node[:20], word, language
1459 ),
1460 sortid="page/1708/20221215",
1461 )
1462 if isinstance(node, WikiNode): 1462 ↛ 1474line 1462 didn't jump to line 1474 because the condition on line 1462 was always true
1463 wxr.wtp.debug(
1464 "later head without list of senses,"
1465 "template node "
1466 "{}, {}/{}".format(
1467 node.sarg if node.sarg else node.largs,
1468 word,
1469 language,
1470 ),
1471 sortid="page/1713/20221215",
1472 )
1473 else:
1474 wxr.wtp.debug(
1475 "later head without list of senses, "
1476 "{}/{}".format(word, language),
1477 sortid="page/1719/20221215",
1478 )
1479 break
1480 head_group = i + 1 if there_are_many_heads else None
1481 # print("parse_part_of_speech: {}: {}: pre={}"
1482 # .format(wxr.wtp.section, wxr.wtp.subsection, pre1))
1484 if previous_head_had_list:
1485 # We use a boolean flag here because we want to be able
1486 # let the header_tags data pass through after the loop
1487 # is over without accidentally emptying it, if there are
1488 # no pos_datas and we need a dummy data.
1489 header_tags.clear()
1490 header_topics.clear()
1492 # print(f"{pre1=}")
1493 process_gloss_header(
1494 pre1, pos, head_group, pos_data, header_tags, header_topics
1495 )
1496 for ln in ls:
1497 # Parse each list associated with this head.
1498 for node in ln.children:
1499 # Parse nodes in l.children recursively.
1500 # The recursion function uses push_sense() to
1501 # add stuff into sense_datas, and returns True or
1502 # False if something is added, which bubbles upward.
1503 # If the bubble is "True", then higher levels of
1504 # the recursion will not push_sense(), because
1505 # the data is already pushed into a sub-gloss
1506 # downstream, unless the higher level has examples
1507 # that need to be put somewhere.
1508 common_data: SenseData = {
1509 "tags": list(header_tags),
1510 "topics": list(header_topics),
1511 }
1512 if head_group:
1513 common_data["head_nr"] = head_group
1514 parse_sense_node(node, common_data, pos) # type: ignore[arg-type]
1516 if len(ls) > 0:
1517 previous_head_had_list = True
1518 else:
1519 previous_head_had_list = False
1521 # If there are no senses extracted, add a dummy sense. We want to
1522 # keep tags extracted from the head for the dummy sense.
1523 push_sense() # Make sure unfinished data pushed, and start clean sense
1524 if len(sense_datas) == 0:
1525 data_extend(sense_data, "tags", header_tags)
1526 data_extend(sense_data, "topics", header_topics)
1527 data_append(sense_data, "tags", "no-gloss")
1528 push_sense()
1530 sense_datas.sort(key=lambda x: x.get("__temp_sense_sorting_ordinal", 0)) # type: ignore
1532 for sd in sense_datas:
1533 if "__temp_sense_sorting_ordinal" in sd: 1533 ↛ 1532line 1533 didn't jump to line 1532 because the condition on line 1533 was always true
1534 del sd["__temp_sense_sorting_ordinal"] # type: ignore
1536 term_label_templates: list[TemplateData] = []
1537 normal_label_templates: list[TemplateData] = []
1539 def head_post_template_fn(
1540 name: str, ht: TemplateArgs, expansion: str
1541 ) -> Optional[str]:
1542 """Handles special templates in the head section of a word. Head
1543 section is the text after part-of-speech subtitle and before word
1544 sense list. Typically it generates the bold line for the word, but
1545 may also contain other useful information that often ends in
1546 side boxes. We want to capture some of that additional information."""
1547 # print("HEAD_POST_TEMPLATE_FN", name, ht)
1548 if is_panel_template(wxr, name): 1548 ↛ 1551line 1548 didn't jump to line 1551 because the condition on line 1548 was never true
1549 # Completely ignore these templates (not even recorded in
1550 # head_templates)
1551 return ""
1552 if name == "head":
1553 # XXX are these also captured in forms? Should this special case
1554 # be removed?
1555 t = ht.get(2, "")
1556 if t == "pinyin": 1556 ↛ 1557line 1556 didn't jump to line 1557 because the condition on line 1556 was never true
1557 data_append(pos_data, "tags", "Pinyin")
1558 elif t == "romanization": 1558 ↛ 1559line 1558 didn't jump to line 1559 because the condition on line 1558 was never true
1559 data_append(pos_data, "tags", "romanization")
1560 if (
1561 HEAD_TAG_RE.search(name) is not None
1562 or name in PROBLEMATIC_TEMPLATES_CLUMP
1563 ):
1564 args_ht = clean_template_args(wxr, ht)
1565 cleaned_expansion = clean_node(wxr, None, expansion)
1566 dt: TemplateData = {
1567 "name": name,
1568 "args": args_ht,
1569 "expansion": cleaned_expansion,
1570 }
1571 if name in ETYMOLOGY_TEMPLATES_IN_HEADS:
1572 etymology_template_append(
1573 pos_data, name, args_ht, cleaned_expansion
1574 )
1575 else:
1576 data_append(pos_data, "head_templates", dt)
1577 if name in WORD_LEVEL_HEAD_TEMPLATES:
1578 term_label_templates.append(dt)
1579 # Squash these, their tags are applied to the whole word,
1580 # and some cause problems like "term-label"
1581 return ""
1583 # The following are both captured in head_templates and parsed
1584 # separately
1586 if name in wikipedia_templates:
1587 # Note: various places expect to have content from wikipedia
1588 # templates, so cannot convert this to empty
1589 parse_wikipedia_template(wxr, pos_data, ht)
1590 return None
1592 if name == "number box": 1592 ↛ 1594line 1592 didn't jump to line 1594 because the condition on line 1592 was never true
1593 # XXX extract numeric value?
1594 return ""
1595 if name == "enum":
1596 # XXX extract?
1597 return ""
1598 if name == "cardinalbox": 1598 ↛ 1601line 1598 didn't jump to line 1601 because the condition on line 1598 was never true
1599 # XXX extract similar to enum?
1600 # XXX this can also occur in top-level under language
1601 return ""
1602 if name == "Han simplified forms": 1602 ↛ 1604line 1602 didn't jump to line 1604 because the condition on line 1602 was never true
1603 # XXX extract?
1604 return ""
1605 # if name == "ja-kanji forms":
1606 # # XXX extract?
1607 # return ""
1608 # if name == "vi-readings":
1609 # # XXX extract?
1610 # return ""
1611 # if name == "ja-kanji":
1612 # # XXX extract?
1613 # return ""
1614 if name == "picdic" or name == "picdicimg" or name == "picdiclabel": 1614 ↛ 1616line 1614 didn't jump to line 1616 because the condition on line 1614 was never true
1615 # XXX extract?
1616 return ""
1617 if name == "defdate": 1617 ↛ 1619line 1617 didn't jump to line 1619 because the condition on line 1617 was never true
1618 # the one exampe I saw of this in a head was weird.
1619 return ""
1620 if name in ("lb", "lbl", "label"):
1621 args_ht = clean_template_args(wxr, ht)
1622 cleaned_expansion = clean_node(wxr, None, expansion).strip("()")
1623 dt = {
1624 "name": name,
1625 "args": args_ht,
1626 "expansion": cleaned_expansion,
1627 }
1628 normal_label_templates.append(dt)
1629 # The parens around __LABEL... below is meaningful: label
1630 # templates generate text with parens, so if we add the magical
1631 # phrase here with parens, it will look like a normal label that
1632 # will be handled as a parenthetical text; only when handling
1633 # parenthetical text do we need to actually actually access
1634 # the contents of the label.
1635 return f"(__LABEL_TEMPLATE_{len(normal_label_templates) - 1}__)"
1637 return None
1639 def process_gloss_header(
1640 header_nodes: list[Union[WikiNode, str]],
1641 pos_type: str,
1642 header_group: Optional[int],
1643 pos_data: WordData,
1644 header_tags: list[str],
1645 header_topics: list[str],
1646 ) -> None:
1647 ruby = []
1649 # process template parse nodes here
1650 new_nodes = []
1651 info_template_data = []
1652 for node in header_nodes:
1653 # print(f"{node=}")
1654 info_data, info_out = parse_info_template_node(wxr, node, "head")
1655 if info_data or info_out:
1656 if info_data: 1656 ↛ 1658line 1656 didn't jump to line 1658 because the condition on line 1656 was always true
1657 info_template_data.append(info_data)
1658 if info_out: # including just the original node 1658 ↛ 1659line 1658 didn't jump to line 1659 because the condition on line 1658 was never true
1659 new_nodes.append(info_out)
1660 else:
1661 new_nodes.append(node)
1662 header_nodes = new_nodes
1664 if info_template_data:
1665 if "info_templates" not in pos_data: 1665 ↛ 1668line 1665 didn't jump to line 1668 because the condition on line 1665 was always true
1666 pos_data["info_templates"] = info_template_data
1667 else:
1668 pos_data["info_templates"].extend(info_template_data)
1670 if lang_code == "ja":
1671 exp = wxr.wtp.parse(
1672 wxr.wtp.node_to_wikitext(header_nodes), expand_all=True
1673 )
1674 rub, _ = recursively_extract(
1675 exp.children,
1676 lambda x: (
1677 isinstance(x, WikiNode)
1678 and x.kind == NodeKind.HTML
1679 and x.sarg == "ruby"
1680 ),
1681 )
1682 if rub is not None: 1682 ↛ 1726line 1682 didn't jump to line 1726 because the condition on line 1682 was always true
1683 for r in rub:
1684 if TYPE_CHECKING:
1685 # we know the lambda above in recursively_extract
1686 # returns only WikiNodes in rub
1687 assert isinstance(r, WikiNode)
1688 rt = parse_ruby(wxr, r)
1689 if rt is not None: 1689 ↛ 1683line 1689 didn't jump to line 1683 because the condition on line 1689 was always true
1690 ruby.append(rt)
1691 elif lang_code == "vi":
1692 # Handle vi-readings templates that have a weird structures for
1693 # Chu Nom vietnamese characters heads
1694 # https://en.wiktionary.org/wiki/Template:vi-readings
1695 new_header_nodes = []
1696 related_readings: list[LinkageData] = []
1697 for node in header_nodes:
1698 if ( 1698 ↛ 1721line 1698 didn't jump to line 1721 because the condition on line 1698 was always true
1699 isinstance(node, TemplateNode)
1700 and node.template_name == "vi-readings"
1701 ):
1702 for parameter, tag in (
1703 ("hanviet", "han-viet-reading"),
1704 ("nom", "nom-reading"),
1705 # we ignore the fanqie parameter "phienthiet"
1706 ):
1707 arg = node.template_parameters.get(parameter)
1708 if arg is not None: 1708 ↛ 1702line 1708 didn't jump to line 1702 because the condition on line 1708 was always true
1709 text = clean_node(wxr, None, arg)
1710 for w in text.split(","):
1711 # ignore - separated references
1712 if "-" in w:
1713 w = w[: w.index("-")]
1714 w = w.strip()
1715 related_readings.append(
1716 LinkageData(word=w, tags=[tag])
1717 )
1718 continue
1720 # Skip the vi-reading template for the rest of the head parsing
1721 new_header_nodes.append(node)
1722 if len(related_readings) > 0: 1722 ↛ 1726line 1722 didn't jump to line 1726 because the condition on line 1722 was always true
1723 data_extend(pos_data, "related", related_readings)
1724 header_nodes = new_header_nodes
1726 header_text = clean_node(
1727 wxr,
1728 pos_data,
1729 header_nodes,
1730 post_template_fn=head_post_template_fn,
1731 collect_links=True,
1732 remove_anchors_from_links=True,
1733 )
1734 if "links" in pos_data:
1735 # WordData doesn't use `links`, so we can use `collect_links=True`
1736 # above without special handling and smuggle link data.
1737 extracted_links = pos_data["links"] # type: ignore
1738 del pos_data["links"] # type: ignore
1739 else:
1740 extracted_links = None
1741 # print(f"{header_text=}, {extracted_links=}")
1743 header_text = re.sub(r"\s+", " ", header_text).strip()
1745 if not header_text:
1746 return
1748 term_label_tags: list[str] = []
1749 term_label_topics: list[str] = []
1750 if len(term_label_templates) > 0:
1751 # parse term label templates; if there are other similar kinds
1752 # of templates in headers that you want to squash and apply as
1753 # tags, you can add them to WORD_LEVEL_HEAD_TEMPLATES
1754 for templ_data in term_label_templates:
1755 # print(templ_data)
1756 expan = templ_data.get("expansion", "").strip("().,; ")
1757 if not expan: 1757 ↛ 1758line 1757 didn't jump to line 1758 because the condition on line 1757 was never true
1758 continue
1759 tlb_tagsets, tlb_topics = decode_tags(expan)
1760 for tlb_tags in tlb_tagsets:
1761 if len(tlb_tags) > 0 and not any(
1762 t.startswith("error-") for t in tlb_tags
1763 ):
1764 term_label_tags.extend(tlb_tags)
1765 term_label_topics.extend(tlb_topics)
1766 # print(f"{tlb_tagsets=}, {tlb_topicsets=}")
1768 # print(f"{header_text=}")
1769 parse_word_head(
1770 wxr,
1771 word,
1772 pos_type,
1773 header_text,
1774 pos_data,
1775 is_reconstruction,
1776 header_group,
1777 header_nodes,
1778 ruby=ruby,
1779 links=extracted_links,
1780 label_templates=normal_label_templates,
1781 )
1782 if "tags" in pos_data:
1783 # pos_data can get "tags" data from some source; type-checkers
1784 # doesn't like it, so let's ignore it.
1785 header_tags.extend(pos_data["tags"]) # type: ignore[typeddict-item]
1786 del pos_data["tags"] # type: ignore[typeddict-item]
1787 if len(term_label_tags) > 0:
1788 header_tags.extend(term_label_tags)
1789 if len(term_label_topics) > 0:
1790 header_topics.extend(term_label_topics)
1792 def process_gloss_without_list(
1793 nodes: list[Union[WikiNode, str]],
1794 pos_type: str,
1795 pos_data: WordData,
1796 header_tags: list[str],
1797 header_topics: list[str],
1798 ) -> None:
1799 # gloss text might not inside a list
1800 header_nodes: list[Union[str, WikiNode]] = []
1801 gloss_nodes: list[Union[str, WikiNode]] = []
1802 for node in strip_nodes(nodes):
1803 if isinstance(node, WikiNode):
1804 if isinstance(node, TemplateNode):
1805 if node.template_name in (
1806 "zh-see",
1807 "ja-see",
1808 "ja-see-kango",
1809 ):
1810 continue # soft redirect
1811 elif (
1812 node.template_name == "head"
1813 or node.template_name.startswith(f"{lang_code}-")
1814 ):
1815 header_nodes.append(node)
1816 continue
1817 elif node.kind in LEVEL_KINDS: # following nodes are not gloss 1817 ↛ 1819line 1817 didn't jump to line 1819 because the condition on line 1817 was always true
1818 break
1819 gloss_nodes.append(node)
1821 if len(header_nodes) > 0:
1822 process_gloss_header(
1823 header_nodes,
1824 pos_type,
1825 None,
1826 pos_data,
1827 header_tags,
1828 header_topics,
1829 )
1830 if len(gloss_nodes) > 0:
1831 process_gloss_contents(
1832 gloss_nodes,
1833 pos_type,
1834 {"tags": list(header_tags), "topics": list(header_topics)},
1835 )
1837 def parse_sense_node(
1838 node: Union[str, WikiNode], # never receives str
1839 sense_base: SenseData,
1840 pos: str,
1841 ) -> bool:
1842 """Recursively (depth first) parse LIST_ITEM nodes for sense data.
1843 Uses push_sense() to attempt adding data to pos_data in the scope
1844 of parse_language() when it reaches deep in the recursion. push_sense()
1845 returns True if it succeeds, and that is bubbled up the stack; if
1846 a sense was added downstream, the higher levels (whose shared data
1847 was already added by a subsense) do not push_sense(), unless it
1848 has examples that need to be put somewhere.
1849 """
1850 assert isinstance(sense_base, dict) # Added to every sense deeper in
1852 nonlocal sense_ordinal
1853 my_ordinal = sense_ordinal # copies, not a reference
1854 sense_ordinal += 1 # only use for sorting
1856 if not isinstance(node, WikiNode): 1856 ↛ 1858line 1856 didn't jump to line 1858 because the condition on line 1856 was never true
1857 # This doesn't seem to ever happen in practice.
1858 wxr.wtp.debug(
1859 "{}: parse_sense_node called with"
1860 "something that isn't a WikiNode".format(pos),
1861 sortid="page/1287/20230119",
1862 )
1863 return False
1865 if node.kind != NodeKind.LIST_ITEM: 1865 ↛ 1866line 1865 didn't jump to line 1866 because the condition on line 1865 was never true
1866 wxr.wtp.debug(
1867 "{}: non-list-item inside list".format(pos), sortid="page/1678"
1868 )
1869 return False
1871 if node.sarg == ":":
1872 # Skip example entries at the highest level, ones without
1873 # a sense ("...#") above them.
1874 # If node.sarg is exactly and only ":", then it's at
1875 # the highest level; lower levels would have more
1876 # "indentation", like "#:" or "##:"
1877 return False
1879 # If a recursion call succeeds in push_sense(), bubble it up with
1880 # `added`.
1881 # added |= push_sense() or added |= parse_sense_node(...) to OR.
1882 added = False
1884 gloss_template_args: set[str] = set()
1886 # For LISTs and LIST_ITEMS, their argument is something like
1887 # "##" or "##:", and using that we can rudimentally determine
1888 # list 'depth' if need be, and also what kind of list or
1889 # entry it is; # is for normal glosses, : for examples (indent)
1890 # and * is used for quotations on wiktionary.
1891 current_depth = node.sarg
1893 children = node.children
1895 # subentries, (presumably) a list
1896 # of subglosses below this. The list's
1897 # argument ends with #, and its depth should
1898 # be bigger than parent node.
1899 subentries = [
1900 x
1901 for x in children
1902 if isinstance(x, WikiNode)
1903 and x.kind == NodeKind.LIST
1904 and x.sarg == current_depth + "#"
1905 ]
1907 # sublists of examples and quotations. .sarg
1908 # does not end with "#".
1909 others = [
1910 x
1911 for x in children
1912 if isinstance(x, WikiNode)
1913 and x.kind == NodeKind.LIST
1914 and x.sarg != current_depth + "#"
1915 ]
1917 # the actual contents of this particular node.
1918 # can be a gloss (or a template that expands into
1919 # many glosses which we can't easily pre-expand)
1920 # or could be an "outer gloss" with more specific
1921 # subglosses, or could be a qualfier for the subglosses.
1922 contents = [
1923 x
1924 for x in children
1925 if not isinstance(x, WikiNode) or x.kind != NodeKind.LIST
1926 ]
1927 # If this entry has sublists of entries, we should combine
1928 # gloss information from both the "outer" and sublist content.
1929 # Sometimes the outer gloss
1930 # is more non-gloss or tags, sometimes it is a coarse sense
1931 # and the inner glosses are more specific. The outer one
1932 # does not seem to have qualifiers.
1934 # If we have one sublist with one element, treat it
1935 # specially as it may be a Wiktionary error; raise
1936 # that nested element to the same level.
1937 # XXX If need be, this block can be easily removed in
1938 # the current recursive logicand the result is one sense entry
1939 # with both glosses in the glosses list, as you would
1940 # expect. If the higher entry has examples, there will
1941 # be a higher entry with some duplicated data.
1942 if len(subentries) == 1:
1943 slc = subentries[0].children
1944 if len(slc) == 1:
1945 # copy current node and modify it so it doesn't
1946 # loop infinitely.
1947 cropped_node = copy.copy(node)
1948 cropped_node.children = [
1949 x
1950 for x in children
1951 if not (
1952 isinstance(x, WikiNode)
1953 and x.kind == NodeKind.LIST
1954 and x.sarg == current_depth + "#"
1955 )
1956 ]
1957 added |= parse_sense_node(cropped_node, sense_base, pos)
1958 nonlocal sense_data # this kludge causes duplicated raw_
1959 # glosses data if this is not done;
1960 # if the top-level (cropped_node)
1961 # does not push_sense() properly or
1962 # parse_sense_node() returns early,
1963 # sense_data is not reset. This happens
1964 # for example when you have a no-gloss
1965 # string like "(intransitive)":
1966 # no gloss, push_sense() returns early
1967 # and sense_data has duplicate data with
1968 # sense_base
1969 sense_data = {}
1970 added |= parse_sense_node(slc[0], sense_base, pos)
1971 return added
1973 return process_gloss_contents(
1974 contents,
1975 pos,
1976 sense_base,
1977 subentries,
1978 others,
1979 gloss_template_args,
1980 added,
1981 my_ordinal,
1982 )
1984 def process_gloss_contents(
1985 contents: list[Union[str, WikiNode]],
1986 pos: str,
1987 sense_base: SenseData,
1988 subentries: list[WikiNode] = [],
1989 others: list[WikiNode] = [],
1990 gloss_template_args: Set[str] = set(),
1991 added: bool = False,
1992 sorting_ordinal: int | None = None,
1993 ) -> bool:
1994 def sense_template_fn(
1995 name: str, ht: TemplateArgs, is_gloss: bool = False
1996 ) -> Optional[str]:
1997 # print(f"sense_template_fn: {name}, {ht}")
1998 if name in wikipedia_templates:
1999 # parse_wikipedia_template(wxr, pos_data, ht)
2000 return None
2001 if is_panel_template(wxr, name):
2002 return ""
2003 if name in INFO_TEMPLATE_FUNCS:
2004 info_data, info_exp = parse_info_template_arguments(
2005 wxr, name, ht, "sense"
2006 )
2007 if info_data or info_exp: 2007 ↛ 2013line 2007 didn't jump to line 2013 because the condition on line 2007 was always true
2008 if info_data: 2008 ↛ 2010line 2008 didn't jump to line 2010 because the condition on line 2008 was always true
2009 data_append(sense_base, "info_templates", info_data)
2010 if info_exp and isinstance(info_exp, str): 2010 ↛ 2012line 2010 didn't jump to line 2012 because the condition on line 2010 was always true
2011 return info_exp
2012 return ""
2013 if name in ("defdate",):
2014 date = clean_node(wxr, None, ht.get(1, ()))
2015 if part_two := ht.get(2): 2015 ↛ 2017line 2015 didn't jump to line 2017 because the condition on line 2015 was never true
2016 # Unicode mdash, not '-'
2017 date += "–" + clean_node(wxr, None, part_two)
2018 refs: dict[str, ReferenceData] = {}
2019 # ref, refn, ref2, ref2n, ref3, ref3n
2020 # ref1 not valid
2021 for k, v in sorted(
2022 (k, v) for k, v in ht.items() if isinstance(k, str)
2023 ):
2024 if m := re.match(r"ref(\d?)(n?)", k): 2024 ↛ 2021line 2024 didn't jump to line 2021 because the condition on line 2024 was always true
2025 ref_v = clean_node(wxr, None, v)
2026 if m.group(1) not in refs: # empty string or digit
2027 refs[m.group(1)] = ReferenceData()
2028 if m.group(2):
2029 refs[m.group(1)]["refn"] = ref_v
2030 else:
2031 refs[m.group(1)]["text"] = ref_v
2032 data_append(
2033 sense_base,
2034 "attestations",
2035 AttestationData(date=date, references=list(refs.values())),
2036 )
2037 return ""
2038 if name == "senseid":
2039 langid = clean_node(wxr, None, ht.get(1, ()))
2040 arg = clean_node(wxr, sense_base, ht.get(2, ()))
2041 if re.match(r"Q\d+$", arg):
2042 data_append(sense_base, "wikidata", arg)
2043 data_append(sense_base, "senseid", langid + ":" + arg)
2044 if name in sense_linkage_templates:
2045 # print(f"SENSE_TEMPLATE_FN: {name}")
2046 parse_sense_linkage(wxr, sense_base, name, ht, pos)
2047 return ""
2048 if name == "†" or name == "zh-obsolete":
2049 data_append(sense_base, "tags", "obsolete")
2050 return ""
2051 if name in {
2052 "ux",
2053 "uxi",
2054 "usex",
2055 "afex",
2056 "prefixusex",
2057 "ko-usex",
2058 "ko-x",
2059 "hi-x",
2060 "ja-usex-inline",
2061 "ja-x",
2062 "quotei",
2063 "he-x",
2064 "hi-x",
2065 "km-x",
2066 "ne-x",
2067 "shn-x",
2068 "th-x",
2069 "ur-x",
2070 }:
2071 # Usage examples are captured separately below. We don't
2072 # want to expand them into glosses even when unusual coding
2073 # is used in the entry.
2074 # These templates may slip through inside another item, but
2075 # currently we're separating out example entries (..#:)
2076 # well enough that there seems to very little contamination.
2077 if is_gloss:
2078 wxr.wtp.wiki_notice(
2079 "Example template is used for gloss text",
2080 sortid="extractor.en.page.sense_template_fn/1415",
2081 )
2082 else:
2083 return ""
2084 if name == "w": 2084 ↛ 2085line 2084 didn't jump to line 2085 because the condition on line 2084 was never true
2085 if ht.get(2) == "Wp":
2086 return ""
2087 for v in ht.values():
2088 v = v.strip()
2089 if v and "<" not in v:
2090 gloss_template_args.add(v)
2091 return None
2093 def extract_link_texts(item: GeneralNode) -> None:
2094 """Recursively extracts link texts from the gloss source. This
2095 information is used to select whether to remove final "." from
2096 form_of/alt_of (e.g., ihm/Hunsrik)."""
2097 if isinstance(item, (list, tuple)):
2098 for x in item:
2099 extract_link_texts(x)
2100 return
2101 if isinstance(item, str):
2102 # There seem to be HTML sections that may futher contain
2103 # unparsed links.
2104 for m in re.finditer(r"\[\[([^]]*)\]\]", item): 2104 ↛ 2105line 2104 didn't jump to line 2105 because the loop on line 2104 never started
2105 print("ITER:", m.group(0))
2106 v = m.group(1).split("|")[-1].strip()
2107 if v:
2108 gloss_template_args.add(v)
2109 return
2110 if not isinstance(item, WikiNode): 2110 ↛ 2111line 2110 didn't jump to line 2111 because the condition on line 2110 was never true
2111 return
2112 if item.kind == NodeKind.LINK:
2113 v = item.largs[-1]
2114 if ( 2114 ↛ 2120line 2114 didn't jump to line 2120 because the condition on line 2114 was always true
2115 isinstance(v, list)
2116 and len(v) == 1
2117 and isinstance(v[0], str)
2118 ):
2119 gloss_template_args.add(v[0].strip())
2120 for x in item.children:
2121 extract_link_texts(x)
2123 extract_link_texts(contents)
2125 # get the raw text of non-list contents of this node, and other stuff
2126 # like tag and category data added to sense_base
2127 # cast = no-op type-setter for the type-checker
2128 partial_template_fn = cast(
2129 TemplateFnCallable,
2130 partial(sense_template_fn, is_gloss=True),
2131 )
2132 rawgloss = clean_node(
2133 wxr,
2134 sense_base,
2135 contents,
2136 template_fn=partial_template_fn,
2137 collect_links=True,
2138 )
2140 if not rawgloss: 2140 ↛ 2141line 2140 didn't jump to line 2141 because the condition on line 2140 was never true
2141 return False
2143 # remove manually typed ordered list text at the start("1. ")
2144 rawgloss = re.sub(r"^\d+\.\s+", "", rawgloss).strip()
2146 # get stuff like synonyms and categories from "others",
2147 # maybe examples and quotations
2148 clean_node(wxr, sense_base, others, template_fn=sense_template_fn)
2150 # The gloss could contain templates that produce more list items.
2151 # This happens commonly with, e.g., {{inflection of|...}}. Split
2152 # to parts. However, e.g. Interlingua generates multiple glosses
2153 # in HTML directly without Wikitext markup, so we must also split
2154 # by just newlines.
2155 subglosses = rawgloss.splitlines()
2157 if len(subglosses) == 0: 2157 ↛ 2158line 2157 didn't jump to line 2158 because the condition on line 2157 was never true
2158 return False
2160 if any(s.startswith("#") for s in subglosses):
2161 subtree = wxr.wtp.parse(rawgloss)
2162 # from wikitextprocessor.parser import print_tree
2163 # print("SUBTREE GENERATED BY TEMPLATE:")
2164 # print_tree(subtree)
2165 new_subentries = [
2166 x
2167 for x in subtree.children
2168 if isinstance(x, WikiNode) and x.kind == NodeKind.LIST
2169 ]
2171 new_others = [
2172 x
2173 for x in subtree.children
2174 if isinstance(x, WikiNode)
2175 and x.kind == NodeKind.LIST
2176 and not x.sarg.endswith("#")
2177 ]
2179 new_contents = [
2180 clean_node(wxr, [], x)
2181 for x in subtree.children
2182 if not isinstance(x, WikiNode) or x.kind != NodeKind.LIST
2183 ]
2185 subentries = subentries or new_subentries
2186 others = others or new_others
2187 subglosses = new_contents
2188 rawgloss = "".join(subglosses)
2189 # Generate no gloss for translation hub pages, but add the
2190 # "translation-hub" tag for them
2191 if rawgloss == "(This entry is a translation hub.)": 2191 ↛ 2192line 2191 didn't jump to line 2192 because the condition on line 2191 was never true
2192 data_append(sense_data, "tags", "translation-hub")
2193 return push_sense(sorting_ordinal)
2195 # Remove certain substrings specific to outer glosses
2196 strip_ends = [", particularly:"]
2197 for x in strip_ends:
2198 if rawgloss.endswith(x):
2199 rawgloss = rawgloss[: -len(x)].strip()
2200 break
2202 # A single gloss, or possibly an outer gloss.
2203 # Check if the possible outer gloss starts with
2204 # parenthesized tags/topics
2206 if rawgloss and rawgloss not in sense_base.get("raw_glosses", ()):
2207 data_append(sense_base, "raw_glosses", subglosses[0].strip())
2208 m = QUALIFIERS_RE.match(rawgloss)
2209 # (...): ... or (...(...)...): ...
2210 if m:
2211 q = m.group(1)
2212 rawgloss = rawgloss[m.end() :].strip()
2213 parse_sense_qualifier(wxr, q, sense_base)
2214 if rawgloss == "A pejorative:": 2214 ↛ 2215line 2214 didn't jump to line 2215 because the condition on line 2214 was never true
2215 data_append(sense_base, "tags", "pejorative")
2216 rawgloss = ""
2217 elif rawgloss == "Short forms.": 2217 ↛ 2218line 2217 didn't jump to line 2218 because the condition on line 2217 was never true
2218 data_append(sense_base, "tags", "abbreviation")
2219 rawgloss = ""
2220 elif rawgloss == "Technical or specialized senses.": 2220 ↛ 2221line 2220 didn't jump to line 2221 because the condition on line 2220 was never true
2221 rawgloss = ""
2222 elif rawgloss.startswith("inflection of "):
2223 parsed = parse_alt_or_inflection_of(wxr, rawgloss, set())
2224 if parsed is not None: 2224 ↛ 2233line 2224 didn't jump to line 2233 because the condition on line 2224 was always true
2225 tags, origins = parsed
2226 if origins is not None: 2226 ↛ 2228line 2226 didn't jump to line 2228 because the condition on line 2226 was always true
2227 data_extend(sense_base, "form_of", origins)
2228 if tags is not None: 2228 ↛ 2231line 2228 didn't jump to line 2231 because the condition on line 2228 was always true
2229 data_extend(sense_base, "tags", tags)
2230 else:
2231 data_append(sense_base, "tags", "form-of")
2232 else:
2233 data_append(sense_base, "tags", "form-of")
2234 if rawgloss: 2234 ↛ 2265line 2234 didn't jump to line 2265 because the condition on line 2234 was always true
2235 # Code duplicating a lot of clean-up operations from later in
2236 # this block. We want to clean up the "supergloss" as much as
2237 # possible, in almost the same way as a normal gloss.
2238 supergloss = rawgloss
2240 if supergloss.startswith("; "): 2240 ↛ 2241line 2240 didn't jump to line 2241 because the condition on line 2240 was never true
2241 supergloss = supergloss[1:].strip()
2243 if supergloss.startswith(("^†", "†")):
2244 data_append(sense_base, "tags", "obsolete")
2245 supergloss = supergloss[2:].strip()
2246 elif supergloss.startswith("^‡"): 2246 ↛ 2247line 2246 didn't jump to line 2247 because the condition on line 2246 was never true
2247 data_extend(sense_base, "tags", ["obsolete", "historical"])
2248 supergloss = supergloss[2:].strip()
2250 # remove [14th century...] style brackets at the end
2251 supergloss = re.sub(r"\s\[[^]]*\]\s*$", "", supergloss)
2253 if supergloss.startswith((",", ":")):
2254 supergloss = supergloss[1:]
2255 supergloss = supergloss.strip()
2256 if supergloss.startswith("N. of "): 2256 ↛ 2257line 2256 didn't jump to line 2257 because the condition on line 2256 was never true
2257 supergloss = "Name of " + supergloss[6:]
2258 supergloss = supergloss[2:]
2259 data_append(sense_base, "glosses", supergloss)
2260 if supergloss in ("A person:",):
2261 data_append(sense_base, "tags", "g-person")
2263 # The main recursive call (except for the exceptions at the
2264 # start of this function).
2265 for sublist in subentries:
2266 if not ( 2266 ↛ 2269line 2266 didn't jump to line 2269 because the condition on line 2266 was never true
2267 isinstance(sublist, WikiNode) and sublist.kind == NodeKind.LIST
2268 ):
2269 wxr.wtp.debug(
2270 f"'{repr(rawgloss[:20])}.' gloss has `subentries`"
2271 f"with items that are not LISTs",
2272 sortid="page/1511/20230119",
2273 )
2274 continue
2275 for item in sublist.children:
2276 if not ( 2276 ↛ 2280line 2276 didn't jump to line 2280 because the condition on line 2276 was never true
2277 isinstance(item, WikiNode)
2278 and item.kind == NodeKind.LIST_ITEM
2279 ):
2280 continue
2281 # copy sense_base to prevent cross-contamination between
2282 # subglosses and other subglosses and superglosses
2283 sense_base2 = copy.deepcopy(sense_base)
2284 if parse_sense_node(item, sense_base2, pos): 2284 ↛ 2275line 2284 didn't jump to line 2275 because the condition on line 2284 was always true
2285 added = True
2287 # Capture examples.
2288 # This is called after the recursive calls above so that
2289 # sense_base is not contaminated with meta-data from
2290 # example entries for *this* gloss.
2291 examples = []
2292 if wxr.config.capture_examples: 2292 ↛ 2296line 2292 didn't jump to line 2296 because the condition on line 2292 was always true
2293 examples = extract_examples(others, sense_base)
2295 # push_sense() succeeded somewhere down-river, so skip this level
2296 if added:
2297 if examples:
2298 # this higher-up gloss has examples that we do not want to skip
2299 wxr.wtp.debug(
2300 "'{}[...]' gloss has examples we want to keep, "
2301 "but there are subglosses.".format(repr(rawgloss[:30])),
2302 sortid="page/1498/20230118",
2303 )
2304 else:
2305 return True
2307 # Some entries, e.g., "iacebam", have weird sentences in quotes
2308 # after the gloss, but these sentences don't seem to be intended
2309 # as glosses. Skip them.
2310 indexed_subglosses = list(
2311 (i, gl)
2312 for i, gl in enumerate(subglosses)
2313 if gl.strip() and not re.match(r'\s*(\([^)]*\)\s*)?"[^"]*"\s*$', gl)
2314 )
2316 if len(indexed_subglosses) > 1 and "form_of" not in sense_base: 2316 ↛ 2317line 2316 didn't jump to line 2317 because the condition on line 2316 was never true
2317 gl = indexed_subglosses[0][1].strip()
2318 if gl.endswith(":"):
2319 gl = gl[:-1].strip()
2320 parsed = parse_alt_or_inflection_of(wxr, gl, gloss_template_args)
2321 if parsed is not None:
2322 infl_tags, infl_dts = parsed
2323 if infl_dts and "form-of" in infl_tags and len(infl_tags) == 1:
2324 # Interpret others as a particular form under
2325 # "inflection of"
2326 data_extend(sense_base, "tags", infl_tags)
2327 data_extend(sense_base, "form_of", infl_dts)
2328 indexed_subglosses = indexed_subglosses[1:]
2329 elif not infl_dts:
2330 data_extend(sense_base, "tags", infl_tags)
2331 indexed_subglosses = indexed_subglosses[1:]
2333 # Create senses for remaining subglosses
2334 for i, (gloss_i, gloss) in enumerate(indexed_subglosses):
2335 gloss = gloss.strip()
2336 if not gloss and len(indexed_subglosses) > 1: 2336 ↛ 2337line 2336 didn't jump to line 2337 because the condition on line 2336 was never true
2337 continue
2338 # Push a new sense (if the last one is not empty)
2339 if push_sense(sorting_ordinal): 2339 ↛ 2340line 2339 didn't jump to line 2340 because the condition on line 2339 was never true
2340 added = True
2341 # if gloss not in sense_data.get("raw_glosses", ()):
2342 # data_append(sense_data, "raw_glosses", gloss)
2343 if i == 0 and examples:
2344 # In a multi-line gloss, associate examples
2345 # with only one of them.
2346 # XXX or you could use gloss_i == len(indexed_subglosses)
2347 # to associate examples with the *last* one.
2348 data_extend(sense_data, "examples", examples)
2349 if gloss.startswith("; ") and gloss_i > 0: 2349 ↛ 2350line 2349 didn't jump to line 2350 because the condition on line 2349 was never true
2350 gloss = gloss[1:].strip()
2351 # If the gloss starts with †, mark as obsolete
2352 if gloss.startswith("^†"): 2352 ↛ 2353line 2352 didn't jump to line 2353 because the condition on line 2352 was never true
2353 data_append(sense_data, "tags", "obsolete")
2354 gloss = gloss[2:].strip()
2355 elif gloss.startswith("^‡"): 2355 ↛ 2356line 2355 didn't jump to line 2356 because the condition on line 2355 was never true
2356 data_extend(sense_data, "tags", ["obsolete", "historical"])
2357 gloss = gloss[2:].strip()
2358 # Copy data for all senses to this sense
2359 for k, v in sense_base.items():
2360 if isinstance(v, (list, tuple)):
2361 if k != "tags":
2362 # Tags handled below (countable/uncountable special)
2363 data_extend(sense_data, k, v)
2364 else:
2365 assert k not in ("tags", "categories", "topics")
2366 sense_data[k] = v # type:ignore[literal-required]
2367 # Parse the gloss for this particular sense
2368 m = QUALIFIERS_RE.match(gloss)
2369 # (...): ... or (...(...)...): ...
2370 if m:
2371 parse_sense_qualifier(wxr, m.group(1), sense_data)
2372 gloss = gloss[m.end() :].strip()
2374 # Remove common suffix "[from 14th c.]" and similar
2375 gloss = re.sub(r"\s\[[^]]*\]\s*$", "", gloss)
2377 # Check to make sure we don't have unhandled list items in gloss
2378 ofs = max(gloss.find("#"), gloss.find("* "))
2379 if ofs > 10 and "(#)" not in gloss:
2380 wxr.wtp.debug(
2381 "gloss may contain unhandled list items: {}".format(gloss),
2382 sortid="page/1412",
2383 )
2384 elif "\n" in gloss: 2384 ↛ 2385line 2384 didn't jump to line 2385 because the condition on line 2384 was never true
2385 wxr.wtp.debug(
2386 "gloss contains newline: {}".format(gloss),
2387 sortid="page/1416",
2388 )
2390 # Kludge, some glosses have a comma after initial qualifiers in
2391 # parentheses
2392 if gloss.startswith((",", ":")):
2393 gloss = gloss[1:]
2394 gloss = gloss.strip()
2395 if gloss.endswith(":"):
2396 gloss = gloss[:-1].strip()
2397 if gloss.startswith("N. of "): 2397 ↛ 2398line 2397 didn't jump to line 2398 because the condition on line 2397 was never true
2398 gloss = "Name of " + gloss[6:]
2399 if gloss.startswith("†"): 2399 ↛ 2400line 2399 didn't jump to line 2400 because the condition on line 2399 was never true
2400 data_append(sense_data, "tags", "obsolete")
2401 gloss = gloss[1:]
2402 elif gloss.startswith("^†"): 2402 ↛ 2403line 2402 didn't jump to line 2403 because the condition on line 2402 was never true
2403 data_append(sense_data, "tags", "obsolete")
2404 gloss = gloss[2:]
2406 # Copy tags from sense_base if any. This will not copy
2407 # countable/uncountable if either was specified in the sense,
2408 # as sometimes both are specified in word head but only one
2409 # in individual senses.
2410 countability_tags = []
2411 base_tags = sense_base.get("tags", ())
2412 sense_tags = sense_data.get("tags", ())
2413 for tag in base_tags:
2414 if tag in ("countable", "uncountable"):
2415 if tag not in countability_tags: 2415 ↛ 2417line 2415 didn't jump to line 2417 because the condition on line 2415 was always true
2416 countability_tags.append(tag)
2417 continue
2418 if tag not in sense_tags:
2419 data_append(sense_data, "tags", tag)
2420 if countability_tags:
2421 if ( 2421 ↛ 2430line 2421 didn't jump to line 2430 because the condition on line 2421 was always true
2422 "countable" not in sense_tags
2423 and "uncountable" not in sense_tags
2424 ):
2425 data_extend(sense_data, "tags", countability_tags)
2427 # If outer gloss specifies a form-of ("inflection of", see
2428 # aquamarine/German), try to parse the inner glosses as
2429 # tags for an inflected form.
2430 if "form-of" in sense_base.get("tags", ()):
2431 parsed = parse_alt_or_inflection_of(
2432 wxr, gloss, gloss_template_args
2433 )
2434 if parsed is not None: 2434 ↛ 2440line 2434 didn't jump to line 2440 because the condition on line 2434 was always true
2435 infl_tags, infl_dts = parsed
2436 if not infl_dts and infl_tags: 2436 ↛ 2440line 2436 didn't jump to line 2440 because the condition on line 2436 was always true
2437 # Interpret as a particular form under "inflection of"
2438 data_extend(sense_data, "tags", infl_tags)
2440 if not gloss: 2440 ↛ 2441line 2440 didn't jump to line 2441 because the condition on line 2440 was never true
2441 data_append(sense_data, "tags", "empty-gloss")
2442 elif gloss != "-" and gloss not in sense_data.get("glosses", []):
2443 if ( 2443 ↛ 2454line 2443 didn't jump to line 2454 because the condition on line 2443 was always true
2444 gloss_i == 0
2445 and len(sense_data.get("glosses", tuple())) >= 1
2446 ):
2447 # If we added a "high-level gloss" from rawgloss, but this
2448 # is that same gloss_i, add this instead of the raw_gloss
2449 # from before if they're different: the rawgloss was not
2450 # cleaned exactly the same as this later gloss
2451 sense_data["glosses"][-1] = gloss
2452 else:
2453 # Add the gloss for the sense.
2454 data_append(sense_data, "glosses", gloss)
2456 # Kludge: there are cases (e.g., etc./Swedish) where there are
2457 # two abbreviations in the same sense, both generated by the
2458 # {{abbreviation of|...}} template. Handle these with some magic.
2459 position = 0
2460 split_glosses = []
2461 for m in re.finditer(r"Abbreviation of ", gloss):
2462 if m.start() != position: 2462 ↛ 2461line 2462 didn't jump to line 2461 because the condition on line 2462 was always true
2463 split_glosses.append(gloss[position : m.start()])
2464 position = m.start()
2465 split_glosses.append(gloss[position:])
2466 for gloss in split_glosses:
2467 # Check if this gloss describes an alt-of or inflection-of
2468 if (
2469 lang_code != "en"
2470 and " " not in gloss
2471 and distw([word], gloss) < 0.3
2472 ):
2473 # Don't try to parse gloss if it is one word
2474 # that is close to the word itself for non-English words
2475 # (probable translations of a tag/form name)
2476 continue
2477 parsed = parse_alt_or_inflection_of(
2478 wxr, gloss, gloss_template_args
2479 )
2480 if parsed is None:
2481 continue
2482 tags, dts = parsed
2483 if not dts and tags:
2484 data_extend(sense_data, "tags", tags)
2485 continue
2486 for dt in dts: # type:ignore[union-attr]
2487 ftags = list(tag for tag in tags if tag != "form-of")
2488 if "alt-of" in tags:
2489 data_extend(sense_data, "tags", ftags)
2490 data_append(sense_data, "alt_of", dt)
2491 elif "compound-of" in tags: 2491 ↛ 2492line 2491 didn't jump to line 2492 because the condition on line 2491 was never true
2492 data_extend(sense_data, "tags", ftags)
2493 data_append(sense_data, "compound_of", dt)
2494 elif "synonym-of" in tags: 2494 ↛ 2495line 2494 didn't jump to line 2495 because the condition on line 2494 was never true
2495 data_extend(dt, "tags", ftags)
2496 data_append(sense_data, "synonyms", dt)
2497 elif tags and dt.get("word", "").startswith("of "): 2497 ↛ 2498line 2497 didn't jump to line 2498 because the condition on line 2497 was never true
2498 dt["word"] = dt["word"][3:]
2499 data_append(sense_data, "tags", "form-of")
2500 data_extend(sense_data, "tags", ftags)
2501 data_append(sense_data, "form_of", dt)
2502 elif "form-of" in tags: 2502 ↛ 2486line 2502 didn't jump to line 2486 because the condition on line 2502 was always true
2503 data_extend(sense_data, "tags", tags)
2504 data_append(sense_data, "form_of", dt)
2506 if len(sense_data) == 0:
2507 if len(sense_base.get("tags", [])) == 0: 2507 ↛ 2509line 2507 didn't jump to line 2509 because the condition on line 2507 was always true
2508 del sense_base["tags"]
2509 sense_data.update(sense_base)
2510 if push_sense(sorting_ordinal): 2510 ↛ 2514line 2510 didn't jump to line 2514 because the condition on line 2510 was always true
2511 # push_sense succeded in adding a sense to pos_data
2512 added = True
2513 # print("PARSE_SENSE DONE:", pos_datas[-1])
2514 return added
2516 def parse_inflection(
2517 node: WikiNode, section: str, pos: Optional[str]
2518 ) -> None:
2519 """Parses inflection data (declension, conjugation) from the given
2520 page. This retrieves the actual inflection template
2521 parameters, which are very useful for applications that need
2522 to learn the inflection classes and generate inflected
2523 forms."""
2524 assert isinstance(node, WikiNode)
2525 assert isinstance(section, str)
2526 assert pos is None or isinstance(pos, str)
2527 # print("parse_inflection:", node)
2529 if pos is None: 2529 ↛ 2530line 2529 didn't jump to line 2530 because the condition on line 2529 was never true
2530 wxr.wtp.debug(
2531 "inflection table outside part-of-speech", sortid="page/1812"
2532 )
2533 return
2535 def inflection_template_fn(
2536 name: str, ht: TemplateArgs
2537 ) -> Optional[str]:
2538 # print("decl_conj_template_fn", name, ht)
2539 if is_panel_template(wxr, name): 2539 ↛ 2540line 2539 didn't jump to line 2540 because the condition on line 2539 was never true
2540 return ""
2541 if name in ("is-u-mutation",): 2541 ↛ 2544line 2541 didn't jump to line 2544 because the condition on line 2541 was never true
2542 # These are not to be captured as an exception to the
2543 # generic code below
2544 return None
2545 m = re.search(
2546 r"-(conj|decl|ndecl|adecl|infl|conjugation|"
2547 r"declension|inflection|mut|mutation)($|-)",
2548 name,
2549 )
2550 if m:
2551 args_ht = clean_template_args(wxr, ht)
2552 dt = {"name": name, "args": args_ht}
2553 data_append(pos_data, "inflection_templates", dt)
2555 return None
2557 # Convert the subtree back to Wikitext, then expand all and parse,
2558 # capturing templates in the process
2559 text = wxr.wtp.node_to_wikitext(node.children)
2561 # Split text into separate sections for each to-level template
2562 brace_matches = re.split(r"((?:^|\n)\s*{\||\n\s*\|}|{{+|}}+)", text)
2563 # ["{{", "template", "}}"] or ["^{|", "table contents", "\n|}"]
2564 # The (?:...) creates a non-capturing regex group; if it was capturing,
2565 # like the group around it, it would create elements in brace_matches,
2566 # including None if it doesn't match.
2567 # 20250114: Added {| and |} into the regex because tables were being
2568 # cut into pieces by this code. Issue #973, introduction of two-part
2569 # book-end templates similar to trans-top and tran-bottom.
2570 template_sections = []
2571 template_nesting = 0 # depth of SINGLE BRACES { { nesting } }
2572 # Because there is the possibility of triple curly braces
2573 # ("{{{", "}}}") in addition to normal ("{{ }}"), we do not
2574 # count nesting depth using pairs of two brackets, but
2575 # instead use singular braces ("{ }").
2576 # Because template delimiters should be balanced, regardless
2577 # of whether {{ or {{{ is used, and because we only care
2578 # about the outer-most delimiters (the highest level template)
2579 # we can just count the single braces when those single
2580 # braces are part of a group.
2581 table_nesting = 0
2582 # However, if we have a stray table ({| ... |}) that should always
2583 # be its own section, and should prevent templates from cutting it
2584 # into sections.
2586 # print(f"Parse inflection: {text=}")
2587 # print(f"Brace matches: {repr('///'.join(brace_matches))}")
2588 if len(brace_matches) > 1:
2589 tsection: list[str] = []
2590 after_templates = False # kludge to keep any text
2591 # before first template
2592 # with the first template;
2593 # otherwise, text
2594 # goes with preceding template
2595 for m in brace_matches:
2596 if m.startswith("\n; ") and after_templates: 2596 ↛ 2597line 2596 didn't jump to line 2597 because the condition on line 2596 was never true
2597 after_templates = False
2598 template_sections.append(tsection)
2599 tsection = []
2600 tsection.append(m)
2601 elif m.startswith("{{") or m.endswith("{|"):
2602 if (
2603 template_nesting == 0
2604 and after_templates
2605 and table_nesting == 0
2606 ):
2607 template_sections.append(tsection)
2608 tsection = []
2609 # start new section
2610 after_templates = True
2611 if m.startswith("{{"):
2612 template_nesting += 1
2613 else:
2614 # m.endswith("{|")
2615 table_nesting += 1
2616 tsection.append(m)
2617 elif m.startswith("}}") or m.endswith("|}"):
2618 if m.startswith("}}"):
2619 template_nesting -= 1
2620 if template_nesting < 0: 2620 ↛ 2621line 2620 didn't jump to line 2621 because the condition on line 2620 was never true
2621 wxr.wtp.error(
2622 "Negatively nested braces, "
2623 "couldn't split inflection templates, "
2624 "{}/{} section {}".format(
2625 word, language, section
2626 ),
2627 sortid="page/1871",
2628 )
2629 template_sections = [] # use whole text
2630 break
2631 else:
2632 table_nesting -= 1
2633 if table_nesting < 0: 2633 ↛ 2634line 2633 didn't jump to line 2634 because the condition on line 2633 was never true
2634 wxr.wtp.error(
2635 "Negatively nested table braces, "
2636 "couldn't split inflection section, "
2637 "{}/{} section {}".format(
2638 word, language, section
2639 ),
2640 sortid="page/20250114",
2641 )
2642 template_sections = [] # use whole text
2643 break
2644 tsection.append(m)
2645 else:
2646 tsection.append(m)
2647 if tsection: # dangling tsection 2647 ↛ 2655line 2647 didn't jump to line 2655 because the condition on line 2647 was always true
2648 template_sections.append(tsection)
2649 # Why do it this way around? The parser has a preference
2650 # to associate bits outside of tables with the preceding
2651 # table (`after`-variable), so a new tsection begins
2652 # at {{ and everything before it belongs to the previous
2653 # template.
2655 texts = []
2656 if not template_sections:
2657 texts = [text]
2658 else:
2659 for tsection in template_sections:
2660 texts.append("".join(tsection))
2661 if template_nesting != 0: 2661 ↛ 2662line 2661 didn't jump to line 2662 because the condition on line 2661 was never true
2662 wxr.wtp.error(
2663 "Template nesting error: "
2664 "template_nesting = {} "
2665 "couldn't split inflection templates, "
2666 "{}/{} section {}".format(
2667 template_nesting, word, language, section
2668 ),
2669 sortid="page/1896",
2670 )
2671 texts = [text]
2672 for text in texts:
2673 tree = wxr.wtp.parse(
2674 text, expand_all=True, template_fn=inflection_template_fn
2675 )
2677 if not text.strip():
2678 continue
2680 # Parse inflection tables from the section. The data is stored
2681 # under "forms".
2682 if wxr.config.capture_inflections: 2682 ↛ 2672line 2682 didn't jump to line 2672 because the condition on line 2682 was always true
2683 tablecontext = None
2684 m = re.search(r"{{([^}{|]+)\|?", text)
2685 if m:
2686 template_name = m.group(1).strip()
2687 tablecontext = TableContext(template_name)
2689 parse_inflection_section(
2690 wxr,
2691 pos_data,
2692 word,
2693 language,
2694 pos,
2695 section,
2696 tree,
2697 tablecontext=tablecontext,
2698 )
2700 def get_subpage_section(
2701 title: str, subtitle: str, seqs: list[Union[list[str], tuple[str, ...]]]
2702 ) -> Optional[Union[WikiNode, str]]:
2703 """Loads a subpage of the given page, and finds the section
2704 for the given language, part-of-speech, and section title. This
2705 is used for finding translations and other sections on subpages."""
2706 assert isinstance(language, str)
2707 assert isinstance(title, str)
2708 assert isinstance(subtitle, str)
2709 assert isinstance(seqs, (list, tuple))
2710 for seq in seqs:
2711 for x in seq:
2712 assert isinstance(x, str)
2713 subpage_title = word + "/" + subtitle
2714 subpage_content = wxr.wtp.get_page_body(subpage_title, 0)
2715 if subpage_content is None:
2716 wxr.wtp.error(
2717 "/translations not found despite "
2718 "{{see translation subpage|...}}",
2719 sortid="page/1934",
2720 )
2721 return None
2723 def recurse(
2724 node: Union[str, WikiNode], seq: Union[list[str], tuple[str, ...]]
2725 ) -> Optional[Union[str, WikiNode]]:
2726 # print(f"seq: {seq}")
2727 if not seq:
2728 return node
2729 if not isinstance(node, WikiNode):
2730 return None
2731 # print(f"node.kind: {node.kind}")
2732 if node.kind in LEVEL_KINDS:
2733 t = clean_node(wxr, None, node.largs[0])
2734 # print(f"t: {t} == seq[0]: {seq[0]}?")
2735 if t.lower() == seq[0].lower():
2736 seq = seq[1:]
2737 if not seq:
2738 return node
2739 for n in node.children:
2740 ret = recurse(n, seq)
2741 if ret is not None:
2742 return ret
2743 return None
2745 tree = wxr.wtp.parse(
2746 subpage_content,
2747 pre_expand=True,
2748 additional_expand=ADDITIONAL_EXPAND_TEMPLATES,
2749 do_not_pre_expand=DO_NOT_PRE_EXPAND_TEMPLATES,
2750 )
2751 assert tree.kind == NodeKind.ROOT
2752 for seq in seqs:
2753 ret = recurse(tree, seq)
2754 if ret is None:
2755 wxr.wtp.debug(
2756 "Failed to find subpage section {}/{} seq {}".format(
2757 title, subtitle, seq
2758 ),
2759 sortid="page/1963",
2760 )
2761 return ret
2763 def parse_translations(data: WordData, xlatnode: WikiNode) -> None:
2764 """Parses translations for a word. This may also pull in translations
2765 from separate translation subpages."""
2766 assert isinstance(data, dict)
2767 assert isinstance(xlatnode, WikiNode)
2768 # print("===== PARSE_TRANSLATIONS {} {} {}"
2769 # .format(wxr.wtp.title, wxr.wtp.section, wxr.wtp.subsection))
2770 # print("parse_translations xlatnode={}".format(xlatnode))
2771 if not wxr.config.capture_translations: 2771 ↛ 2772line 2771 didn't jump to line 2772 because the condition on line 2771 was never true
2772 return
2773 sense_parts: list[Union[WikiNode, str]] = []
2774 sense: Optional[str] = None
2776 def parse_translation_item(
2777 contents: list[Union[WikiNode, str]], lang: Optional[str] = None
2778 ) -> None:
2779 nonlocal sense
2780 assert isinstance(contents, list)
2781 assert lang is None or isinstance(lang, str)
2782 # print("PARSE_TRANSLATION_ITEM:", contents)
2784 langcode: Optional[str] = None
2785 if sense is None:
2786 sense = clean_node(wxr, data, sense_parts).strip()
2787 # print("sense <- clean_node: ", sense)
2788 idx = sense.find("See also translations at")
2789 if idx > 0: 2789 ↛ 2790line 2789 didn't jump to line 2790 because the condition on line 2789 was never true
2790 wxr.wtp.debug(
2791 "Skipping translation see also: {}".format(sense),
2792 sortid="page/2361",
2793 )
2794 sense = sense[:idx].strip()
2795 if sense.endswith(":"): 2795 ↛ 2796line 2795 didn't jump to line 2796 because the condition on line 2795 was never true
2796 sense = sense[:-1].strip()
2797 if sense.endswith("—"): 2797 ↛ 2798line 2797 didn't jump to line 2798 because the condition on line 2797 was never true
2798 sense = sense[:-1].strip()
2799 translations_from_template: list[str] = []
2801 def translation_item_template_fn(
2802 name: str, ht: TemplateArgs
2803 ) -> Optional[str]:
2804 nonlocal langcode
2805 # print("TRANSLATION_ITEM_TEMPLATE_FN:", name, ht)
2806 if is_panel_template(wxr, name):
2807 return ""
2808 if name in ("t+check", "t-check", "t-needed"):
2809 # We ignore these templates. They seem to have outright
2810 # garbage in some entries, and very varying formatting in
2811 # others. These should be transitory and unreliable
2812 # anyway.
2813 return "__IGNORE__"
2814 if name in ("t", "t+", "t-simple", "tt", "tt+"):
2815 code = ht.get(1)
2816 if code: 2816 ↛ 2826line 2816 didn't jump to line 2826 because the condition on line 2816 was always true
2817 if langcode and code != langcode:
2818 wxr.wtp.debug(
2819 "inconsistent language codes {} vs "
2820 "{} in translation item: {!r} {}".format(
2821 langcode, code, name, ht
2822 ),
2823 sortid="page/2386",
2824 )
2825 langcode = code
2826 tr = ht.get(2)
2827 if tr:
2828 tr = clean_node(wxr, None, [tr])
2829 translations_from_template.append(tr)
2830 return None
2831 if name == "t-egy":
2832 langcode = "egy"
2833 return None
2834 if name == "ttbc":
2835 code = ht.get(1)
2836 if code: 2836 ↛ 2838line 2836 didn't jump to line 2838 because the condition on line 2836 was always true
2837 langcode = code
2838 return None
2839 if name == "trans-see": 2839 ↛ 2840line 2839 didn't jump to line 2840 because the condition on line 2839 was never true
2840 wxr.wtp.error(
2841 "UNIMPLEMENTED trans-see template", sortid="page/2405"
2842 )
2843 return ""
2844 if name.endswith("-top"): 2844 ↛ 2845line 2844 didn't jump to line 2845 because the condition on line 2844 was never true
2845 return ""
2846 if name.endswith("-bottom"): 2846 ↛ 2847line 2846 didn't jump to line 2847 because the condition on line 2846 was never true
2847 return ""
2848 if name.endswith("-mid"): 2848 ↛ 2849line 2848 didn't jump to line 2849 because the condition on line 2848 was never true
2849 return ""
2850 # wxr.wtp.debug("UNHANDLED TRANSLATION ITEM TEMPLATE: {!r}"
2851 # .format(name),
2852 # sortid="page/2414")
2853 return None
2855 sublists = list(
2856 x
2857 for x in contents
2858 if isinstance(x, WikiNode) and x.kind == NodeKind.LIST
2859 )
2860 contents = list(
2861 x
2862 for x in contents
2863 if not isinstance(x, WikiNode) or x.kind != NodeKind.LIST
2864 )
2866 item = clean_node(
2867 wxr, data, contents, template_fn=translation_item_template_fn
2868 )
2869 # print(" TRANSLATION ITEM: {!r} [{}]".format(item, sense))
2871 # Parse the translation item.
2872 if item: 2872 ↛ exitline 2872 didn't return from function 'parse_translation_item' because the condition on line 2872 was always true
2873 lang = parse_translation_item_text(
2874 wxr,
2875 word,
2876 data,
2877 item,
2878 sense,
2879 lang,
2880 langcode,
2881 translations_from_template,
2882 is_reconstruction,
2883 )
2885 # Handle sublists. They are frequently used for different
2886 # scripts for the language and different variants of the
2887 # language. We will include the lower-level header as a
2888 # tag in those cases.
2889 for listnode in sublists:
2890 assert listnode.kind == NodeKind.LIST
2891 for node in listnode.children:
2892 if not isinstance(node, WikiNode): 2892 ↛ 2893line 2892 didn't jump to line 2893 because the condition on line 2892 was never true
2893 continue
2894 if node.kind == NodeKind.LIST_ITEM: 2894 ↛ 2891line 2894 didn't jump to line 2891 because the condition on line 2894 was always true
2895 parse_translation_item(node.children, lang=lang)
2897 def parse_translation_template(node: WikiNode) -> None:
2898 assert isinstance(node, WikiNode)
2900 def template_fn(name: str, ht: TemplateArgs) -> Optional[str]:
2901 nonlocal sense_parts
2902 nonlocal sense
2903 if is_panel_template(wxr, name):
2904 return ""
2905 if name == "see also":
2906 # XXX capture
2907 # XXX for example, "/" has top-level list containing
2908 # see also items. So also should parse those.
2909 return ""
2910 if name == "trans-see":
2911 # XXX capture
2912 return ""
2913 if name == "see translation subpage": 2913 ↛ 2914line 2913 didn't jump to line 2914 because the condition on line 2913 was never true
2914 sense_parts = []
2915 sense = None
2916 sub = ht.get(1, "")
2917 if sub:
2918 m = re.match(
2919 r"\s*(([^:\d]*)\s*\d*)\s*:\s*([^:]*)\s*", sub
2920 )
2921 else:
2922 m = None
2923 etym = ""
2924 etym_numbered = ""
2925 pos = ""
2926 if m:
2927 etym_numbered = m.group(1)
2928 etym = m.group(2)
2929 pos = m.group(3)
2930 if not sub:
2931 wxr.wtp.debug(
2932 "no part-of-speech in "
2933 "{{see translation subpage|...}}, "
2934 "defaulting to just wxr.wtp.section "
2935 "(= language)",
2936 sortid="page/2468",
2937 )
2938 # seq sent to get_subpage_section without sub and pos
2939 seq = [
2940 language,
2941 TRANSLATIONS_TITLE,
2942 ]
2943 elif (
2944 m
2945 and etym.lower().strip() in ETYMOLOGY_TITLES
2946 and pos.lower() in POS_TITLES
2947 ):
2948 seq = [
2949 language,
2950 etym_numbered,
2951 pos,
2952 TRANSLATIONS_TITLE,
2953 ]
2954 elif sub.lower() in POS_TITLES:
2955 # seq with sub but not pos
2956 seq = [
2957 language,
2958 sub,
2959 TRANSLATIONS_TITLE,
2960 ]
2961 else:
2962 # seq with sub and pos
2963 pos = wxr.wtp.subsection or "MISSING_SUBSECTION"
2964 if pos.lower() not in POS_TITLES:
2965 wxr.wtp.debug(
2966 "unhandled see translation subpage: "
2967 "language={} sub={} "
2968 "wxr.wtp.subsection={}".format(
2969 language, sub, wxr.wtp.subsection
2970 ),
2971 sortid="page/2478",
2972 )
2973 seq = [language, sub, pos, TRANSLATIONS_TITLE]
2974 subnode = get_subpage_section(
2975 wxr.wtp.title or "MISSING_TITLE",
2976 TRANSLATIONS_TITLE,
2977 [seq],
2978 )
2979 if subnode is None or not isinstance(subnode, WikiNode):
2980 # Failed to find the normal subpage section
2981 # seq with sub and pos
2982 pos = wxr.wtp.subsection or "MISSING_SUBSECTION"
2983 # print(f"{language=}, {pos=}, {TRANSLATIONS_TITLE=}")
2984 seqs: list[list[str] | tuple[str, ...]] = [
2985 [TRANSLATIONS_TITLE],
2986 [language, pos],
2987 ]
2988 subnode = get_subpage_section(
2989 wxr.wtp.title or "MISSING_TITLE",
2990 TRANSLATIONS_TITLE,
2991 seqs,
2992 )
2993 if subnode is not None and isinstance(subnode, WikiNode):
2994 parse_translations(data, subnode)
2995 return ""
2996 if name in (
2997 "c",
2998 "C",
2999 "categorize",
3000 "cat",
3001 "catlangname",
3002 "topics",
3003 "top",
3004 "qualifier",
3005 "cln",
3006 ):
3007 # These are expanded in the default way
3008 return None
3009 if name in (
3010 "trans-top",
3011 "trans-top-see",
3012 ):
3013 # XXX capture id from trans-top? Capture sense here
3014 # instead of trying to parse it from expanded content?
3015 if ht.get(1):
3016 sense_parts = []
3017 sense = ht.get(1)
3018 else:
3019 sense_parts = []
3020 sense = None
3021 return None
3022 if name in (
3023 "trans-bottom",
3024 "trans-mid",
3025 "checktrans-mid",
3026 "checktrans-bottom",
3027 ):
3028 return None
3029 if name == "checktrans-top":
3030 sense_parts = []
3031 sense = None
3032 return ""
3033 if name == "trans-top-also":
3034 # XXX capture?
3035 sense_parts = []
3036 sense = None
3037 return ""
3038 wxr.wtp.error(
3039 "UNIMPLEMENTED parse_translation_template: {} {}".format(
3040 name, ht
3041 ),
3042 sortid="page/2517",
3043 )
3044 return ""
3046 wxr.wtp.expand(
3047 wxr.wtp.node_to_wikitext(node), template_fn=template_fn
3048 )
3050 def parse_translation_recurse(xlatnode: WikiNode) -> None:
3051 nonlocal sense
3052 nonlocal sense_parts
3053 for node in xlatnode.children:
3054 # print(node)
3055 if isinstance(node, str):
3056 if sense:
3057 if not node.isspace():
3058 wxr.wtp.debug(
3059 "skipping string in the middle of "
3060 "translations: {}".format(node),
3061 sortid="page/2530",
3062 )
3063 continue
3064 # Add a part to the sense
3065 sense_parts.append(node)
3066 sense = None
3067 continue
3068 assert isinstance(node, WikiNode)
3069 kind = node.kind
3070 if kind == NodeKind.LIST:
3071 for item in node.children:
3072 if not isinstance(item, WikiNode): 3072 ↛ 3073line 3072 didn't jump to line 3073 because the condition on line 3072 was never true
3073 continue
3074 if item.kind != NodeKind.LIST_ITEM: 3074 ↛ 3075line 3074 didn't jump to line 3075 because the condition on line 3074 was never true
3075 continue
3076 if item.sarg == ":": 3076 ↛ 3077line 3076 didn't jump to line 3077 because the condition on line 3076 was never true
3077 continue
3078 parse_translation_item(item.children)
3079 elif kind == NodeKind.LIST_ITEM and node.sarg == ":": 3079 ↛ 3083line 3079 didn't jump to line 3083 because the condition on line 3079 was never true
3080 # Silently skip list items that are just indented; these
3081 # are used for text between translations, such as indicating
3082 # translations that need to be checked.
3083 pass
3084 elif kind == NodeKind.TEMPLATE:
3085 parse_translation_template(node)
3086 elif kind in ( 3086 ↛ 3091line 3086 didn't jump to line 3091 because the condition on line 3086 was never true
3087 NodeKind.TABLE,
3088 NodeKind.TABLE_ROW,
3089 NodeKind.TABLE_CELL,
3090 ):
3091 parse_translation_recurse(node)
3092 elif kind == NodeKind.HTML:
3093 if node.attrs.get("class") == "NavFrame": 3093 ↛ 3099line 3093 didn't jump to line 3099 because the condition on line 3093 was never true
3094 # Reset ``sense_parts`` (and force recomputing
3095 # by clearing ``sense``) as each NavFrame specifies
3096 # its own sense. This helps eliminate garbage coming
3097 # from text at the beginning at the translations
3098 # section.
3099 sense_parts = []
3100 sense = None
3101 # for item in node.children:
3102 # if not isinstance(item, WikiNode):
3103 # continue
3104 # parse_translation_recurse(item)
3105 parse_translation_recurse(node)
3106 elif kind in LEVEL_KINDS: 3106 ↛ 3108line 3106 didn't jump to line 3108 because the condition on line 3106 was never true
3107 # Sub-levels will be recursed elsewhere
3108 pass
3109 elif kind in (NodeKind.ITALIC, NodeKind.BOLD):
3110 parse_translation_recurse(node)
3111 elif kind == NodeKind.PREFORMATTED: 3111 ↛ 3112line 3111 didn't jump to line 3112 because the condition on line 3111 was never true
3112 print("parse_translation_recurse: PREFORMATTED:", node)
3113 elif kind == NodeKind.LINK: 3113 ↛ 3167line 3113 didn't jump to line 3167 because the condition on line 3113 was always true
3114 arg0 = node.largs[0]
3115 # Kludge: I've seen occasional normal links to translation
3116 # subpages from main pages (e.g., language/English/Noun
3117 # in July 2021) instead of the normal
3118 # {{see translation subpage|...}} template. This should
3119 # handle them. Note: must be careful not to read other
3120 # links, particularly things like in "human being":
3121 # "a human being -- see [[man/translations]]" (group title)
3122 if ( 3122 ↛ 3130line 3122 didn't jump to line 3130 because the condition on line 3122 was never true
3123 isinstance(arg0, (list, tuple))
3124 and arg0
3125 and isinstance(arg0[0], str)
3126 and arg0[0].endswith("/" + TRANSLATIONS_TITLE)
3127 and arg0[0][: -(1 + len(TRANSLATIONS_TITLE))]
3128 == wxr.wtp.title
3129 ):
3130 wxr.wtp.debug(
3131 "translations subpage link found on main "
3132 "page instead "
3133 "of normal {{see translation subpage|...}}",
3134 sortid="page/2595",
3135 )
3136 sub = wxr.wtp.subsection or "MISSING_SUBSECTION"
3137 if sub.lower() in POS_TITLES:
3138 seq = [
3139 language,
3140 sub,
3141 TRANSLATIONS_TITLE,
3142 ]
3143 subnode = get_subpage_section(
3144 wxr.wtp.title,
3145 TRANSLATIONS_TITLE,
3146 [seq],
3147 )
3148 if subnode is not None and isinstance(
3149 subnode, WikiNode
3150 ):
3151 parse_translations(data, subnode)
3152 else:
3153 wxr.wtp.error(
3154 "/translations link outside part-of-speech"
3155 )
3157 if (
3158 len(arg0) >= 1
3159 and isinstance(arg0[0], str)
3160 and not arg0[0].lower().startswith("category:")
3161 ):
3162 for x in node.largs[-1]:
3163 if isinstance(x, str): 3163 ↛ 3166line 3163 didn't jump to line 3166 because the condition on line 3163 was always true
3164 sense_parts.append(x)
3165 else:
3166 parse_translation_recurse(x)
3167 elif not sense:
3168 sense_parts.append(node)
3169 else:
3170 wxr.wtp.debug(
3171 "skipping text between translation items/senses: "
3172 "{}".format(node),
3173 sortid="page/2621",
3174 )
3176 # Main code of parse_translation(). We want ``sense`` to be assigned
3177 # regardless of recursion levels, and thus the code is structured
3178 # to define at this level and recurse in parse_translation_recurse().
3179 parse_translation_recurse(xlatnode)
3181 def parse_etymology(data: WordData, node: LevelNode) -> None:
3182 """Parses an etymology section."""
3183 assert isinstance(data, dict)
3184 assert isinstance(node, WikiNode)
3186 templates: list[TemplateData] = []
3188 # Counter for preventing the capture of etymology templates
3189 # when we are inside templates that we want to ignore (i.e.,
3190 # not capture).
3191 ignore_count = 0
3193 def etym_template_fn(name: str, ht: TemplateArgs) -> Optional[str]:
3194 nonlocal ignore_count
3195 if is_panel_template(wxr, name) or name in ["zh-x", "zh-q"]:
3196 return ""
3197 if re.match(ignored_etymology_templates_re, name):
3198 ignore_count += 1
3199 return None
3201 def etym_post_template_fn(
3202 name: str, ht: TemplateArgs, expansion: str
3203 ) -> None:
3204 nonlocal ignore_count
3205 if name in wikipedia_templates:
3206 parse_wikipedia_template(wxr, data, ht)
3207 return None
3208 if re.match(ignored_etymology_templates_re, name):
3209 ignore_count -= 1
3210 return None
3211 if ignore_count == 0: 3211 ↛ 3217line 3211 didn't jump to line 3217 because the condition on line 3211 was always true
3212 ht = clean_template_args(wxr, ht)
3213 expansion = clean_node(wxr, None, expansion)
3214 templates.append(
3215 {"name": name, "args": ht, "expansion": expansion}
3216 )
3217 return None
3219 # Remove any subsections
3220 contents = list(
3221 x
3222 for x in node.children
3223 if not isinstance(x, WikiNode) or x.kind not in LEVEL_KINDS
3224 )
3225 # Collect expanded links separately from templates. Generic linking
3226 # templates such as m/l remain ignored in etymology_templates, but
3227 # their destinations (and ordinary wikilinks) are still useful.
3228 links: list[tuple[str, str]] = []
3229 # Convert to text, also capturing templates using post_template_fn
3230 text = clean_node(
3231 wxr,
3232 None,
3233 contents,
3234 template_fn=etym_template_fn,
3235 post_template_fn=etym_post_template_fn,
3236 link_collector=links,
3237 ).strip(": \n") # remove ":" indent wikitext before zh-x template
3238 # Save the collected information.
3239 if len(text) > 0:
3240 data["etymology_text"] = text
3241 if links:
3242 data["etymology_links"] = links
3243 if len(templates) > 0:
3244 # Some etymology templates, like Template:root do not generate
3245 # text, so they should be added here. Elsewhere, we check
3246 # for Template:root and add some text to the expansion to please
3247 # the validation.
3248 data["etymology_templates"] = templates
3250 for child_node in node.find_child_recursively( 3250 ↛ exitline 3250 didn't return from function 'parse_etymology' because the loop on line 3250 didn't complete
3251 LEVEL_KIND_FLAGS | NodeKind.TEMPLATE
3252 ):
3253 if child_node.kind in LEVEL_KIND_FLAGS:
3254 break
3255 elif isinstance( 3255 ↛ 3258line 3255 didn't jump to line 3258 because the condition on line 3255 was never true
3256 child_node, TemplateNode
3257 ) and child_node.template_name in ["zh-x", "zh-q"]:
3258 if "etymology_examples" not in data:
3259 data["etymology_examples"] = []
3260 data["etymology_examples"].extend(
3261 extract_template_zh_x(
3262 wxr, child_node, None, ExampleData(raw_tags=[], tags=[])
3263 )
3264 )
3266 def process_children(treenode: WikiNode, pos: Optional[str]) -> None:
3267 """This recurses into a subtree in the parse tree for a page."""
3268 nonlocal etym_data
3269 nonlocal pos_data
3270 nonlocal inside_level_four
3272 redirect_list: list[str] = [] # for `zh-see` template
3274 def skip_template_fn(name: str, ht: TemplateArgs) -> Optional[str]:
3275 """This is called for otherwise unprocessed parts of the page.
3276 We still expand them so that e.g. Category links get captured."""
3277 if name in wikipedia_templates:
3278 data = select_data()
3279 parse_wikipedia_template(wxr, data, ht)
3280 return None
3281 if is_panel_template(wxr, name):
3282 return ""
3283 return None
3285 for node in treenode.children:
3286 if not isinstance(node, WikiNode):
3287 # print(" X{}".format(repr(node)[:40]))
3288 continue
3289 if isinstance(node, TemplateNode):
3290 if process_soft_redirect_template(wxr, node, redirect_list):
3291 continue
3292 elif node.template_name == "zh-forms":
3293 extract_zh_forms_template(wxr, node, select_data())
3294 elif (
3295 node.template_name.endswith("-kanjitab")
3296 or node.template_name == "ja-kt"
3297 ):
3298 extract_ja_kanjitab_template(wxr, node, select_data())
3299 elif node.template_name in ETYMOLOGY_TEMPLATES_IN_HEADS:
3300 args_ht = clean_template_args(wxr, node.template_parameters)
3301 expansion = clean_node(wxr, etym_data, node)
3302 etymology_template_append(
3303 etym_data, node.template_name, args_ht, expansion
3304 )
3306 if not isinstance(node, LevelNode):
3307 # XXX handle e.g. wikipedia links at the top of a language
3308 # XXX should at least capture "also" at top of page
3309 if node.kind in (
3310 NodeKind.HLINE,
3311 NodeKind.LIST,
3312 NodeKind.LIST_ITEM,
3313 ):
3314 continue
3315 # print(" UNEXPECTED: {}".format(node))
3316 # Clean the node to collect category links
3317 clean_node(wxr, etym_data, node, template_fn=skip_template_fn)
3318 continue
3319 t = clean_node(
3320 wxr, etym_data, node.sarg if node.sarg else node.largs
3321 )
3322 t = t.lower()
3323 # XXX these counts were never implemented fully, and even this
3324 # gets discarded: Search STATISTICS_IMPLEMENTATION
3325 wxr.config.section_counts[t] += 1
3326 # print("PROCESS_CHILDREN: T:", repr(t))
3327 if t in IGNORED_TITLES:
3328 pass
3329 elif t.startswith(PRONUNCIATION_TITLE):
3330 # Chinese Pronunciation section kludge; we demote these to
3331 # be level 4 instead of 3 so that they're part of a larger
3332 # etymology hierarchy; usually the data here is empty and
3333 # acts as an inbetween between POS and Etymology data
3334 if lang_code in ("zh",):
3335 inside_level_four = True
3336 if t.startswith(PRONUNCIATION_TITLE + " "):
3337 # Pronunciation 1, etc, are used in Chinese Glyphs,
3338 # and each of them may have senses under Definition
3339 push_level_four_section(True)
3340 wxr.wtp.start_subsection(None)
3341 if wxr.config.capture_pronunciation: 3341 ↛ 3449line 3341 didn't jump to line 3449 because the condition on line 3341 was always true
3342 data = select_data()
3343 parse_pronunciation(
3344 wxr,
3345 node,
3346 data,
3347 etym_data,
3348 have_etym,
3349 base_data,
3350 lang_code,
3351 )
3352 elif t.startswith(tuple(ETYMOLOGY_TITLES)):
3353 push_etym()
3354 wxr.wtp.start_subsection(None)
3355 if wxr.config.capture_etymologies: 3355 ↛ 3449line 3355 didn't jump to line 3449 because the condition on line 3355 was always true
3356 m = re.search(r"\s(\d+(\.\d+)?)$", t)
3357 if m:
3358 etym_data["etymology_number"] = m.group(1)
3359 parse_etymology(etym_data, node)
3360 elif t == DESCENDANTS_TITLE and wxr.config.capture_descendants:
3361 data = select_data()
3362 extract_descendant_section(wxr, data, node, False)
3363 elif (
3364 t in PROTO_ROOT_DERIVED_TITLES
3365 and pos == "root"
3366 and is_reconstruction
3367 and wxr.config.capture_descendants
3368 ):
3369 data = select_data()
3370 extract_descendant_section(wxr, data, node, True)
3371 elif t == TRANSLATIONS_TITLE:
3372 data = select_data()
3373 parse_translations(data, node)
3374 elif t in INFLECTION_TITLES:
3375 parse_inflection(node, t, pos)
3376 elif t == "alternative forms":
3377 extract_alt_form_section(wxr, select_data(), node)
3378 else:
3379 lst = t.split()
3380 while len(lst) > 1 and lst[-1].isdigit():
3381 lst = lst[:-1]
3382 t_no_number = " ".join(lst).lower()
3383 if t_no_number in POS_TITLES:
3384 push_pos()
3385 dt = POS_TITLES[t_no_number] # type:ignore[literal-required]
3386 pos = dt["pos"] or "MISSING_POS"
3387 wxr.wtp.start_subsection(t)
3388 if "debug" in dt:
3389 wxr.wtp.debug(
3390 "{} in section {}".format(dt["debug"], t),
3391 sortid="page/2755",
3392 )
3393 if "warning" in dt: 3393 ↛ 3394line 3393 didn't jump to line 3394 because the condition on line 3393 was never true
3394 wxr.wtp.wiki_notice(
3395 "{} in section {}".format(dt["warning"], t),
3396 sortid="page/2759",
3397 )
3398 if "error" in dt: 3398 ↛ 3399line 3398 didn't jump to line 3399 because the condition on line 3398 was never true
3399 wxr.wtp.error(
3400 "{} in section {}".format(dt["error"], t),
3401 sortid="page/2763",
3402 )
3403 if "note" in dt: 3403 ↛ 3404line 3403 didn't jump to line 3404 because the condition on line 3403 was never true
3404 wxr.wtp.note(
3405 "{} in section {}".format(dt["note"], t),
3406 sortid="page/20251017a",
3407 )
3408 if "wiki_notice" in dt: 3408 ↛ 3409line 3408 didn't jump to line 3409 because the condition on line 3408 was never true
3409 wxr.wtp.wiki_notice(
3410 "{} in section {}".format(dt["wiki_notices"], t),
3411 sortid="page/20251017b",
3412 )
3413 # Parse word senses for the part-of-speech
3414 parse_part_of_speech(node, pos)
3415 if "tags" in dt:
3416 for pdata in sense_datas:
3417 data_extend(pdata, "tags", dt["tags"])
3418 elif t_no_number in LINKAGE_TITLES:
3419 # print(f"LINKAGE_TITLES NODE {node=}")
3420 rel = LINKAGE_TITLES[t_no_number]
3421 data = select_data()
3422 parse_linkage(
3423 wxr,
3424 data,
3425 rel,
3426 node,
3427 word,
3428 sense_datas,
3429 is_reconstruction,
3430 )
3431 elif t_no_number == COMPOUNDS_TITLE:
3432 data = select_data()
3433 if wxr.config.capture_compounds: 3433 ↛ 3449line 3433 didn't jump to line 3449 because the condition on line 3433 was always true
3434 parse_linkage(
3435 wxr,
3436 data,
3437 "derived",
3438 node,
3439 word,
3440 sense_datas,
3441 is_reconstruction,
3442 )
3444 # XXX parse interesting templates also from other sections. E.g.,
3445 # {{Letter|...}} in ===See also===
3446 # Also <gallery>
3448 # Recurse to children of this node, processing subtitles therein
3449 stack.append(t)
3450 process_children(node, pos)
3451 stack.pop()
3453 if len(redirect_list) > 0:
3454 if len(pos_data) > 0:
3455 pos_data["redirects"] = redirect_list
3456 if "pos" not in pos_data: 3456 ↛ 3457line 3456 didn't jump to line 3457 because the condition on line 3456 was never true
3457 pos_data["pos"] = "soft-redirect"
3458 else:
3459 new_page_data = copy.deepcopy(base_data)
3460 new_page_data["redirects"] = redirect_list
3461 if "pos" not in new_page_data: 3461 ↛ 3463line 3461 didn't jump to line 3463 because the condition on line 3461 was always true
3462 new_page_data["pos"] = "soft-redirect"
3463 new_page_data["senses"] = [{"tags": ["no-gloss"]}]
3464 page_datas.append(new_page_data)
3466 def extract_examples(
3467 others: list[WikiNode], sense_base: SenseData
3468 ) -> list[ExampleData]:
3469 """Parses through a list of definitions and quotes to find examples.
3470 Returns a list of example dicts to be added to sense data. Adds
3471 meta-data, mostly categories, into sense_base."""
3472 assert isinstance(others, list)
3473 examples: list[ExampleData] = []
3475 for sub in others:
3476 if not sub.sarg.endswith((":", "*")): 3476 ↛ 3477line 3476 didn't jump to line 3477 because the condition on line 3476 was never true
3477 continue
3478 for item in sub.children:
3479 if not isinstance(item, WikiNode): 3479 ↛ 3480line 3479 didn't jump to line 3480 because the condition on line 3479 was never true
3480 continue
3481 if item.kind != NodeKind.LIST_ITEM: 3481 ↛ 3482line 3481 didn't jump to line 3482 because the condition on line 3481 was never true
3482 continue
3483 usex_type = None
3484 example_template_args = []
3485 example_template_names = []
3486 taxons = set()
3488 # Bypass this function when parsing Chinese, Japanese and
3489 # quotation templates.
3490 new_example_lists = extract_example_list_item(
3491 wxr, item, sense_base, ExampleData(raw_tags=[], tags=[])
3492 )
3493 if len(new_example_lists) > 0:
3494 examples.extend(new_example_lists)
3495 continue
3497 def usex_template_fn(
3498 name: str, ht: TemplateArgs
3499 ) -> Optional[str]:
3500 nonlocal usex_type
3501 if is_panel_template(wxr, name):
3502 return ""
3503 if name in usex_templates:
3504 usex_type = "example"
3505 example_template_args.append(ht)
3506 example_template_names.append(name)
3507 elif name in quotation_templates:
3508 usex_type = "quotation"
3509 elif name in taxonomy_templates: 3509 ↛ 3510line 3509 didn't jump to line 3510 because the condition on line 3509 was never true
3510 taxons.update(ht.get(1, "").split())
3511 for prefix in template_linkages_to_ignore_in_examples:
3512 if re.search(
3513 r"(^|[-/\s]){}($|\b|[0-9])".format(prefix), name
3514 ):
3515 return ""
3516 return None
3518 # bookmark
3519 ruby: list[tuple[str, str]] = []
3520 contents = item.children
3521 if lang_code == "ja":
3522 # Capture ruby contents if this is a Japanese language
3523 # example.
3524 # print(contents)
3525 if ( 3525 ↛ 3530line 3525 didn't jump to line 3530 because the condition on line 3525 was never true
3526 contents
3527 and isinstance(contents, str)
3528 and re.match(r"\s*$", contents[0])
3529 ):
3530 contents = contents[1:]
3531 exp = wxr.wtp.parse(
3532 wxr.wtp.node_to_wikitext(contents),
3533 # post_template_fn=head_post_template_fn,
3534 expand_all=True,
3535 )
3536 rub, rest = extract_ruby(wxr, exp.children)
3537 if rub:
3538 for rtup in rub:
3539 ruby.append(rtup)
3540 contents = rest
3541 subtext = clean_node(
3542 wxr, sense_base, contents, template_fn=usex_template_fn
3543 )
3545 frozen_taxons = frozenset(taxons)
3546 classify_desc2 = partial(classify_desc, accepted=frozen_taxons)
3548 # print(f"{subtext=}")
3549 subtext = re.sub(
3550 r"\s*\(please add an English "
3551 r"translation of this "
3552 r"(example|usage example|quote)\)",
3553 "",
3554 subtext,
3555 ).strip()
3556 subtext = re.sub(r"\^\([^)]*\)", "", subtext)
3557 subtext = re.sub(r"\s*[―—]+$", "", subtext)
3558 # print("subtext:", repr(subtext))
3560 lines = subtext.splitlines()
3561 # print(lines)
3563 lines = list(re.sub(r"^[#:*]*", "", x).strip() for x in lines)
3564 lines = list(
3565 x
3566 for x in lines
3567 if not re.match(
3568 r"(Synonyms: |Antonyms: |Hyponyms: |"
3569 r"Synonym: |Antonym: |Hyponym: |"
3570 r"Hypernyms: |Derived terms: |"
3571 r"Related terms: |"
3572 r"Hypernym: |Derived term: |"
3573 r"Coordinate terms:|"
3574 r"Related term: |"
3575 r"For more quotations using )",
3576 x,
3577 )
3578 )
3579 tr = ""
3580 ref = ""
3581 roman = ""
3582 # for line in lines:
3583 # print("LINE:", repr(line))
3584 # print(classify_desc(line))
3585 if len(lines) == 1 and lang_code != "en":
3586 parts = example_splitter_re.split(lines[0])
3587 if ( 3587 ↛ 3595line 3587 didn't jump to line 3595 because the condition on line 3587 was never true
3588 len(parts) > 2
3589 and len(example_template_args) == 1
3590 and any(
3591 ("―" in s) or ("—" in s)
3592 for s in example_template_args[0].values()
3593 )
3594 ):
3595 if nparts := synch_splits_with_args(
3596 lines[0], example_template_args[0]
3597 ):
3598 parts = nparts
3599 if ( 3599 ↛ 3604line 3599 didn't jump to line 3604 because the condition on line 3599 was never true
3600 len(example_template_args) == 1
3601 and "lit" in example_template_args[0]
3602 ):
3603 # ugly brute-force kludge in case there's a lit= arg
3604 literally = example_template_args[0].get("lit", "")
3605 if literally:
3606 literally = (
3607 " (literally, “"
3608 + clean_value(wxr, literally)
3609 + "”)"
3610 )
3611 else:
3612 literally = ""
3613 if ( 3613 ↛ 3652line 3613 didn't jump to line 3652 because the condition on line 3613 was never true
3614 len(example_template_args) == 1
3615 and len(parts) == 2
3616 and len(example_template_args[0])
3617 - (
3618 # horrible kludge to ignore these arguments
3619 # when calculating how many there are
3620 sum(
3621 s in example_template_args[0]
3622 for s in (
3623 "lit", # generates text, but we handle it
3624 "inline",
3625 "noenum",
3626 "nocat",
3627 "sort",
3628 )
3629 )
3630 )
3631 == 3
3632 and clean_value(
3633 wxr, example_template_args[0].get(2, "")
3634 )
3635 == parts[0].strip()
3636 and clean_value(
3637 wxr,
3638 (
3639 example_template_args[0].get(3)
3640 or example_template_args[0].get("translation")
3641 or example_template_args[0].get("t", "")
3642 )
3643 + literally, # in case there's a lit= argument
3644 )
3645 == parts[1].strip()
3646 ):
3647 # {{exampletemplate|ex|Foo bar baz|English translation}}
3648 # is a pretty reliable 'heuristic', so we use it here
3649 # before the others. To be extra sure the template
3650 # doesn't do anything weird, we compare the arguments
3651 # and the output to each other.
3652 lines = [parts[0].strip()]
3653 tr = parts[1].strip()
3654 elif (
3655 len(parts) == 2
3656 and classify_desc2(parts[1]) in ENGLISH_TEXTS
3657 ):
3658 # These other branches just do some simple heuristics w/
3659 # the expanded output of the template (if applicable).
3660 lines = [parts[0].strip()]
3661 tr = parts[1].strip()
3662 elif ( 3662 ↛ 3668line 3662 didn't jump to line 3668 because the condition on line 3662 was never true
3663 len(parts) == 3
3664 and classify_desc2(parts[1])
3665 in ("romanization", "english")
3666 and classify_desc2(parts[2]) in ENGLISH_TEXTS
3667 ):
3668 lines = [parts[0].strip()]
3669 roman = parts[1].strip()
3670 tr = parts[2].strip()
3671 else:
3672 parts = re.split(r"\s+-\s+", lines[0])
3673 if ( 3673 ↛ 3677line 3673 didn't jump to line 3677 because the condition on line 3673 was never true
3674 len(parts) == 2
3675 and classify_desc2(parts[1]) in ENGLISH_TEXTS
3676 ):
3677 lines = [parts[0].strip()]
3678 tr = parts[1].strip()
3679 elif len(lines) > 1:
3680 if any(
3681 re.search(r"[]\d:)]\s*$", x) for x in lines[:-1]
3682 ) and not (len(example_template_names) == 1):
3683 refs: list[str] = []
3684 for i in range(len(lines)): 3684 ↛ 3690line 3684 didn't jump to line 3690 because the loop on line 3684 didn't complete
3685 if re.match(r"^[#*]*:+(\s*$|\s+)", lines[i]): 3685 ↛ 3686line 3685 didn't jump to line 3686 because the condition on line 3685 was never true
3686 break
3687 refs.append(lines[i].strip())
3688 if re.search(r"[]\d:)]\s*$", lines[i]):
3689 break
3690 ref = " ".join(refs)
3691 lines = lines[i + 1 :]
3692 if (
3693 lang_code != "en"
3694 and len(lines) >= 2
3695 and classify_desc2(lines[-1]) in ENGLISH_TEXTS
3696 ):
3697 i = len(lines) - 1
3698 while ( 3698 ↛ 3703line 3698 didn't jump to line 3703 because the condition on line 3698 was never true
3699 i > 1
3700 and classify_desc2(lines[i - 1])
3701 in ENGLISH_TEXTS
3702 ):
3703 i -= 1
3704 tr = "\n".join(lines[i:])
3705 lines = lines[:i]
3706 if len(lines) >= 2:
3707 if classify_desc2(lines[-1]) == "romanization":
3708 roman = lines[-1].strip()
3709 lines = lines[:-1]
3711 elif lang_code == "en" and re.match(r"^[#*]*:+", lines[1]):
3712 ref = lines[0]
3713 lines = lines[1:]
3714 elif lang_code != "en" and len(lines) == 2:
3715 cls1 = classify_desc2(lines[0])
3716 cls2 = classify_desc2(lines[1])
3717 if cls2 in ENGLISH_TEXTS and cls1 != "english":
3718 tr = lines[1]
3719 lines = [lines[0]]
3720 elif cls1 in ENGLISH_TEXTS and cls2 != "english": 3720 ↛ 3721line 3720 didn't jump to line 3721 because the condition on line 3720 was never true
3721 tr = lines[0]
3722 lines = [lines[1]]
3723 elif ( 3723 ↛ 3730line 3723 didn't jump to line 3730 because the condition on line 3723 was never true
3724 re.match(r"^[#*]*:+", lines[1])
3725 and classify_desc2(
3726 re.sub(r"^[#*:]+\s*", "", lines[1])
3727 )
3728 in ENGLISH_TEXTS
3729 ):
3730 tr = re.sub(r"^[#*:]+\s*", "", lines[1])
3731 lines = [lines[0]]
3732 elif cls1 == "english" and cls2 in ENGLISH_TEXTS:
3733 # Both were classified as English, but
3734 # presumably one is not. Assume first is
3735 # non-English, as that seems more common.
3736 tr = lines[1]
3737 lines = [lines[0]]
3738 elif (
3739 usex_type != "quotation"
3740 and lang_code != "en"
3741 and len(lines) == 3
3742 ):
3743 cls1 = classify_desc2(lines[0])
3744 cls2 = classify_desc2(lines[1])
3745 cls3 = classify_desc2(lines[2])
3746 if (
3747 cls3 == "english"
3748 and cls2 in ("english", "romanization")
3749 and cls1 != "english"
3750 ):
3751 tr = lines[2].strip()
3752 roman = lines[1].strip()
3753 lines = [lines[0].strip()]
3754 elif ( 3754 ↛ 3762line 3754 didn't jump to line 3762 because the condition on line 3754 was never true
3755 usex_type == "quotation"
3756 and lang_code != "en"
3757 and len(lines) > 2
3758 ):
3759 # for x in lines:
3760 # print(" LINE: {}: {}"
3761 # .format(classify_desc2(x), x))
3762 if re.match(r"^[#*]*:+\s*$", lines[1]):
3763 ref = lines[0]
3764 lines = lines[2:]
3765 cls1 = classify_desc2(lines[-1])
3766 if cls1 == "english":
3767 i = len(lines) - 1
3768 while (
3769 i > 1
3770 and classify_desc2(lines[i - 1])
3771 == ENGLISH_TEXTS
3772 ):
3773 i -= 1
3774 tr = "\n".join(lines[i:])
3775 lines = lines[:i]
3777 roman = re.sub(r"[ \t\r]+", " ", roman).strip()
3778 roman = re.sub(r"\[\s*…\s*\]", "[…]", roman)
3779 tr = re.sub(r"^[#*:]+\s*", "", tr)
3780 tr = re.sub(r"[ \t\r]+", " ", tr).strip()
3781 tr = re.sub(r"\[\s*…\s*\]", "[…]", tr)
3782 ref = re.sub(r"^[#*:]+\s*", "", ref)
3783 ref = re.sub(
3784 r", (volume |number |page )?“?"
3785 r"\(please specify ([^)]|\(s\))*\)”?|"
3786 ", text here$",
3787 "",
3788 ref,
3789 )
3790 ref = re.sub(r"\[\s*…\s*\]", "[…]", ref)
3791 lines = list(re.sub(r"^[#*:]+\s*", "", x) for x in lines)
3792 subtext = "\n".join(x for x in lines if x)
3793 if not tr and lang_code != "en":
3794 m = re.search(r"([.!?])\s+\(([^)]+)\)\s*$", subtext)
3795 if m and classify_desc2(m.group(2)) in ENGLISH_TEXTS: 3795 ↛ 3796line 3795 didn't jump to line 3796 because the condition on line 3795 was never true
3796 tr = m.group(2)
3797 subtext = subtext[: m.start()] + m.group(1)
3798 elif lines:
3799 parts = re.split(r"\s*[―—]+\s*", lines[0])
3800 if ( 3800 ↛ 3804line 3800 didn't jump to line 3804 because the condition on line 3800 was never true
3801 len(parts) == 2
3802 and classify_desc2(parts[1]) in ENGLISH_TEXTS
3803 ):
3804 subtext = parts[0].strip()
3805 tr = parts[1].strip()
3806 subtext = re.sub(r'^[“"`]([^“"`”\']*)[”"\']$', r"\1", subtext)
3807 subtext = re.sub(
3808 r"(please add an English translation of "
3809 r"this (quote|usage example))",
3810 "",
3811 subtext,
3812 )
3813 subtext = re.sub(
3814 r"\s*→New International Version " "translation$",
3815 "",
3816 subtext,
3817 ) # e.g. pis/Tok Pisin (Bible)
3818 subtext = re.sub(r"[ \t\r]+", " ", subtext).strip()
3819 subtext = re.sub(r"\[\s*…\s*\]", "[…]", subtext)
3820 note = None
3821 m = re.match(r"^\(([^)]*)\):\s+", subtext)
3822 if ( 3822 ↛ 3830line 3822 didn't jump to line 3830 because the condition on line 3822 was never true
3823 m is not None
3824 and lang_code != "en"
3825 and (
3826 m.group(1).startswith("with ")
3827 or classify_desc2(m.group(1)) == "english"
3828 )
3829 ):
3830 note = m.group(1)
3831 subtext = subtext[m.end() :]
3832 ref = re.sub(r"\s*\(→ISBN\)", "", ref)
3833 ref = re.sub(r",\s*→ISBN", "", ref)
3834 ref = ref.strip()
3835 if ref.endswith(":") or ref.endswith(","):
3836 ref = ref[:-1].strip()
3837 ref = re.sub(r"\s+,\s+", ", ", ref)
3838 ref = re.sub(r"\s+", " ", ref)
3839 if ref and not subtext: 3839 ↛ 3840line 3839 didn't jump to line 3840 because the condition on line 3839 was never true
3840 subtext = ref
3841 ref = ""
3842 if subtext:
3843 dt: ExampleData = {"text": subtext}
3844 if ref:
3845 dt["ref"] = ref
3846 if tr:
3847 dt["english"] = tr # DEPRECATED for "translation"
3848 dt["translation"] = tr
3849 if usex_type:
3850 dt["type"] = usex_type
3851 if note: 3851 ↛ 3852line 3851 didn't jump to line 3852 because the condition on line 3851 was never true
3852 dt["note"] = note
3853 if roman:
3854 dt["roman"] = roman
3855 if ruby:
3856 dt["ruby"] = ruby
3857 examples.append(dt)
3859 return examples
3861 # Main code of parse_language()
3862 # Process the section
3863 stack.append(language)
3864 process_children(langnode, None)
3865 stack.pop()
3867 # Finalize word entires
3868 push_etym()
3869 ret = []
3870 for data in page_datas:
3871 merge_base(data, base_data)
3872 ret.append(data)
3874 # Copy all tags to word senses
3875 for data in ret:
3876 if "senses" not in data: 3876 ↛ 3877line 3876 didn't jump to line 3877 because the condition on line 3876 was never true
3877 continue
3878 # WordData should not have a 'tags' field, but if it does, it's
3879 # deleted and its contents removed and placed in each sense;
3880 # that's why the type ignores.
3881 tags: Iterable = data.get("tags", ()) # type: ignore[assignment]
3882 if "tags" in data:
3883 del data["tags"] # type: ignore[typeddict-item]
3884 for sense in data["senses"]:
3885 data_extend(sense, "tags", tags)
3887 return ret
3890def parse_wikipedia_template(
3891 wxr: WiktextractContext, data: WordData, ht: TemplateArgs
3892) -> None:
3893 """Helper function for parsing {{wikipedia|...}} and related templates."""
3894 assert isinstance(wxr, WiktextractContext)
3895 assert isinstance(data, dict)
3896 assert isinstance(ht, dict)
3897 langid = clean_node(wxr, data, ht.get("lang", ()))
3898 pagename = (
3899 clean_node(wxr, data, ht.get(1, ()))
3900 or wxr.wtp.title
3901 or "MISSING_PAGE_TITLE"
3902 )
3903 if langid:
3904 data_append(data, "wikipedia", langid + ":" + pagename)
3905 else:
3906 data_append(data, "wikipedia", pagename)
3909def parse_top_template(
3910 wxr: WiktextractContext, node: WikiNode, data: WordData
3911) -> None:
3912 """Parses a template that occurs on the top-level in a page, before any
3913 language subtitles."""
3914 assert isinstance(wxr, WiktextractContext)
3915 assert isinstance(node, WikiNode)
3916 assert isinstance(data, dict)
3918 def top_template_fn(name: str, ht: TemplateArgs) -> Optional[str]:
3919 if name in wikipedia_templates:
3920 parse_wikipedia_template(wxr, data, ht)
3921 return None
3922 if is_panel_template(wxr, name):
3923 return ""
3924 if name in ("reconstruction",): 3924 ↛ 3925line 3924 didn't jump to line 3925 because the condition on line 3924 was never true
3925 return ""
3926 if name.lower() == "also" or name.lower().startswith("also/"):
3927 # XXX shows related words that might really have been the intended
3928 # word, capture them
3929 return ""
3930 if name == "see also": 3930 ↛ 3932line 3930 didn't jump to line 3932 because the condition on line 3930 was never true
3931 # XXX capture
3932 return ""
3933 if name == "cardinalbox": 3933 ↛ 3935line 3933 didn't jump to line 3935 because the condition on line 3933 was never true
3934 # XXX capture
3935 return ""
3936 if name == "character info": 3936 ↛ 3938line 3936 didn't jump to line 3938 because the condition on line 3936 was never true
3937 # XXX capture
3938 return ""
3939 if name == "commonscat": 3939 ↛ 3941line 3939 didn't jump to line 3941 because the condition on line 3939 was never true
3940 # XXX capture link to Wikimedia commons
3941 return ""
3942 if name == "wrongtitle": 3942 ↛ 3945line 3942 didn't jump to line 3945 because the condition on line 3942 was never true
3943 # XXX this should be captured to replace page title with the
3944 # correct title. E.g. ⿰亻革家
3945 return ""
3946 if name == "wikidata": 3946 ↛ 3947line 3946 didn't jump to line 3947 because the condition on line 3946 was never true
3947 arg = clean_node(wxr, data, ht.get(1, ()))
3948 if arg.startswith("Q") or arg.startswith("Lexeme:L"):
3949 data_append(data, "wikidata", arg)
3950 return ""
3951 wxr.wtp.debug(
3952 "UNIMPLEMENTED top-level template: {} {}".format(name, ht),
3953 sortid="page/2870",
3954 )
3955 return ""
3957 clean_node(wxr, None, [node], template_fn=top_template_fn)
3960def fix_subtitle_hierarchy(wxr: WiktextractContext, text: str) -> str:
3961 """Fix subtitle hierarchy to be strict Language -> Etymology ->
3962 Part-of-Speech -> Translation/Linkage. Also merge Etymology sections
3963 that are next to each other."""
3965 # Wiktextract issue #620, Chinese Glyph Origin before an etymology
3966 # section get overwritten. In this case, let's just combine the two.
3968 # In Chinese entries, Pronunciation can be preceded on the
3969 # same level 3 by its Etymology *and* Glyph Origin sections:
3970 # ===Glyph Origin===
3971 # ===Etymology===
3972 # ===Pronunciation===
3973 # Tatu suggested adding a new 'level' between 3 and 4, so Pronunciation
3974 # is now Level 4, POS is shifted to Level 5 and the rest (incl. 'default')
3975 # are now level 6
3977 # Known lowercase PoS names are in part_of_speech_map
3978 # Known lowercase linkage section names are in linkage_map
3980 old = re.split(
3981 r"(?m)^(==+)[ \t]*([^= \t]([^=\n]|=[^=])*?)" r"[ \t]*(==+)[ \t]*$", text
3982 )
3984 parts = []
3985 npar = 4 # Number of parentheses in above expression
3986 parts.append(old[0])
3987 prev_level = None
3988 level = None
3989 skip_level_title = False # When combining etymology sections
3990 for i in range(1, len(old), npar + 1):
3991 left = old[i]
3992 right = old[i + npar - 1]
3993 # remove Wikilinks in title
3994 title = re.sub(r"^\[\[", "", old[i + 1])
3995 title = re.sub(r"\]\]$", "", title)
3996 prev_level = level
3997 level = len(left)
3998 part = old[i + npar]
3999 if level != len(right): 3999 ↛ 4000line 3999 didn't jump to line 4000 because the condition on line 3999 was never true
4000 wxr.wtp.debug(
4001 "subtitle has unbalanced levels: "
4002 "{!r} has {} on the left and {} on the right".format(
4003 title, left, right
4004 ),
4005 sortid="page/2904",
4006 )
4007 lc = title.lower()
4008 if name_to_code(title, "en") != "":
4009 if level > 2: 4009 ↛ 4010line 4009 didn't jump to line 4010 because the condition on line 4009 was never true
4010 wxr.wtp.debug(
4011 "subtitle has language name {} at level {}".format(
4012 title, level
4013 ),
4014 sortid="page/2911",
4015 )
4016 level = 2
4017 elif lc.startswith(tuple(ETYMOLOGY_TITLES)):
4018 if level > 3: 4018 ↛ 4019line 4018 didn't jump to line 4019 because the condition on line 4018 was never true
4019 wxr.wtp.debug(
4020 "etymology section {} at level {}".format(title, level),
4021 sortid="page/2917",
4022 )
4023 if prev_level == 3: # Two etymology (Glyph Origin + Etymology)
4024 # sections cheek-to-cheek
4025 skip_level_title = True
4026 # Modify the title of previous ("Glyph Origin") section, in
4027 # case we have a meaningful title like "Etymology 1"
4028 parts[-2] = "{}{}{}".format("=" * level, title, "=" * level)
4029 level = 3
4030 elif lc.startswith(PRONUNCIATION_TITLE):
4031 # Pronunciation is now a level between POS and Etymology, so
4032 # we need to shift everything down by one
4033 level = 4
4034 elif lc in POS_TITLES:
4035 level = 5
4036 elif lc == TRANSLATIONS_TITLE:
4037 level = 6
4038 elif lc in LINKAGE_TITLES or lc == COMPOUNDS_TITLE:
4039 level = 6
4040 elif lc in INFLECTION_TITLES:
4041 level = 6
4042 elif lc == DESCENDANTS_TITLE:
4043 level = 6
4044 elif title in PROTO_ROOT_DERIVED_TITLES: 4044 ↛ 4045line 4044 didn't jump to line 4045 because the condition on line 4044 was never true
4045 level = 6
4046 elif lc in IGNORED_TITLES:
4047 level = 6
4048 else:
4049 level = 6
4050 if skip_level_title:
4051 skip_level_title = False
4052 parts.append(part)
4053 else:
4054 parts.append("{}{}{}".format("=" * level, title, "=" * level))
4055 parts.append(part)
4056 # print("=" * level, title)
4057 # if level != len(left):
4058 # print(" FIXED LEVEL OF {} {} -> {}"
4059 # .format(title, len(left), level))
4061 text = "".join(parts)
4062 # print(text)
4063 return text
4066def parse_page(wxr: WiktextractContext, word: str, text: str) -> list[WordData]:
4067 # Skip translation pages
4068 if word.endswith("/" + TRANSLATIONS_TITLE): 4068 ↛ 4069line 4068 didn't jump to line 4069 because the condition on line 4068 was never true
4069 return []
4071 if wxr.config.verbose: 4071 ↛ 4072line 4071 didn't jump to line 4072 because the condition on line 4071 was never true
4072 logger.info(f"Parsing page: {word}")
4074 wxr.config.word = word
4075 wxr.wtp.start_page(word)
4077 # Remove <noinclude> and similar tags from main pages. They
4078 # should not appear there, but at least net/Elfdala has one and it
4079 # is probably not the only one.
4080 text = re.sub(r"(?si)<(/)?noinclude\s*>", "", text)
4081 text = re.sub(r"(?si)<(/)?onlyinclude\s*>", "", text)
4082 text = re.sub(r"(?si)<(/)?includeonly\s*>", "", text)
4084 # Fix up the subtitle hierarchy. There are hundreds if not thousands of
4085 # pages that have, for example, Translations section under Linkage, or
4086 # Translations section on the same level as Noun. Enforce a proper
4087 # hierarchy by manipulating the subtitle levels in certain cases.
4088 text = fix_subtitle_hierarchy(wxr, text)
4090 # Parse the page, pre-expanding those templates that are likely to
4091 # influence parsing
4092 tree = wxr.wtp.parse(
4093 text,
4094 pre_expand=True,
4095 additional_expand=ADDITIONAL_EXPAND_TEMPLATES,
4096 do_not_pre_expand=DO_NOT_PRE_EXPAND_TEMPLATES,
4097 )
4098 # from wikitextprocessor.parser import print_tree
4099 # print("PAGE PARSE:", print_tree(tree))
4101 top_data: WordData = {}
4103 # Iterate over top-level titles, which should be languages for normal
4104 # pages
4105 by_lang = defaultdict(list)
4106 for langnode in tree.children:
4107 if not isinstance(langnode, WikiNode):
4108 continue
4109 if langnode.kind == NodeKind.TEMPLATE:
4110 parse_top_template(wxr, langnode, top_data)
4111 continue
4112 if langnode.kind == NodeKind.LINK:
4113 # Some pages have links at top level, e.g., "trees" in Wiktionary
4114 continue
4115 if langnode.kind != NodeKind.LEVEL2: 4115 ↛ 4116line 4115 didn't jump to line 4116 because the condition on line 4115 was never true
4116 wxr.wtp.debug(
4117 f"unexpected top-level node: {langnode}", sortid="page/3014"
4118 )
4119 continue
4120 lang = clean_node(
4121 wxr, None, langnode.sarg if langnode.sarg else langnode.largs
4122 )
4123 lang_code = name_to_code(lang, "en")
4124 if lang_code == "": 4124 ↛ 4125line 4124 didn't jump to line 4125 because the condition on line 4124 was never true
4125 wxr.wtp.debug(
4126 f"unrecognized language name: {lang}", sortid="page/3019"
4127 )
4128 if (
4129 wxr.config.capture_language_codes
4130 and lang_code not in wxr.config.capture_language_codes
4131 ):
4132 continue
4133 wxr.wtp.start_section(lang)
4135 # Collect all words from the page.
4136 # print(f"{langnode=}")
4137 datas = parse_language(wxr, langnode, lang, lang_code)
4139 # Propagate fields resulting from top-level templates to this
4140 # part-of-speech.
4141 for data in datas:
4142 if "lang" not in data: 4142 ↛ 4143line 4142 didn't jump to line 4143 because the condition on line 4142 was never true
4143 wxr.wtp.debug(
4144 "internal error -- no lang in data: {}".format(data),
4145 sortid="page/3034",
4146 )
4147 continue
4148 for k, v in top_data.items():
4149 assert isinstance(v, (list, tuple))
4150 data_extend(data, k, v)
4151 by_lang[data["lang"]].append(data)
4153 # XXX this code is clearly out of date. There is no longer a "conjugation"
4154 # field. FIX OR REMOVE.
4155 # Do some post-processing on the words. For example, we may distribute
4156 # conjugation information to all the words.
4157 ret = []
4158 for lang, lang_datas in by_lang.items():
4159 ret.extend(lang_datas)
4161 for x in ret:
4162 if x["word"] != word:
4163 if word.startswith("Unsupported titles/"):
4164 wxr.wtp.debug(
4165 f"UNSUPPORTED TITLE: '{word}' -> '{x['word']}'",
4166 sortid="20231101/3578page.py",
4167 )
4168 else:
4169 wxr.wtp.debug(
4170 f"DIFFERENT ORIGINAL TITLE: '{word}' -> '{x['word']}'",
4171 sortid="20231101/3582page.py",
4172 )
4173 x["original_title"] = word
4174 # validate tag data
4175 recursively_separate_raw_tags(wxr, x) # type:ignore[arg-type]
4176 return ret
4179def recursively_separate_raw_tags(
4180 wxr: WiktextractContext, data: dict[str, Any]
4181) -> None:
4182 if not isinstance(data, dict): 4182 ↛ 4183line 4182 didn't jump to line 4183 because the condition on line 4182 was never true
4183 wxr.wtp.error(
4184 "'data' is not dict; most probably "
4185 "data has a list that contains at least one dict and "
4186 "at least one non-dict item",
4187 sortid="en/page-4016/20240419",
4188 )
4189 return
4190 new_tags: list[str] = []
4191 raw_tags: list[str] = data.get("raw_tags", [])
4192 for field, val in data.items():
4193 if field == "tags":
4194 for tag in val:
4195 if tag not in valid_tags:
4196 raw_tags.append(tag)
4197 else:
4198 new_tags.append(tag)
4199 if isinstance(val, list):
4200 if len(val) > 0 and isinstance(val[0], dict):
4201 for d in val:
4202 recursively_separate_raw_tags(wxr, d)
4203 if "tags" in data and not new_tags:
4204 del data["tags"]
4205 elif new_tags:
4206 data["tags"] = new_tags
4207 if raw_tags:
4208 data["raw_tags"] = raw_tags
4211def process_soft_redirect_template(
4212 wxr: WiktextractContext,
4213 template_node: TemplateNode,
4214 redirect_pages: list[str],
4215) -> bool:
4216 # return `True` if the template is soft redirect template
4217 if template_node.template_name == "zh-see":
4218 # https://en.wiktionary.org/wiki/Template:zh-see
4219 title = clean_node(
4220 wxr, None, template_node.template_parameters.get(1, "")
4221 )
4222 if title != "": 4222 ↛ 4224line 4222 didn't jump to line 4224 because the condition on line 4222 was always true
4223 redirect_pages.append(title)
4224 return True
4225 elif template_node.template_name in ["ja-see", "ja-see-kango"]:
4226 # https://en.wiktionary.org/wiki/Template:ja-see
4227 for key, value in template_node.template_parameters.items():
4228 if isinstance(key, int): 4228 ↛ 4227line 4228 didn't jump to line 4227 because the condition on line 4228 was always true
4229 title = clean_node(wxr, None, value)
4230 if title != "": 4230 ↛ 4227line 4230 didn't jump to line 4227 because the condition on line 4230 was always true
4231 redirect_pages.append(title)
4232 return True
4233 return False
4236ZH_FORMS_TAGS = {
4237 "trad.": "Traditional-Chinese",
4238 "simp.": "Simplified-Chinese",
4239 "alternative forms": "alternative",
4240 "2nd round simp.": "Second-Round-Simplified-Chinese",
4241}
4244def extract_zh_forms_template(
4245 wxr: WiktextractContext, t_node: TemplateNode, base_data: WordData
4246):
4247 # https://en.wiktionary.org/wiki/Template:zh-forms
4248 lit_meaning = clean_node(
4249 wxr, None, t_node.template_parameters.get("lit", "")
4250 )
4251 if lit_meaning != "":
4252 base_data["literal_meaning"] = lit_meaning
4253 expanded_node = wxr.wtp.parse(
4254 wxr.wtp.node_to_wikitext(t_node), expand_all=True
4255 )
4256 for table in expanded_node.find_child(NodeKind.TABLE):
4257 for row in table.find_child(NodeKind.TABLE_ROW):
4258 row_header = ""
4259 row_header_tags: list[str] = []
4260 header_has_span = False
4261 for cell in row.find_child(
4262 NodeKind.TABLE_HEADER_CELL | NodeKind.TABLE_CELL
4263 ):
4264 if cell.kind == NodeKind.TABLE_HEADER_CELL:
4265 row_header, row_header_tags, header_has_span = (
4266 extract_zh_forms_header_cell(wxr, base_data, cell)
4267 )
4268 elif not header_has_span:
4269 extract_zh_forms_data_cell(
4270 wxr, base_data, cell, row_header, row_header_tags
4271 )
4273 if "forms" in base_data and len(base_data["forms"]) == 0: 4273 ↛ 4274line 4273 didn't jump to line 4274 because the condition on line 4273 was never true
4274 del base_data["forms"]
4277def extract_zh_forms_header_cell(
4278 wxr: WiktextractContext, base_data: WordData, header_cell: WikiNode
4279) -> tuple[str, list[str], bool]:
4280 row_header = ""
4281 row_header_tags = []
4282 header_has_span = False
4283 first_span_index = len(header_cell.children)
4284 for index, span_tag in header_cell.find_html("span", with_index=True):
4285 if index < first_span_index: 4285 ↛ 4287line 4285 didn't jump to line 4287 because the condition on line 4285 was always true
4286 first_span_index = index
4287 header_has_span = True
4288 row_header = clean_node(wxr, None, header_cell.children[:first_span_index])
4289 for raw_tag in row_header.split(" and "):
4290 raw_tag = raw_tag.strip()
4291 if raw_tag != "":
4292 row_header_tags.append(raw_tag)
4293 for span_tag in header_cell.find_html_recursively("span"):
4294 span_lang = span_tag.attrs.get("lang", "")
4295 form_nodes = []
4296 sup_title = ""
4297 for node in span_tag.children:
4298 if isinstance(node, HTMLNode) and node.tag == "sup": 4298 ↛ 4299line 4298 didn't jump to line 4299 because the condition on line 4298 was never true
4299 for sup_span in node.find_html("span"):
4300 sup_title = sup_span.attrs.get("title", "")
4301 else:
4302 form_nodes.append(node)
4303 if span_lang in ["zh-Hant", "zh-Hans"]:
4304 for word in clean_node(wxr, None, form_nodes).split("/"):
4305 if word not in [wxr.wtp.title, ""]:
4306 form = {"form": word}
4307 for raw_tag in row_header_tags:
4308 if raw_tag in ZH_FORMS_TAGS: 4308 ↛ 4311line 4308 didn't jump to line 4311 because the condition on line 4308 was always true
4309 data_append(form, "tags", ZH_FORMS_TAGS[raw_tag])
4310 else:
4311 data_append(form, "raw_tags", raw_tag)
4312 if sup_title != "": 4312 ↛ 4313line 4312 didn't jump to line 4313 because the condition on line 4312 was never true
4313 data_append(form, "raw_tags", sup_title)
4314 data_append(base_data, "forms", form)
4315 return row_header, row_header_tags, header_has_span
4318TagLiteral = Literal["tags", "raw_tags"]
4319TAG_LITERALS_TUPLE: tuple[TagLiteral, ...] = ("tags", "raw_tags")
4322def extract_zh_forms_data_cell(
4323 wxr: WiktextractContext,
4324 base_data: WordData,
4325 cell: WikiNode,
4326 row_header: str,
4327 row_header_tags: list[str],
4328) -> None:
4329 from .zh_pron_tags import ZH_PRON_TAGS
4331 forms: list[FormData] = []
4332 for top_span_tag in cell.find_html("span"):
4333 span_style = top_span_tag.attrs.get("style", "")
4334 span_lang = top_span_tag.attrs.get("lang", "")
4335 if span_style == "white-space:nowrap;":
4336 extract_zh_forms_data_cell(
4337 wxr, base_data, top_span_tag, row_header, row_header_tags
4338 )
4339 elif "font-size:80%" in span_style:
4340 raw_tag = clean_node(wxr, None, top_span_tag)
4341 if raw_tag != "": 4341 ↛ 4332line 4341 didn't jump to line 4332 because the condition on line 4341 was always true
4342 for form in forms:
4343 if raw_tag in ZH_PRON_TAGS: 4343 ↛ 4349line 4343 didn't jump to line 4349 because the condition on line 4343 was always true
4344 tr_tag = ZH_PRON_TAGS[raw_tag]
4345 if isinstance(tr_tag, list): 4345 ↛ 4346line 4345 didn't jump to line 4346 because the condition on line 4345 was never true
4346 data_extend(form, "tags", tr_tag)
4347 elif isinstance(tr_tag, str): 4347 ↛ 4342line 4347 didn't jump to line 4342 because the condition on line 4347 was always true
4348 data_append(form, "tags", tr_tag)
4349 elif raw_tag in valid_tags:
4350 data_append(form, "tags", raw_tag)
4351 else:
4352 data_append(form, "raw_tags", raw_tag)
4353 elif span_lang in ["zh-Hant", "zh-Hans", "zh"]: 4353 ↛ 4332line 4353 didn't jump to line 4332 because the condition on line 4353 was always true
4354 word = clean_node(wxr, None, top_span_tag)
4355 if word not in ["", "／", wxr.wtp.title]:
4356 form = {"form": word}
4357 if row_header != "anagram": 4357 ↛ 4363line 4357 didn't jump to line 4363 because the condition on line 4357 was always true
4358 for raw_tag in row_header_tags:
4359 if raw_tag in ZH_FORMS_TAGS: 4359 ↛ 4362line 4359 didn't jump to line 4362 because the condition on line 4359 was always true
4360 data_append(form, "tags", ZH_FORMS_TAGS[raw_tag])
4361 else:
4362 data_append(form, "raw_tags", raw_tag)
4363 if span_lang == "zh-Hant":
4364 data_append(form, "tags", "Traditional-Chinese")
4365 elif span_lang == "zh-Hans":
4366 data_append(form, "tags", "Simplified-Chinese")
4367 forms.append(form)
4369 if row_header == "anagram": 4369 ↛ 4370line 4369 didn't jump to line 4370 because the condition on line 4369 was never true
4370 for form in forms:
4371 l_data: LinkageData = {"word": form["form"]}
4372 for key in TAG_LITERALS_TUPLE:
4373 if key in form:
4374 l_data[key] = form[key]
4375 data_append(base_data, "anagrams", l_data)
4376 else:
4377 data_extend(base_data, "forms", forms)
4380def extract_ja_kanjitab_template(
4381 wxr: WiktextractContext, t_node: TemplateNode, base_data: WordData
4382):
4383 # https://en.wiktionary.org/wiki/Template:ja-kanjitab
4384 expanded_node = wxr.wtp.parse(
4385 wxr.wtp.node_to_wikitext(t_node), expand_all=True
4386 )
4387 for table in expanded_node.find_child(NodeKind.TABLE):
4388 is_alt_form_table = False
4389 for row in table.find_child(NodeKind.TABLE_ROW):
4390 for header_node in row.find_child(NodeKind.TABLE_HEADER_CELL):
4391 header_text = clean_node(wxr, None, header_node)
4392 if header_text.startswith("Alternative spelling"):
4393 is_alt_form_table = True
4394 if not is_alt_form_table:
4395 continue
4396 forms = []
4397 for row in table.find_child(NodeKind.TABLE_ROW):
4398 for cell_node in row.find_child(NodeKind.TABLE_CELL):
4399 for child_node in cell_node.children:
4400 if isinstance(child_node, HTMLNode):
4401 if child_node.tag == "span":
4402 word = clean_node(wxr, None, child_node)
4403 if word != "": 4403 ↛ 4399line 4403 didn't jump to line 4399 because the condition on line 4403 was always true
4404 forms.append(
4405 {
4406 "form": word,
4407 "tags": ["alternative", "kanji"],
4408 }
4409 )
4410 elif child_node.tag == "small":
4411 raw_tag = clean_node(wxr, None, child_node).strip(
4412 "()"
4413 )
4414 if raw_tag != "" and len(forms) > 0: 4414 ↛ 4399line 4414 didn't jump to line 4399 because the condition on line 4414 was always true
4415 data_append(
4416 forms[-1],
4417 "tags"
4418 if raw_tag in valid_tags
4419 else "raw_tags",
4420 raw_tag,
4421 )
4422 data_extend(base_data, "forms", forms)
4423 for link_node in expanded_node.find_child(NodeKind.LINK):
4424 clean_node(wxr, base_data, link_node)