Coverage for src/wiktextract/extractor/en/page.py: 79%
1839 statements
« prev ^ index » next coverage.py v7.15.0, created at 2026-07-07 09:23 +0000
1# Code for parsing information from a single Wiktionary page.
2#
3# Copyright (c) 2018-2022 Tatu Ylonen. See file LICENSE and https://ylonen.org
5import copy
6import html
7import re
8from collections import defaultdict
9from functools import partial
10from typing import (
11 TYPE_CHECKING,
12 Any,
13 Iterable,
14 Literal,
15 Optional,
16 Set,
17 Union,
18 cast,
19)
21from mediawiki_langcodes import get_all_names, name_to_code
22from wikitextprocessor.core import TemplateArgs, TemplateFnCallable
23from wikitextprocessor.parser import (
24 LEVEL_KIND_FLAGS,
25 GeneralNode,
26 HTMLNode,
27 LevelNode,
28 NodeKind,
29 TemplateNode,
30 WikiNode,
31)
33from ...clean import clean_template_args, clean_value
34from ...datautils import (
35 data_append,
36 data_extend,
37 ns_title_prefix_tuple,
38)
39from ...page import (
40 LEVEL_KINDS,
41 clean_node,
42 is_panel_template,
43 recursively_extract,
44)
45from ...tags import valid_tags
46from ...wxr_context import WiktextractContext
47from ...wxr_logging import logger
48from ..ruby import extract_ruby, parse_ruby
49from ..share import strip_nodes
50from .descendant import extract_descendant_section
51from .example import extract_example_list_item, extract_template_zh_x
52from .form_descriptions import (
53 classify_desc,
54 decode_tags,
55 distw,
56 parse_alt_or_inflection_of,
57 parse_sense_qualifier,
58 parse_word_head,
59)
60from .inflection import TableContext, parse_inflection_section
61from .info_templates import (
62 INFO_TEMPLATE_FUNCS,
63 parse_info_template_arguments,
64 parse_info_template_node,
65)
66from .linkages import (
67 extract_alt_form_section,
68 parse_linkage,
69)
70from .parts_of_speech import PARTS_OF_SPEECH
71from .section_titles import (
72 COMPOUNDS_TITLE,
73 DESCENDANTS_TITLE,
74 ETYMOLOGY_TITLES,
75 IGNORED_TITLES,
76 INFLECTION_TITLES,
77 LINKAGE_TITLES,
78 POS_TITLES,
79 PRONUNCIATION_TITLE,
80 PROTO_ROOT_DERIVED_TITLES,
81 TRANSLATIONS_TITLE,
82)
83from .translations import parse_translation_item_text
84from .type_utils import (
85 AttestationData,
86 ExampleData,
87 FormData,
88 LinkageData,
89 ReferenceData,
90 SenseData,
91 SoundData,
92 TemplateData,
93 WordData,
94)
95from .unsupported_titles import unsupported_title_map
97# When determining whether a string is 'english', classify_desc
98# might return 'taxonomic' which is English text 99% of the time.
99ENGLISH_TEXTS = ("english", "taxonomic")
101# Matches head tag
102HEAD_TAG_RE = re.compile(
103 r"^(head|Han char|arabic-noun|arabic-noun-form|"
104 r"hangul-symbol|syllable-hangul)$|"
105 + r"^(latin|"
106 + "|".join(lang_code for lang_code, *_ in get_all_names("en"))
107 + r")-("
108 + "|".join(
109 [
110 "abbr",
111 "adj",
112 "adjective",
113 "adjective form",
114 "adjective-form",
115 "adv",
116 "adverb",
117 "affix",
118 "animal command",
119 "art",
120 "article",
121 "aux",
122 "bound pronoun",
123 "bound-pronoun",
124 "Buyla",
125 "card num",
126 "card-num",
127 "cardinal",
128 "chunom",
129 "classifier",
130 "clitic",
131 "cls",
132 "cmene",
133 "cmavo",
134 "colloq-verb",
135 "colverbform",
136 "combining form",
137 "combining-form",
138 "comparative",
139 "con",
140 "concord",
141 "conj",
142 "conjunction",
143 "conjug",
144 "cont",
145 "contr",
146 "converb",
147 "daybox",
148 "decl",
149 "decl noun",
150 "def",
151 "dem",
152 "det",
153 "determ",
154 "Deva",
155 "ending",
156 "entry",
157 "form",
158 "fuhivla",
159 "gerund",
160 "gismu",
161 "hanja",
162 "hantu",
163 "hanzi",
164 "head",
165 "ideophone",
166 "idiom",
167 "inf",
168 "indef",
169 "infixed pronoun",
170 "infixed-pronoun",
171 "infl",
172 "inflection",
173 "initialism",
174 "int",
175 "interfix",
176 "interj",
177 "interjection",
178 "jyut",
179 "latin",
180 "letter",
181 "locative",
182 "lujvo",
183 "monthbox",
184 "mutverb",
185 "name",
186 "nisba",
187 "nom",
188 "noun",
189 "noun form",
190 "noun-form",
191 "noun plural",
192 "noun-plural",
193 "nounprefix",
194 "num",
195 "number",
196 "numeral",
197 "ord",
198 "ordinal",
199 "par",
200 "part",
201 "part form",
202 "part-form",
203 "participle",
204 "particle",
205 "past",
206 "past neg",
207 "past-neg",
208 "past participle",
209 "past-participle",
210 "perfect participle",
211 "perfect-participle",
212 "personal pronoun",
213 "personal-pronoun",
214 "pref",
215 "prefix",
216 "phrase",
217 "pinyin",
218 "plural noun",
219 "plural-noun",
220 "pos",
221 "poss-noun",
222 "post",
223 "postp",
224 "postposition",
225 "PP",
226 "pp",
227 "ppron",
228 "pred",
229 "predicative",
230 "prep",
231 "prep phrase",
232 "prep-phrase",
233 "preposition",
234 "present participle",
235 "present-participle",
236 "pron",
237 "prondem",
238 "pronindef",
239 "pronoun",
240 "prop",
241 "proper noun",
242 "proper-noun",
243 "proper noun form",
244 "proper-noun form",
245 "proper noun-form",
246 "proper-noun-form",
247 "prov",
248 "proverb",
249 "prpn",
250 "prpr",
251 "punctuation mark",
252 "punctuation-mark",
253 "regnoun",
254 "rel",
255 "rom",
256 "romanji",
257 "root",
258 "sign",
259 "suff",
260 "suffix",
261 "syllable",
262 "symbol",
263 "verb",
264 "verb form",
265 "verb-form",
266 "verbal noun",
267 "verbal-noun",
268 "verbnec",
269 "vform",
270 ]
271 )
272 + r")(-|/|\+|$)"
273)
275# Head-templates causing problems (like newlines) that can be squashed into
276# an empty string in the template handler while saving their template
277# data for later.
278WORD_LEVEL_HEAD_TEMPLATES = {"term-label", "tlb"}
280# Annoying templates that should be in etymology sections, but sometimes
281# are thrown in heads because the etymology section is missing, like at
282# the oldest level of a reconstruction: see wiktextract#1658
283ETYMOLOGY_TEMPLATES_IN_HEADS = {
284 "ety",
285 "etymon",
286}
288PROBLEMATIC_TEMPLATES_CLUMP = (
289 WORD_LEVEL_HEAD_TEMPLATES | ETYMOLOGY_TEMPLATES_IN_HEADS
290)
292FLOATING_TABLE_TEMPLATES: set[str] = {
293 # az-suffix-form creates a style=floatright div that is otherwise
294 # deleted; if it is not pre-expanded, we can intercept the template
295 # so we add this set into do_not_pre_expand, and intercept the
296 # templates in parse_part_of_speech
297 "az-suffix-forms",
298 "az-inf-p",
299 "kk-suffix-forms",
300 "ky-suffix-forms",
301 "tr-inf-p",
302 "tr-suffix-forms",
303 "tt-suffix-forms",
304 "uz-suffix-forms",
305}
306# These two should contain template names that should always be
307# pre-expanded when *first* processing the tree, or not pre-expanded
308# so that the template are left in place with their identifying
309# name intact for later filtering.
311DO_NOT_PRE_EXPAND_TEMPLATES: set[str] = set()
312DO_NOT_PRE_EXPAND_TEMPLATES.update(FLOATING_TABLE_TEMPLATES)
314# Additional templates to be expanded in the pre-expand phase
315ADDITIONAL_EXPAND_TEMPLATES: set[str] = {
316 "multitrans",
317 "multitrans-nowiki",
318 "trans-top",
319 "trans-top-also",
320 "trans-bottom",
321 "checktrans-top",
322 "checktrans-bottom",
323 "col",
324 "col1",
325 "col2",
326 "col3",
327 "col4",
328 "col5",
329 "col1-u",
330 "col2-u",
331 "col3-u",
332 "col4-u",
333 "col5-u",
334 "check deprecated lang param usage",
335 "deprecated code",
336 "ru-verb-alt-ё",
337 "ru-noun-alt-ё",
338 "ru-adj-alt-ё",
339 "ru-proper noun-alt-ё",
340 "ru-pos-alt-ё",
341 "ru-alt-ё",
342 "inflection of",
343 "no deprecated lang param usage",
344 "transclude", # these produce sense entries (or other lists)
345 "tcl",
346}
348# Inverse linkage for those that have them
349linkage_inverses: dict[str, str] = {
350 # XXX this is not currently used, move to post-processing
351 "synonyms": "synonyms",
352 "hypernyms": "hyponyms",
353 "hyponyms": "hypernyms",
354 "holonyms": "meronyms",
355 "meronyms": "holonyms",
356 "derived": "derived_from",
357 "coordinate_terms": "coordinate_terms",
358 "troponyms": "hypernyms",
359 "antonyms": "antonyms",
360 "instances": "instance_of",
361 "related": "related",
362}
364# Templates that are used to form panels on pages and that
365# should be ignored in various positions
366PANEL_TEMPLATES: set[str] = {
367 "Character info",
368 "CJKV",
369 "French personal pronouns",
370 "French possessive adjectives",
371 "French possessive pronouns",
372 "Han etym",
373 "Han etyl", # this redirects to Han etym and would cause Lua errors,
374 # and I don't know why, but I'm putting it here because
375 # we should be ignoring it anyhow.
376 "Japanese demonstratives",
377 "Latn-script",
378 "LDL",
379 "MW1913Abbr",
380 "Number-encoding",
381 "Nuttall",
382 "Spanish possessive adjectives",
383 "Spanish possessive pronouns",
384 "USRegionDisputed",
385 "Webster 1913",
386 "ase-rfr",
387 "attention",
388 "attn",
389 "beer",
390 "broken ref",
391 "ca-compass",
392 "character info",
393 "character info/var",
394 "checksense",
395 "compass-fi",
396 "copyvio suspected",
397 "delete",
398 "dial syn", # Currently ignore these, but could be useful in Chinese/Korean
399 "etystub",
400 "examples",
401 "hu-corr",
402 "hu-suff-pron",
403 "interwiktionary",
404 "ja-kanjitab",
405 "ja-kt",
406 "ko-hanja-search",
407 "look",
408 "maintenance box",
409 "maintenance line",
410 "mediagenic terms",
411 "merge",
412 "missing template",
413 "morse links",
414 "move",
415 "multiple images",
416 "no inline",
417 "picdic",
418 "picdicimg",
419 "picdiclabel",
420 "polyominoes",
421 "predidential nomics",
422 "punctuation", # This actually gets pre-expanded
423 "reconstructed",
424 "request box",
425 "rf-sound example",
426 "rfaccents",
427 "rfap",
428 "rfaspect",
429 "rfc",
430 "rfc-auto",
431 "rfc-header",
432 "rfc-level",
433 "rfc-pron-n",
434 "rfc-sense",
435 "rfclarify",
436 "rfd",
437 "rfd-redundant",
438 "rfd-sense",
439 "rfdate",
440 "rfdatek",
441 "rfdef",
442 "rfe",
443 "rfe/dowork",
444 "rfex",
445 "rfexp",
446 "rfform",
447 "rfgender",
448 "rfi",
449 "rfinfl",
450 "rfm",
451 "rfm-sense",
452 "rfp",
453 "rfp-old",
454 "rfquote",
455 "rfquote-sense",
456 "rfquotek",
457 "rfref",
458 "rfscript",
459 "rft2",
460 "rftaxon",
461 "rftone",
462 "rftranslit",
463 "rfv",
464 "rfv-etym",
465 "rfv-pron",
466 "rfv-quote",
467 "rfv-sense",
468 "selfref",
469 "split",
470 "stroke order", # XXX consider capturing this?
471 "stub entry",
472 "t-needed",
473 "tbot entry",
474 "tea room",
475 "tea room sense",
476 # "ttbc", - XXX needed in at least on/Preposition/Translation page
477 "unblock",
478 "unsupportedpage",
479 "video frames",
480 "was wotd",
481 "wrongtitle",
482 "zh-forms",
483 "zh-hanzi-box",
484 "no entry",
485}
487# Template name prefixes used for language-specific panel templates (i.e.,
488# templates that create side boxes or notice boxes or that should generally
489# be ignored).
490PANEL_PREFIXES: set[str] = {
491 "list:compass points/",
492 "list:Gregorian calendar months/",
493 "RQ:",
494}
496# Templates used for wikipedia links.
497wikipedia_templates: set[str] = {
498 "wikipedia",
499 "slim-wikipedia",
500 "w",
501 "W",
502 "swp",
503 "wiki",
504 "Wikipedia",
505 "wtorw",
506}
507for x in PANEL_PREFIXES & wikipedia_templates: 507 ↛ 508line 507 didn't jump to line 508 because the loop on line 507 never started
508 print(
509 "WARNING: {!r} in both panel_templates and wikipedia_templates".format(
510 x
511 )
512 )
514# Mapping from a template name (without language prefix) for the main word
515# (e.g., fi-noun, fi-adj, en-verb) to permitted parts-of-speech in which
516# it could validly occur. This is used as just a sanity check to give
517# warnings about probably incorrect coding in Wiktionary.
518template_allowed_pos_map: dict[str, list[str]] = {
519 "abbr": ["abbrev"],
520 "noun": ["noun", "abbrev", "pron", "name", "num", "adj_noun"],
521 "plural noun": ["noun", "name"],
522 "plural-noun": ["noun", "name"],
523 "proper noun": ["noun", "name"],
524 "proper-noun": ["name", "noun"],
525 "prop": ["name", "noun"],
526 "verb": ["verb", "phrase"],
527 "gerund": ["verb"],
528 "particle": ["adv", "particle"],
529 "adj": ["adj", "adj_noun"],
530 "pron": ["pron", "noun"],
531 "name": ["name", "noun"],
532 "adv": ["adv", "intj", "conj", "particle"],
533 "phrase": ["phrase", "prep_phrase"],
534 "noun phrase": ["phrase"],
535 "ordinal": ["num"],
536 "number": ["num"],
537 "pos": ["affix", "name", "num"],
538 "suffix": ["suffix", "affix"],
539 "character": ["character"],
540 "letter": ["character"],
541 "kanji": ["character"],
542 "cont": ["abbrev"],
543 "interj": ["intj"],
544 "con": ["conj"],
545 "part": ["particle"],
546 "prep": ["prep", "postp"],
547 "postp": ["postp"],
548 "misspelling": ["noun", "adj", "verb", "adv"],
549 "part-form": ["verb"],
550}
551for k, v in template_allowed_pos_map.items():
552 for x in v:
553 if x not in PARTS_OF_SPEECH: 553 ↛ 554line 553 didn't jump to line 554 because the condition on line 553 was never true
554 print(
555 "BAD PART OF SPEECH {!r} IN template_allowed_pos_map: {}={}"
556 "".format(x, k, v)
557 )
558 assert False
561# Templates ignored during etymology extraction, i.e., these will not be listed
562# in the extracted etymology templates.
563ignored_etymology_templates: list[str] = [
564 "...",
565 "IPAchar",
566 "ipachar",
567 "ISBN",
568 "isValidPageName",
569 "redlink category",
570 "deprecated code",
571 "check deprecated lang param usage",
572 "para",
573 "p",
574 "cite",
575 "Cite news",
576 "Cite newsgroup",
577 "cite paper",
578 "cite MLLM 1976",
579 "cite journal",
580 "cite news/documentation",
581 "cite paper/documentation",
582 "cite video game",
583 "cite video game/documentation",
584 "cite newsgroup",
585 "cite newsgroup/documentation",
586 "cite web/documentation",
587 "cite news",
588 "Cite book",
589 "Cite-book",
590 "cite book",
591 "cite web",
592 "cite-usenet",
593 "cite-video/documentation",
594 "Cite-journal",
595 "rfe",
596 "catlangname",
597 "cln",
598 "langname-lite",
599 "no deprecated lang param usage",
600 "mention",
601 "m",
602 "m-self",
603 "link",
604 "l",
605 "ll",
606 "l-self",
607]
608# Regexp for matching ignored etymology template names. This adds certain
609# prefixes to the names listed above.
610ignored_etymology_templates_re = re.compile(
611 r"^((cite-|R:|RQ:).*|"
612 + r"|".join(re.escape(x) for x in ignored_etymology_templates)
613 + r")$"
614)
616# Regexp for matching ignored descendants template names. Right now we just
617# copy the ignored etymology templates
618ignored_descendants_templates_re = ignored_etymology_templates_re
620# Set of template names that are used to define usage examples. If the usage
621# example contains one of these templates, then it its type is set to
622# "example"
623usex_templates: set[str] = {
624 "afex",
625 "affixusex",
626 "co", # {{collocation}} acts like a example template, specifically for
627 # pairs of combinations of words that are more common than you'd
628 # except would be randomly; hlavní#Czech
629 "coi",
630 "collocation",
631 "el-example",
632 "el-x",
633 "example",
634 "examples",
635 "he-usex",
636 "he-x",
637 "hi-usex",
638 "hi-x",
639 "ja-usex-inline",
640 "ja-usex",
641 "ja-x",
642 "jbo-example",
643 "jbo-x",
644 "km-usex",
645 "km-x",
646 "ko-usex",
647 "ko-x",
648 "lo-usex",
649 "lo-x",
650 "ne-x",
651 "ne-usex",
652 "prefixusex",
653 "ryu-usex",
654 "ryu-x",
655 "shn-usex",
656 "shn-x",
657 "suffixusex",
658 "th-usex",
659 "th-x",
660 "ur-usex",
661 "ur-x",
662 "usex",
663 "usex-suffix",
664 "ux",
665 "uxi",
666}
668stop_head_at_these_templates: set[str] = {
669 "category",
670 "cat",
671 "topics",
672 "catlangname",
673 "c",
674 "C",
675 "top",
676 "cln",
677}
679# Set of template names that are used to define quotation examples. If the
680# usage example contains one of these templates, then its type is set to
681# "quotation".
682quotation_templates: set[str] = {
683 "collapse-quote",
684 "quote-av",
685 "quote-book",
686 "quote-GYLD",
687 "quote-hansard",
688 "quotei",
689 "quote-journal",
690 "quotelite",
691 "quote-mailing list",
692 "quote-meta",
693 "quote-newsgroup",
694 "quote-song",
695 "quote-text",
696 "quote",
697 "quote-us-patent",
698 "quote-video game",
699 "quote-web",
700 "quote-wikipedia",
701 "wikiquote",
702 "Wikiquote",
703 "Q",
704}
706taxonomy_templates = {
707 # argument 1 should be the taxonomic name, frex. "Lupus lupus"
708 "taxfmt",
709 "taxlink",
710 "taxlink2",
711 "taxlinknew",
712 "taxlook",
713}
715# Template names, this was exctracted from template_linkage_mappings,
716# because the code using template_linkage_mappings was actually not used
717# (but not removed).
718template_linkages_to_ignore_in_examples: set[str] = {
719 "syn",
720 "synonyms",
721 "ant",
722 "antonyms",
723 "hyp",
724 "hyponyms",
725 "der",
726 "derived terms",
727 "coordinate terms",
728 "cot",
729 "rel",
730 "col",
731 "inline alt forms",
732 "alti",
733 "comeronyms",
734 "holonyms",
735 "holo",
736 "hypernyms",
737 "hyper",
738 "meronyms",
739 "mero",
740 "troponyms",
741 "perfectives",
742 "pf",
743 "imperfectives",
744 "impf",
745 "syndiff",
746 "synsee",
747 # not linkage nor example templates
748 "sense",
749 "s",
750 "color panel",
751 "colour panel",
752}
754# Maps template name used in a word sense to a linkage field that it adds.
755sense_linkage_templates: dict[str, str] = {
756 "syn": "synonyms",
757 "synonyms": "synonyms",
758 "synsee": "synonyms",
759 "syndiff": "synonyms",
760 "hyp": "hyponyms",
761 "hyponyms": "hyponyms",
762 "ant": "antonyms",
763 "antonyms": "antonyms",
764 "alti": "related",
765 "inline alt forms": "related",
766 "coordinate terms": "coordinate_terms",
767 "cot": "coordinate_terms",
768 "comeronyms": "related",
769 "holonyms": "holonyms",
770 "holo": "holonyms",
771 "hypernyms": "hypernyms",
772 "hyper": "hypernyms",
773 "meronyms": "meronyms",
774 "mero": "meronyms",
775 "troponyms": "troponyms",
776 "perfectives": "related",
777 "pf": "related",
778 "imperfectives": "related",
779 "impf": "related",
780 "parasynonyms": "synonyms",
781 "par": "synonyms",
782 "parasyn": "synonyms",
783 "nearsyn": "synonyms",
784 "near-syn": "synonyms",
785}
787sense_linkage_templates_tags: dict[str, list[str]] = {
788 "alti": ["alternative"],
789 "inline alt forms": ["alternative"],
790 "comeronyms": ["comeronym"],
791 "perfectives": ["perfective"],
792 "pf": ["perfective"],
793 "imperfectives": ["imperfective"],
794 "impf": ["imperfective"],
795}
798def decode_html_entities(v: Union[str, int]) -> str:
799 """Decodes HTML entities from a value, converting them to the respective
800 Unicode characters/strings."""
801 if isinstance(v, int):
802 # I changed this to return str(v) instead of v = str(v),
803 # but there might have been the intention to have more logic
804 # here. html.unescape would not do anything special with an integer,
805 # it needs html escape symbols (&xx;).
806 return str(v)
807 return html.unescape(v)
810def parse_sense_linkage(
811 wxr: WiktextractContext,
812 data: SenseData,
813 name: str,
814 ht: TemplateArgs,
815 pos: str,
816) -> None:
817 """Parses a linkage (synonym, etc) specified in a word sense."""
818 assert isinstance(wxr, WiktextractContext)
819 assert isinstance(data, dict)
820 assert isinstance(name, str)
821 assert isinstance(ht, dict)
822 field = sense_linkage_templates[name]
823 field_tags = sense_linkage_templates_tags.get(name, [])
824 for i in range(2, 20):
825 if i not in ht:
826 break
827 w = clean_node(wxr, data, ht[i])
828 if "#" in w:
829 w = w[: w.index("#")]
830 if w in ["", "<"]: # `<` used in "hypernyms" template
831 continue
832 if ( 832 ↛ 837line 832 didn't jump to line 837 because the condition on line 832 was never true
833 i > 2
834 and w in (",", "or", ";")
835 or w.startswith(("see also", "See also"))
836 ):
837 continue
838 is_thesaurus = False
839 for alias in ns_title_prefix_tuple(wxr, "Thesaurus"):
840 if w.startswith(alias):
841 is_thesaurus = True
842 w = w[len(alias) :]
843 if w != wxr.wtp.title: 843 ↛ 863line 843 didn't jump to line 863 because the condition on line 843 was always true
844 from ...thesaurus import search_thesaurus
846 lang_code = clean_node(wxr, None, ht.get(1, ""))
847 for t_data in search_thesaurus(
848 wxr.thesaurus_db_conn, # type: ignore
849 w,
850 lang_code,
851 pos,
852 "synonyms", # GH issue #1570
853 ):
854 l_data: LinkageData = {
855 "word": t_data.term,
856 "source": "Thesaurus:" + w,
857 }
858 if len(t_data.tags) > 0: 858 ↛ 859line 858 didn't jump to line 859 because the condition on line 858 was never true
859 l_data["tags"] = t_data.tags
860 if len(t_data.raw_tags) > 0: 860 ↛ 861line 860 didn't jump to line 861 because the condition on line 860 was never true
861 l_data["raw_tags"] = t_data.raw_tags
862 data_append(data, field, l_data)
863 break
864 if is_thesaurus:
865 continue
866 tags: list[str] = []
867 topics: list[str] = []
868 english: Optional[str] = None
869 # Try to find qualifiers for this synonym
870 q = ht.get("q{}".format(i - 1))
871 if q:
872 cls = classify_desc(q)
873 if cls == "tags":
874 tagsets1, topics1 = decode_tags(q)
875 for ts in tagsets1:
876 tags.extend(ts)
877 topics.extend(topics1)
878 elif cls == "english": 878 ↛ 884line 878 didn't jump to line 884 because the condition on line 878 was always true
879 if english: 879 ↛ 880line 879 didn't jump to line 880 because the condition on line 879 was never true
880 english += "; " + q
881 else:
882 english = q
883 # Try to find English translation for this synonym
884 t = ht.get("t{}".format(i - 1))
885 if t: 885 ↛ 886line 885 didn't jump to line 886 because the condition on line 885 was never true
886 if english:
887 english += "; " + t
888 else:
889 english = t
891 # See if the linkage contains a parenthesized alt
892 alt = None
893 m = re.search(r"\(([^)]+)\)$", w)
894 if m: 894 ↛ 895line 894 didn't jump to line 895 because the condition on line 894 was never true
895 w = w[: m.start()].strip()
896 alt = m.group(1)
898 dt = {"word": w}
899 if field_tags: 899 ↛ 900line 899 didn't jump to line 900 because the condition on line 899 was never true
900 data_extend(dt, "tags", field_tags)
901 if tags:
902 data_extend(dt, "tags", tags)
903 if topics: 903 ↛ 904line 903 didn't jump to line 904 because the condition on line 903 was never true
904 data_extend(dt, "topics", topics)
905 if english:
906 dt["english"] = english # DEPRECATED for "translation"
907 dt["translation"] = english
908 if alt: 908 ↛ 909line 908 didn't jump to line 909 because the condition on line 908 was never true
909 dt["alt"] = alt
910 data_append(data, field, dt)
913EXAMPLE_SPLITTERS = r"\s*[―—]+\s*"
914example_splitter_re = re.compile(EXAMPLE_SPLITTERS)
915captured_splitters_re = re.compile(r"(" + EXAMPLE_SPLITTERS + r")")
918def synch_splits_with_args(
919 line: str, targs: TemplateArgs
920) -> Optional[list[str]]:
921 """If it looks like there's something weird with how a line of example
922 text has been split, this function will do the splitting after counting
923 occurences of the splitting regex inside the two main template arguments
924 containing the string data for the original language example and the
925 English translations.
926 """
927 # Previously, we split without capturing groups, but here we want to
928 # keep the original splitting hyphen regex intact.
929 fparts = captured_splitters_re.split(line)
930 new_parts = []
931 # ["First", " – ", "second", " – ", "third..."] from OL argument
932 first = 1 + (2 * len(example_splitter_re.findall(targs.get(2, ""))))
933 new_parts.append("".join(fparts[:first]))
934 # Translation argument
935 tr_arg = targs.get(3) or targs.get("translation") or targs.get("t", "")
936 # +2 = + 1 to skip the "expected" hyphen, + 1 as the `1 +` above.
937 second = first + 2 + (2 * len(example_splitter_re.findall(tr_arg)))
938 new_parts.append("".join(fparts[first + 1 : second]))
940 if all(new_parts): # no empty strings from the above spaghetti
941 new_parts.extend(fparts[second + 1 :: 2]) # skip rest of hyphens
942 return new_parts
943 else:
944 return None
947QUALIFIERS = r"^\((([^()]|\([^()]*\))*)\):?\s*"
948QUALIFIERS_RE = re.compile(QUALIFIERS)
949# (...): ... or (...(...)...): ...
952def parse_language(
953 wxr: WiktextractContext, langnode: WikiNode, language: str, lang_code: str
954) -> list[WordData]:
955 """Iterates over the text of the page, returning words (parts-of-speech)
956 defined on the page one at a time. (Individual word senses for the
957 same part-of-speech are typically encoded in the same entry.)"""
958 # imported here to avoid circular import
959 from .pronunciation import parse_pronunciation
961 assert isinstance(wxr, WiktextractContext)
962 assert isinstance(langnode, WikiNode)
963 assert isinstance(language, str)
964 assert isinstance(lang_code, str)
965 # print("parse_language", language)
967 is_reconstruction = False
968 word: str = wxr.wtp.title # type: ignore[assignment]
969 unsupported_prefix = "Unsupported titles/"
970 if word.startswith(unsupported_prefix):
971 w = word[len(unsupported_prefix) :]
972 if w in unsupported_title_map: 972 ↛ 975line 972 didn't jump to line 975 because the condition on line 972 was always true
973 word = unsupported_title_map[w]
974 else:
975 wxr.wtp.error(
976 "Unimplemented unsupported title: {}".format(word),
977 sortid="page/870",
978 )
979 word = w
980 elif word.startswith("Reconstruction:"):
981 word = word[word.find("/") + 1 :]
982 is_reconstruction = True
983 elif word.startswith("a/languages"): 983 ↛ 985line 983 didn't jump to line 985 because the condition on line 983 was never true
984 # ATM there's only one "mammoth page" in English wiktionary, 'a'
985 word = "a"
987 base_data: WordData = {
988 "word": word,
989 "lang": language,
990 "lang_code": lang_code,
991 }
992 if is_reconstruction:
993 data_append(base_data, "tags", "reconstruction")
994 sense_data: SenseData = {}
995 pos_data: WordData = {} # For a current part-of-speech
996 level_four_data: WordData = {} # Chinese Pronunciation-sections in-between
997 etym_data: WordData = {} # For one etymology
998 sense_datas: list[SenseData] = []
999 sense_ordinal = 0 # The recursive sense parsing messes up the ordering
1000 # Never reset, do not use as data
1001 level_four_datas: list[WordData] = []
1002 etym_datas: list[WordData] = []
1003 page_datas: list[WordData] = []
1004 have_etym = False
1005 inside_level_four = False # This is for checking if the etymology section
1006 # or article has a Pronunciation section, for Chinese mostly; because
1007 # Chinese articles can have three level three sections (two etymology
1008 # sections and pronunciation sections) one after another, we need a kludge
1009 # to better keep track of whether we're in a normal "etym" or inside a
1010 # "level four" (which is what we've turned the level three Pron sections
1011 # into in the fix_subtitle_hierarchy(); all other sections are demoted by
1012 # a step.
1013 stack: list[str] = [] # names of items on the "stack"
1015 def merge_base(data: WordData, base: WordData) -> None:
1016 for k, v in base.items():
1017 # Copy the value to ensure that we don't share lists or
1018 # dicts between structures (even nested ones).
1019 v = copy.deepcopy(v)
1020 if k not in data:
1021 # The list was copied above, so this will not create shared ref
1022 data[k] = v # type: ignore[literal-required]
1023 continue
1024 if data[k] == v: # type: ignore[literal-required]
1025 continue
1026 if ( 1026 ↛ 1034line 1026 didn't jump to line 1034 because the condition on line 1026 was always true
1027 isinstance(data[k], (list, tuple)) # type: ignore[literal-required]
1028 or isinstance(
1029 v,
1030 (list, tuple), # Should this be "and"?
1031 )
1032 ):
1033 data[k] = list(data[k]) + list(v) # type: ignore
1034 elif data[k] != v: # type: ignore[literal-required]
1035 wxr.wtp.warning(
1036 "conflicting values for {} in merge_base: "
1037 "{!r} vs {!r}".format(k, data[k], v), # type: ignore[literal-required]
1038 sortid="page/904",
1039 )
1041 def complementary_pop(pron: SoundData, key: str) -> SoundData:
1042 """Remove unnecessary keys from dict values
1043 in a list comprehension..."""
1044 if key in pron:
1045 pron.pop(key) # type: ignore
1046 return pron
1048 def sound_matches_pos(sound: SoundData, pos: str) -> bool:
1049 if "pos" not in sound:
1050 return True
1051 sound_pos = sound["pos"] # type: ignore[typeddict-item]
1052 return pos in sound_pos
1054 def strip_sound_pos(sound: SoundData) -> SoundData:
1055 complementary_pop(sound, "pos")
1056 return sound
1058 # If the result has sounds, eliminate sounds that have a prefix that
1059 # does not match "word" or one of "forms"
1060 if "sounds" in data and "word" in data:
1061 accepted = [data["word"]]
1062 accepted.extend(f["form"] for f in data.get("forms", dict()))
1063 data["sounds"] = list(
1064 s
1065 for s in data["sounds"]
1066 if "form" not in s or s["form"] in accepted
1067 )
1068 # If the result has sounds, eliminate sounds that have a pos that
1069 # does not match "pos"
1070 if "sounds" in data and "pos" in data:
1071 data["sounds"] = list(
1072 strip_sound_pos(s)
1073 for s in data["sounds"]
1074 # "pos" is not a field of SoundData, correctly, so we're
1075 # removing it here. It's a kludge on a kludge on a kludge.
1076 if sound_matches_pos(s, data["pos"])
1077 )
1078 elif "sounds" in data: 1078 ↛ 1079line 1078 didn't jump to line 1079 because the condition on line 1078 was never true
1079 data["sounds"] = [strip_sound_pos(s) for s in data["sounds"]]
1081 def push_sense(sorting_ordinal: int | None = None) -> bool:
1082 """Starts collecting data for a new word sense. This returns True
1083 if a sense was added."""
1084 nonlocal sense_data
1085 if sorting_ordinal is None:
1086 sorting_ordinal = sense_ordinal
1087 tags = sense_data.get("tags", ())
1088 if (
1089 not sense_data.get("glosses")
1090 and "translation-hub" not in tags
1091 and "no-gloss" not in tags
1092 ):
1093 return False
1095 if ( 1095 ↛ 1105line 1095 didn't jump to line 1105 because the condition on line 1095 was never true
1096 (
1097 "participle" in sense_data.get("tags", ())
1098 or "infinitive" in sense_data.get("tags", ())
1099 )
1100 and "alt_of" not in sense_data
1101 and "form_of" not in sense_data
1102 and "etymology_text" in etym_data
1103 and etym_data["etymology_text"] != ""
1104 ):
1105 etym = etym_data["etymology_text"]
1106 etym = etym.split(". ")[0]
1107 ret = parse_alt_or_inflection_of(wxr, etym, set())
1108 if ret is not None:
1109 tags, lst = ret
1110 assert isinstance(lst, (list, tuple))
1111 if "form-of" in tags:
1112 data_extend(sense_data, "form_of", lst)
1113 data_extend(sense_data, "tags", tags)
1114 elif "alt-of" in tags:
1115 data_extend(sense_data, "alt_of", lst)
1116 data_extend(sense_data, "tags", tags)
1118 if not sense_data.get("glosses") and "no-gloss" not in sense_data.get( 1118 ↛ 1121line 1118 didn't jump to line 1121 because the condition on line 1118 was never true
1119 "tags", ()
1120 ):
1121 data_append(sense_data, "tags", "no-gloss")
1123 sense_data["__temp_sense_sorting_ordinal"] = sorting_ordinal # type: ignore
1124 sense_datas.append(sense_data)
1125 sense_data = {}
1126 return True
1128 def push_pos(sorting_ordinal: int | None = None) -> None:
1129 """Starts collecting data for a new part-of-speech."""
1130 nonlocal pos_data
1131 nonlocal sense_datas
1132 push_sense(sorting_ordinal)
1133 if wxr.wtp.subsection:
1134 data: WordData = {"senses": sense_datas}
1135 merge_base(data, pos_data)
1136 level_four_datas.append(data)
1137 pos_data = {}
1138 sense_datas = []
1139 wxr.wtp.start_subsection(None)
1141 def push_level_four_section(clear_sound_data: bool) -> None:
1142 """Starts collecting data for a new level four sections, which
1143 is usually virtual and empty, unless the article has Chinese
1144 'Pronunciation' sections that are etymology-section-like but
1145 under etymology, and at the same level in the source. We modify
1146 the source to demote Pronunciation sections like that to level
1147 4, and other sections one step lower."""
1148 nonlocal level_four_data
1149 nonlocal level_four_datas
1150 nonlocal etym_datas
1151 push_pos()
1152 # print(f"======\n{etym_data=}")
1153 # print(f"======\n{etym_datas=}")
1154 # print(f"======\n{level_four_data=}")
1155 # print(f"======\n{level_four_datas=}")
1156 for data in level_four_datas:
1157 merge_base(data, level_four_data)
1158 etym_datas.append(data)
1159 for data in etym_datas:
1160 merge_base(data, etym_data)
1161 page_datas.append(data)
1162 if clear_sound_data:
1163 level_four_data = {}
1164 level_four_datas = []
1165 etym_datas = []
1167 def push_etym() -> None:
1168 """Starts collecting data for a new etymology."""
1169 nonlocal etym_data
1170 nonlocal etym_datas
1171 nonlocal have_etym
1172 nonlocal inside_level_four
1173 have_etym = True
1174 push_level_four_section(False)
1175 inside_level_four = False
1176 # etymology section could under pronunciation section
1177 etym_data = (
1178 copy.deepcopy(level_four_data) if len(level_four_data) > 0 else {}
1179 )
1181 def select_data() -> WordData:
1182 """Selects where to store data (pos or etym) based on whether we
1183 are inside a pos (part-of-speech)."""
1184 # print(f"{wxr.wtp.subsection=}")
1185 # print(f"{stack=}")
1186 if wxr.wtp.subsection is not None:
1187 return pos_data
1188 if inside_level_four:
1189 return level_four_data
1190 if stack[-1] == language:
1191 return base_data
1192 return etym_data
1194 def parse_part_of_speech(posnode: WikiNode, pos: str) -> None:
1195 """Parses the subsection for a part-of-speech under a language on
1196 a page."""
1197 assert isinstance(posnode, WikiNode)
1198 assert isinstance(pos, str)
1199 # print("parse_part_of_speech", pos)
1200 pos_data["pos"] = pos
1201 pre: list[list[Union[str, WikiNode]]] = [[]] # list of lists
1202 lists: list[list[WikiNode]] = [[]] # list of lists
1203 first_para = True
1204 first_head_tmplt = True
1205 collecting_head = True
1206 start_of_paragraph = True
1208 # XXX extract templates from posnode with recursively_extract
1209 # that break stuff, like ja-kanji or az-suffix-form.
1210 # Do the extraction with a list of template names, combined from
1211 # different lists, then separate out them into different lists
1212 # that are handled at different points of the POS section.
1213 # First, extract az-suffix-form, put it in `inflection`,
1214 # and parse `inflection`'s content when appropriate later.
1215 # The contents of az-suffix-form (and ja-kanji) that generate
1216 # divs with "floatright" in their style gets deleted by
1217 # clean_value, so templates that slip through from here won't
1218 # break anything.
1219 # XXX bookmark
1220 # print("===================")
1221 # print(posnode.children)
1223 floaters, poschildren = recursively_extract(
1224 posnode.children,
1225 lambda x: (
1226 isinstance(x, WikiNode)
1227 and (
1228 (
1229 isinstance(x, TemplateNode)
1230 and x.template_name in FLOATING_TABLE_TEMPLATES
1231 )
1232 or (
1233 x.kind == NodeKind.LINK
1234 # Need to check for stringiness because some links are
1235 # broken; for example, if a template is missing an
1236 # argument, a link might look like `[[{{{1}}}...]]`
1237 and len(x.largs) > 0
1238 and len(x.largs[0]) > 0
1239 and isinstance(x.largs[0][0], str)
1240 and x.largs[0][0].lower().startswith("file:") # type:ignore[union-attr]
1241 )
1242 )
1243 ),
1244 )
1245 tempnode = WikiNode(NodeKind.LEVEL6, 0)
1246 tempnode.largs = [["Inflection"]]
1247 tempnode.children = floaters
1248 parse_inflection(tempnode, "Floating Div", pos)
1249 # print(poschildren)
1250 # XXX new above
1252 if not poschildren: 1252 ↛ 1253line 1252 didn't jump to line 1253 because the condition on line 1252 was never true
1253 if not floaters:
1254 wxr.wtp.debug(
1255 "PoS section without contents",
1256 sortid="en/page/1051/20230612",
1257 )
1258 else:
1259 wxr.wtp.debug(
1260 "PoS section without contents except for a floating table",
1261 sortid="en/page/1056/20230612",
1262 )
1263 return
1265 for node in poschildren:
1266 if isinstance(node, str):
1267 for m in re.finditer(r"\n+|[^\n]+", node):
1268 p = m.group(0)
1269 if p.startswith("\n\n") and pre:
1270 first_para = False
1271 start_of_paragraph = True
1272 break
1273 if p and collecting_head:
1274 pre[-1].append(p)
1275 continue
1276 assert isinstance(node, WikiNode)
1277 kind = node.kind
1278 if kind == NodeKind.LIST:
1279 lists[-1].append(node)
1280 collecting_head = False
1281 start_of_paragraph = True
1282 continue
1283 elif kind in LEVEL_KINDS:
1284 # Stop parsing section if encountering any kind of
1285 # level header (like ===Noun=== or ====Further Reading====).
1286 # At a quick glance, this should be the default behavior,
1287 # but if some kinds of source articles have sub-sub-sections
1288 # that should be parsed XXX it should be handled by changing
1289 # this break.
1290 break
1291 elif collecting_head and kind == NodeKind.LINK:
1292 # We might collect relevant links as they are often pictures
1293 # relating to the word
1294 if len(node.largs[0]) >= 1 and isinstance( 1294 ↛ 1309line 1294 didn't jump to line 1309 because the condition on line 1294 was always true
1295 node.largs[0][0], str
1296 ):
1297 if node.largs[0][0].startswith( 1297 ↛ 1303line 1297 didn't jump to line 1303 because the condition on line 1297 was never true
1298 ns_title_prefix_tuple(wxr, "Category")
1299 ):
1300 # [[Category:...]]
1301 # We're at the end of the file, probably, so stop
1302 # here. Otherwise the head will get garbage.
1303 break
1304 if node.largs[0][0].startswith(
1305 ns_title_prefix_tuple(wxr, "File")
1306 ):
1307 # Skips file links
1308 continue
1309 start_of_paragraph = False
1310 pre[-1].append(node)
1311 elif kind == NodeKind.HTML:
1312 if node.sarg == "br":
1313 if pre[-1]: 1313 ↛ 1265line 1313 didn't jump to line 1265 because the condition on line 1313 was always true
1314 pre.append([]) # Switch to next head
1315 lists.append([]) # Lists parallels pre
1316 collecting_head = True
1317 start_of_paragraph = True
1318 elif collecting_head and node.sarg not in ( 1318 ↛ 1324line 1318 didn't jump to line 1324 because the condition on line 1318 was never true
1319 "gallery",
1320 "ref",
1321 "cite",
1322 "caption",
1323 ):
1324 start_of_paragraph = False
1325 pre[-1].append(node)
1326 else:
1327 start_of_paragraph = False
1328 elif isinstance(node, TemplateNode):
1329 # XXX Insert code here that disambiguates between
1330 # templates that generate word heads and templates
1331 # that don't.
1332 # There's head_tag_re that seems like a regex meant
1333 # to identify head templates. Too bad it's None.
1335 # ignore {{category}}, {{cat}}... etc.
1336 if node.template_name in stop_head_at_these_templates:
1337 # we've reached a template that should be at the end,
1338 continue
1340 # skip these templates; panel_templates is already used
1341 # to skip certain templates else, but it also applies to
1342 # head parsing quite well.
1343 # node.largs[0][0] should always be str, but can't type-check
1344 # that.
1345 if is_panel_template(wxr, node.template_name):
1346 continue
1347 # skip these templates
1348 # if node.largs[0][0] in skip_these_templates_in_head:
1349 # first_head_tmplt = False # no first_head_tmplt at all
1350 # start_of_paragraph = False
1351 # continue
1353 if first_head_tmplt and pre[-1]:
1354 first_head_tmplt = False
1355 start_of_paragraph = False
1356 pre[-1].append(node)
1357 elif pre[-1] and start_of_paragraph:
1358 pre.append([]) # Switch to the next head
1359 lists.append([]) # lists parallel pre
1360 collecting_head = True
1361 start_of_paragraph = False
1362 pre[-1].append(node)
1363 else:
1364 pre[-1].append(node)
1365 elif first_para:
1366 start_of_paragraph = False
1367 if collecting_head: 1367 ↛ 1265line 1367 didn't jump to line 1265 because the condition on line 1367 was always true
1368 pre[-1].append(node)
1369 # XXX use template_fn in clean_node to check that the head macro
1370 # is compatible with the current part-of-speech and generate warning
1371 # if not. Use template_allowed_pos_map.
1373 # Clean up empty pairs, and fix messes with extra newlines that
1374 # separate templates that are followed by lists wiktextract issue #314
1376 cleaned_pre: list[list[Union[str, WikiNode]]] = []
1377 cleaned_lists: list[list[WikiNode]] = []
1378 pairless_pre_index = None
1380 for pre1, ls in zip(pre, lists):
1381 if pre1 and not ls:
1382 pairless_pre_index = len(cleaned_pre)
1383 if not pre1 and not ls: 1383 ↛ 1385line 1383 didn't jump to line 1385 because the condition on line 1383 was never true
1384 # skip [] + []
1385 continue
1386 if not ls and all(
1387 (isinstance(x, str) and not x.strip()) for x in pre1
1388 ):
1389 # skip ["\n", " "] + []
1390 continue
1391 if ls and not pre1:
1392 if pairless_pre_index is not None: 1392 ↛ 1393line 1392 didn't jump to line 1393 because the condition on line 1392 was never true
1393 cleaned_lists[pairless_pre_index] = ls
1394 pairless_pre_index = None
1395 continue
1396 cleaned_pre.append(pre1)
1397 cleaned_lists.append(ls)
1399 pre = cleaned_pre
1400 lists = cleaned_lists
1402 there_are_many_heads = len(pre) > 1
1403 header_tags: list[str] = []
1404 header_topics: list[str] = []
1405 previous_head_had_list = False
1407 if not any(g for g in lists):
1408 process_gloss_without_list(
1409 poschildren, pos, pos_data, header_tags, header_topics
1410 )
1411 else:
1412 for i, (pre1, ls) in enumerate(zip(pre, lists)):
1413 # if len(ls) == 0:
1414 # # don't have gloss list
1415 # # XXX add code here to filter out 'garbage', like text
1416 # # that isn't a head template or head.
1417 # continue
1419 if all(not sl for sl in lists[i:]):
1420 if i == 0: 1420 ↛ 1421line 1420 didn't jump to line 1421 because the condition on line 1420 was never true
1421 if isinstance(node, str):
1422 wxr.wtp.debug(
1423 "first head without list of senses,"
1424 "string: '{}[...]', {}/{}".format(
1425 node[:20], word, language
1426 ),
1427 sortid="page/1689/20221215",
1428 )
1429 if isinstance(node, WikiNode):
1430 if node.largs and node.largs[0][0] in [
1431 "Han char",
1432 ]:
1433 # just ignore these templates
1434 pass
1435 else:
1436 wxr.wtp.debug(
1437 "first head without "
1438 "list of senses, "
1439 "template node "
1440 "{}, {}/{}".format(
1441 node.largs, word, language
1442 ),
1443 sortid="page/1694/20221215",
1444 )
1445 else:
1446 wxr.wtp.debug(
1447 "first head without list of senses, "
1448 "{}/{}".format(word, language),
1449 sortid="page/1700/20221215",
1450 )
1451 # no break here so that the first head always
1452 # gets processed.
1453 else:
1454 if isinstance(node, str): 1454 ↛ 1455line 1454 didn't jump to line 1455 because the condition on line 1454 was never true
1455 wxr.wtp.debug(
1456 "later head without list of senses,"
1457 "string: '{}[...]', {}/{}".format(
1458 node[:20], word, language
1459 ),
1460 sortid="page/1708/20221215",
1461 )
1462 if isinstance(node, WikiNode): 1462 ↛ 1474line 1462 didn't jump to line 1474 because the condition on line 1462 was always true
1463 wxr.wtp.debug(
1464 "later head without list of senses,"
1465 "template node "
1466 "{}, {}/{}".format(
1467 node.sarg if node.sarg else node.largs,
1468 word,
1469 language,
1470 ),
1471 sortid="page/1713/20221215",
1472 )
1473 else:
1474 wxr.wtp.debug(
1475 "later head without list of senses, "
1476 "{}/{}".format(word, language),
1477 sortid="page/1719/20221215",
1478 )
1479 break
1480 head_group = i + 1 if there_are_many_heads else None
1481 # print("parse_part_of_speech: {}: {}: pre={}"
1482 # .format(wxr.wtp.section, wxr.wtp.subsection, pre1))
1484 if previous_head_had_list:
1485 # We use a boolean flag here because we want to be able
1486 # let the header_tags data pass through after the loop
1487 # is over without accidentally emptying it, if there are
1488 # no pos_datas and we need a dummy data.
1489 header_tags.clear()
1490 header_topics.clear()
1492 # print(f"{pre1=}")
1493 process_gloss_header(
1494 pre1, pos, head_group, pos_data, header_tags, header_topics
1495 )
1496 for ln in ls:
1497 # Parse each list associated with this head.
1498 for node in ln.children:
1499 # Parse nodes in l.children recursively.
1500 # The recursion function uses push_sense() to
1501 # add stuff into sense_datas, and returns True or
1502 # False if something is added, which bubbles upward.
1503 # If the bubble is "True", then higher levels of
1504 # the recursion will not push_sense(), because
1505 # the data is already pushed into a sub-gloss
1506 # downstream, unless the higher level has examples
1507 # that need to be put somewhere.
1508 common_data: SenseData = {
1509 "tags": list(header_tags),
1510 "topics": list(header_topics),
1511 }
1512 if head_group:
1513 common_data["head_nr"] = head_group
1514 parse_sense_node(node, common_data, pos) # type: ignore[arg-type]
1516 if len(ls) > 0:
1517 previous_head_had_list = True
1518 else:
1519 previous_head_had_list = False
1521 # If there are no senses extracted, add a dummy sense. We want to
1522 # keep tags extracted from the head for the dummy sense.
1523 push_sense() # Make sure unfinished data pushed, and start clean sense
1524 if len(sense_datas) == 0:
1525 data_extend(sense_data, "tags", header_tags)
1526 data_extend(sense_data, "topics", header_topics)
1527 data_append(sense_data, "tags", "no-gloss")
1528 push_sense()
1530 sense_datas.sort(key=lambda x: x.get("__temp_sense_sorting_ordinal", 0)) # type: ignore
1532 for sd in sense_datas:
1533 if "__temp_sense_sorting_ordinal" in sd: 1533 ↛ 1532line 1533 didn't jump to line 1532 because the condition on line 1533 was always true
1534 del sd["__temp_sense_sorting_ordinal"] # type: ignore
1536 term_label_templates: list[TemplateData] = []
1537 normal_label_templates: list[TemplateData] = []
1539 def head_post_template_fn(
1540 name: str, ht: TemplateArgs, expansion: str
1541 ) -> Optional[str]:
1542 """Handles special templates in the head section of a word. Head
1543 section is the text after part-of-speech subtitle and before word
1544 sense list. Typically it generates the bold line for the word, but
1545 may also contain other useful information that often ends in
1546 side boxes. We want to capture some of that additional information."""
1547 # print("HEAD_POST_TEMPLATE_FN", name, ht)
1548 if is_panel_template(wxr, name): 1548 ↛ 1551line 1548 didn't jump to line 1551 because the condition on line 1548 was never true
1549 # Completely ignore these templates (not even recorded in
1550 # head_templates)
1551 return ""
1552 if name == "head":
1553 # XXX are these also captured in forms? Should this special case
1554 # be removed?
1555 t = ht.get(2, "")
1556 if t == "pinyin": 1556 ↛ 1557line 1556 didn't jump to line 1557 because the condition on line 1556 was never true
1557 data_append(pos_data, "tags", "Pinyin")
1558 elif t == "romanization": 1558 ↛ 1559line 1558 didn't jump to line 1559 because the condition on line 1558 was never true
1559 data_append(pos_data, "tags", "romanization")
1560 if (
1561 HEAD_TAG_RE.search(name) is not None
1562 or name in PROBLEMATIC_TEMPLATES_CLUMP
1563 ):
1564 args_ht = clean_template_args(wxr, ht)
1565 cleaned_expansion = clean_node(wxr, None, expansion)
1566 dt: TemplateData = {
1567 "name": name,
1568 "args": args_ht,
1569 "expansion": cleaned_expansion,
1570 }
1571 if name in ETYMOLOGY_TEMPLATES_IN_HEADS:
1572 data_append(pos_data, "etymology_templates", dt)
1573 else:
1574 data_append(pos_data, "head_templates", dt)
1575 if name in WORD_LEVEL_HEAD_TEMPLATES:
1576 term_label_templates.append(dt)
1577 # Squash these, their tags are applied to the whole word,
1578 # and some cause problems like "term-label"
1579 return ""
1581 # The following are both captured in head_templates and parsed
1582 # separately
1584 if name in wikipedia_templates:
1585 # Note: various places expect to have content from wikipedia
1586 # templates, so cannot convert this to empty
1587 parse_wikipedia_template(wxr, pos_data, ht)
1588 return None
1590 if name == "number box": 1590 ↛ 1592line 1590 didn't jump to line 1592 because the condition on line 1590 was never true
1591 # XXX extract numeric value?
1592 return ""
1593 if name == "enum":
1594 # XXX extract?
1595 return ""
1596 if name == "cardinalbox": 1596 ↛ 1599line 1596 didn't jump to line 1599 because the condition on line 1596 was never true
1597 # XXX extract similar to enum?
1598 # XXX this can also occur in top-level under language
1599 return ""
1600 if name == "Han simplified forms": 1600 ↛ 1602line 1600 didn't jump to line 1602 because the condition on line 1600 was never true
1601 # XXX extract?
1602 return ""
1603 # if name == "ja-kanji forms":
1604 # # XXX extract?
1605 # return ""
1606 # if name == "vi-readings":
1607 # # XXX extract?
1608 # return ""
1609 # if name == "ja-kanji":
1610 # # XXX extract?
1611 # return ""
1612 if name == "picdic" or name == "picdicimg" or name == "picdiclabel": 1612 ↛ 1614line 1612 didn't jump to line 1614 because the condition on line 1612 was never true
1613 # XXX extract?
1614 return ""
1615 if name == "defdate": 1615 ↛ 1617line 1615 didn't jump to line 1617 because the condition on line 1615 was never true
1616 # the one exampe I saw of this in a head was weird.
1617 return ""
1618 if name in ("lb", "lbl", "label"):
1619 args_ht = clean_template_args(wxr, ht)
1620 cleaned_expansion = clean_node(wxr, None, expansion).strip("()")
1621 dt = {
1622 "name": name,
1623 "args": args_ht,
1624 "expansion": cleaned_expansion,
1625 }
1626 normal_label_templates.append(dt)
1627 # The parens around __LABEL... below is meaningful: label
1628 # templates generate text with parens, so if we add the magical
1629 # phrase here with parens, it will look like a normal label that
1630 # will be handled as a parenthetical text; only when handling
1631 # parenthetical text do we need to actually actually access
1632 # the contents of the label.
1633 return f"(__LABEL_TEMPLATE_{len(normal_label_templates) - 1}__)"
1635 return None
1637 def process_gloss_header(
1638 header_nodes: list[Union[WikiNode, str]],
1639 pos_type: str,
1640 header_group: Optional[int],
1641 pos_data: WordData,
1642 header_tags: list[str],
1643 header_topics: list[str],
1644 ) -> None:
1645 ruby = []
1647 # process template parse nodes here
1648 new_nodes = []
1649 info_template_data = []
1650 for node in header_nodes:
1651 # print(f"{node=}")
1652 info_data, info_out = parse_info_template_node(wxr, node, "head")
1653 if info_data or info_out:
1654 if info_data: 1654 ↛ 1656line 1654 didn't jump to line 1656 because the condition on line 1654 was always true
1655 info_template_data.append(info_data)
1656 if info_out: # including just the original node 1656 ↛ 1657line 1656 didn't jump to line 1657 because the condition on line 1656 was never true
1657 new_nodes.append(info_out)
1658 else:
1659 new_nodes.append(node)
1660 header_nodes = new_nodes
1662 if info_template_data:
1663 if "info_templates" not in pos_data: 1663 ↛ 1666line 1663 didn't jump to line 1666 because the condition on line 1663 was always true
1664 pos_data["info_templates"] = info_template_data
1665 else:
1666 pos_data["info_templates"].extend(info_template_data)
1668 if lang_code == "ja":
1669 exp = wxr.wtp.parse(
1670 wxr.wtp.node_to_wikitext(header_nodes), expand_all=True
1671 )
1672 rub, _ = recursively_extract(
1673 exp.children,
1674 lambda x: (
1675 isinstance(x, WikiNode)
1676 and x.kind == NodeKind.HTML
1677 and x.sarg == "ruby"
1678 ),
1679 )
1680 if rub is not None: 1680 ↛ 1724line 1680 didn't jump to line 1724 because the condition on line 1680 was always true
1681 for r in rub:
1682 if TYPE_CHECKING:
1683 # we know the lambda above in recursively_extract
1684 # returns only WikiNodes in rub
1685 assert isinstance(r, WikiNode)
1686 rt = parse_ruby(wxr, r)
1687 if rt is not None: 1687 ↛ 1681line 1687 didn't jump to line 1681 because the condition on line 1687 was always true
1688 ruby.append(rt)
1689 elif lang_code == "vi":
1690 # Handle vi-readings templates that have a weird structures for
1691 # Chu Nom vietnamese characters heads
1692 # https://en.wiktionary.org/wiki/Template:vi-readings
1693 new_header_nodes = []
1694 related_readings: list[LinkageData] = []
1695 for node in header_nodes:
1696 if ( 1696 ↛ 1719line 1696 didn't jump to line 1719 because the condition on line 1696 was always true
1697 isinstance(node, TemplateNode)
1698 and node.template_name == "vi-readings"
1699 ):
1700 for parameter, tag in (
1701 ("hanviet", "han-viet-reading"),
1702 ("nom", "nom-reading"),
1703 # we ignore the fanqie parameter "phienthiet"
1704 ):
1705 arg = node.template_parameters.get(parameter)
1706 if arg is not None: 1706 ↛ 1700line 1706 didn't jump to line 1700 because the condition on line 1706 was always true
1707 text = clean_node(wxr, None, arg)
1708 for w in text.split(","):
1709 # ignore - separated references
1710 if "-" in w:
1711 w = w[: w.index("-")]
1712 w = w.strip()
1713 related_readings.append(
1714 LinkageData(word=w, tags=[tag])
1715 )
1716 continue
1718 # Skip the vi-reading template for the rest of the head parsing
1719 new_header_nodes.append(node)
1720 if len(related_readings) > 0: 1720 ↛ 1724line 1720 didn't jump to line 1724 because the condition on line 1720 was always true
1721 data_extend(pos_data, "related", related_readings)
1722 header_nodes = new_header_nodes
1724 header_text = clean_node(
1725 wxr,
1726 pos_data,
1727 header_nodes,
1728 post_template_fn=head_post_template_fn,
1729 collect_links=True,
1730 remove_anchors_from_links=True,
1731 )
1732 if "links" in pos_data:
1733 # WordData doesn't use `links`, so we can use `collect_links=True`
1734 # above without special handling and smuggle link data.
1735 extracted_links = pos_data["links"] # type: ignore
1736 del pos_data["links"] # type: ignore
1737 else:
1738 extracted_links = None
1739 # print(f"{header_text=}, {extracted_links=}")
1741 header_text = re.sub(r"\s+", " ", header_text).strip()
1743 if not header_text:
1744 return
1746 term_label_tags: list[str] = []
1747 term_label_topics: list[str] = []
1748 if len(term_label_templates) > 0:
1749 # parse term label templates; if there are other similar kinds
1750 # of templates in headers that you want to squash and apply as
1751 # tags, you can add them to WORD_LEVEL_HEAD_TEMPLATES
1752 for templ_data in term_label_templates:
1753 # print(templ_data)
1754 expan = templ_data.get("expansion", "").strip("().,; ")
1755 if not expan: 1755 ↛ 1756line 1755 didn't jump to line 1756 because the condition on line 1755 was never true
1756 continue
1757 tlb_tagsets, tlb_topics = decode_tags(expan)
1758 for tlb_tags in tlb_tagsets:
1759 if len(tlb_tags) > 0 and not any(
1760 t.startswith("error-") for t in tlb_tags
1761 ):
1762 term_label_tags.extend(tlb_tags)
1763 term_label_topics.extend(tlb_topics)
1764 # print(f"{tlb_tagsets=}, {tlb_topicsets=}")
1766 # print(f"{header_text=}")
1767 parse_word_head(
1768 wxr,
1769 word,
1770 pos_type,
1771 header_text,
1772 pos_data,
1773 is_reconstruction,
1774 header_group,
1775 header_nodes,
1776 ruby=ruby,
1777 links=extracted_links,
1778 label_templates=normal_label_templates,
1779 )
1780 if "tags" in pos_data:
1781 # pos_data can get "tags" data from some source; type-checkers
1782 # doesn't like it, so let's ignore it.
1783 header_tags.extend(pos_data["tags"]) # type: ignore[typeddict-item]
1784 del pos_data["tags"] # type: ignore[typeddict-item]
1785 if len(term_label_tags) > 0:
1786 header_tags.extend(term_label_tags)
1787 if len(term_label_topics) > 0:
1788 header_topics.extend(term_label_topics)
1790 def process_gloss_without_list(
1791 nodes: list[Union[WikiNode, str]],
1792 pos_type: str,
1793 pos_data: WordData,
1794 header_tags: list[str],
1795 header_topics: list[str],
1796 ) -> None:
1797 # gloss text might not inside a list
1798 header_nodes: list[Union[str, WikiNode]] = []
1799 gloss_nodes: list[Union[str, WikiNode]] = []
1800 for node in strip_nodes(nodes):
1801 if isinstance(node, WikiNode):
1802 if isinstance(node, TemplateNode):
1803 if node.template_name in (
1804 "zh-see",
1805 "ja-see",
1806 "ja-see-kango",
1807 ):
1808 continue # soft redirect
1809 elif (
1810 node.template_name == "head"
1811 or node.template_name.startswith(f"{lang_code}-")
1812 ):
1813 header_nodes.append(node)
1814 continue
1815 elif node.kind in LEVEL_KINDS: # following nodes are not gloss 1815 ↛ 1817line 1815 didn't jump to line 1817 because the condition on line 1815 was always true
1816 break
1817 gloss_nodes.append(node)
1819 if len(header_nodes) > 0:
1820 process_gloss_header(
1821 header_nodes,
1822 pos_type,
1823 None,
1824 pos_data,
1825 header_tags,
1826 header_topics,
1827 )
1828 if len(gloss_nodes) > 0:
1829 process_gloss_contents(
1830 gloss_nodes,
1831 pos_type,
1832 {"tags": list(header_tags), "topics": list(header_topics)},
1833 )
1835 def parse_sense_node(
1836 node: Union[str, WikiNode], # never receives str
1837 sense_base: SenseData,
1838 pos: str,
1839 ) -> bool:
1840 """Recursively (depth first) parse LIST_ITEM nodes for sense data.
1841 Uses push_sense() to attempt adding data to pos_data in the scope
1842 of parse_language() when it reaches deep in the recursion. push_sense()
1843 returns True if it succeeds, and that is bubbled up the stack; if
1844 a sense was added downstream, the higher levels (whose shared data
1845 was already added by a subsense) do not push_sense(), unless it
1846 has examples that need to be put somewhere.
1847 """
1848 assert isinstance(sense_base, dict) # Added to every sense deeper in
1850 nonlocal sense_ordinal
1851 my_ordinal = sense_ordinal # copies, not a reference
1852 sense_ordinal += 1 # only use for sorting
1854 if not isinstance(node, WikiNode): 1854 ↛ 1856line 1854 didn't jump to line 1856 because the condition on line 1854 was never true
1855 # This doesn't seem to ever happen in practice.
1856 wxr.wtp.debug(
1857 "{}: parse_sense_node called with"
1858 "something that isn't a WikiNode".format(pos),
1859 sortid="page/1287/20230119",
1860 )
1861 return False
1863 if node.kind != NodeKind.LIST_ITEM: 1863 ↛ 1864line 1863 didn't jump to line 1864 because the condition on line 1863 was never true
1864 wxr.wtp.debug(
1865 "{}: non-list-item inside list".format(pos), sortid="page/1678"
1866 )
1867 return False
1869 if node.sarg == ":":
1870 # Skip example entries at the highest level, ones without
1871 # a sense ("...#") above them.
1872 # If node.sarg is exactly and only ":", then it's at
1873 # the highest level; lower levels would have more
1874 # "indentation", like "#:" or "##:"
1875 return False
1877 # If a recursion call succeeds in push_sense(), bubble it up with
1878 # `added`.
1879 # added |= push_sense() or added |= parse_sense_node(...) to OR.
1880 added = False
1882 gloss_template_args: set[str] = set()
1884 # For LISTs and LIST_ITEMS, their argument is something like
1885 # "##" or "##:", and using that we can rudimentally determine
1886 # list 'depth' if need be, and also what kind of list or
1887 # entry it is; # is for normal glosses, : for examples (indent)
1888 # and * is used for quotations on wiktionary.
1889 current_depth = node.sarg
1891 children = node.children
1893 # subentries, (presumably) a list
1894 # of subglosses below this. The list's
1895 # argument ends with #, and its depth should
1896 # be bigger than parent node.
1897 subentries = [
1898 x
1899 for x in children
1900 if isinstance(x, WikiNode)
1901 and x.kind == NodeKind.LIST
1902 and x.sarg == current_depth + "#"
1903 ]
1905 # sublists of examples and quotations. .sarg
1906 # does not end with "#".
1907 others = [
1908 x
1909 for x in children
1910 if isinstance(x, WikiNode)
1911 and x.kind == NodeKind.LIST
1912 and x.sarg != current_depth + "#"
1913 ]
1915 # the actual contents of this particular node.
1916 # can be a gloss (or a template that expands into
1917 # many glosses which we can't easily pre-expand)
1918 # or could be an "outer gloss" with more specific
1919 # subglosses, or could be a qualfier for the subglosses.
1920 contents = [
1921 x
1922 for x in children
1923 if not isinstance(x, WikiNode) or x.kind != NodeKind.LIST
1924 ]
1925 # If this entry has sublists of entries, we should combine
1926 # gloss information from both the "outer" and sublist content.
1927 # Sometimes the outer gloss
1928 # is more non-gloss or tags, sometimes it is a coarse sense
1929 # and the inner glosses are more specific. The outer one
1930 # does not seem to have qualifiers.
1932 # If we have one sublist with one element, treat it
1933 # specially as it may be a Wiktionary error; raise
1934 # that nested element to the same level.
1935 # XXX If need be, this block can be easily removed in
1936 # the current recursive logicand the result is one sense entry
1937 # with both glosses in the glosses list, as you would
1938 # expect. If the higher entry has examples, there will
1939 # be a higher entry with some duplicated data.
1940 if len(subentries) == 1:
1941 slc = subentries[0].children
1942 if len(slc) == 1:
1943 # copy current node and modify it so it doesn't
1944 # loop infinitely.
1945 cropped_node = copy.copy(node)
1946 cropped_node.children = [
1947 x
1948 for x in children
1949 if not (
1950 isinstance(x, WikiNode)
1951 and x.kind == NodeKind.LIST
1952 and x.sarg == current_depth + "#"
1953 )
1954 ]
1955 added |= parse_sense_node(cropped_node, sense_base, pos)
1956 nonlocal sense_data # this kludge causes duplicated raw_
1957 # glosses data if this is not done;
1958 # if the top-level (cropped_node)
1959 # does not push_sense() properly or
1960 # parse_sense_node() returns early,
1961 # sense_data is not reset. This happens
1962 # for example when you have a no-gloss
1963 # string like "(intransitive)":
1964 # no gloss, push_sense() returns early
1965 # and sense_data has duplicate data with
1966 # sense_base
1967 sense_data = {}
1968 added |= parse_sense_node(slc[0], sense_base, pos)
1969 return added
1971 return process_gloss_contents(
1972 contents,
1973 pos,
1974 sense_base,
1975 subentries,
1976 others,
1977 gloss_template_args,
1978 added,
1979 my_ordinal,
1980 )
1982 def process_gloss_contents(
1983 contents: list[Union[str, WikiNode]],
1984 pos: str,
1985 sense_base: SenseData,
1986 subentries: list[WikiNode] = [],
1987 others: list[WikiNode] = [],
1988 gloss_template_args: Set[str] = set(),
1989 added: bool = False,
1990 sorting_ordinal: int | None = None,
1991 ) -> bool:
1992 def sense_template_fn(
1993 name: str, ht: TemplateArgs, is_gloss: bool = False
1994 ) -> Optional[str]:
1995 # print(f"sense_template_fn: {name}, {ht}")
1996 if name in wikipedia_templates:
1997 # parse_wikipedia_template(wxr, pos_data, ht)
1998 return None
1999 if is_panel_template(wxr, name):
2000 return ""
2001 if name in INFO_TEMPLATE_FUNCS:
2002 info_data, info_exp = parse_info_template_arguments(
2003 wxr, name, ht, "sense"
2004 )
2005 if info_data or info_exp: 2005 ↛ 2011line 2005 didn't jump to line 2011 because the condition on line 2005 was always true
2006 if info_data: 2006 ↛ 2008line 2006 didn't jump to line 2008 because the condition on line 2006 was always true
2007 data_append(sense_base, "info_templates", info_data)
2008 if info_exp and isinstance(info_exp, str): 2008 ↛ 2010line 2008 didn't jump to line 2010 because the condition on line 2008 was always true
2009 return info_exp
2010 return ""
2011 if name in ("defdate",):
2012 date = clean_node(wxr, None, ht.get(1, ()))
2013 if part_two := ht.get(2): 2013 ↛ 2015line 2013 didn't jump to line 2015 because the condition on line 2013 was never true
2014 # Unicode mdash, not '-'
2015 date += "–" + clean_node(wxr, None, part_two)
2016 refs: dict[str, ReferenceData] = {}
2017 # ref, refn, ref2, ref2n, ref3, ref3n
2018 # ref1 not valid
2019 for k, v in sorted(
2020 (k, v) for k, v in ht.items() if isinstance(k, str)
2021 ):
2022 if m := re.match(r"ref(\d?)(n?)", k): 2022 ↛ 2019line 2022 didn't jump to line 2019 because the condition on line 2022 was always true
2023 ref_v = clean_node(wxr, None, v)
2024 if m.group(1) not in refs: # empty string or digit
2025 refs[m.group(1)] = ReferenceData()
2026 if m.group(2):
2027 refs[m.group(1)]["refn"] = ref_v
2028 else:
2029 refs[m.group(1)]["text"] = ref_v
2030 data_append(
2031 sense_base,
2032 "attestations",
2033 AttestationData(date=date, references=list(refs.values())),
2034 )
2035 return ""
2036 if name == "senseid":
2037 langid = clean_node(wxr, None, ht.get(1, ()))
2038 arg = clean_node(wxr, sense_base, ht.get(2, ()))
2039 if re.match(r"Q\d+$", arg):
2040 data_append(sense_base, "wikidata", arg)
2041 data_append(sense_base, "senseid", langid + ":" + arg)
2042 if name in sense_linkage_templates:
2043 # print(f"SENSE_TEMPLATE_FN: {name}")
2044 parse_sense_linkage(wxr, sense_base, name, ht, pos)
2045 return ""
2046 if name == "†" or name == "zh-obsolete":
2047 data_append(sense_base, "tags", "obsolete")
2048 return ""
2049 if name in {
2050 "ux",
2051 "uxi",
2052 "usex",
2053 "afex",
2054 "prefixusex",
2055 "ko-usex",
2056 "ko-x",
2057 "hi-x",
2058 "ja-usex-inline",
2059 "ja-x",
2060 "quotei",
2061 "he-x",
2062 "hi-x",
2063 "km-x",
2064 "ne-x",
2065 "shn-x",
2066 "th-x",
2067 "ur-x",
2068 }:
2069 # Usage examples are captured separately below. We don't
2070 # want to expand them into glosses even when unusual coding
2071 # is used in the entry.
2072 # These templates may slip through inside another item, but
2073 # currently we're separating out example entries (..#:)
2074 # well enough that there seems to very little contamination.
2075 if is_gloss:
2076 wxr.wtp.wiki_notice(
2077 "Example template is used for gloss text",
2078 sortid="extractor.en.page.sense_template_fn/1415",
2079 )
2080 else:
2081 return ""
2082 if name == "w": 2082 ↛ 2083line 2082 didn't jump to line 2083 because the condition on line 2082 was never true
2083 if ht.get(2) == "Wp":
2084 return ""
2085 for v in ht.values():
2086 v = v.strip()
2087 if v and "<" not in v:
2088 gloss_template_args.add(v)
2089 return None
2091 def extract_link_texts(item: GeneralNode) -> None:
2092 """Recursively extracts link texts from the gloss source. This
2093 information is used to select whether to remove final "." from
2094 form_of/alt_of (e.g., ihm/Hunsrik)."""
2095 if isinstance(item, (list, tuple)):
2096 for x in item:
2097 extract_link_texts(x)
2098 return
2099 if isinstance(item, str):
2100 # There seem to be HTML sections that may futher contain
2101 # unparsed links.
2102 for m in re.finditer(r"\[\[([^]]*)\]\]", item): 2102 ↛ 2103line 2102 didn't jump to line 2103 because the loop on line 2102 never started
2103 print("ITER:", m.group(0))
2104 v = m.group(1).split("|")[-1].strip()
2105 if v:
2106 gloss_template_args.add(v)
2107 return
2108 if not isinstance(item, WikiNode): 2108 ↛ 2109line 2108 didn't jump to line 2109 because the condition on line 2108 was never true
2109 return
2110 if item.kind == NodeKind.LINK:
2111 v = item.largs[-1]
2112 if ( 2112 ↛ 2118line 2112 didn't jump to line 2118 because the condition on line 2112 was always true
2113 isinstance(v, list)
2114 and len(v) == 1
2115 and isinstance(v[0], str)
2116 ):
2117 gloss_template_args.add(v[0].strip())
2118 for x in item.children:
2119 extract_link_texts(x)
2121 extract_link_texts(contents)
2123 # get the raw text of non-list contents of this node, and other stuff
2124 # like tag and category data added to sense_base
2125 # cast = no-op type-setter for the type-checker
2126 partial_template_fn = cast(
2127 TemplateFnCallable,
2128 partial(sense_template_fn, is_gloss=True),
2129 )
2130 rawgloss = clean_node(
2131 wxr,
2132 sense_base,
2133 contents,
2134 template_fn=partial_template_fn,
2135 collect_links=True,
2136 )
2138 if not rawgloss: 2138 ↛ 2139line 2138 didn't jump to line 2139 because the condition on line 2138 was never true
2139 return False
2141 # remove manually typed ordered list text at the start("1. ")
2142 rawgloss = re.sub(r"^\d+\.\s+", "", rawgloss).strip()
2144 # get stuff like synonyms and categories from "others",
2145 # maybe examples and quotations
2146 clean_node(wxr, sense_base, others, template_fn=sense_template_fn)
2148 # The gloss could contain templates that produce more list items.
2149 # This happens commonly with, e.g., {{inflection of|...}}. Split
2150 # to parts. However, e.g. Interlingua generates multiple glosses
2151 # in HTML directly without Wikitext markup, so we must also split
2152 # by just newlines.
2153 subglosses = rawgloss.splitlines()
2155 if len(subglosses) == 0: 2155 ↛ 2156line 2155 didn't jump to line 2156 because the condition on line 2155 was never true
2156 return False
2158 if any(s.startswith("#") for s in subglosses):
2159 subtree = wxr.wtp.parse(rawgloss)
2160 # from wikitextprocessor.parser import print_tree
2161 # print("SUBTREE GENERATED BY TEMPLATE:")
2162 # print_tree(subtree)
2163 new_subentries = [
2164 x
2165 for x in subtree.children
2166 if isinstance(x, WikiNode) and x.kind == NodeKind.LIST
2167 ]
2169 new_others = [
2170 x
2171 for x in subtree.children
2172 if isinstance(x, WikiNode)
2173 and x.kind == NodeKind.LIST
2174 and not x.sarg.endswith("#")
2175 ]
2177 new_contents = [
2178 clean_node(wxr, [], x)
2179 for x in subtree.children
2180 if not isinstance(x, WikiNode) or x.kind != NodeKind.LIST
2181 ]
2183 subentries = subentries or new_subentries
2184 others = others or new_others
2185 subglosses = new_contents
2186 rawgloss = "".join(subglosses)
2187 # Generate no gloss for translation hub pages, but add the
2188 # "translation-hub" tag for them
2189 if rawgloss == "(This entry is a translation hub.)": 2189 ↛ 2190line 2189 didn't jump to line 2190 because the condition on line 2189 was never true
2190 data_append(sense_data, "tags", "translation-hub")
2191 return push_sense(sorting_ordinal)
2193 # Remove certain substrings specific to outer glosses
2194 strip_ends = [", particularly:"]
2195 for x in strip_ends:
2196 if rawgloss.endswith(x):
2197 rawgloss = rawgloss[: -len(x)].strip()
2198 break
2200 # A single gloss, or possibly an outer gloss.
2201 # Check if the possible outer gloss starts with
2202 # parenthesized tags/topics
2204 if rawgloss and rawgloss not in sense_base.get("raw_glosses", ()):
2205 data_append(sense_base, "raw_glosses", subglosses[0].strip())
2206 m = QUALIFIERS_RE.match(rawgloss)
2207 # (...): ... or (...(...)...): ...
2208 if m:
2209 q = m.group(1)
2210 rawgloss = rawgloss[m.end() :].strip()
2211 parse_sense_qualifier(wxr, q, sense_base)
2212 if rawgloss == "A pejorative:": 2212 ↛ 2213line 2212 didn't jump to line 2213 because the condition on line 2212 was never true
2213 data_append(sense_base, "tags", "pejorative")
2214 rawgloss = ""
2215 elif rawgloss == "Short forms.": 2215 ↛ 2216line 2215 didn't jump to line 2216 because the condition on line 2215 was never true
2216 data_append(sense_base, "tags", "abbreviation")
2217 rawgloss = ""
2218 elif rawgloss == "Technical or specialized senses.": 2218 ↛ 2219line 2218 didn't jump to line 2219 because the condition on line 2218 was never true
2219 rawgloss = ""
2220 elif rawgloss.startswith("inflection of "):
2221 parsed = parse_alt_or_inflection_of(wxr, rawgloss, set())
2222 if parsed is not None: 2222 ↛ 2231line 2222 didn't jump to line 2231 because the condition on line 2222 was always true
2223 tags, origins = parsed
2224 if origins is not None: 2224 ↛ 2226line 2224 didn't jump to line 2226 because the condition on line 2224 was always true
2225 data_extend(sense_base, "form_of", origins)
2226 if tags is not None: 2226 ↛ 2229line 2226 didn't jump to line 2229 because the condition on line 2226 was always true
2227 data_extend(sense_base, "tags", tags)
2228 else:
2229 data_append(sense_base, "tags", "form-of")
2230 else:
2231 data_append(sense_base, "tags", "form-of")
2232 if rawgloss: 2232 ↛ 2263line 2232 didn't jump to line 2263 because the condition on line 2232 was always true
2233 # Code duplicating a lot of clean-up operations from later in
2234 # this block. We want to clean up the "supergloss" as much as
2235 # possible, in almost the same way as a normal gloss.
2236 supergloss = rawgloss
2238 if supergloss.startswith("; "): 2238 ↛ 2239line 2238 didn't jump to line 2239 because the condition on line 2238 was never true
2239 supergloss = supergloss[1:].strip()
2241 if supergloss.startswith(("^†", "†")):
2242 data_append(sense_base, "tags", "obsolete")
2243 supergloss = supergloss[2:].strip()
2244 elif supergloss.startswith("^‡"): 2244 ↛ 2245line 2244 didn't jump to line 2245 because the condition on line 2244 was never true
2245 data_extend(sense_base, "tags", ["obsolete", "historical"])
2246 supergloss = supergloss[2:].strip()
2248 # remove [14th century...] style brackets at the end
2249 supergloss = re.sub(r"\s\[[^]]*\]\s*$", "", supergloss)
2251 if supergloss.startswith((",", ":")):
2252 supergloss = supergloss[1:]
2253 supergloss = supergloss.strip()
2254 if supergloss.startswith("N. of "): 2254 ↛ 2255line 2254 didn't jump to line 2255 because the condition on line 2254 was never true
2255 supergloss = "Name of " + supergloss[6:]
2256 supergloss = supergloss[2:]
2257 data_append(sense_base, "glosses", supergloss)
2258 if supergloss in ("A person:",):
2259 data_append(sense_base, "tags", "g-person")
2261 # The main recursive call (except for the exceptions at the
2262 # start of this function).
2263 for sublist in subentries:
2264 if not ( 2264 ↛ 2267line 2264 didn't jump to line 2267 because the condition on line 2264 was never true
2265 isinstance(sublist, WikiNode) and sublist.kind == NodeKind.LIST
2266 ):
2267 wxr.wtp.debug(
2268 f"'{repr(rawgloss[:20])}.' gloss has `subentries`"
2269 f"with items that are not LISTs",
2270 sortid="page/1511/20230119",
2271 )
2272 continue
2273 for item in sublist.children:
2274 if not ( 2274 ↛ 2278line 2274 didn't jump to line 2278 because the condition on line 2274 was never true
2275 isinstance(item, WikiNode)
2276 and item.kind == NodeKind.LIST_ITEM
2277 ):
2278 continue
2279 # copy sense_base to prevent cross-contamination between
2280 # subglosses and other subglosses and superglosses
2281 sense_base2 = copy.deepcopy(sense_base)
2282 if parse_sense_node(item, sense_base2, pos): 2282 ↛ 2273line 2282 didn't jump to line 2273 because the condition on line 2282 was always true
2283 added = True
2285 # Capture examples.
2286 # This is called after the recursive calls above so that
2287 # sense_base is not contaminated with meta-data from
2288 # example entries for *this* gloss.
2289 examples = []
2290 if wxr.config.capture_examples: 2290 ↛ 2294line 2290 didn't jump to line 2294 because the condition on line 2290 was always true
2291 examples = extract_examples(others, sense_base)
2293 # push_sense() succeeded somewhere down-river, so skip this level
2294 if added:
2295 if examples:
2296 # this higher-up gloss has examples that we do not want to skip
2297 wxr.wtp.debug(
2298 "'{}[...]' gloss has examples we want to keep, "
2299 "but there are subglosses.".format(repr(rawgloss[:30])),
2300 sortid="page/1498/20230118",
2301 )
2302 else:
2303 return True
2305 # Some entries, e.g., "iacebam", have weird sentences in quotes
2306 # after the gloss, but these sentences don't seem to be intended
2307 # as glosses. Skip them.
2308 indexed_subglosses = list(
2309 (i, gl)
2310 for i, gl in enumerate(subglosses)
2311 if gl.strip() and not re.match(r'\s*(\([^)]*\)\s*)?"[^"]*"\s*$', gl)
2312 )
2314 if len(indexed_subglosses) > 1 and "form_of" not in sense_base: 2314 ↛ 2315line 2314 didn't jump to line 2315 because the condition on line 2314 was never true
2315 gl = indexed_subglosses[0][1].strip()
2316 if gl.endswith(":"):
2317 gl = gl[:-1].strip()
2318 parsed = parse_alt_or_inflection_of(wxr, gl, gloss_template_args)
2319 if parsed is not None:
2320 infl_tags, infl_dts = parsed
2321 if infl_dts and "form-of" in infl_tags and len(infl_tags) == 1:
2322 # Interpret others as a particular form under
2323 # "inflection of"
2324 data_extend(sense_base, "tags", infl_tags)
2325 data_extend(sense_base, "form_of", infl_dts)
2326 indexed_subglosses = indexed_subglosses[1:]
2327 elif not infl_dts:
2328 data_extend(sense_base, "tags", infl_tags)
2329 indexed_subglosses = indexed_subglosses[1:]
2331 # Create senses for remaining subglosses
2332 for i, (gloss_i, gloss) in enumerate(indexed_subglosses):
2333 gloss = gloss.strip()
2334 if not gloss and len(indexed_subglosses) > 1: 2334 ↛ 2335line 2334 didn't jump to line 2335 because the condition on line 2334 was never true
2335 continue
2336 # Push a new sense (if the last one is not empty)
2337 if push_sense(sorting_ordinal): 2337 ↛ 2338line 2337 didn't jump to line 2338 because the condition on line 2337 was never true
2338 added = True
2339 # if gloss not in sense_data.get("raw_glosses", ()):
2340 # data_append(sense_data, "raw_glosses", gloss)
2341 if i == 0 and examples:
2342 # In a multi-line gloss, associate examples
2343 # with only one of them.
2344 # XXX or you could use gloss_i == len(indexed_subglosses)
2345 # to associate examples with the *last* one.
2346 data_extend(sense_data, "examples", examples)
2347 if gloss.startswith("; ") and gloss_i > 0: 2347 ↛ 2348line 2347 didn't jump to line 2348 because the condition on line 2347 was never true
2348 gloss = gloss[1:].strip()
2349 # If the gloss starts with †, mark as obsolete
2350 if gloss.startswith("^†"): 2350 ↛ 2351line 2350 didn't jump to line 2351 because the condition on line 2350 was never true
2351 data_append(sense_data, "tags", "obsolete")
2352 gloss = gloss[2:].strip()
2353 elif gloss.startswith("^‡"): 2353 ↛ 2354line 2353 didn't jump to line 2354 because the condition on line 2353 was never true
2354 data_extend(sense_data, "tags", ["obsolete", "historical"])
2355 gloss = gloss[2:].strip()
2356 # Copy data for all senses to this sense
2357 for k, v in sense_base.items():
2358 if isinstance(v, (list, tuple)):
2359 if k != "tags":
2360 # Tags handled below (countable/uncountable special)
2361 data_extend(sense_data, k, v)
2362 else:
2363 assert k not in ("tags", "categories", "topics")
2364 sense_data[k] = v # type:ignore[literal-required]
2365 # Parse the gloss for this particular sense
2366 m = QUALIFIERS_RE.match(gloss)
2367 # (...): ... or (...(...)...): ...
2368 if m:
2369 parse_sense_qualifier(wxr, m.group(1), sense_data)
2370 gloss = gloss[m.end() :].strip()
2372 # Remove common suffix "[from 14th c.]" and similar
2373 gloss = re.sub(r"\s\[[^]]*\]\s*$", "", gloss)
2375 # Check to make sure we don't have unhandled list items in gloss
2376 ofs = max(gloss.find("#"), gloss.find("* "))
2377 if ofs > 10 and "(#)" not in gloss:
2378 wxr.wtp.debug(
2379 "gloss may contain unhandled list items: {}".format(gloss),
2380 sortid="page/1412",
2381 )
2382 elif "\n" in gloss: 2382 ↛ 2383line 2382 didn't jump to line 2383 because the condition on line 2382 was never true
2383 wxr.wtp.debug(
2384 "gloss contains newline: {}".format(gloss),
2385 sortid="page/1416",
2386 )
2388 # Kludge, some glosses have a comma after initial qualifiers in
2389 # parentheses
2390 if gloss.startswith((",", ":")):
2391 gloss = gloss[1:]
2392 gloss = gloss.strip()
2393 if gloss.endswith(":"):
2394 gloss = gloss[:-1].strip()
2395 if gloss.startswith("N. of "): 2395 ↛ 2396line 2395 didn't jump to line 2396 because the condition on line 2395 was never true
2396 gloss = "Name of " + gloss[6:]
2397 if gloss.startswith("†"): 2397 ↛ 2398line 2397 didn't jump to line 2398 because the condition on line 2397 was never true
2398 data_append(sense_data, "tags", "obsolete")
2399 gloss = gloss[1:]
2400 elif gloss.startswith("^†"): 2400 ↛ 2401line 2400 didn't jump to line 2401 because the condition on line 2400 was never true
2401 data_append(sense_data, "tags", "obsolete")
2402 gloss = gloss[2:]
2404 # Copy tags from sense_base if any. This will not copy
2405 # countable/uncountable if either was specified in the sense,
2406 # as sometimes both are specified in word head but only one
2407 # in individual senses.
2408 countability_tags = []
2409 base_tags = sense_base.get("tags", ())
2410 sense_tags = sense_data.get("tags", ())
2411 for tag in base_tags:
2412 if tag in ("countable", "uncountable"):
2413 if tag not in countability_tags: 2413 ↛ 2415line 2413 didn't jump to line 2415 because the condition on line 2413 was always true
2414 countability_tags.append(tag)
2415 continue
2416 if tag not in sense_tags:
2417 data_append(sense_data, "tags", tag)
2418 if countability_tags:
2419 if ( 2419 ↛ 2428line 2419 didn't jump to line 2428 because the condition on line 2419 was always true
2420 "countable" not in sense_tags
2421 and "uncountable" not in sense_tags
2422 ):
2423 data_extend(sense_data, "tags", countability_tags)
2425 # If outer gloss specifies a form-of ("inflection of", see
2426 # aquamarine/German), try to parse the inner glosses as
2427 # tags for an inflected form.
2428 if "form-of" in sense_base.get("tags", ()):
2429 parsed = parse_alt_or_inflection_of(
2430 wxr, gloss, gloss_template_args
2431 )
2432 if parsed is not None: 2432 ↛ 2438line 2432 didn't jump to line 2438 because the condition on line 2432 was always true
2433 infl_tags, infl_dts = parsed
2434 if not infl_dts and infl_tags: 2434 ↛ 2438line 2434 didn't jump to line 2438 because the condition on line 2434 was always true
2435 # Interpret as a particular form under "inflection of"
2436 data_extend(sense_data, "tags", infl_tags)
2438 if not gloss: 2438 ↛ 2439line 2438 didn't jump to line 2439 because the condition on line 2438 was never true
2439 data_append(sense_data, "tags", "empty-gloss")
2440 elif gloss != "-" and gloss not in sense_data.get("glosses", []):
2441 if ( 2441 ↛ 2452line 2441 didn't jump to line 2452 because the condition on line 2441 was always true
2442 gloss_i == 0
2443 and len(sense_data.get("glosses", tuple())) >= 1
2444 ):
2445 # If we added a "high-level gloss" from rawgloss, but this
2446 # is that same gloss_i, add this instead of the raw_gloss
2447 # from before if they're different: the rawgloss was not
2448 # cleaned exactly the same as this later gloss
2449 sense_data["glosses"][-1] = gloss
2450 else:
2451 # Add the gloss for the sense.
2452 data_append(sense_data, "glosses", gloss)
2454 # Kludge: there are cases (e.g., etc./Swedish) where there are
2455 # two abbreviations in the same sense, both generated by the
2456 # {{abbreviation of|...}} template. Handle these with some magic.
2457 position = 0
2458 split_glosses = []
2459 for m in re.finditer(r"Abbreviation of ", gloss):
2460 if m.start() != position: 2460 ↛ 2459line 2460 didn't jump to line 2459 because the condition on line 2460 was always true
2461 split_glosses.append(gloss[position : m.start()])
2462 position = m.start()
2463 split_glosses.append(gloss[position:])
2464 for gloss in split_glosses:
2465 # Check if this gloss describes an alt-of or inflection-of
2466 if (
2467 lang_code != "en"
2468 and " " not in gloss
2469 and distw([word], gloss) < 0.3
2470 ):
2471 # Don't try to parse gloss if it is one word
2472 # that is close to the word itself for non-English words
2473 # (probable translations of a tag/form name)
2474 continue
2475 parsed = parse_alt_or_inflection_of(
2476 wxr, gloss, gloss_template_args
2477 )
2478 if parsed is None:
2479 continue
2480 tags, dts = parsed
2481 if not dts and tags:
2482 data_extend(sense_data, "tags", tags)
2483 continue
2484 for dt in dts: # type:ignore[union-attr]
2485 ftags = list(tag for tag in tags if tag != "form-of")
2486 if "alt-of" in tags:
2487 data_extend(sense_data, "tags", ftags)
2488 data_append(sense_data, "alt_of", dt)
2489 elif "compound-of" in tags: 2489 ↛ 2490line 2489 didn't jump to line 2490 because the condition on line 2489 was never true
2490 data_extend(sense_data, "tags", ftags)
2491 data_append(sense_data, "compound_of", dt)
2492 elif "synonym-of" in tags: 2492 ↛ 2493line 2492 didn't jump to line 2493 because the condition on line 2492 was never true
2493 data_extend(dt, "tags", ftags)
2494 data_append(sense_data, "synonyms", dt)
2495 elif tags and dt.get("word", "").startswith("of "): 2495 ↛ 2496line 2495 didn't jump to line 2496 because the condition on line 2495 was never true
2496 dt["word"] = dt["word"][3:]
2497 data_append(sense_data, "tags", "form-of")
2498 data_extend(sense_data, "tags", ftags)
2499 data_append(sense_data, "form_of", dt)
2500 elif "form-of" in tags: 2500 ↛ 2484line 2500 didn't jump to line 2484 because the condition on line 2500 was always true
2501 data_extend(sense_data, "tags", tags)
2502 data_append(sense_data, "form_of", dt)
2504 if len(sense_data) == 0:
2505 if len(sense_base.get("tags", [])) == 0: 2505 ↛ 2507line 2505 didn't jump to line 2507 because the condition on line 2505 was always true
2506 del sense_base["tags"]
2507 sense_data.update(sense_base)
2508 if push_sense(sorting_ordinal): 2508 ↛ 2512line 2508 didn't jump to line 2512 because the condition on line 2508 was always true
2509 # push_sense succeded in adding a sense to pos_data
2510 added = True
2511 # print("PARSE_SENSE DONE:", pos_datas[-1])
2512 return added
2514 def parse_inflection(
2515 node: WikiNode, section: str, pos: Optional[str]
2516 ) -> None:
2517 """Parses inflection data (declension, conjugation) from the given
2518 page. This retrieves the actual inflection template
2519 parameters, which are very useful for applications that need
2520 to learn the inflection classes and generate inflected
2521 forms."""
2522 assert isinstance(node, WikiNode)
2523 assert isinstance(section, str)
2524 assert pos is None or isinstance(pos, str)
2525 # print("parse_inflection:", node)
2527 if pos is None: 2527 ↛ 2528line 2527 didn't jump to line 2528 because the condition on line 2527 was never true
2528 wxr.wtp.debug(
2529 "inflection table outside part-of-speech", sortid="page/1812"
2530 )
2531 return
2533 def inflection_template_fn(
2534 name: str, ht: TemplateArgs
2535 ) -> Optional[str]:
2536 # print("decl_conj_template_fn", name, ht)
2537 if is_panel_template(wxr, name): 2537 ↛ 2538line 2537 didn't jump to line 2538 because the condition on line 2537 was never true
2538 return ""
2539 if name in ("is-u-mutation",): 2539 ↛ 2542line 2539 didn't jump to line 2542 because the condition on line 2539 was never true
2540 # These are not to be captured as an exception to the
2541 # generic code below
2542 return None
2543 m = re.search(
2544 r"-(conj|decl|ndecl|adecl|infl|conjugation|"
2545 r"declension|inflection|mut|mutation)($|-)",
2546 name,
2547 )
2548 if m:
2549 args_ht = clean_template_args(wxr, ht)
2550 dt = {"name": name, "args": args_ht}
2551 data_append(pos_data, "inflection_templates", dt)
2553 return None
2555 # Convert the subtree back to Wikitext, then expand all and parse,
2556 # capturing templates in the process
2557 text = wxr.wtp.node_to_wikitext(node.children)
2559 # Split text into separate sections for each to-level template
2560 brace_matches = re.split(r"((?:^|\n)\s*{\||\n\s*\|}|{{+|}}+)", text)
2561 # ["{{", "template", "}}"] or ["^{|", "table contents", "\n|}"]
2562 # The (?:...) creates a non-capturing regex group; if it was capturing,
2563 # like the group around it, it would create elements in brace_matches,
2564 # including None if it doesn't match.
2565 # 20250114: Added {| and |} into the regex because tables were being
2566 # cut into pieces by this code. Issue #973, introduction of two-part
2567 # book-end templates similar to trans-top and tran-bottom.
2568 template_sections = []
2569 template_nesting = 0 # depth of SINGLE BRACES { { nesting } }
2570 # Because there is the possibility of triple curly braces
2571 # ("{{{", "}}}") in addition to normal ("{{ }}"), we do not
2572 # count nesting depth using pairs of two brackets, but
2573 # instead use singular braces ("{ }").
2574 # Because template delimiters should be balanced, regardless
2575 # of whether {{ or {{{ is used, and because we only care
2576 # about the outer-most delimiters (the highest level template)
2577 # we can just count the single braces when those single
2578 # braces are part of a group.
2579 table_nesting = 0
2580 # However, if we have a stray table ({| ... |}) that should always
2581 # be its own section, and should prevent templates from cutting it
2582 # into sections.
2584 # print(f"Parse inflection: {text=}")
2585 # print(f"Brace matches: {repr('///'.join(brace_matches))}")
2586 if len(brace_matches) > 1:
2587 tsection: list[str] = []
2588 after_templates = False # kludge to keep any text
2589 # before first template
2590 # with the first template;
2591 # otherwise, text
2592 # goes with preceding template
2593 for m in brace_matches:
2594 if m.startswith("\n; ") and after_templates: 2594 ↛ 2595line 2594 didn't jump to line 2595 because the condition on line 2594 was never true
2595 after_templates = False
2596 template_sections.append(tsection)
2597 tsection = []
2598 tsection.append(m)
2599 elif m.startswith("{{") or m.endswith("{|"):
2600 if (
2601 template_nesting == 0
2602 and after_templates
2603 and table_nesting == 0
2604 ):
2605 template_sections.append(tsection)
2606 tsection = []
2607 # start new section
2608 after_templates = True
2609 if m.startswith("{{"):
2610 template_nesting += 1
2611 else:
2612 # m.endswith("{|")
2613 table_nesting += 1
2614 tsection.append(m)
2615 elif m.startswith("}}") or m.endswith("|}"):
2616 if m.startswith("}}"):
2617 template_nesting -= 1
2618 if template_nesting < 0: 2618 ↛ 2619line 2618 didn't jump to line 2619 because the condition on line 2618 was never true
2619 wxr.wtp.error(
2620 "Negatively nested braces, "
2621 "couldn't split inflection templates, "
2622 "{}/{} section {}".format(
2623 word, language, section
2624 ),
2625 sortid="page/1871",
2626 )
2627 template_sections = [] # use whole text
2628 break
2629 else:
2630 table_nesting -= 1
2631 if table_nesting < 0: 2631 ↛ 2632line 2631 didn't jump to line 2632 because the condition on line 2631 was never true
2632 wxr.wtp.error(
2633 "Negatively nested table braces, "
2634 "couldn't split inflection section, "
2635 "{}/{} section {}".format(
2636 word, language, section
2637 ),
2638 sortid="page/20250114",
2639 )
2640 template_sections = [] # use whole text
2641 break
2642 tsection.append(m)
2643 else:
2644 tsection.append(m)
2645 if tsection: # dangling tsection 2645 ↛ 2653line 2645 didn't jump to line 2653 because the condition on line 2645 was always true
2646 template_sections.append(tsection)
2647 # Why do it this way around? The parser has a preference
2648 # to associate bits outside of tables with the preceding
2649 # table (`after`-variable), so a new tsection begins
2650 # at {{ and everything before it belongs to the previous
2651 # template.
2653 texts = []
2654 if not template_sections:
2655 texts = [text]
2656 else:
2657 for tsection in template_sections:
2658 texts.append("".join(tsection))
2659 if template_nesting != 0: 2659 ↛ 2660line 2659 didn't jump to line 2660 because the condition on line 2659 was never true
2660 wxr.wtp.error(
2661 "Template nesting error: "
2662 "template_nesting = {} "
2663 "couldn't split inflection templates, "
2664 "{}/{} section {}".format(
2665 template_nesting, word, language, section
2666 ),
2667 sortid="page/1896",
2668 )
2669 texts = [text]
2670 for text in texts:
2671 tree = wxr.wtp.parse(
2672 text, expand_all=True, template_fn=inflection_template_fn
2673 )
2675 if not text.strip():
2676 continue
2678 # Parse inflection tables from the section. The data is stored
2679 # under "forms".
2680 if wxr.config.capture_inflections: 2680 ↛ 2670line 2680 didn't jump to line 2670 because the condition on line 2680 was always true
2681 tablecontext = None
2682 m = re.search(r"{{([^}{|]+)\|?", text)
2683 if m:
2684 template_name = m.group(1).strip()
2685 tablecontext = TableContext(template_name)
2687 parse_inflection_section(
2688 wxr,
2689 pos_data,
2690 word,
2691 language,
2692 pos,
2693 section,
2694 tree,
2695 tablecontext=tablecontext,
2696 )
2698 def get_subpage_section(
2699 title: str, subtitle: str, seqs: list[Union[list[str], tuple[str, ...]]]
2700 ) -> Optional[Union[WikiNode, str]]:
2701 """Loads a subpage of the given page, and finds the section
2702 for the given language, part-of-speech, and section title. This
2703 is used for finding translations and other sections on subpages."""
2704 assert isinstance(language, str)
2705 assert isinstance(title, str)
2706 assert isinstance(subtitle, str)
2707 assert isinstance(seqs, (list, tuple))
2708 for seq in seqs:
2709 for x in seq:
2710 assert isinstance(x, str)
2711 subpage_title = word + "/" + subtitle
2712 subpage_content = wxr.wtp.get_page_body(subpage_title, 0)
2713 if subpage_content is None:
2714 wxr.wtp.error(
2715 "/translations not found despite "
2716 "{{see translation subpage|...}}",
2717 sortid="page/1934",
2718 )
2719 return None
2721 def recurse(
2722 node: Union[str, WikiNode], seq: Union[list[str], tuple[str, ...]]
2723 ) -> Optional[Union[str, WikiNode]]:
2724 # print(f"seq: {seq}")
2725 if not seq:
2726 return node
2727 if not isinstance(node, WikiNode):
2728 return None
2729 # print(f"node.kind: {node.kind}")
2730 if node.kind in LEVEL_KINDS:
2731 t = clean_node(wxr, None, node.largs[0])
2732 # print(f"t: {t} == seq[0]: {seq[0]}?")
2733 if t.lower() == seq[0].lower():
2734 seq = seq[1:]
2735 if not seq:
2736 return node
2737 for n in node.children:
2738 ret = recurse(n, seq)
2739 if ret is not None:
2740 return ret
2741 return None
2743 tree = wxr.wtp.parse(
2744 subpage_content,
2745 pre_expand=True,
2746 additional_expand=ADDITIONAL_EXPAND_TEMPLATES,
2747 do_not_pre_expand=DO_NOT_PRE_EXPAND_TEMPLATES,
2748 )
2749 assert tree.kind == NodeKind.ROOT
2750 for seq in seqs:
2751 ret = recurse(tree, seq)
2752 if ret is None:
2753 wxr.wtp.debug(
2754 "Failed to find subpage section {}/{} seq {}".format(
2755 title, subtitle, seq
2756 ),
2757 sortid="page/1963",
2758 )
2759 return ret
2761 def parse_translations(data: WordData, xlatnode: WikiNode) -> None:
2762 """Parses translations for a word. This may also pull in translations
2763 from separate translation subpages."""
2764 assert isinstance(data, dict)
2765 assert isinstance(xlatnode, WikiNode)
2766 # print("===== PARSE_TRANSLATIONS {} {} {}"
2767 # .format(wxr.wtp.title, wxr.wtp.section, wxr.wtp.subsection))
2768 # print("parse_translations xlatnode={}".format(xlatnode))
2769 if not wxr.config.capture_translations: 2769 ↛ 2770line 2769 didn't jump to line 2770 because the condition on line 2769 was never true
2770 return
2771 sense_parts: list[Union[WikiNode, str]] = []
2772 sense: Optional[str] = None
2774 def parse_translation_item(
2775 contents: list[Union[WikiNode, str]], lang: Optional[str] = None
2776 ) -> None:
2777 nonlocal sense
2778 assert isinstance(contents, list)
2779 assert lang is None or isinstance(lang, str)
2780 # print("PARSE_TRANSLATION_ITEM:", contents)
2782 langcode: Optional[str] = None
2783 if sense is None:
2784 sense = clean_node(wxr, data, sense_parts).strip()
2785 # print("sense <- clean_node: ", sense)
2786 idx = sense.find("See also translations at")
2787 if idx > 0: 2787 ↛ 2788line 2787 didn't jump to line 2788 because the condition on line 2787 was never true
2788 wxr.wtp.debug(
2789 "Skipping translation see also: {}".format(sense),
2790 sortid="page/2361",
2791 )
2792 sense = sense[:idx].strip()
2793 if sense.endswith(":"): 2793 ↛ 2794line 2793 didn't jump to line 2794 because the condition on line 2793 was never true
2794 sense = sense[:-1].strip()
2795 if sense.endswith("—"): 2795 ↛ 2796line 2795 didn't jump to line 2796 because the condition on line 2795 was never true
2796 sense = sense[:-1].strip()
2797 translations_from_template: list[str] = []
2799 def translation_item_template_fn(
2800 name: str, ht: TemplateArgs
2801 ) -> Optional[str]:
2802 nonlocal langcode
2803 # print("TRANSLATION_ITEM_TEMPLATE_FN:", name, ht)
2804 if is_panel_template(wxr, name):
2805 return ""
2806 if name in ("t+check", "t-check", "t-needed"):
2807 # We ignore these templates. They seem to have outright
2808 # garbage in some entries, and very varying formatting in
2809 # others. These should be transitory and unreliable
2810 # anyway.
2811 return "__IGNORE__"
2812 if name in ("t", "t+", "t-simple", "tt", "tt+"):
2813 code = ht.get(1)
2814 if code: 2814 ↛ 2824line 2814 didn't jump to line 2824 because the condition on line 2814 was always true
2815 if langcode and code != langcode:
2816 wxr.wtp.debug(
2817 "inconsistent language codes {} vs "
2818 "{} in translation item: {!r} {}".format(
2819 langcode, code, name, ht
2820 ),
2821 sortid="page/2386",
2822 )
2823 langcode = code
2824 tr = ht.get(2)
2825 if tr:
2826 tr = clean_node(wxr, None, [tr])
2827 translations_from_template.append(tr)
2828 return None
2829 if name == "t-egy":
2830 langcode = "egy"
2831 return None
2832 if name == "ttbc":
2833 code = ht.get(1)
2834 if code: 2834 ↛ 2836line 2834 didn't jump to line 2836 because the condition on line 2834 was always true
2835 langcode = code
2836 return None
2837 if name == "trans-see": 2837 ↛ 2838line 2837 didn't jump to line 2838 because the condition on line 2837 was never true
2838 wxr.wtp.error(
2839 "UNIMPLEMENTED trans-see template", sortid="page/2405"
2840 )
2841 return ""
2842 if name.endswith("-top"): 2842 ↛ 2843line 2842 didn't jump to line 2843 because the condition on line 2842 was never true
2843 return ""
2844 if name.endswith("-bottom"): 2844 ↛ 2845line 2844 didn't jump to line 2845 because the condition on line 2844 was never true
2845 return ""
2846 if name.endswith("-mid"): 2846 ↛ 2847line 2846 didn't jump to line 2847 because the condition on line 2846 was never true
2847 return ""
2848 # wxr.wtp.debug("UNHANDLED TRANSLATION ITEM TEMPLATE: {!r}"
2849 # .format(name),
2850 # sortid="page/2414")
2851 return None
2853 sublists = list(
2854 x
2855 for x in contents
2856 if isinstance(x, WikiNode) and x.kind == NodeKind.LIST
2857 )
2858 contents = list(
2859 x
2860 for x in contents
2861 if not isinstance(x, WikiNode) or x.kind != NodeKind.LIST
2862 )
2864 item = clean_node(
2865 wxr, data, contents, template_fn=translation_item_template_fn
2866 )
2867 # print(" TRANSLATION ITEM: {!r} [{}]".format(item, sense))
2869 # Parse the translation item.
2870 if item: 2870 ↛ exitline 2870 didn't return from function 'parse_translation_item' because the condition on line 2870 was always true
2871 lang = parse_translation_item_text(
2872 wxr,
2873 word,
2874 data,
2875 item,
2876 sense,
2877 lang,
2878 langcode,
2879 translations_from_template,
2880 is_reconstruction,
2881 )
2883 # Handle sublists. They are frequently used for different
2884 # scripts for the language and different variants of the
2885 # language. We will include the lower-level header as a
2886 # tag in those cases.
2887 for listnode in sublists:
2888 assert listnode.kind == NodeKind.LIST
2889 for node in listnode.children:
2890 if not isinstance(node, WikiNode): 2890 ↛ 2891line 2890 didn't jump to line 2891 because the condition on line 2890 was never true
2891 continue
2892 if node.kind == NodeKind.LIST_ITEM: 2892 ↛ 2889line 2892 didn't jump to line 2889 because the condition on line 2892 was always true
2893 parse_translation_item(node.children, lang=lang)
2895 def parse_translation_template(node: WikiNode) -> None:
2896 assert isinstance(node, WikiNode)
2898 def template_fn(name: str, ht: TemplateArgs) -> Optional[str]:
2899 nonlocal sense_parts
2900 nonlocal sense
2901 if is_panel_template(wxr, name):
2902 return ""
2903 if name == "see also":
2904 # XXX capture
2905 # XXX for example, "/" has top-level list containing
2906 # see also items. So also should parse those.
2907 return ""
2908 if name == "trans-see":
2909 # XXX capture
2910 return ""
2911 if name == "see translation subpage": 2911 ↛ 2912line 2911 didn't jump to line 2912 because the condition on line 2911 was never true
2912 sense_parts = []
2913 sense = None
2914 sub = ht.get(1, "")
2915 if sub:
2916 m = re.match(
2917 r"\s*(([^:\d]*)\s*\d*)\s*:\s*([^:]*)\s*", sub
2918 )
2919 else:
2920 m = None
2921 etym = ""
2922 etym_numbered = ""
2923 pos = ""
2924 if m:
2925 etym_numbered = m.group(1)
2926 etym = m.group(2)
2927 pos = m.group(3)
2928 if not sub:
2929 wxr.wtp.debug(
2930 "no part-of-speech in "
2931 "{{see translation subpage|...}}, "
2932 "defaulting to just wxr.wtp.section "
2933 "(= language)",
2934 sortid="page/2468",
2935 )
2936 # seq sent to get_subpage_section without sub and pos
2937 seq = [
2938 language,
2939 TRANSLATIONS_TITLE,
2940 ]
2941 elif (
2942 m
2943 and etym.lower().strip() in ETYMOLOGY_TITLES
2944 and pos.lower() in POS_TITLES
2945 ):
2946 seq = [
2947 language,
2948 etym_numbered,
2949 pos,
2950 TRANSLATIONS_TITLE,
2951 ]
2952 elif sub.lower() in POS_TITLES:
2953 # seq with sub but not pos
2954 seq = [
2955 language,
2956 sub,
2957 TRANSLATIONS_TITLE,
2958 ]
2959 else:
2960 # seq with sub and pos
2961 pos = wxr.wtp.subsection or "MISSING_SUBSECTION"
2962 if pos.lower() not in POS_TITLES:
2963 wxr.wtp.debug(
2964 "unhandled see translation subpage: "
2965 "language={} sub={} "
2966 "wxr.wtp.subsection={}".format(
2967 language, sub, wxr.wtp.subsection
2968 ),
2969 sortid="page/2478",
2970 )
2971 seq = [language, sub, pos, TRANSLATIONS_TITLE]
2972 subnode = get_subpage_section(
2973 wxr.wtp.title or "MISSING_TITLE",
2974 TRANSLATIONS_TITLE,
2975 [seq],
2976 )
2977 if subnode is None or not isinstance(subnode, WikiNode):
2978 # Failed to find the normal subpage section
2979 # seq with sub and pos
2980 pos = wxr.wtp.subsection or "MISSING_SUBSECTION"
2981 # print(f"{language=}, {pos=}, {TRANSLATIONS_TITLE=}")
2982 seqs: list[list[str] | tuple[str, ...]] = [
2983 [TRANSLATIONS_TITLE],
2984 [language, pos],
2985 ]
2986 subnode = get_subpage_section(
2987 wxr.wtp.title or "MISSING_TITLE",
2988 TRANSLATIONS_TITLE,
2989 seqs,
2990 )
2991 if subnode is not None and isinstance(subnode, WikiNode):
2992 parse_translations(data, subnode)
2993 return ""
2994 if name in (
2995 "c",
2996 "C",
2997 "categorize",
2998 "cat",
2999 "catlangname",
3000 "topics",
3001 "top",
3002 "qualifier",
3003 "cln",
3004 ):
3005 # These are expanded in the default way
3006 return None
3007 if name in (
3008 "trans-top",
3009 "trans-top-see",
3010 ):
3011 # XXX capture id from trans-top? Capture sense here
3012 # instead of trying to parse it from expanded content?
3013 if ht.get(1):
3014 sense_parts = []
3015 sense = ht.get(1)
3016 else:
3017 sense_parts = []
3018 sense = None
3019 return None
3020 if name in (
3021 "trans-bottom",
3022 "trans-mid",
3023 "checktrans-mid",
3024 "checktrans-bottom",
3025 ):
3026 return None
3027 if name == "checktrans-top":
3028 sense_parts = []
3029 sense = None
3030 return ""
3031 if name == "trans-top-also":
3032 # XXX capture?
3033 sense_parts = []
3034 sense = None
3035 return ""
3036 wxr.wtp.error(
3037 "UNIMPLEMENTED parse_translation_template: {} {}".format(
3038 name, ht
3039 ),
3040 sortid="page/2517",
3041 )
3042 return ""
3044 wxr.wtp.expand(
3045 wxr.wtp.node_to_wikitext(node), template_fn=template_fn
3046 )
3048 def parse_translation_recurse(xlatnode: WikiNode) -> None:
3049 nonlocal sense
3050 nonlocal sense_parts
3051 for node in xlatnode.children:
3052 # print(node)
3053 if isinstance(node, str):
3054 if sense:
3055 if not node.isspace():
3056 wxr.wtp.debug(
3057 "skipping string in the middle of "
3058 "translations: {}".format(node),
3059 sortid="page/2530",
3060 )
3061 continue
3062 # Add a part to the sense
3063 sense_parts.append(node)
3064 sense = None
3065 continue
3066 assert isinstance(node, WikiNode)
3067 kind = node.kind
3068 if kind == NodeKind.LIST:
3069 for item in node.children:
3070 if not isinstance(item, WikiNode): 3070 ↛ 3071line 3070 didn't jump to line 3071 because the condition on line 3070 was never true
3071 continue
3072 if item.kind != NodeKind.LIST_ITEM: 3072 ↛ 3073line 3072 didn't jump to line 3073 because the condition on line 3072 was never true
3073 continue
3074 if item.sarg == ":": 3074 ↛ 3075line 3074 didn't jump to line 3075 because the condition on line 3074 was never true
3075 continue
3076 parse_translation_item(item.children)
3077 elif kind == NodeKind.LIST_ITEM and node.sarg == ":": 3077 ↛ 3081line 3077 didn't jump to line 3081 because the condition on line 3077 was never true
3078 # Silently skip list items that are just indented; these
3079 # are used for text between translations, such as indicating
3080 # translations that need to be checked.
3081 pass
3082 elif kind == NodeKind.TEMPLATE:
3083 parse_translation_template(node)
3084 elif kind in ( 3084 ↛ 3089line 3084 didn't jump to line 3089 because the condition on line 3084 was never true
3085 NodeKind.TABLE,
3086 NodeKind.TABLE_ROW,
3087 NodeKind.TABLE_CELL,
3088 ):
3089 parse_translation_recurse(node)
3090 elif kind == NodeKind.HTML:
3091 if node.attrs.get("class") == "NavFrame": 3091 ↛ 3097line 3091 didn't jump to line 3097 because the condition on line 3091 was never true
3092 # Reset ``sense_parts`` (and force recomputing
3093 # by clearing ``sense``) as each NavFrame specifies
3094 # its own sense. This helps eliminate garbage coming
3095 # from text at the beginning at the translations
3096 # section.
3097 sense_parts = []
3098 sense = None
3099 # for item in node.children:
3100 # if not isinstance(item, WikiNode):
3101 # continue
3102 # parse_translation_recurse(item)
3103 parse_translation_recurse(node)
3104 elif kind in LEVEL_KINDS: 3104 ↛ 3106line 3104 didn't jump to line 3106 because the condition on line 3104 was never true
3105 # Sub-levels will be recursed elsewhere
3106 pass
3107 elif kind in (NodeKind.ITALIC, NodeKind.BOLD):
3108 parse_translation_recurse(node)
3109 elif kind == NodeKind.PREFORMATTED: 3109 ↛ 3110line 3109 didn't jump to line 3110 because the condition on line 3109 was never true
3110 print("parse_translation_recurse: PREFORMATTED:", node)
3111 elif kind == NodeKind.LINK: 3111 ↛ 3165line 3111 didn't jump to line 3165 because the condition on line 3111 was always true
3112 arg0 = node.largs[0]
3113 # Kludge: I've seen occasional normal links to translation
3114 # subpages from main pages (e.g., language/English/Noun
3115 # in July 2021) instead of the normal
3116 # {{see translation subpage|...}} template. This should
3117 # handle them. Note: must be careful not to read other
3118 # links, particularly things like in "human being":
3119 # "a human being -- see [[man/translations]]" (group title)
3120 if ( 3120 ↛ 3128line 3120 didn't jump to line 3128 because the condition on line 3120 was never true
3121 isinstance(arg0, (list, tuple))
3122 and arg0
3123 and isinstance(arg0[0], str)
3124 and arg0[0].endswith("/" + TRANSLATIONS_TITLE)
3125 and arg0[0][: -(1 + len(TRANSLATIONS_TITLE))]
3126 == wxr.wtp.title
3127 ):
3128 wxr.wtp.debug(
3129 "translations subpage link found on main "
3130 "page instead "
3131 "of normal {{see translation subpage|...}}",
3132 sortid="page/2595",
3133 )
3134 sub = wxr.wtp.subsection or "MISSING_SUBSECTION"
3135 if sub.lower() in POS_TITLES:
3136 seq = [
3137 language,
3138 sub,
3139 TRANSLATIONS_TITLE,
3140 ]
3141 subnode = get_subpage_section(
3142 wxr.wtp.title,
3143 TRANSLATIONS_TITLE,
3144 [seq],
3145 )
3146 if subnode is not None and isinstance(
3147 subnode, WikiNode
3148 ):
3149 parse_translations(data, subnode)
3150 else:
3151 wxr.wtp.error(
3152 "/translations link outside part-of-speech"
3153 )
3155 if (
3156 len(arg0) >= 1
3157 and isinstance(arg0[0], str)
3158 and not arg0[0].lower().startswith("category:")
3159 ):
3160 for x in node.largs[-1]:
3161 if isinstance(x, str): 3161 ↛ 3164line 3161 didn't jump to line 3164 because the condition on line 3161 was always true
3162 sense_parts.append(x)
3163 else:
3164 parse_translation_recurse(x)
3165 elif not sense:
3166 sense_parts.append(node)
3167 else:
3168 wxr.wtp.debug(
3169 "skipping text between translation items/senses: "
3170 "{}".format(node),
3171 sortid="page/2621",
3172 )
3174 # Main code of parse_translation(). We want ``sense`` to be assigned
3175 # regardless of recursion levels, and thus the code is structured
3176 # to define at this level and recurse in parse_translation_recurse().
3177 parse_translation_recurse(xlatnode)
3179 def parse_etymology(data: WordData, node: LevelNode) -> None:
3180 """Parses an etymology section."""
3181 assert isinstance(data, dict)
3182 assert isinstance(node, WikiNode)
3184 templates: list[TemplateData] = []
3186 # Counter for preventing the capture of etymology templates
3187 # when we are inside templates that we want to ignore (i.e.,
3188 # not capture).
3189 ignore_count = 0
3191 def etym_template_fn(name: str, ht: TemplateArgs) -> Optional[str]:
3192 nonlocal ignore_count
3193 if is_panel_template(wxr, name) or name in ["zh-x", "zh-q"]:
3194 return ""
3195 if re.match(ignored_etymology_templates_re, name):
3196 ignore_count += 1
3197 return None
3199 # CONTINUE_HERE
3201 def etym_post_template_fn(
3202 name: str, ht: TemplateArgs, expansion: str
3203 ) -> None:
3204 nonlocal ignore_count
3205 if name in wikipedia_templates:
3206 parse_wikipedia_template(wxr, data, ht)
3207 return None
3208 if re.match(ignored_etymology_templates_re, name):
3209 ignore_count -= 1
3210 return None
3211 if ignore_count == 0: 3211 ↛ 3217line 3211 didn't jump to line 3217 because the condition on line 3211 was always true
3212 ht = clean_template_args(wxr, ht)
3213 expansion = clean_node(wxr, None, expansion)
3214 templates.append(
3215 {"name": name, "args": ht, "expansion": expansion}
3216 )
3217 return None
3219 # Remove any subsections
3220 contents = list(
3221 x
3222 for x in node.children
3223 if not isinstance(x, WikiNode) or x.kind not in LEVEL_KINDS
3224 )
3225 # Convert to text, also capturing templates using post_template_fn
3226 text = clean_node(
3227 wxr,
3228 None,
3229 contents,
3230 template_fn=etym_template_fn,
3231 post_template_fn=etym_post_template_fn,
3232 ).strip(": \n") # remove ":" indent wikitext before zh-x template
3233 # Save the collected information.
3234 if len(text) > 0:
3235 data["etymology_text"] = text
3236 if len(templates) > 0:
3237 # Some etymology templates, like Template:root do not generate
3238 # text, so they should be added here. Elsewhere, we check
3239 # for Template:root and add some text to the expansion to please
3240 # the validation.
3241 data["etymology_templates"] = templates
3243 for child_node in node.find_child_recursively( 3243 ↛ exitline 3243 didn't return from function 'parse_etymology' because the loop on line 3243 didn't complete
3244 LEVEL_KIND_FLAGS | NodeKind.TEMPLATE
3245 ):
3246 if child_node.kind in LEVEL_KIND_FLAGS:
3247 break
3248 elif isinstance( 3248 ↛ 3251line 3248 didn't jump to line 3251 because the condition on line 3248 was never true
3249 child_node, TemplateNode
3250 ) and child_node.template_name in ["zh-x", "zh-q"]:
3251 if "etymology_examples" not in data:
3252 data["etymology_examples"] = []
3253 data["etymology_examples"].extend(
3254 extract_template_zh_x(
3255 wxr, child_node, None, ExampleData(raw_tags=[], tags=[])
3256 )
3257 )
3259 def process_children(treenode: WikiNode, pos: Optional[str]) -> None:
3260 """This recurses into a subtree in the parse tree for a page."""
3261 nonlocal etym_data
3262 nonlocal pos_data
3263 nonlocal inside_level_four
3265 redirect_list: list[str] = [] # for `zh-see` template
3267 def skip_template_fn(name: str, ht: TemplateArgs) -> Optional[str]:
3268 """This is called for otherwise unprocessed parts of the page.
3269 We still expand them so that e.g. Category links get captured."""
3270 if name in wikipedia_templates:
3271 data = select_data()
3272 parse_wikipedia_template(wxr, data, ht)
3273 return None
3274 if is_panel_template(wxr, name):
3275 return ""
3276 return None
3278 for node in treenode.children:
3279 if not isinstance(node, WikiNode):
3280 # print(" X{}".format(repr(node)[:40]))
3281 continue
3282 if isinstance(node, TemplateNode):
3283 if process_soft_redirect_template(wxr, node, redirect_list):
3284 continue
3285 elif node.template_name == "zh-forms":
3286 extract_zh_forms_template(wxr, node, select_data())
3287 elif (
3288 node.template_name.endswith("-kanjitab")
3289 or node.template_name == "ja-kt"
3290 ):
3291 extract_ja_kanjitab_template(wxr, node, select_data())
3293 if not isinstance(node, LevelNode):
3294 # XXX handle e.g. wikipedia links at the top of a language
3295 # XXX should at least capture "also" at top of page
3296 if node.kind in (
3297 NodeKind.HLINE,
3298 NodeKind.LIST,
3299 NodeKind.LIST_ITEM,
3300 ):
3301 continue
3302 # print(" UNEXPECTED: {}".format(node))
3303 # Clean the node to collect category links
3304 clean_node(wxr, etym_data, node, template_fn=skip_template_fn)
3305 continue
3306 t = clean_node(
3307 wxr, etym_data, node.sarg if node.sarg else node.largs
3308 )
3309 t = t.lower()
3310 # XXX these counts were never implemented fully, and even this
3311 # gets discarded: Search STATISTICS_IMPLEMENTATION
3312 wxr.config.section_counts[t] += 1
3313 # print("PROCESS_CHILDREN: T:", repr(t))
3314 if t in IGNORED_TITLES:
3315 pass
3316 elif t.startswith(PRONUNCIATION_TITLE):
3317 # Chinese Pronunciation section kludge; we demote these to
3318 # be level 4 instead of 3 so that they're part of a larger
3319 # etymology hierarchy; usually the data here is empty and
3320 # acts as an inbetween between POS and Etymology data
3321 if lang_code in ("zh",):
3322 inside_level_four = True
3323 if t.startswith(PRONUNCIATION_TITLE + " "):
3324 # Pronunciation 1, etc, are used in Chinese Glyphs,
3325 # and each of them may have senses under Definition
3326 push_level_four_section(True)
3327 wxr.wtp.start_subsection(None)
3328 if wxr.config.capture_pronunciation: 3328 ↛ 3436line 3328 didn't jump to line 3436 because the condition on line 3328 was always true
3329 data = select_data()
3330 parse_pronunciation(
3331 wxr,
3332 node,
3333 data,
3334 etym_data,
3335 have_etym,
3336 base_data,
3337 lang_code,
3338 )
3339 elif t.startswith(tuple(ETYMOLOGY_TITLES)):
3340 push_etym()
3341 wxr.wtp.start_subsection(None)
3342 if wxr.config.capture_etymologies: 3342 ↛ 3436line 3342 didn't jump to line 3436 because the condition on line 3342 was always true
3343 m = re.search(r"\s(\d+(\.\d+)?)$", t)
3344 if m:
3345 etym_data["etymology_number"] = m.group(1)
3346 parse_etymology(etym_data, node)
3347 elif t == DESCENDANTS_TITLE and wxr.config.capture_descendants:
3348 data = select_data()
3349 extract_descendant_section(wxr, data, node, False)
3350 elif (
3351 t in PROTO_ROOT_DERIVED_TITLES
3352 and pos == "root"
3353 and is_reconstruction
3354 and wxr.config.capture_descendants
3355 ):
3356 data = select_data()
3357 extract_descendant_section(wxr, data, node, True)
3358 elif t == TRANSLATIONS_TITLE:
3359 data = select_data()
3360 parse_translations(data, node)
3361 elif t in INFLECTION_TITLES:
3362 parse_inflection(node, t, pos)
3363 elif t == "alternative forms":
3364 extract_alt_form_section(wxr, select_data(), node)
3365 else:
3366 lst = t.split()
3367 while len(lst) > 1 and lst[-1].isdigit():
3368 lst = lst[:-1]
3369 t_no_number = " ".join(lst).lower()
3370 if t_no_number in POS_TITLES:
3371 push_pos()
3372 dt = POS_TITLES[t_no_number] # type:ignore[literal-required]
3373 pos = dt["pos"] or "MISSING_POS"
3374 wxr.wtp.start_subsection(t)
3375 if "debug" in dt:
3376 wxr.wtp.debug(
3377 "{} in section {}".format(dt["debug"], t),
3378 sortid="page/2755",
3379 )
3380 if "warning" in dt: 3380 ↛ 3381line 3380 didn't jump to line 3381 because the condition on line 3380 was never true
3381 wxr.wtp.wiki_notice(
3382 "{} in section {}".format(dt["warning"], t),
3383 sortid="page/2759",
3384 )
3385 if "error" in dt: 3385 ↛ 3386line 3385 didn't jump to line 3386 because the condition on line 3385 was never true
3386 wxr.wtp.error(
3387 "{} in section {}".format(dt["error"], t),
3388 sortid="page/2763",
3389 )
3390 if "note" in dt: 3390 ↛ 3391line 3390 didn't jump to line 3391 because the condition on line 3390 was never true
3391 wxr.wtp.note(
3392 "{} in section {}".format(dt["note"], t),
3393 sortid="page/20251017a",
3394 )
3395 if "wiki_notice" in dt: 3395 ↛ 3396line 3395 didn't jump to line 3396 because the condition on line 3395 was never true
3396 wxr.wtp.wiki_notice(
3397 "{} in section {}".format(dt["wiki_notices"], t),
3398 sortid="page/20251017b",
3399 )
3400 # Parse word senses for the part-of-speech
3401 parse_part_of_speech(node, pos)
3402 if "tags" in dt:
3403 for pdata in sense_datas:
3404 data_extend(pdata, "tags", dt["tags"])
3405 elif t_no_number in LINKAGE_TITLES:
3406 # print(f"LINKAGE_TITLES NODE {node=}")
3407 rel = LINKAGE_TITLES[t_no_number]
3408 data = select_data()
3409 parse_linkage(
3410 wxr,
3411 data,
3412 rel,
3413 node,
3414 word,
3415 sense_datas,
3416 is_reconstruction,
3417 )
3418 elif t_no_number == COMPOUNDS_TITLE:
3419 data = select_data()
3420 if wxr.config.capture_compounds: 3420 ↛ 3436line 3420 didn't jump to line 3436 because the condition on line 3420 was always true
3421 parse_linkage(
3422 wxr,
3423 data,
3424 "derived",
3425 node,
3426 word,
3427 sense_datas,
3428 is_reconstruction,
3429 )
3431 # XXX parse interesting templates also from other sections. E.g.,
3432 # {{Letter|...}} in ===See also===
3433 # Also <gallery>
3435 # Recurse to children of this node, processing subtitles therein
3436 stack.append(t)
3437 process_children(node, pos)
3438 stack.pop()
3440 if len(redirect_list) > 0:
3441 if len(pos_data) > 0:
3442 pos_data["redirects"] = redirect_list
3443 if "pos" not in pos_data: 3443 ↛ 3444line 3443 didn't jump to line 3444 because the condition on line 3443 was never true
3444 pos_data["pos"] = "soft-redirect"
3445 else:
3446 new_page_data = copy.deepcopy(base_data)
3447 new_page_data["redirects"] = redirect_list
3448 if "pos" not in new_page_data: 3448 ↛ 3450line 3448 didn't jump to line 3450 because the condition on line 3448 was always true
3449 new_page_data["pos"] = "soft-redirect"
3450 new_page_data["senses"] = [{"tags": ["no-gloss"]}]
3451 page_datas.append(new_page_data)
3453 def extract_examples(
3454 others: list[WikiNode], sense_base: SenseData
3455 ) -> list[ExampleData]:
3456 """Parses through a list of definitions and quotes to find examples.
3457 Returns a list of example dicts to be added to sense data. Adds
3458 meta-data, mostly categories, into sense_base."""
3459 assert isinstance(others, list)
3460 examples: list[ExampleData] = []
3462 for sub in others:
3463 if not sub.sarg.endswith((":", "*")): 3463 ↛ 3464line 3463 didn't jump to line 3464 because the condition on line 3463 was never true
3464 continue
3465 for item in sub.children:
3466 if not isinstance(item, WikiNode): 3466 ↛ 3467line 3466 didn't jump to line 3467 because the condition on line 3466 was never true
3467 continue
3468 if item.kind != NodeKind.LIST_ITEM: 3468 ↛ 3469line 3468 didn't jump to line 3469 because the condition on line 3468 was never true
3469 continue
3470 usex_type = None
3471 example_template_args = []
3472 example_template_names = []
3473 taxons = set()
3475 # Bypass this function when parsing Chinese, Japanese and
3476 # quotation templates.
3477 new_example_lists = extract_example_list_item(
3478 wxr, item, sense_base, ExampleData(raw_tags=[], tags=[])
3479 )
3480 if len(new_example_lists) > 0:
3481 examples.extend(new_example_lists)
3482 continue
3484 def usex_template_fn(
3485 name: str, ht: TemplateArgs
3486 ) -> Optional[str]:
3487 nonlocal usex_type
3488 if is_panel_template(wxr, name):
3489 return ""
3490 if name in usex_templates:
3491 usex_type = "example"
3492 example_template_args.append(ht)
3493 example_template_names.append(name)
3494 elif name in quotation_templates:
3495 usex_type = "quotation"
3496 elif name in taxonomy_templates: 3496 ↛ 3497line 3496 didn't jump to line 3497 because the condition on line 3496 was never true
3497 taxons.update(ht.get(1, "").split())
3498 for prefix in template_linkages_to_ignore_in_examples:
3499 if re.search(
3500 r"(^|[-/\s]){}($|\b|[0-9])".format(prefix), name
3501 ):
3502 return ""
3503 return None
3505 # bookmark
3506 ruby: list[tuple[str, str]] = []
3507 contents = item.children
3508 if lang_code == "ja":
3509 # Capture ruby contents if this is a Japanese language
3510 # example.
3511 # print(contents)
3512 if ( 3512 ↛ 3517line 3512 didn't jump to line 3517 because the condition on line 3512 was never true
3513 contents
3514 and isinstance(contents, str)
3515 and re.match(r"\s*$", contents[0])
3516 ):
3517 contents = contents[1:]
3518 exp = wxr.wtp.parse(
3519 wxr.wtp.node_to_wikitext(contents),
3520 # post_template_fn=head_post_template_fn,
3521 expand_all=True,
3522 )
3523 rub, rest = extract_ruby(wxr, exp.children)
3524 if rub:
3525 for rtup in rub:
3526 ruby.append(rtup)
3527 contents = rest
3528 subtext = clean_node(
3529 wxr, sense_base, contents, template_fn=usex_template_fn
3530 )
3532 frozen_taxons = frozenset(taxons)
3533 classify_desc2 = partial(classify_desc, accepted=frozen_taxons)
3535 # print(f"{subtext=}")
3536 subtext = re.sub(
3537 r"\s*\(please add an English "
3538 r"translation of this "
3539 r"(example|usage example|quote)\)",
3540 "",
3541 subtext,
3542 ).strip()
3543 subtext = re.sub(r"\^\([^)]*\)", "", subtext)
3544 subtext = re.sub(r"\s*[―—]+$", "", subtext)
3545 # print("subtext:", repr(subtext))
3547 lines = subtext.splitlines()
3548 # print(lines)
3550 lines = list(re.sub(r"^[#:*]*", "", x).strip() for x in lines)
3551 lines = list(
3552 x
3553 for x in lines
3554 if not re.match(
3555 r"(Synonyms: |Antonyms: |Hyponyms: |"
3556 r"Synonym: |Antonym: |Hyponym: |"
3557 r"Hypernyms: |Derived terms: |"
3558 r"Related terms: |"
3559 r"Hypernym: |Derived term: |"
3560 r"Coordinate terms:|"
3561 r"Related term: |"
3562 r"For more quotations using )",
3563 x,
3564 )
3565 )
3566 tr = ""
3567 ref = ""
3568 roman = ""
3569 # for line in lines:
3570 # print("LINE:", repr(line))
3571 # print(classify_desc(line))
3572 if len(lines) == 1 and lang_code != "en":
3573 parts = example_splitter_re.split(lines[0])
3574 if ( 3574 ↛ 3582line 3574 didn't jump to line 3582 because the condition on line 3574 was never true
3575 len(parts) > 2
3576 and len(example_template_args) == 1
3577 and any(
3578 ("―" in s) or ("—" in s)
3579 for s in example_template_args[0].values()
3580 )
3581 ):
3582 if nparts := synch_splits_with_args(
3583 lines[0], example_template_args[0]
3584 ):
3585 parts = nparts
3586 if ( 3586 ↛ 3591line 3586 didn't jump to line 3591 because the condition on line 3586 was never true
3587 len(example_template_args) == 1
3588 and "lit" in example_template_args[0]
3589 ):
3590 # ugly brute-force kludge in case there's a lit= arg
3591 literally = example_template_args[0].get("lit", "")
3592 if literally:
3593 literally = (
3594 " (literally, “"
3595 + clean_value(wxr, literally)
3596 + "”)"
3597 )
3598 else:
3599 literally = ""
3600 if ( 3600 ↛ 3639line 3600 didn't jump to line 3639 because the condition on line 3600 was never true
3601 len(example_template_args) == 1
3602 and len(parts) == 2
3603 and len(example_template_args[0])
3604 - (
3605 # horrible kludge to ignore these arguments
3606 # when calculating how many there are
3607 sum(
3608 s in example_template_args[0]
3609 for s in (
3610 "lit", # generates text, but we handle it
3611 "inline",
3612 "noenum",
3613 "nocat",
3614 "sort",
3615 )
3616 )
3617 )
3618 == 3
3619 and clean_value(
3620 wxr, example_template_args[0].get(2, "")
3621 )
3622 == parts[0].strip()
3623 and clean_value(
3624 wxr,
3625 (
3626 example_template_args[0].get(3)
3627 or example_template_args[0].get("translation")
3628 or example_template_args[0].get("t", "")
3629 )
3630 + literally, # in case there's a lit= argument
3631 )
3632 == parts[1].strip()
3633 ):
3634 # {{exampletemplate|ex|Foo bar baz|English translation}}
3635 # is a pretty reliable 'heuristic', so we use it here
3636 # before the others. To be extra sure the template
3637 # doesn't do anything weird, we compare the arguments
3638 # and the output to each other.
3639 lines = [parts[0].strip()]
3640 tr = parts[1].strip()
3641 elif (
3642 len(parts) == 2
3643 and classify_desc2(parts[1]) in ENGLISH_TEXTS
3644 ):
3645 # These other branches just do some simple heuristics w/
3646 # the expanded output of the template (if applicable).
3647 lines = [parts[0].strip()]
3648 tr = parts[1].strip()
3649 elif ( 3649 ↛ 3655line 3649 didn't jump to line 3655 because the condition on line 3649 was never true
3650 len(parts) == 3
3651 and classify_desc2(parts[1])
3652 in ("romanization", "english")
3653 and classify_desc2(parts[2]) in ENGLISH_TEXTS
3654 ):
3655 lines = [parts[0].strip()]
3656 roman = parts[1].strip()
3657 tr = parts[2].strip()
3658 else:
3659 parts = re.split(r"\s+-\s+", lines[0])
3660 if ( 3660 ↛ 3664line 3660 didn't jump to line 3664 because the condition on line 3660 was never true
3661 len(parts) == 2
3662 and classify_desc2(parts[1]) in ENGLISH_TEXTS
3663 ):
3664 lines = [parts[0].strip()]
3665 tr = parts[1].strip()
3666 elif len(lines) > 1:
3667 if any(
3668 re.search(r"[]\d:)]\s*$", x) for x in lines[:-1]
3669 ) and not (len(example_template_names) == 1):
3670 refs: list[str] = []
3671 for i in range(len(lines)): 3671 ↛ 3677line 3671 didn't jump to line 3677 because the loop on line 3671 didn't complete
3672 if re.match(r"^[#*]*:+(\s*$|\s+)", lines[i]): 3672 ↛ 3673line 3672 didn't jump to line 3673 because the condition on line 3672 was never true
3673 break
3674 refs.append(lines[i].strip())
3675 if re.search(r"[]\d:)]\s*$", lines[i]):
3676 break
3677 ref = " ".join(refs)
3678 lines = lines[i + 1 :]
3679 if (
3680 lang_code != "en"
3681 and len(lines) >= 2
3682 and classify_desc2(lines[-1]) in ENGLISH_TEXTS
3683 ):
3684 i = len(lines) - 1
3685 while ( 3685 ↛ 3690line 3685 didn't jump to line 3690 because the condition on line 3685 was never true
3686 i > 1
3687 and classify_desc2(lines[i - 1])
3688 in ENGLISH_TEXTS
3689 ):
3690 i -= 1
3691 tr = "\n".join(lines[i:])
3692 lines = lines[:i]
3693 if len(lines) >= 2:
3694 if classify_desc2(lines[-1]) == "romanization":
3695 roman = lines[-1].strip()
3696 lines = lines[:-1]
3698 elif lang_code == "en" and re.match(r"^[#*]*:+", lines[1]):
3699 ref = lines[0]
3700 lines = lines[1:]
3701 elif lang_code != "en" and len(lines) == 2:
3702 cls1 = classify_desc2(lines[0])
3703 cls2 = classify_desc2(lines[1])
3704 if cls2 in ENGLISH_TEXTS and cls1 != "english":
3705 tr = lines[1]
3706 lines = [lines[0]]
3707 elif cls1 in ENGLISH_TEXTS and cls2 != "english": 3707 ↛ 3708line 3707 didn't jump to line 3708 because the condition on line 3707 was never true
3708 tr = lines[0]
3709 lines = [lines[1]]
3710 elif ( 3710 ↛ 3717line 3710 didn't jump to line 3717 because the condition on line 3710 was never true
3711 re.match(r"^[#*]*:+", lines[1])
3712 and classify_desc2(
3713 re.sub(r"^[#*:]+\s*", "", lines[1])
3714 )
3715 in ENGLISH_TEXTS
3716 ):
3717 tr = re.sub(r"^[#*:]+\s*", "", lines[1])
3718 lines = [lines[0]]
3719 elif cls1 == "english" and cls2 in ENGLISH_TEXTS:
3720 # Both were classified as English, but
3721 # presumably one is not. Assume first is
3722 # non-English, as that seems more common.
3723 tr = lines[1]
3724 lines = [lines[0]]
3725 elif (
3726 usex_type != "quotation"
3727 and lang_code != "en"
3728 and len(lines) == 3
3729 ):
3730 cls1 = classify_desc2(lines[0])
3731 cls2 = classify_desc2(lines[1])
3732 cls3 = classify_desc2(lines[2])
3733 if (
3734 cls3 == "english"
3735 and cls2 in ("english", "romanization")
3736 and cls1 != "english"
3737 ):
3738 tr = lines[2].strip()
3739 roman = lines[1].strip()
3740 lines = [lines[0].strip()]
3741 elif ( 3741 ↛ 3749line 3741 didn't jump to line 3749 because the condition on line 3741 was never true
3742 usex_type == "quotation"
3743 and lang_code != "en"
3744 and len(lines) > 2
3745 ):
3746 # for x in lines:
3747 # print(" LINE: {}: {}"
3748 # .format(classify_desc2(x), x))
3749 if re.match(r"^[#*]*:+\s*$", lines[1]):
3750 ref = lines[0]
3751 lines = lines[2:]
3752 cls1 = classify_desc2(lines[-1])
3753 if cls1 == "english":
3754 i = len(lines) - 1
3755 while (
3756 i > 1
3757 and classify_desc2(lines[i - 1])
3758 == ENGLISH_TEXTS
3759 ):
3760 i -= 1
3761 tr = "\n".join(lines[i:])
3762 lines = lines[:i]
3764 roman = re.sub(r"[ \t\r]+", " ", roman).strip()
3765 roman = re.sub(r"\[\s*…\s*\]", "[…]", roman)
3766 tr = re.sub(r"^[#*:]+\s*", "", tr)
3767 tr = re.sub(r"[ \t\r]+", " ", tr).strip()
3768 tr = re.sub(r"\[\s*…\s*\]", "[…]", tr)
3769 ref = re.sub(r"^[#*:]+\s*", "", ref)
3770 ref = re.sub(
3771 r", (volume |number |page )?“?"
3772 r"\(please specify ([^)]|\(s\))*\)”?|"
3773 ", text here$",
3774 "",
3775 ref,
3776 )
3777 ref = re.sub(r"\[\s*…\s*\]", "[…]", ref)
3778 lines = list(re.sub(r"^[#*:]+\s*", "", x) for x in lines)
3779 subtext = "\n".join(x for x in lines if x)
3780 if not tr and lang_code != "en":
3781 m = re.search(r"([.!?])\s+\(([^)]+)\)\s*$", subtext)
3782 if m and classify_desc2(m.group(2)) in ENGLISH_TEXTS: 3782 ↛ 3783line 3782 didn't jump to line 3783 because the condition on line 3782 was never true
3783 tr = m.group(2)
3784 subtext = subtext[: m.start()] + m.group(1)
3785 elif lines:
3786 parts = re.split(r"\s*[―—]+\s*", lines[0])
3787 if ( 3787 ↛ 3791line 3787 didn't jump to line 3791 because the condition on line 3787 was never true
3788 len(parts) == 2
3789 and classify_desc2(parts[1]) in ENGLISH_TEXTS
3790 ):
3791 subtext = parts[0].strip()
3792 tr = parts[1].strip()
3793 subtext = re.sub(r'^[“"`]([^“"`”\']*)[”"\']$', r"\1", subtext)
3794 subtext = re.sub(
3795 r"(please add an English translation of "
3796 r"this (quote|usage example))",
3797 "",
3798 subtext,
3799 )
3800 subtext = re.sub(
3801 r"\s*→New International Version " "translation$",
3802 "",
3803 subtext,
3804 ) # e.g. pis/Tok Pisin (Bible)
3805 subtext = re.sub(r"[ \t\r]+", " ", subtext).strip()
3806 subtext = re.sub(r"\[\s*…\s*\]", "[…]", subtext)
3807 note = None
3808 m = re.match(r"^\(([^)]*)\):\s+", subtext)
3809 if ( 3809 ↛ 3817line 3809 didn't jump to line 3817 because the condition on line 3809 was never true
3810 m is not None
3811 and lang_code != "en"
3812 and (
3813 m.group(1).startswith("with ")
3814 or classify_desc2(m.group(1)) == "english"
3815 )
3816 ):
3817 note = m.group(1)
3818 subtext = subtext[m.end() :]
3819 ref = re.sub(r"\s*\(→ISBN\)", "", ref)
3820 ref = re.sub(r",\s*→ISBN", "", ref)
3821 ref = ref.strip()
3822 if ref.endswith(":") or ref.endswith(","):
3823 ref = ref[:-1].strip()
3824 ref = re.sub(r"\s+,\s+", ", ", ref)
3825 ref = re.sub(r"\s+", " ", ref)
3826 if ref and not subtext: 3826 ↛ 3827line 3826 didn't jump to line 3827 because the condition on line 3826 was never true
3827 subtext = ref
3828 ref = ""
3829 if subtext:
3830 dt: ExampleData = {"text": subtext}
3831 if ref:
3832 dt["ref"] = ref
3833 if tr:
3834 dt["english"] = tr # DEPRECATED for "translation"
3835 dt["translation"] = tr
3836 if usex_type:
3837 dt["type"] = usex_type
3838 if note: 3838 ↛ 3839line 3838 didn't jump to line 3839 because the condition on line 3838 was never true
3839 dt["note"] = note
3840 if roman:
3841 dt["roman"] = roman
3842 if ruby:
3843 dt["ruby"] = ruby
3844 examples.append(dt)
3846 return examples
3848 # Main code of parse_language()
3849 # Process the section
3850 stack.append(language)
3851 process_children(langnode, None)
3852 stack.pop()
3854 # Finalize word entires
3855 push_etym()
3856 ret = []
3857 for data in page_datas:
3858 merge_base(data, base_data)
3859 ret.append(data)
3861 # Copy all tags to word senses
3862 for data in ret:
3863 if "senses" not in data: 3863 ↛ 3864line 3863 didn't jump to line 3864 because the condition on line 3863 was never true
3864 continue
3865 # WordData should not have a 'tags' field, but if it does, it's
3866 # deleted and its contents removed and placed in each sense;
3867 # that's why the type ignores.
3868 tags: Iterable = data.get("tags", ()) # type: ignore[assignment]
3869 if "tags" in data:
3870 del data["tags"] # type: ignore[typeddict-item]
3871 for sense in data["senses"]:
3872 data_extend(sense, "tags", tags)
3874 return ret
3877def parse_wikipedia_template(
3878 wxr: WiktextractContext, data: WordData, ht: TemplateArgs
3879) -> None:
3880 """Helper function for parsing {{wikipedia|...}} and related templates."""
3881 assert isinstance(wxr, WiktextractContext)
3882 assert isinstance(data, dict)
3883 assert isinstance(ht, dict)
3884 langid = clean_node(wxr, data, ht.get("lang", ()))
3885 pagename = (
3886 clean_node(wxr, data, ht.get(1, ()))
3887 or wxr.wtp.title
3888 or "MISSING_PAGE_TITLE"
3889 )
3890 if langid:
3891 data_append(data, "wikipedia", langid + ":" + pagename)
3892 else:
3893 data_append(data, "wikipedia", pagename)
3896def parse_top_template(
3897 wxr: WiktextractContext, node: WikiNode, data: WordData
3898) -> None:
3899 """Parses a template that occurs on the top-level in a page, before any
3900 language subtitles."""
3901 assert isinstance(wxr, WiktextractContext)
3902 assert isinstance(node, WikiNode)
3903 assert isinstance(data, dict)
3905 def top_template_fn(name: str, ht: TemplateArgs) -> Optional[str]:
3906 if name in wikipedia_templates:
3907 parse_wikipedia_template(wxr, data, ht)
3908 return None
3909 if is_panel_template(wxr, name):
3910 return ""
3911 if name in ("reconstruction",): 3911 ↛ 3912line 3911 didn't jump to line 3912 because the condition on line 3911 was never true
3912 return ""
3913 if name.lower() == "also" or name.lower().startswith("also/"):
3914 # XXX shows related words that might really have been the intended
3915 # word, capture them
3916 return ""
3917 if name == "see also": 3917 ↛ 3919line 3917 didn't jump to line 3919 because the condition on line 3917 was never true
3918 # XXX capture
3919 return ""
3920 if name == "cardinalbox": 3920 ↛ 3922line 3920 didn't jump to line 3922 because the condition on line 3920 was never true
3921 # XXX capture
3922 return ""
3923 if name == "character info": 3923 ↛ 3925line 3923 didn't jump to line 3925 because the condition on line 3923 was never true
3924 # XXX capture
3925 return ""
3926 if name == "commonscat": 3926 ↛ 3928line 3926 didn't jump to line 3928 because the condition on line 3926 was never true
3927 # XXX capture link to Wikimedia commons
3928 return ""
3929 if name == "wrongtitle": 3929 ↛ 3932line 3929 didn't jump to line 3932 because the condition on line 3929 was never true
3930 # XXX this should be captured to replace page title with the
3931 # correct title. E.g. ⿰亻革家
3932 return ""
3933 if name == "wikidata": 3933 ↛ 3934line 3933 didn't jump to line 3934 because the condition on line 3933 was never true
3934 arg = clean_node(wxr, data, ht.get(1, ()))
3935 if arg.startswith("Q") or arg.startswith("Lexeme:L"):
3936 data_append(data, "wikidata", arg)
3937 return ""
3938 wxr.wtp.debug(
3939 "UNIMPLEMENTED top-level template: {} {}".format(name, ht),
3940 sortid="page/2870",
3941 )
3942 return ""
3944 clean_node(wxr, None, [node], template_fn=top_template_fn)
3947def fix_subtitle_hierarchy(wxr: WiktextractContext, text: str) -> str:
3948 """Fix subtitle hierarchy to be strict Language -> Etymology ->
3949 Part-of-Speech -> Translation/Linkage. Also merge Etymology sections
3950 that are next to each other."""
3952 # Wiktextract issue #620, Chinese Glyph Origin before an etymology
3953 # section get overwritten. In this case, let's just combine the two.
3955 # In Chinese entries, Pronunciation can be preceded on the
3956 # same level 3 by its Etymology *and* Glyph Origin sections:
3957 # ===Glyph Origin===
3958 # ===Etymology===
3959 # ===Pronunciation===
3960 # Tatu suggested adding a new 'level' between 3 and 4, so Pronunciation
3961 # is now Level 4, POS is shifted to Level 5 and the rest (incl. 'default')
3962 # are now level 6
3964 # Known lowercase PoS names are in part_of_speech_map
3965 # Known lowercase linkage section names are in linkage_map
3967 old = re.split(
3968 r"(?m)^(==+)[ \t]*([^= \t]([^=\n]|=[^=])*?)" r"[ \t]*(==+)[ \t]*$", text
3969 )
3971 parts = []
3972 npar = 4 # Number of parentheses in above expression
3973 parts.append(old[0])
3974 prev_level = None
3975 level = None
3976 skip_level_title = False # When combining etymology sections
3977 for i in range(1, len(old), npar + 1):
3978 left = old[i]
3979 right = old[i + npar - 1]
3980 # remove Wikilinks in title
3981 title = re.sub(r"^\[\[", "", old[i + 1])
3982 title = re.sub(r"\]\]$", "", title)
3983 prev_level = level
3984 level = len(left)
3985 part = old[i + npar]
3986 if level != len(right): 3986 ↛ 3987line 3986 didn't jump to line 3987 because the condition on line 3986 was never true
3987 wxr.wtp.debug(
3988 "subtitle has unbalanced levels: "
3989 "{!r} has {} on the left and {} on the right".format(
3990 title, left, right
3991 ),
3992 sortid="page/2904",
3993 )
3994 lc = title.lower()
3995 if name_to_code(title, "en") != "":
3996 if level > 2: 3996 ↛ 3997line 3996 didn't jump to line 3997 because the condition on line 3996 was never true
3997 wxr.wtp.debug(
3998 "subtitle has language name {} at level {}".format(
3999 title, level
4000 ),
4001 sortid="page/2911",
4002 )
4003 level = 2
4004 elif lc.startswith(tuple(ETYMOLOGY_TITLES)):
4005 if level > 3: 4005 ↛ 4006line 4005 didn't jump to line 4006 because the condition on line 4005 was never true
4006 wxr.wtp.debug(
4007 "etymology section {} at level {}".format(title, level),
4008 sortid="page/2917",
4009 )
4010 if prev_level == 3: # Two etymology (Glyph Origin + Etymology)
4011 # sections cheek-to-cheek
4012 skip_level_title = True
4013 # Modify the title of previous ("Glyph Origin") section, in
4014 # case we have a meaningful title like "Etymology 1"
4015 parts[-2] = "{}{}{}".format("=" * level, title, "=" * level)
4016 level = 3
4017 elif lc.startswith(PRONUNCIATION_TITLE):
4018 # Pronunciation is now a level between POS and Etymology, so
4019 # we need to shift everything down by one
4020 level = 4
4021 elif lc in POS_TITLES:
4022 level = 5
4023 elif lc == TRANSLATIONS_TITLE:
4024 level = 6
4025 elif lc in LINKAGE_TITLES or lc == COMPOUNDS_TITLE:
4026 level = 6
4027 elif lc in INFLECTION_TITLES:
4028 level = 6
4029 elif lc == DESCENDANTS_TITLE:
4030 level = 6
4031 elif title in PROTO_ROOT_DERIVED_TITLES: 4031 ↛ 4032line 4031 didn't jump to line 4032 because the condition on line 4031 was never true
4032 level = 6
4033 elif lc in IGNORED_TITLES:
4034 level = 6
4035 else:
4036 level = 6
4037 if skip_level_title:
4038 skip_level_title = False
4039 parts.append(part)
4040 else:
4041 parts.append("{}{}{}".format("=" * level, title, "=" * level))
4042 parts.append(part)
4043 # print("=" * level, title)
4044 # if level != len(left):
4045 # print(" FIXED LEVEL OF {} {} -> {}"
4046 # .format(title, len(left), level))
4048 text = "".join(parts)
4049 # print(text)
4050 return text
4053def parse_page(wxr: WiktextractContext, word: str, text: str) -> list[WordData]:
4054 # Skip translation pages
4055 if word.endswith("/" + TRANSLATIONS_TITLE): 4055 ↛ 4056line 4055 didn't jump to line 4056 because the condition on line 4055 was never true
4056 return []
4058 if wxr.config.verbose: 4058 ↛ 4059line 4058 didn't jump to line 4059 because the condition on line 4058 was never true
4059 logger.info(f"Parsing page: {word}")
4061 wxr.config.word = word
4062 wxr.wtp.start_page(word)
4064 # Remove <noinclude> and similar tags from main pages. They
4065 # should not appear there, but at least net/Elfdala has one and it
4066 # is probably not the only one.
4067 text = re.sub(r"(?si)<(/)?noinclude\s*>", "", text)
4068 text = re.sub(r"(?si)<(/)?onlyinclude\s*>", "", text)
4069 text = re.sub(r"(?si)<(/)?includeonly\s*>", "", text)
4071 # Fix up the subtitle hierarchy. There are hundreds if not thousands of
4072 # pages that have, for example, Translations section under Linkage, or
4073 # Translations section on the same level as Noun. Enforce a proper
4074 # hierarchy by manipulating the subtitle levels in certain cases.
4075 text = fix_subtitle_hierarchy(wxr, text)
4077 # Parse the page, pre-expanding those templates that are likely to
4078 # influence parsing
4079 tree = wxr.wtp.parse(
4080 text,
4081 pre_expand=True,
4082 additional_expand=ADDITIONAL_EXPAND_TEMPLATES,
4083 do_not_pre_expand=DO_NOT_PRE_EXPAND_TEMPLATES,
4084 )
4085 # from wikitextprocessor.parser import print_tree
4086 # print("PAGE PARSE:", print_tree(tree))
4088 top_data: WordData = {}
4090 # Iterate over top-level titles, which should be languages for normal
4091 # pages
4092 by_lang = defaultdict(list)
4093 for langnode in tree.children:
4094 if not isinstance(langnode, WikiNode):
4095 continue
4096 if langnode.kind == NodeKind.TEMPLATE:
4097 parse_top_template(wxr, langnode, top_data)
4098 continue
4099 if langnode.kind == NodeKind.LINK:
4100 # Some pages have links at top level, e.g., "trees" in Wiktionary
4101 continue
4102 if langnode.kind != NodeKind.LEVEL2: 4102 ↛ 4103line 4102 didn't jump to line 4103 because the condition on line 4102 was never true
4103 wxr.wtp.debug(
4104 f"unexpected top-level node: {langnode}", sortid="page/3014"
4105 )
4106 continue
4107 lang = clean_node(
4108 wxr, None, langnode.sarg if langnode.sarg else langnode.largs
4109 )
4110 lang_code = name_to_code(lang, "en")
4111 if lang_code == "": 4111 ↛ 4112line 4111 didn't jump to line 4112 because the condition on line 4111 was never true
4112 wxr.wtp.debug(
4113 f"unrecognized language name: {lang}", sortid="page/3019"
4114 )
4115 if (
4116 wxr.config.capture_language_codes
4117 and lang_code not in wxr.config.capture_language_codes
4118 ):
4119 continue
4120 wxr.wtp.start_section(lang)
4122 # Collect all words from the page.
4123 # print(f"{langnode=}")
4124 datas = parse_language(wxr, langnode, lang, lang_code)
4126 # Propagate fields resulting from top-level templates to this
4127 # part-of-speech.
4128 for data in datas:
4129 if "lang" not in data: 4129 ↛ 4130line 4129 didn't jump to line 4130 because the condition on line 4129 was never true
4130 wxr.wtp.debug(
4131 "internal error -- no lang in data: {}".format(data),
4132 sortid="page/3034",
4133 )
4134 continue
4135 for k, v in top_data.items():
4136 assert isinstance(v, (list, tuple))
4137 data_extend(data, k, v)
4138 by_lang[data["lang"]].append(data)
4140 # XXX this code is clearly out of date. There is no longer a "conjugation"
4141 # field. FIX OR REMOVE.
4142 # Do some post-processing on the words. For example, we may distribute
4143 # conjugation information to all the words.
4144 ret = []
4145 for lang, lang_datas in by_lang.items():
4146 ret.extend(lang_datas)
4148 for x in ret:
4149 if x["word"] != word:
4150 if word.startswith("Unsupported titles/"):
4151 wxr.wtp.debug(
4152 f"UNSUPPORTED TITLE: '{word}' -> '{x['word']}'",
4153 sortid="20231101/3578page.py",
4154 )
4155 else:
4156 wxr.wtp.debug(
4157 f"DIFFERENT ORIGINAL TITLE: '{word}' -> '{x['word']}'",
4158 sortid="20231101/3582page.py",
4159 )
4160 x["original_title"] = word
4161 # validate tag data
4162 recursively_separate_raw_tags(wxr, x) # type:ignore[arg-type]
4163 return ret
4166def recursively_separate_raw_tags(
4167 wxr: WiktextractContext, data: dict[str, Any]
4168) -> None:
4169 if not isinstance(data, dict): 4169 ↛ 4170line 4169 didn't jump to line 4170 because the condition on line 4169 was never true
4170 wxr.wtp.error(
4171 "'data' is not dict; most probably "
4172 "data has a list that contains at least one dict and "
4173 "at least one non-dict item",
4174 sortid="en/page-4016/20240419",
4175 )
4176 return
4177 new_tags: list[str] = []
4178 raw_tags: list[str] = data.get("raw_tags", [])
4179 for field, val in data.items():
4180 if field == "tags":
4181 for tag in val:
4182 if tag not in valid_tags:
4183 raw_tags.append(tag)
4184 else:
4185 new_tags.append(tag)
4186 if isinstance(val, list):
4187 if len(val) > 0 and isinstance(val[0], dict):
4188 for d in val:
4189 recursively_separate_raw_tags(wxr, d)
4190 if "tags" in data and not new_tags:
4191 del data["tags"]
4192 elif new_tags:
4193 data["tags"] = new_tags
4194 if raw_tags:
4195 data["raw_tags"] = raw_tags
4198def process_soft_redirect_template(
4199 wxr: WiktextractContext,
4200 template_node: TemplateNode,
4201 redirect_pages: list[str],
4202) -> bool:
4203 # return `True` if the template is soft redirect template
4204 if template_node.template_name == "zh-see":
4205 # https://en.wiktionary.org/wiki/Template:zh-see
4206 title = clean_node(
4207 wxr, None, template_node.template_parameters.get(1, "")
4208 )
4209 if title != "": 4209 ↛ 4211line 4209 didn't jump to line 4211 because the condition on line 4209 was always true
4210 redirect_pages.append(title)
4211 return True
4212 elif template_node.template_name in ["ja-see", "ja-see-kango"]:
4213 # https://en.wiktionary.org/wiki/Template:ja-see
4214 for key, value in template_node.template_parameters.items():
4215 if isinstance(key, int): 4215 ↛ 4214line 4215 didn't jump to line 4214 because the condition on line 4215 was always true
4216 title = clean_node(wxr, None, value)
4217 if title != "": 4217 ↛ 4214line 4217 didn't jump to line 4214 because the condition on line 4217 was always true
4218 redirect_pages.append(title)
4219 return True
4220 return False
4223ZH_FORMS_TAGS = {
4224 "trad.": "Traditional-Chinese",
4225 "simp.": "Simplified-Chinese",
4226 "alternative forms": "alternative",
4227 "2nd round simp.": "Second-Round-Simplified-Chinese",
4228}
4231def extract_zh_forms_template(
4232 wxr: WiktextractContext, t_node: TemplateNode, base_data: WordData
4233):
4234 # https://en.wiktionary.org/wiki/Template:zh-forms
4235 lit_meaning = clean_node(
4236 wxr, None, t_node.template_parameters.get("lit", "")
4237 )
4238 if lit_meaning != "":
4239 base_data["literal_meaning"] = lit_meaning
4240 expanded_node = wxr.wtp.parse(
4241 wxr.wtp.node_to_wikitext(t_node), expand_all=True
4242 )
4243 for table in expanded_node.find_child(NodeKind.TABLE):
4244 for row in table.find_child(NodeKind.TABLE_ROW):
4245 row_header = ""
4246 row_header_tags: list[str] = []
4247 header_has_span = False
4248 for cell in row.find_child(
4249 NodeKind.TABLE_HEADER_CELL | NodeKind.TABLE_CELL
4250 ):
4251 if cell.kind == NodeKind.TABLE_HEADER_CELL:
4252 row_header, row_header_tags, header_has_span = (
4253 extract_zh_forms_header_cell(wxr, base_data, cell)
4254 )
4255 elif not header_has_span:
4256 extract_zh_forms_data_cell(
4257 wxr, base_data, cell, row_header, row_header_tags
4258 )
4260 if "forms" in base_data and len(base_data["forms"]) == 0: 4260 ↛ 4261line 4260 didn't jump to line 4261 because the condition on line 4260 was never true
4261 del base_data["forms"]
4264def extract_zh_forms_header_cell(
4265 wxr: WiktextractContext, base_data: WordData, header_cell: WikiNode
4266) -> tuple[str, list[str], bool]:
4267 row_header = ""
4268 row_header_tags = []
4269 header_has_span = False
4270 first_span_index = len(header_cell.children)
4271 for index, span_tag in header_cell.find_html("span", with_index=True):
4272 if index < first_span_index: 4272 ↛ 4274line 4272 didn't jump to line 4274 because the condition on line 4272 was always true
4273 first_span_index = index
4274 header_has_span = True
4275 row_header = clean_node(wxr, None, header_cell.children[:first_span_index])
4276 for raw_tag in row_header.split(" and "):
4277 raw_tag = raw_tag.strip()
4278 if raw_tag != "":
4279 row_header_tags.append(raw_tag)
4280 for span_tag in header_cell.find_html_recursively("span"):
4281 span_lang = span_tag.attrs.get("lang", "")
4282 form_nodes = []
4283 sup_title = ""
4284 for node in span_tag.children:
4285 if isinstance(node, HTMLNode) and node.tag == "sup": 4285 ↛ 4286line 4285 didn't jump to line 4286 because the condition on line 4285 was never true
4286 for sup_span in node.find_html("span"):
4287 sup_title = sup_span.attrs.get("title", "")
4288 else:
4289 form_nodes.append(node)
4290 if span_lang in ["zh-Hant", "zh-Hans"]:
4291 for word in clean_node(wxr, None, form_nodes).split("/"):
4292 if word not in [wxr.wtp.title, ""]:
4293 form = {"form": word}
4294 for raw_tag in row_header_tags:
4295 if raw_tag in ZH_FORMS_TAGS: 4295 ↛ 4298line 4295 didn't jump to line 4298 because the condition on line 4295 was always true
4296 data_append(form, "tags", ZH_FORMS_TAGS[raw_tag])
4297 else:
4298 data_append(form, "raw_tags", raw_tag)
4299 if sup_title != "": 4299 ↛ 4300line 4299 didn't jump to line 4300 because the condition on line 4299 was never true
4300 data_append(form, "raw_tags", sup_title)
4301 data_append(base_data, "forms", form)
4302 return row_header, row_header_tags, header_has_span
4305TagLiteral = Literal["tags", "raw_tags"]
4306TAG_LITERALS_TUPLE: tuple[TagLiteral, ...] = ("tags", "raw_tags")
4309def extract_zh_forms_data_cell(
4310 wxr: WiktextractContext,
4311 base_data: WordData,
4312 cell: WikiNode,
4313 row_header: str,
4314 row_header_tags: list[str],
4315) -> None:
4316 from .zh_pron_tags import ZH_PRON_TAGS
4318 forms: list[FormData] = []
4319 for top_span_tag in cell.find_html("span"):
4320 span_style = top_span_tag.attrs.get("style", "")
4321 span_lang = top_span_tag.attrs.get("lang", "")
4322 if span_style == "white-space:nowrap;":
4323 extract_zh_forms_data_cell(
4324 wxr, base_data, top_span_tag, row_header, row_header_tags
4325 )
4326 elif "font-size:80%" in span_style:
4327 raw_tag = clean_node(wxr, None, top_span_tag)
4328 if raw_tag != "": 4328 ↛ 4319line 4328 didn't jump to line 4319 because the condition on line 4328 was always true
4329 for form in forms:
4330 if raw_tag in ZH_PRON_TAGS: 4330 ↛ 4336line 4330 didn't jump to line 4336 because the condition on line 4330 was always true
4331 tr_tag = ZH_PRON_TAGS[raw_tag]
4332 if isinstance(tr_tag, list): 4332 ↛ 4333line 4332 didn't jump to line 4333 because the condition on line 4332 was never true
4333 data_extend(form, "tags", tr_tag)
4334 elif isinstance(tr_tag, str): 4334 ↛ 4329line 4334 didn't jump to line 4329 because the condition on line 4334 was always true
4335 data_append(form, "tags", tr_tag)
4336 elif raw_tag in valid_tags:
4337 data_append(form, "tags", raw_tag)
4338 else:
4339 data_append(form, "raw_tags", raw_tag)
4340 elif span_lang in ["zh-Hant", "zh-Hans", "zh"]: 4340 ↛ 4319line 4340 didn't jump to line 4319 because the condition on line 4340 was always true
4341 word = clean_node(wxr, None, top_span_tag)
4342 if word not in ["", "／", wxr.wtp.title]:
4343 form = {"form": word}
4344 if row_header != "anagram": 4344 ↛ 4350line 4344 didn't jump to line 4350 because the condition on line 4344 was always true
4345 for raw_tag in row_header_tags:
4346 if raw_tag in ZH_FORMS_TAGS: 4346 ↛ 4349line 4346 didn't jump to line 4349 because the condition on line 4346 was always true
4347 data_append(form, "tags", ZH_FORMS_TAGS[raw_tag])
4348 else:
4349 data_append(form, "raw_tags", raw_tag)
4350 if span_lang == "zh-Hant":
4351 data_append(form, "tags", "Traditional-Chinese")
4352 elif span_lang == "zh-Hans":
4353 data_append(form, "tags", "Simplified-Chinese")
4354 forms.append(form)
4356 if row_header == "anagram": 4356 ↛ 4357line 4356 didn't jump to line 4357 because the condition on line 4356 was never true
4357 for form in forms:
4358 l_data: LinkageData = {"word": form["form"]}
4359 for key in TAG_LITERALS_TUPLE:
4360 if key in form:
4361 l_data[key] = form[key]
4362 data_append(base_data, "anagrams", l_data)
4363 else:
4364 data_extend(base_data, "forms", forms)
4367def extract_ja_kanjitab_template(
4368 wxr: WiktextractContext, t_node: TemplateNode, base_data: WordData
4369):
4370 # https://en.wiktionary.org/wiki/Template:ja-kanjitab
4371 expanded_node = wxr.wtp.parse(
4372 wxr.wtp.node_to_wikitext(t_node), expand_all=True
4373 )
4374 for table in expanded_node.find_child(NodeKind.TABLE):
4375 is_alt_form_table = False
4376 for row in table.find_child(NodeKind.TABLE_ROW):
4377 for header_node in row.find_child(NodeKind.TABLE_HEADER_CELL):
4378 header_text = clean_node(wxr, None, header_node)
4379 if header_text.startswith("Alternative spelling"):
4380 is_alt_form_table = True
4381 if not is_alt_form_table:
4382 continue
4383 forms = []
4384 for row in table.find_child(NodeKind.TABLE_ROW):
4385 for cell_node in row.find_child(NodeKind.TABLE_CELL):
4386 for child_node in cell_node.children:
4387 if isinstance(child_node, HTMLNode):
4388 if child_node.tag == "span":
4389 word = clean_node(wxr, None, child_node)
4390 if word != "": 4390 ↛ 4386line 4390 didn't jump to line 4386 because the condition on line 4390 was always true
4391 forms.append(
4392 {
4393 "form": word,
4394 "tags": ["alternative", "kanji"],
4395 }
4396 )
4397 elif child_node.tag == "small":
4398 raw_tag = clean_node(wxr, None, child_node).strip(
4399 "()"
4400 )
4401 if raw_tag != "" and len(forms) > 0: 4401 ↛ 4386line 4401 didn't jump to line 4386 because the condition on line 4401 was always true
4402 data_append(
4403 forms[-1],
4404 "tags"
4405 if raw_tag in valid_tags
4406 else "raw_tags",
4407 raw_tag,
4408 )
4409 data_extend(base_data, "forms", forms)
4410 for link_node in expanded_node.find_child(NodeKind.LINK):
4411 clean_node(wxr, base_data, link_node)