Coverage for src/wiktextract/extractor/en/inflection.py: 87%
1549 statements
« prev ^ index » next coverage.py v7.15.0, created at 2026-07-07 09:23 +0000
1# Code for parsing inflection tables.
2#
3# Copyright (c) 2021-2022 Tatu Ylonen. See file LICENSE and https://ylonen.org.
5import collections
6import copy
7import functools
8import html
9import re
10import unicodedata
11from typing import TYPE_CHECKING, Generator, Literal, Optional, Union
13from mediawiki_langcodes import code_to_name, name_to_code
14from wikitextprocessor import MAGIC_FIRST, HTMLNode, NodeKind, WikiNode
16from ...clean import clean_value
17from ...datautils import data_append, freeze, split_at_comma_semi
18from ...tags import valid_tags
19from ...wxr_context import WiktextractContext
20from .form_descriptions import (
21 classify_desc,
22 decode_tags,
23 distw,
24 match_links_to_form,
25 parse_head_final_tags,
26)
27from .inflection_kludges import ka_decl_noun_template_cell
28from .inflectiondata import infl_map, infl_start_map, infl_start_re
29from .lang_specific_configs import get_lang_conf, lang_specific_tags
30from .table_headers_heuristics_data import LANGUAGES_WITH_CELLS_AS_HEADERS
31from .type_utils import FormData, WordData
33# --debug-text-cell WORD
34# Command-line parameter for debugging. When parsing inflection tables,
35# print out debug messages when encountering this text.
36debug_cell_text: Optional[str] = None
39def set_debug_cell_text(text: str) -> None:
40 global debug_cell_text
41 debug_cell_text = text
44TagSets = list[tuple[str, ...]]
46# Column texts that are interpreted as an empty column.
47IGNORED_COLVALUES = {
48 "-",
49 "־",
50 "᠆",
51 "‐",
52 "‑",
53 "‒",
54 "–",
55 "—",
56 "―",
57 "−",
58 "⸺",
59 "⸻",
60 "﹘",
61 "﹣",
62 "－",
63 "/",
64 "?",
65 "not used",
66 "not applicable",
67}
69# These tags are never inherited from above
70# XXX merge with lang_specific
71noinherit_tags = {
72 "infinitive-i",
73 "infinitive-i-long",
74 "infinitive-ii",
75 "infinitive-iii",
76 "infinitive-iv",
77 "infinitive-v",
78}
80# Subject->object transformation mapping, when using dummy-object-concord
81# to replace subject concord tags with object concord tags
82object_concord_replacements = {
83 "first-person": "object-first-person",
84 "second-person": "object-second-person",
85 "third-person": "object-third-person",
86 "singular": "object-singular",
87 "plural": "object-plural",
88 "definite": "object-definite",
89 "indefinite": "object-indefinite",
90 "class-1": "object-class-1",
91 "class-2": "object-class-2",
92 "class-3": "object-class-3",
93 "class-4": "object-class-4",
94 "class-5": "object-class-5",
95 "class-6": "object-class-6",
96 "class-7": "object-class-7",
97 "class-8": "object-class-8",
98 "class-9": "object-class-9",
99 "class-10": "object-class-10",
100 "class-11": "object-class-11",
101 "class-12": "object-class-12",
102 "class-13": "object-class-13",
103 "class-14": "object-class-14",
104 "class-15": "object-class-15",
105 "class-16": "object-class-16",
106 "class-17": "object-class-17",
107 "class-18": "object-class-18",
108 "masculine": "object-masculine",
109 "feminine": "object-feminine",
110}
112# Words in title that cause addition of tags in all entries
113title_contains_global_map = {
114 "possessive": "possessive",
115 "possessed forms of": "possessive",
116 "predicative forms of": "predicative",
117 "negative": "negative",
118 "positive definite forms": "positive definite",
119 "positive indefinite forms": "positive indefinite",
120 "comparative": "comparative",
121 "superlative": "superlative",
122 "combined forms": "combined-form",
123 "mutation": "mutation",
124 "definite article": "definite",
125 "indefinite article": "indefinite",
126 "indefinite declension": "indefinite",
127 "bare forms": "indefinite", # e.g., cois/Irish
128 "definite declension": "definite",
129 "pre-reform": "dated",
130 "personal pronouns": "personal pronoun",
131 "composed forms of": "multiword-construction",
132 "subordinate-clause forms of": "subordinate-clause",
133 "participles of": "participle",
134 "variation of": "dummy-skip-this", # a'/Scottish Gaelic
135 "command form of": "imperative", # a راتلل/Pashto
136 "historical inflection of": "dummy-skip-this", # kork/Norwegian Nynorsk
137 "obsolete declension": "obsolete", # März/German 20241111
138 # käyminen/Finnish 20260707
139 "first-person singular possessor": "first-person singular singular-possessive",
140 "second-person singular possessor": "second-person singular singular-possessive",
141 "first-person plural possessor": "first-person plural plural-possessive",
142 "second-person plural possessor": "second-person plural plural-possessive",
143 "third-person possessor": "third-person possessive",
144}
145for k, v in title_contains_global_map.items():
146 if any(t not in valid_tags for t in v.split()): 146 ↛ 147line 146 didn't jump to line 147 because the condition on line 146 was never true
147 print("TITLE_CONTAINS_GLOBAL_MAP UNRECOGNIZED TAG: {}: {}".format(k, v))
148table_hdr_ign_part = r"(Inflection|Conjugation|Declension|Mutation) of [^\s]"
150table_hdr_ign_part_re = re.compile(r"(?i)(" + table_hdr_ign_part + ")")
151# (?i) python regex extension, ignore case
152title_contains_global_re = re.compile(
153 r"(?i)(^|\b)({}|{})($|\b)".format(
154 table_hdr_ign_part,
155 "|".join(re.escape(x) for x in title_contains_global_map.keys()),
156 )
157)
159# Words in title that cause addition of tags to table-tags "form"
160title_contains_wordtags_map = {
161 "pf": "perfective",
162 "impf": "imperfective",
163 "strong": "strong",
164 "weak": "weak",
165 "countable": "countable",
166 "uncountable": "uncountable",
167 "inanimate": "inanimate",
168 "animate": "animate",
169 "transitive": "transitive",
170 "intransitive": "intransitive",
171 "ditransitive": "ditransitive",
172 "ambitransitive": "ambitransitive",
173 "archaic": "archaic",
174 "dated": "dated",
175 "affirmative": "affirmative",
176 "negative": "negative",
177 "subject pronouns": "subjective",
178 "object pronouns": "objective",
179 "emphatic": "emphatic",
180 "proper noun": "proper-noun",
181 "no plural": "no-plural",
182 "imperfective": "imperfective",
183 "perfective": "perfective",
184 "no supine stem": "no-supine",
185 "no perfect stem": "no-perfect",
186 "deponent": "deponent",
187 "irregular": "irregular",
188 "no short forms": "no-short-form",
189 "iō-variant": "iō-variant",
190 "1st declension": "declension-1",
191 "2nd declension": "declension-2",
192 "3rd declension": "declension-3",
193 "4th declension": "declension-4",
194 "5th declension": "declension-5",
195 "6th declension": "declension-6",
196 "first declension": "declension-1",
197 "second declension": "declension-2",
198 "third declension": "declension-3",
199 "fourth declension": "declension-4",
200 "fifth declension": "declension-5",
201 "sixth declension": "declension-6",
202 "1st conjugation": "conjugation-1",
203 "2nd conjugation": "conjugation-2",
204 "3rd conjugation": "conjugation-3",
205 "4th conjugation": "conjugation-4",
206 "5th conjugation": "conjugation-5",
207 "6th conjugation": "conjugation-6",
208 "7th conjugation": "conjugation-7",
209 "first conjugation": "conjugation-1",
210 "second conjugation": "conjugation-2",
211 "third conjugation": "conjugation-3",
212 "fourth conjugation": "conjugation-4",
213 "fifth conjugation": "conjugation-5",
214 "sixth conjugation": "conjugation-6",
215 "seventh conjugation": "conjugation-7",
216 # Corsican regional tags in table header
217 "cismontane": "Cismontane",
218 "ultramontane": "Ultramontane",
219 "western lombard": "Western-Lombard",
220 "eastern lombard": "Eastern-Lombard",
221 "contracted": "contracted",
222 "present": "present",
223 "perfect": "perfect",
224 "imperfect": "imperfect",
225 "pluperfect": "pluperfect",
226 "future": "future",
227 "aorist": "aorist",
228 "eastern armenian": "Eastern-Armenian",
229 "western armenian": "Western-Armenian",
230 "-al conjugation": "-al-conjugation",
231 "-al negative conjugation": "-al-conjugation",
232 "-il conjugation": "-il-conjugation",
233 "-il negative conjugation": "-il-conjugation",
234 "-el conjugation": "-el-conjugation",
235 "-el negative conjugation": "-el-conjugation",
236 "-ul conjugation": "-ul-conjugation",
237 "-ul negative conjugation": "-ul-conjugation",
238 "u-type": "u-type",
239 "nominalized infinitive": "noun infinitive",
240}
241for k, v in title_contains_wordtags_map.items():
242 if any(t not in valid_tags for t in v.split()): 242 ↛ 243line 242 didn't jump to line 243 because the condition on line 242 was never true
243 print(
244 "TITLE_CONTAINS_WORDTAGS_MAP UNRECOGNIZED TAG: {}: {}".format(k, v)
245 )
246title_contains_wordtags_re = re.compile(
247 r"(?i)(^|\b)({}|{})($|\b)".format(
248 table_hdr_ign_part,
249 "|".join(
250 re.escape(x)
251 for x in reversed(
252 sorted(title_contains_wordtags_map.keys(), key=len)
253 )
254 ),
255 )
256)
258# Parenthesized elements in title that are converted to tags in
259# "table-tags" form
260title_elements_map = {
261 "weak": "weak",
262 "strong": "strong",
263 "separable": "separable",
264 "masculine": "masculine",
265 "feminine": "feminine",
266 "neuter": "neuter",
267 "singular": "singular",
268 "plural": "plural",
269 "archaic": "archaic",
270 "dated": "dated",
271 "iterative": "iterative",
272 "poetic": "poetic",
273 "Attic": "Attic",
274 "Epic": "Epic",
275 "Aeolic": "Aeolic",
276 "Arcadocypriot": "Arcadocypriot",
277 "Old Attic": "Old-Attic",
278 "Boeotian": "Boeotian",
279 "Byzantine": "Byzantine",
280 "Choral Doric": "Choral-Doric",
281 "Doric": "Doric",
282 "Elean": "Elean",
283 "Epirote": "Epirote",
284 "Ionic": "Ionic",
285 "Koine": "Koine",
286 "Cretan": "Cretan",
287 "Corinthian": "Corinthian",
288 "Laconian": "Laconian",
289 "Later poetic": "Later-poetic-Ancient-Greek",
290 "Lesbian": "Lesbian",
291 "Locrian": "Locrian",
292 "Lyric": "Lyric-Ancient-Greek",
293 "Thessalian": "Thessalian",
294 "Tragic": "Tragic-Ancient-Greek",
295}
296for k, v in title_elements_map.items():
297 if any(t not in valid_tags for t in v.split()): 297 ↛ 298line 297 didn't jump to line 298 because the condition on line 297 was never true
298 print("TITLE_ELEMENTS_MAP UNRECOGNIZED TAG: {}: {}".format(k, v))
300# Parenthized element starts to map them to tags for form for the rest of
301# the element
302title_elemstart_map = {
303 "auxiliary": "auxiliary",
304 "Kotus type": "class",
305 "ÕS type": "class",
306 "class": "class",
307 "short class": "class",
308 "type": "class",
309 "strong class": "class",
310 "weak class": "class",
311 "accent paradigm": "accent-paradigm",
312 "stem in": "class",
313}
314for k, v in title_elemstart_map.items():
315 if any(t not in valid_tags for t in v.split()): 315 ↛ 316line 315 didn't jump to line 316 because the condition on line 315 was never true
316 print("TITLE_ELEMSTART_MAP UNRECOGNIZED TAG: {}: {}".format(k, v))
317title_elemstart_re = re.compile(
318 r"^({}) ".format("|".join(re.escape(x) for x in title_elemstart_map.keys()))
319)
322# Regexp for cell starts that are likely definitions of reference symbols.
323# See also nondef_re.
324def_re = re.compile(
325 r"(\s*•?\s+)?"
326 r"((\*+|[△†0123456789⁰¹²³⁴⁵⁶⁷⁸⁹⁺⁻]+)([⁾):]|\s|(?=[A-Z]))|"
327 r"\^(\*+|[△†])|"
328 r"([¹²³⁴⁵⁶⁷⁸⁹])|"
329 r"([ᴬᴮᴰᴱᴳᴴᴵᴶᴷᴸᴹᴺᴼᴾᴿᵀᵁⱽᵂᵃᵇᶜᵈᵉᶠᵍʰⁱʲᵏˡᵐⁿᵒᵖʳˢᵗᵘᵛʷˣʸᶻᵝᵞᵟᶿᶥᵠᵡ]))"
330)
331# ᴺᴸᴴ persan/Old Irish
333# Regexp for cell starts that are exceptions to def_re and do not actually
334# start a definition.
335nondef_re = re.compile(
336 r"(^\s*(1|2|3)\s+(sg|pl)\s*$|" # 1s or 3p etc.
337 r"\s*\d\d?\s*/\s*\d\d?\s*$)"
338) # taka/Swahili "15 / 17"
341class InflCell:
342 """Cell in an inflection table."""
344 __slots__ = (
345 "text",
346 "is_title",
347 "colspan",
348 "rowspan",
349 "target",
350 "links",
351 )
353 def __init__(
354 self,
355 text: str,
356 is_title: bool,
357 colspan: int,
358 rowspan: int,
359 target: str | None,
360 cell_links: list[tuple[str, str]] | None = None,
361 ) -> None:
362 assert isinstance(text, str)
363 assert is_title in (True, False)
364 assert isinstance(colspan, int) and colspan >= 1
365 assert isinstance(rowspan, int) and rowspan >= 1
366 assert target is None or isinstance(target, str)
367 self.text = text.strip()
368 self.is_title = text and is_title
369 self.colspan = colspan
370 self.rowspan = rowspan
371 self.target = target
372 self.links = cell_links
374 def __str__(self) -> str:
375 v = "{}/{}/{}/{!r}".format(
376 self.text, self.is_title, self.colspan, self.rowspan
377 )
378 if self.target:
379 v += ": {!r}".format(self.target)
380 return v
382 def __repr__(self) -> str:
383 return str(self)
386class HdrSpan:
387 """Saved information about a header cell/span during the parsing
388 of a table."""
390 __slots__ = (
391 "start",
392 "colspan",
393 "rowspan",
394 "rownum", # Row number where this occurred
395 "tagsets", # list of tuples
396 "text", # For debugging
397 "all_headers_row",
398 "expanded", # The header has been expanded to cover whole row/part
399 )
401 def __init__(
402 self,
403 start: int,
404 colspan: int,
405 rowspan: int,
406 rownum: int,
407 tagsets: TagSets,
408 text: str,
409 all_headers_row: bool,
410 ) -> None:
411 assert isinstance(start, int) and start >= 0
412 assert isinstance(colspan, int) and colspan >= 1
413 assert isinstance(rownum, int)
414 assert isinstance(tagsets, list)
415 for x in tagsets:
416 assert isinstance(x, tuple)
417 assert all_headers_row in (True, False)
418 self.start = start
419 self.colspan = colspan
420 self.rowspan = rowspan
421 self.rownum = rownum
422 self.tagsets = list(tuple(sorted(set(tags))) for tags in tagsets)
423 self.text = text
424 self.all_headers_row = all_headers_row
425 self.expanded = False
428def is_superscript(ch: str) -> bool:
429 """Returns True if the argument is a superscript character."""
430 assert isinstance(ch, str) and len(ch) == 1
431 try:
432 name = unicodedata.name(ch)
433 except ValueError:
434 return False
435 return (
436 re.match(
437 r"SUPERSCRIPT |"
438 r"MODIFIER LETTER SMALL |"
439 r"MODIFIER LETTER CAPITAL ",
440 name,
441 )
442 is not None
443 )
446def remove_useless_tags(lang: str, pos: str, tags: set[str]) -> None:
447 """Remove certain tag combinations from ``tags`` when they serve no purpose
448 together (cover all options)."""
449 assert isinstance(lang, str)
450 assert isinstance(pos, str)
451 assert isinstance(tags, set)
452 if (
453 "animate" in tags
454 and "inanimate" in tags
455 and get_lang_conf(lang, "animate_inanimate_remove")
456 ):
457 tags.remove("animate")
458 tags.remove("inanimate")
459 if (
460 "virile" in tags
461 and "nonvirile" in tags
462 and get_lang_conf(lang, "virile_nonvirile_remove")
463 ):
464 tags.remove("virile")
465 tags.remove("nonvirile")
466 # If all numbers in the language are listed, remove them all
467 numbers = get_lang_conf(lang, "numbers")
468 if numbers and all(x in tags for x in numbers):
469 for x in numbers:
470 tags.remove(x)
471 # If all genders in the language are listed, remove them all
472 genders = get_lang_conf(lang, "genders")
473 if genders and all(x in tags for x in genders):
474 for x in genders:
475 tags.remove(x)
476 # If all voices in the language are listed, remove them all
477 voices = get_lang_conf(lang, "voices")
478 if voices and all(x in tags for x in voices):
479 for x in voices:
480 tags.remove(x)
481 # If all strengths of the language are listed, remove them all
482 strengths = get_lang_conf(lang, "strengths")
483 if strengths and all(x in tags for x in strengths):
484 for x in strengths:
485 tags.remove(x)
486 # If all persons of the language are listed, remove them all
487 persons = get_lang_conf(lang, "persons")
488 if persons and all(x in tags for x in persons):
489 for x in persons:
490 tags.remove(x)
491 # If all definitenesses of the language are listed, remove them all
492 definitenesses = get_lang_conf(lang, "definitenesses")
493 if definitenesses and all(x in tags for x in definitenesses):
494 for x in definitenesses:
495 tags.remove(x)
498def tagset_cats(tagset: TagSets) -> set[str]:
499 """Returns a set of tag categories for the tagset (merged from all
500 alternatives)."""
501 return set(valid_tags[t] for ts in tagset for t in ts)
504def or_tagsets(
505 lang: str, pos: str, tagsets1: TagSets, tagsets2: TagSets
506) -> TagSets:
507 """Merges two tagsets (the new tagset just merges the tags from both, in
508 all combinations). If they contain simple alternatives (differ in
509 only one category), they are simply merged; otherwise they are split to
510 more alternatives. The tagsets are assumed be sets of sorted tuples."""
511 assert isinstance(tagsets1, list)
512 assert all(isinstance(x, tuple) for x in tagsets1)
513 assert isinstance(tagsets2, list)
514 assert all(isinstance(x, tuple) for x in tagsets1)
515 tagsets: TagSets = [] # This will be the result
517 def add_tags(tags1: tuple[str, ...]) -> None:
518 # CONTINUE
519 if not tags1:
520 return # empty set would merge with anything, won't change result
521 if not tagsets:
522 tagsets.append(tags1)
523 return
524 for tags2 in tagsets:
525 # Determine if tags1 can be merged with tags2
526 num_differ = 0
527 if tags1 and tags2: 527 ↛ 545line 527 didn't jump to line 545 because the condition on line 527 was always true
528 cats1 = set(valid_tags[t] for t in tags1)
529 cats2 = set(valid_tags[t] for t in tags2)
530 cats = cats1 | cats2
531 for cat in cats:
532 tags1_in_cat = set(t for t in tags1 if valid_tags[t] == cat)
533 tags2_in_cat = set(t for t in tags2 if valid_tags[t] == cat)
534 if (
535 tags1_in_cat != tags2_in_cat
536 or not tags1_in_cat
537 or not tags2_in_cat
538 ):
539 num_differ += 1
540 if not tags1_in_cat or not tags2_in_cat:
541 # Prevent merging if one is empty
542 num_differ += 1
543 # print("tags1={} tags2={} num_differ={}"
544 # .format(tags1, tags2, num_differ))
545 if num_differ <= 1:
546 # Yes, they can be merged
547 tagsets.remove(tags2)
548 tags_s = set(tags1) | set(tags2)
549 remove_useless_tags(lang, pos, tags_s)
550 tags_t = tuple(sorted(tags_s))
551 add_tags(tags_t) # Could result in further merging
552 return
553 # If we could not merge, add to tagsets
554 tagsets.append(tags1)
556 for tags in tagsets1:
557 add_tags(tags)
558 for tags in tagsets2:
559 add_tags(tags)
560 if not tagsets:
561 tagsets.append(())
563 # print("or_tagsets: {} + {} -> {}"
564 # .format(tagsets1, tagsets2, tagsets))
565 return tagsets
568def and_tagsets(
569 lang: str,
570 pos: str,
571 tagsets1: list[tuple[str, ...]],
572 tagsets2: list[tuple[str, ...]],
573) -> list[tuple[str, ...]]:
574 """Merges tagsets by taking union of all cobinations, without trying
575 to determine whether they are compatible."""
576 assert isinstance(tagsets1, list) and len(tagsets1) >= 1
577 assert all(isinstance(x, tuple) for x in tagsets1)
578 assert isinstance(tagsets2, list) and len(tagsets2) >= 1
579 assert all(isinstance(x, tuple) for x in tagsets1)
580 new_tagsets = []
581 tags: Union[set[str], tuple[str, ...]]
582 for tags1 in tagsets1:
583 for tags2 in tagsets2:
584 tags = set(tags1) | set(tags2)
585 remove_useless_tags(lang, pos, tags)
586 if "dummy-ignored-text-cell" in tags: 586 ↛ 587line 586 didn't jump to line 587 because the condition on line 586 was never true
587 tags.remove("dummy-ignored-text-cell")
588 tags = tuple(sorted(tags))
589 if tags not in new_tagsets: 589 ↛ 583line 589 didn't jump to line 583 because the condition on line 589 was always true
590 new_tagsets.append(tags)
591 # print("and_tagsets: {} + {} -> {}"
592 # .format(tagsets1, tagsets2, new_tagsets))
593 return new_tagsets
596@functools.lru_cache(65536)
597def extract_cell_content(
598 lang: str, word: str, col: str
599) -> tuple[str, list[str], list[tuple[str, str]], list[str]]:
600 """Cleans a row/column header for later processing. This returns
601 (cleaned, refs, defs, tags)."""
602 # print("EXTRACT_CELL_CONTENT {!r}".format(col))
603 hdr_tags = []
604 col = re.sub(r"(?s)\s*,\s*$", "", col)
605 col = re.sub(r"(?s)\s*•\s*$", "", col)
606 col = re.sub(r"\s+", " ", col)
607 col = col.strip()
608 if re.search(
609 r"^\s*(There are |"
610 r"\* |"
611 r"see |"
612 r"See |"
613 r"Use |"
614 r"use the |"
615 r"Only used |"
616 r"The forms in |"
617 r"these are also written |"
618 r"The genitive can be |"
619 r"Genitive forms are rare or non-existant|"
620 r"Accusative Note: |"
621 r"Classifier Note: |"
622 r"Noun: Assamese nouns are |"
623 r"the active conjugation|"
624 r"the instrumenal singular|"
625 r"Note:|"
626 r"\^* Note:|"
627 r"possible mutated form |"
628 r"The future tense: )",
629 col,
630 ):
631 return "dummy-ignored-text-cell", [], [], []
633 # Temporarily remove final parenthesized part (if separated by whitespace),
634 # so that we can extract reference markers before it.
635 final_paren = ""
636 m = re.search(r"\s+\([^)]*\)$", col)
637 if m is not None:
638 final_paren = m.group(0)
639 col = col[: m.start()]
641 # Extract references and tag markers
642 refs = []
643 special_references = get_lang_conf(lang, "special_references")
644 while True:
645 m = re.search(r"\^(.|\([^)]*\))$", col)
646 if not m:
647 break
648 r = m.group(1)
649 if r.startswith("(") and r.endswith(")"):
650 r = r[1:-1]
651 for r1 in r.split(","):
652 if r1 == "rare": 652 ↛ 653line 652 didn't jump to line 653 because the condition on line 652 was never true
653 hdr_tags.append("rare")
654 elif special_references and r1 in special_references:
655 hdr_tags.extend(special_references[r1].split())
656 else:
657 # v = m.group(1)
658 if r1.startswith("(") and r1.endswith(")"): 658 ↛ 659line 658 didn't jump to line 659 because the condition on line 658 was never true
659 r1 = r1[1:-1]
660 refs.append(unicodedata.normalize("NFKD", r1))
661 col = col[: m.start()]
662 # See if it is a ref definition
663 # print("BEFORE REF CHECK: {!r}".format(col))
664 m = def_re.match(col)
665 # print(f"Before def_re: {refs=}")
666 if m and not nondef_re.match(col):
667 ofs = 0
668 ref = None
669 deflst = []
670 for m in re.finditer(def_re, col):
671 if ref:
672 deflst.append((ref, col[ofs : m.start()].strip()))
673 ref = unicodedata.normalize(
674 "NFKD", m.group(3) or m.group(5) or m.group(6) or ""
675 )
676 ofs = m.end()
677 if ref: 677 ↛ 680line 677 didn't jump to line 680 because the condition on line 677 was always true
678 deflst.append((ref, col[ofs:].strip()))
679 # print("deflst:", deflst)
680 return "", [], deflst, []
681 # See if it *looks* like a reference to a definition
682 # print(f"After def_re: {refs=}")
683 while col:
684 if is_superscript(col[-1]) or col[-1] in ("†",):
685 if col.endswith("ʳᵃʳᵉ"):
686 hdr_tags.append("rare")
687 col = col[:-4].strip()
688 continue
689 if special_references:
690 stop_flag = False
691 for r in special_references:
692 if col.endswith(r):
693 hdr_tags.extend(special_references[r].split())
694 col = col[: -len(r)].strip()
695 stop_flag = True
696 break # this for loop
697 if stop_flag:
698 continue # this while loop
699 # Numbers and H/L/N are useful information
700 refs.append(unicodedata.normalize("NFKD", col[-1]))
701 col = col[:-1]
702 else:
703 break
705 # Check for another form of note definition
706 if ( 706 ↛ 712line 706 didn't jump to line 712 because the condition on line 706 was never true
707 len(col) > 2
708 and col[1] in (")", " ", ":")
709 and col[0].isdigit()
710 and not re.match(nondef_re, col)
711 ):
712 return "", [], [(col[0], col[2:].strip())], []
713 col = col.strip()
715 # Extract final "*" reference symbols. Sometimes there are multiple.
716 m = re.search(r"\*+$", col)
717 if m is not None:
718 col = col[: m.start()]
719 refs.append(unicodedata.normalize("NFKD", m.group(0)))
720 if col.endswith("(*)"): 720 ↛ 721line 720 didn't jump to line 721 because the condition on line 720 was never true
721 col = col[:-3].strip()
722 refs.append("*")
724 # Put back the final parenthesized part
725 col = col.strip() + final_paren
726 # print("EXTRACT_CELL_CONTENT: orig_col={!r} col={!r} refs={!r} hdr_tags={}"
727 # .format(orig_col, col, refs, hdr_tags))
728 return col.strip(), refs, [], hdr_tags
731@functools.lru_cache(10000)
732def parse_title(
733 title: str, source: str
734) -> tuple[list[str], list[str], list[FormData]]:
735 """Parses inflection table title. This returns (global_tags, table_tags,
736 extra_forms), where ``global_tags`` is tags to be added to each inflection
737 entry, ``table_tags`` are tags for the word but not to be added to every
738 form, and ``extra_forms`` is dictionary describing additional forms to be
739 included in the part-of-speech entry)."""
740 assert isinstance(title, str)
741 assert isinstance(source, str)
742 title = html.unescape(title)
743 title = re.sub(r"(?i)<[^>]*>", "", title).strip()
744 title = re.sub(r"\s+", " ", title)
745 # print("PARSE_TITLE:", title)
746 global_tags: list[str] = []
747 table_tags: list[str] = []
748 extra_forms = []
749 # Add certain global tags based on contained words
750 for m in re.finditer(title_contains_global_re, title):
751 v = m.group(0).lower()
752 if re.match(table_hdr_ign_part_re, v): 752 ↛ 753line 752 didn't jump to line 753 because the condition on line 752 was never true
753 continue
754 global_tags.extend(title_contains_global_map[v].split())
755 # Add certain tags to table-tags "form" based on contained words
756 for m in re.finditer(title_contains_wordtags_re, title):
757 v = m.group(0).lower()
758 if re.match(table_hdr_ign_part_re, v): 758 ↛ 759line 758 didn't jump to line 759 because the condition on line 758 was never true
759 continue
760 table_tags.extend(title_contains_wordtags_map[v].split())
761 if re.search(r"Conjugation of (s’|se ).*French verbs", title): 761 ↛ 762line 761 didn't jump to line 762 because the condition on line 761 was never true
762 global_tags.append("reflexive")
763 # Check for <x>-type at the beginning of title (e.g., Armenian) and various
764 # other ways of specifying an inflection class.
765 for m in re.finditer(
766 r"\b("
767 r"[\w/]+-type|"
768 r"accent-\w+|"
769 r"[\w/]+-stem|"
770 r"[^ ]+ gradation|"
771 r"\b(stem in [\w/ ]+)|"
772 r"[^ ]+ alternation|"
773 r"(First|Second|Third|Fourth|Fifth|Sixth|Seventh) "
774 r"(Conjugation|declension)|"
775 r"First and second declension|"
776 r"(1st|2nd|3rd|4th|5th|6th) declension|"
777 r"\w[\w/ ]* harmony"
778 r")\b",
779 title,
780 ):
781 dt: FormData = {"form": m.group(1), "source": source, "tags": ["class"]}
782 extra_forms.append(dt)
783 # Parse parenthesized part from title
784 for m in re.finditer(r"\(([^)]*)\)", title):
785 for elem in m.group(1).split(","):
786 # group(0) is the whole string, group(1) first parens
787 elem = elem.strip()
788 if elem in title_elements_map:
789 table_tags.extend(title_elements_map[elem].split())
790 else:
791 m1 = re.match(title_elemstart_re, elem)
792 if m1:
793 tags = title_elemstart_map[m1.group(1)].split()
794 dt = {
795 "form": elem[m1.end() :],
796 "source": source,
797 "tags": tags,
798 }
799 extra_forms.append(dt)
800 # For titles that contains no parenthesized parts, do some special
801 # handling to still interpret parts from them
802 if "(" not in title:
803 # No parenthesized parts
804 m1 = re.search(r"\b(Portuguese) (-.* verb) ", title)
805 if m1 is not None:
806 dt = {"form": m1.group(2), "tags": ["class"], "source": source}
807 extra_forms.append(dt)
808 for elem in title.split(","):
809 elem = elem.strip()
810 if elem in title_elements_map: 810 ↛ 811line 810 didn't jump to line 811 because the condition on line 810 was never true
811 table_tags.extend(title_elements_map[elem].split())
812 elif elem.endswith("-stem"): 812 ↛ 813line 812 didn't jump to line 813 because the condition on line 812 was never true
813 dt = {"form": elem, "tags": ["class"], "source": source}
814 extra_forms.append(dt)
815 return global_tags, table_tags, extra_forms
818def expand_header(
819 wxr: WiktextractContext,
820 tablecontext: "TableContext",
821 word: str,
822 lang: str,
823 pos: str,
824 text: str,
825 base_tags: Union[list[str], set[str], tuple[str, ...]],
826 silent=False,
827 ignore_tags=False,
828 depth=0,
829 column_number: int | None = None,
830) -> list[tuple[str, ...]]:
831 """Expands a cell header to tagset, handling conditional expressions
832 in infl_map. This returns list of tuples of tags, each list element
833 describing an alternative interpretation. ``base_tags`` is combined
834 column and row tags for the cell in which the text is being interpreted
835 (conditional expressions in inflection data may depend on it).
836 If ``silent`` is True, then no warnings will be printed. If ``ignore_tags``
837 is True, then tags listed in "if" will be ignored in the test (this is
838 used when trying to heuristically detect whether a non-<th> cell is anyway
839 a header)."""
840 assert isinstance(wxr, WiktextractContext)
841 assert isinstance(word, str)
842 assert isinstance(lang, str)
843 assert isinstance(pos, str)
844 assert isinstance(text, str)
845 assert isinstance(base_tags, (list, tuple, set))
846 assert silent in (True, False)
847 assert isinstance(depth, int)
848 # print("EXPAND_HDR: text={!r} base_tags={!r}".format(text, base_tags))
849 # First map the text using the inflection map
850 text = clean_value(wxr, text)
851 combined_return: list[tuple[str, ...]] = []
852 parts = split_at_comma_semi(text, separators=[";"])
853 for text in parts:
854 if not text: 854 ↛ 855line 854 didn't jump to line 855 because the condition on line 854 was never true
855 continue
856 if text in infl_map:
857 v = infl_map[text] # list or string
858 else:
859 m = re.match(infl_start_re, text)
860 if m is not None: 860 ↛ 861line 860 didn't jump to line 861 because the condition on line 860 was never true
861 v = infl_start_map[m.group(1)]
862 # print("INFL_START {} -> {}".format(text, v))
863 elif re.match(r"Notes", text):
864 # Ignored header
865 # print("IGNORING NOTES")
866 combined_return = or_tagsets(
867 lang, pos, combined_return, [("dummy-skip-this",)]
868 )
869 # this just adds dummy-skip-this
870 continue
871 elif text in IGNORED_COLVALUES:
872 combined_return = or_tagsets(
873 lang, pos, combined_return, [("dummy-ignore-skipped",)]
874 )
875 continue
876 # Try without final parenthesized part
877 text_without_parens = re.sub(r"[,/]?\s+\([^)]*\)\s*$", "", text)
878 if text_without_parens in infl_map:
879 v = infl_map[text_without_parens]
880 elif m is None: 880 ↛ 896line 880 didn't jump to line 896 because the condition on line 880 was always true
881 if not silent:
882 wxr.wtp.debug(
883 "inflection table: unrecognized header: {}".format(
884 repr(text)
885 ),
886 sortid="inflection/735",
887 )
888 # Unrecognized header
889 combined_return = or_tagsets(
890 lang, pos, combined_return, [("error-unrecognized-form",)]
891 )
892 continue
894 # Then loop interpreting the value, until the value is a simple string.
895 # This may evaluate nested conditional expressions.
896 default_else = None
897 while True:
898 # If it is a string, we are done.
899 if isinstance(v, str):
900 tags = set(v.split())
901 remove_useless_tags(lang, pos, tags)
902 tagset = [tuple(sorted(tags))]
903 break
904 # For a list, just interpret it as alternatives. (Currently the
905 # alternatives must directly be strings.)
906 if isinstance(v, (list, tuple)):
907 tagset = []
908 for x in v:
909 tags = set(x.split())
910 remove_useless_tags(lang, pos, tags)
911 tags_t = tuple(sorted(tags))
912 if tags_t not in tagset: 912 ↛ 908line 912 didn't jump to line 908 because the condition on line 912 was always true
913 tagset.append(tags_t)
914 break
915 # Otherwise the value should be a dictionary describing a
916 # conditional expression.
917 if not isinstance(v, dict): 917 ↛ 918line 917 didn't jump to line 918 because the condition on line 917 was never true
918 wxr.wtp.debug(
919 "inflection table: internal: "
920 "UNIMPLEMENTED INFL_MAP VALUE: {}".format(infl_map[text]),
921 sortid="inflection/767",
922 )
923 tagset = [()]
924 break
925 # Evaluate the conditional expression.
926 assert isinstance(v, dict)
927 cond: Union[bool, str] = "default-true"
928 c: Union[str, list[str], set[str]] = ""
929 # Handle "lang" condition. The value must be either a
930 # single language or a list of languages, and the
931 # condition evaluates to True if the table is one of
932 # those languages.
933 if "lang" in v:
934 c = v["lang"]
935 # check if it's a code and transform if necessary
936 if isinstance(c, str):
937 if c != lang:
938 cond = lang == code_to_name(c, "en")
939 else:
940 cond = True
941 else:
942 assert isinstance(c, (list, tuple, set))
943 if lang not in c:
944 cond = name_to_code(lang, "en") in c
945 else:
946 cond = True
947 # Handle "nested-table-depth" condition. The value must
948 # be an int or list of ints, and the condition evaluates
949 # True if the depth is one of those values.
950 # "depth" is how deep into a nested table tree the current
951 # table lies. It is first started in handle_wikitext_table,
952 # so only applies to tables-within-tables, not other
953 # WikiNode content. `depth` is currently only passed as a
954 # parameter down the table parsing stack, and not stored.
955 if cond and "nested-table-depth" in v: 955 ↛ 956line 955 didn't jump to line 956 because the condition on line 955 was never true
956 d = v["nested-table-depth"]
957 if isinstance(d, int):
958 cond = d == depth
959 else:
960 assert isinstance(d, (list, tuple, set))
961 cond = depth in d
962 # Column index: check if we're in position X of the row
963 if cond and "column-index" in v:
964 index = v["column-index"]
965 if isinstance(index, int): 965 ↛ 968line 965 didn't jump to line 968 because the condition on line 965 was always true
966 cond = index == column_number
967 else:
968 assert isinstance(index, (list, tuple, set))
969 cond = column_number in index
970 # Handle inflection-template condition. Must be a string
971 # or list of strings, and if tablecontext.template_name is in
972 # those, accept the condition.
973 # TableContext.template_name is passed down from page/
974 # parse_inflection, before parsing and expanding itself
975 # has begun.
976 if cond and tablecontext and "inflection-template" in v:
977 d1 = v["inflection-template"]
978 if isinstance(d1, str): 978 ↛ 981line 978 didn't jump to line 981 because the condition on line 978 was always true
979 cond = d1 == tablecontext.template_name
980 else:
981 assert isinstance(d1, (list, tuple, set))
982 cond = tablecontext.template_name in d1
983 # Handle "pos" condition. The value must be either a single
984 # part-of-speech or a list of them, and the condition evaluates to
985 # True if the part-of-speech is any of those listed.
986 if cond and "pos" in v:
987 c = v["pos"]
988 if isinstance(c, str):
989 cond = c == pos
990 else:
991 assert isinstance(c, (list, tuple, set))
992 cond = pos in c
993 # Handle "if" condition. The value must be a string containing a
994 # space-separated list of tags. The condition evaluates to True if
995 # ``base_tags`` contains all of the listed tags. If the condition
996 # is of the form "any: ...tags...", then any of the tags will be
997 # enough.
998 if cond and "if" in v and not ignore_tags:
999 c = v["if"]
1000 assert isinstance(c, str)
1001 # "if" condition is true if any of the listed tags is present if
1002 # it starts with "any:", otherwise all must be present
1003 if c.startswith("any: "):
1004 cond = any(t in base_tags for t in c[5:].split())
1005 else:
1006 cond = all(t in base_tags for t in c.split())
1008 # Handle "default" assignment. Store the value to be used
1009 # as a default later.
1010 if "default" in v:
1011 assert isinstance(v["default"], str)
1012 default_else = v["default"]
1014 # Warning message about missing conditions for debugging.
1016 if cond == "default-true" and not default_else and not silent:
1017 wxr.wtp.debug(
1018 "inflection table: IF MISSING COND: word={} "
1019 "lang={} text={} base_tags={} c={} cond={}".format(
1020 word, lang, text, base_tags, c, cond
1021 ),
1022 sortid="inflection/851",
1023 )
1024 # Based on the result of evaluating the condition, select either
1025 # "then" part or "else" part.
1026 if cond:
1027 v = v.get("then", "")
1028 else:
1029 v1 = v.get("else")
1030 if v1 is None:
1031 if default_else is not None:
1032 v = default_else
1033 else:
1034 if not silent:
1035 wxr.wtp.debug(
1036 "inflection table: IF WITHOUT ELSE EVALS "
1037 "False: "
1038 "{}/{} {!r} base_tags={}".format(
1039 word, lang, text, base_tags
1040 ),
1041 sortid="inflection/865",
1042 )
1043 v = "error-unrecognized-form"
1044 else:
1045 v = v1
1047 # Merge the resulting tagset from this header part with the other
1048 # tagsets from the whole header
1049 combined_return = or_tagsets(lang, pos, combined_return, tagset)
1051 # Return the combined tagsets, or empty tagset if we got no tagsets
1052 if not combined_return:
1053 combined_return = [()]
1054 return combined_return
1057def compute_coltags(
1058 lang: str,
1059 pos: str,
1060 hdrspans: list[HdrSpan],
1061 start: int,
1062 colspan: int,
1063 celltext: str,
1064) -> list[tuple[str, ...]]:
1065 """Computes column tags for a column of the given width based on the
1066 current header spans."""
1067 assert isinstance(lang, str)
1068 assert isinstance(pos, str)
1069 assert isinstance(hdrspans, list)
1070 assert isinstance(start, int) and start >= 0
1071 assert isinstance(colspan, int) and colspan >= 1
1072 assert isinstance(celltext, str) # For debugging only
1073 # print("COMPUTE_COLTAGS CALLED start={} colspan={} celltext={!r}"
1074 # .format(start, colspan, celltext))
1075 # For debugging, set this to the form for whose cell you want debug prints
1076 if celltext == debug_cell_text: 1076 ↛ 1077line 1076 didn't jump to line 1077 because the condition on line 1076 was never true
1077 print(
1078 "COMPUTE_COLTAGS CALLED start={} colspan={} celltext={!r}".format(
1079 start, colspan, celltext
1080 )
1081 )
1082 for hdrspan in hdrspans:
1083 print(
1084 " row={} start={} colspans={} tagsets={}".format(
1085 hdrspan.rownum,
1086 hdrspan.start,
1087 hdrspan.colspan,
1088 hdrspan.tagsets,
1089 )
1090 )
1091 used = set()
1092 coltags: list[tuple[str, ...]] = [()]
1093 last_header_row = 1000000
1094 # Iterate through the headers in reverse order, i.e., headers lower in the
1095 # table (closer to the cell) first.
1096 row_tagsets: list[tuple[str, ...]] = [()]
1097 row_tagsets_rownum = 1000000
1098 used_hdrspans = set()
1099 for hdrspan in reversed(hdrspans):
1100 if (
1101 hdrspan.start + hdrspan.colspan <= start
1102 or hdrspan.start >= start + colspan
1103 ):
1104 # Does not horizontally overlap current cell. Ignore this hdrspan.
1105 if celltext == debug_cell_text: 1105 ↛ 1106line 1105 didn't jump to line 1106 because the condition on line 1105 was never true
1106 print(
1107 "Ignoring row={} start={} colspan={} tagsets={}".format(
1108 hdrspan.rownum,
1109 hdrspan.start,
1110 hdrspan.colspan,
1111 hdrspan.tagsets,
1112 )
1113 )
1114 continue
1115 # If the cell partially overlaps the current cell, assume we have
1116 # reached something unrelated and abort.
1117 if (
1118 hdrspan.start < start
1119 and hdrspan.start + hdrspan.colspan > start
1120 and hdrspan.start + hdrspan.colspan < start + colspan
1121 ):
1122 if celltext == debug_cell_text: 1122 ↛ 1123line 1122 didn't jump to line 1123 because the condition on line 1122 was never true
1123 print(
1124 "break on partial overlap at start {} {} {}".format(
1125 hdrspan.start, hdrspan.colspan, hdrspan.tagsets
1126 )
1127 )
1128 break
1129 if (
1130 hdrspan.start < start + colspan
1131 and hdrspan.start > start
1132 and hdrspan.start + hdrspan.colspan > start + colspan
1133 and not hdrspan.expanded
1134 ):
1135 if celltext == debug_cell_text: 1135 ↛ 1136line 1135 didn't jump to line 1136 because the condition on line 1135 was never true
1136 print(
1137 "break on partial overlap at end {} {} {}".format(
1138 hdrspan.start, hdrspan.colspan, hdrspan.tagsets
1139 )
1140 )
1141 break
1142 # Check if we have already used this cell.
1143 if id(hdrspan) in used_hdrspans:
1144 continue
1145 # We are going to use this cell.
1146 used_hdrspans.add(id(hdrspan))
1147 tagsets = hdrspan.tagsets
1148 # If the hdrspan is fully inside the current cell and does not cover
1149 # it fully, check if we should merge information from multiple cells.
1150 if not hdrspan.expanded and (
1151 hdrspan.start > start
1152 or hdrspan.start + hdrspan.colspan < start + colspan
1153 ):
1154 # Multiple columns apply to the current cell, only
1155 # gender/number/case tags present
1156 # If there are no tags outside the range in any of the
1157 # categories included in these cells, don't add anything
1158 # (assume all choices valid in the language are possible).
1159 in_cats = set(
1160 valid_tags[t]
1161 for x in hdrspans
1162 if x.rownum == hdrspan.rownum
1163 and x.start >= start
1164 and x.start + x.colspan <= start + colspan
1165 for tt in x.tagsets
1166 for t in tt
1167 )
1168 if celltext == debug_cell_text: 1168 ↛ 1169line 1168 didn't jump to line 1169 because the condition on line 1168 was never true
1169 print("in_cats={} tagsets={}".format(in_cats, tagsets))
1170 # Merge the tagsets into existing tagsets. This merges
1171 # alternatives into the same tagset if there is only one
1172 # category different; otherwise this splits the tagset into
1173 # more alternatives.
1174 includes_all_on_row = True
1175 for x in hdrspans:
1176 # print("X: x.rownum={} x.start={}".format(x.rownum, x.start))
1177 if x.rownum != hdrspan.rownum:
1178 continue
1179 if x.start < start or x.start + x.colspan > start + colspan:
1180 if celltext == debug_cell_text: 1180 ↛ 1181line 1180 didn't jump to line 1181 because the condition on line 1180 was never true
1181 print(
1182 "NOT IN RANGE: {} {} {}".format(
1183 x.start, x.colspan, x.tagsets
1184 )
1185 )
1186 includes_all_on_row = False
1187 continue
1188 if id(x) in used_hdrspans:
1189 if celltext == debug_cell_text: 1189 ↛ 1190line 1189 didn't jump to line 1190 because the condition on line 1189 was never true
1190 print(
1191 "ALREADY USED: {} {} {}".format(
1192 x.start, x.colspan, x.tagsets
1193 )
1194 )
1195 continue
1196 used_hdrspans.add(id(x))
1197 if celltext == debug_cell_text: 1197 ↛ 1198line 1197 didn't jump to line 1198 because the condition on line 1197 was never true
1198 print(
1199 "Merging into wide col: x.rownum={} "
1200 "x.start={} x.colspan={} "
1201 "start={} colspan={} tagsets={} x.tagsets={}".format(
1202 x.rownum,
1203 x.start,
1204 x.colspan,
1205 start,
1206 colspan,
1207 tagsets,
1208 x.tagsets,
1209 )
1210 )
1211 tagsets = or_tagsets(lang, pos, tagsets, x.tagsets)
1212 # If all headers on the row were included, ignore them.
1213 # See e.g. kunna/Swedish/Verb.
1214 ts_cats = tagset_cats(tagsets)
1215 if (
1216 includes_all_on_row
1217 or
1218 # Kludge, see fut/Hungarian/Verb
1219 ("tense" in ts_cats and "object" in ts_cats)
1220 ):
1221 tagsets = [()]
1222 # For limited categories, if the category doesn't appear
1223 # outside, we won't include the category
1224 if not in_cats - set(
1225 ("gender", "number", "person", "case", "category", "voice")
1226 ):
1227 # Sometimes we have masc, fem, neut and plural, so treat
1228 # number and gender as the same here (if one given, look for
1229 # the other too)
1230 if "number" in in_cats or "gender" in in_cats:
1231 in_cats.update(("number", "gender"))
1232 # Determine which categories occur outside on
1233 # the same row. Ignore headers that have been expanded
1234 # to cover the whole row/part of it.
1235 out_cats = set(
1236 valid_tags[t]
1237 for x in hdrspans
1238 if x.rownum == hdrspan.rownum
1239 and not x.expanded
1240 and (
1241 x.start < start or x.start + x.colspan > start + colspan
1242 )
1243 for tt in x.tagsets
1244 for t in tt
1245 )
1246 if celltext == debug_cell_text: 1246 ↛ 1247line 1246 didn't jump to line 1247 because the condition on line 1246 was never true
1247 print("in_cats={} out_cats={}".format(in_cats, out_cats))
1248 # Remove all inside categories that do not appear outside
1250 new_tagsets = []
1251 for ts in tagsets:
1252 tags = tuple(
1253 sorted(t for t in ts if valid_tags[t] in out_cats)
1254 )
1255 if tags not in new_tagsets: 1255 ↛ 1251line 1255 didn't jump to line 1251 because the condition on line 1255 was always true
1256 new_tagsets.append(tags)
1257 if celltext == debug_cell_text and new_tagsets != tagsets: 1257 ↛ 1258line 1257 didn't jump to line 1258 because the condition on line 1257 was never true
1258 print(
1259 "Removed tags that do not "
1260 "appear outside {} -> {}".format(
1261 # have_hdr never used?
1262 tagsets,
1263 new_tagsets,
1264 )
1265 )
1266 tagsets = new_tagsets
1267 key = (hdrspan.start, hdrspan.colspan)
1268 if key in used:
1269 if celltext == debug_cell_text: 1269 ↛ 1270line 1269 didn't jump to line 1270 because the condition on line 1269 was never true
1270 print(
1271 "Cellspan already used: start={} "
1272 "colspan={} rownum={} {}".format(
1273 hdrspan.start,
1274 hdrspan.colspan,
1275 hdrspan.rownum,
1276 hdrspan.tagsets,
1277 )
1278 )
1279 action = get_lang_conf(lang, "reuse_cellspan")
1280 # can be "stop", "skip" or "reuse"
1281 if action == "stop":
1282 break
1283 if action == "skip":
1284 continue
1285 assert action == "reuse"
1286 tcats = tagset_cats(tagsets)
1287 # Most headers block using the same column position above. However,
1288 # "register" tags don't do this (cf. essere/Italian/verb: "formal")
1289 if len(tcats) != 1 or "register" not in tcats:
1290 used.add(key)
1291 # If we have moved to a different row, merge into column tagsets
1292 # (we use different and_tagsets within the row)
1293 if row_tagsets_rownum != hdrspan.rownum:
1294 # row_tagsets_rownum was initialized as 10000000
1295 ret = and_tagsets(lang, pos, coltags, row_tagsets)
1296 if celltext == debug_cell_text: 1296 ↛ 1297line 1296 didn't jump to line 1297 because the condition on line 1296 was never true
1297 print(
1298 "merging rows: {} {} -> {}".format(
1299 coltags, row_tagsets, ret
1300 )
1301 )
1302 coltags = ret
1303 row_tagsets = [()]
1304 row_tagsets_rownum = hdrspan.rownum
1305 # Merge into coltags
1306 if hdrspan.all_headers_row and hdrspan.rownum + 1 == last_header_row:
1307 # If this row is all headers and immediately preceeds the last
1308 # header we accepted, take any header from there.
1309 row_tagsets = and_tagsets(lang, pos, row_tagsets, tagsets)
1310 if celltext == debug_cell_text: 1310 ↛ 1311line 1310 didn't jump to line 1311 because the condition on line 1310 was never true
1311 print("merged (next header row): {}".format(row_tagsets))
1312 else:
1313 # new_cats is for the new tags (higher up in the table)
1314 new_cats = tagset_cats(tagsets)
1315 # cur_cats is for the tags already collected (lower in the table)
1316 cur_cats = tagset_cats(coltags)
1317 if celltext == debug_cell_text: 1317 ↛ 1318line 1317 didn't jump to line 1318 because the condition on line 1317 was never true
1318 print(
1319 "row={} start={} colspan={} tagsets={} coltags={} "
1320 "new_cats={} cur_cats={}".format(
1321 hdrspan.rownum,
1322 hdrspan.start,
1323 hdrspan.colspan,
1324 tagsets,
1325 coltags,
1326 new_cats,
1327 cur_cats,
1328 )
1329 )
1330 if "detail" in new_cats:
1331 if not any(coltags): # Only if no tags so far
1332 coltags = or_tagsets(lang, pos, coltags, tagsets)
1333 if celltext == debug_cell_text: 1333 ↛ 1334line 1333 didn't jump to line 1334 because the condition on line 1333 was never true
1334 print("stopping on detail after merge")
1335 break
1336 # Here, we block bleeding of categories from above
1337 elif "non-finite" in cur_cats and "non-finite" in new_cats:
1338 stop = get_lang_conf(lang, "stop_non_finite_non_finite")
1339 if stop: 1339 ↛ 1365line 1339 didn't jump to line 1365 because the condition on line 1339 was always true
1340 if celltext == debug_cell_text: 1340 ↛ 1341line 1340 didn't jump to line 1341 because the condition on line 1340 was never true
1341 print("stopping on non-finite-non-finite")
1342 break
1343 elif "non-finite" in cur_cats and "voice" in new_cats:
1344 stop = get_lang_conf(lang, "stop_non_finite_voice")
1345 if stop: 1345 ↛ 1365line 1345 didn't jump to line 1365 because the condition on line 1345 was always true
1346 if celltext == debug_cell_text: 1346 ↛ 1347line 1346 didn't jump to line 1347 because the condition on line 1346 was never true
1347 print("stopping on non-finite-voice")
1348 break
1349 elif "non-finite" in new_cats and cur_cats & set(
1350 ("person", "number")
1351 ):
1352 if celltext == debug_cell_text: 1352 ↛ 1353line 1352 didn't jump to line 1353 because the condition on line 1352 was never true
1353 print("stopping on non-finite new")
1354 break
1355 elif "non-finite" in new_cats and "tense" in new_cats:
1356 stop = get_lang_conf(lang, "stop_non_finite_tense")
1357 if stop:
1358 if celltext == debug_cell_text: 1358 ↛ 1359line 1358 didn't jump to line 1359 because the condition on line 1358 was never true
1359 print("stopping on non-finite new")
1360 break
1361 elif "non-finite" in cur_cats and new_cats & set(("mood",)): 1361 ↛ 1362line 1361 didn't jump to line 1362 because the condition on line 1361 was never true
1362 if celltext == debug_cell_text:
1363 print("stopping on non-finite cur")
1364 break
1365 if (
1366 "tense" in new_cats
1367 and any("imperative" in x for x in coltags)
1368 and get_lang_conf(lang, "imperative_no_tense")
1369 ):
1370 if celltext == debug_cell_text: 1370 ↛ 1371line 1370 didn't jump to line 1371 because the condition on line 1370 was never true
1371 print("skipping tense in imperative")
1372 continue
1373 elif (
1374 "mood" in new_cats
1375 and "mood" in cur_cats
1376 and
1377 # Allow if all new tags are already in current set
1378 any(
1379 t not in ts1
1380 for ts1 in coltags # current
1381 for ts2 in tagsets # new (from above)
1382 for t in ts2
1383 )
1384 ):
1385 skip = get_lang_conf(lang, "skip_mood_mood")
1386 if skip:
1387 if celltext == debug_cell_text: 1387 ↛ 1388line 1387 didn't jump to line 1388 because the condition on line 1387 was never true
1388 print("skipping on mood-mood")
1389 # we continue to next header
1390 else:
1391 if celltext == debug_cell_text: 1391 ↛ 1392line 1391 didn't jump to line 1392 because the condition on line 1391 was never true
1392 print("stopping on mood-mood")
1393 break
1394 elif "tense" in new_cats and "tense" in cur_cats:
1395 skip = get_lang_conf(lang, "skip_tense_tense")
1396 if skip:
1397 if celltext == debug_cell_text: 1397 ↛ 1398line 1397 didn't jump to line 1398 because the condition on line 1397 was never true
1398 print("skipping on tense-tense")
1399 # we continue to next header
1400 else:
1401 if celltext == debug_cell_text: 1401 ↛ 1402line 1401 didn't jump to line 1402 because the condition on line 1401 was never true
1402 print("stopping on tense-tense")
1403 break
1404 elif "aspect" in new_cats and "aspect" in cur_cats:
1405 if celltext == debug_cell_text: 1405 ↛ 1406line 1405 didn't jump to line 1406 because the condition on line 1405 was never true
1406 print("skipping on aspect-aspect")
1407 continue
1408 elif "number" in cur_cats and "number" in new_cats:
1409 if celltext == debug_cell_text: 1409 ↛ 1410line 1409 didn't jump to line 1410 because the condition on line 1409 was never true
1410 print("stopping on number-number")
1411 break
1412 elif "number" in cur_cats and "gender" in new_cats:
1413 if celltext == debug_cell_text: 1413 ↛ 1414line 1413 didn't jump to line 1414 because the condition on line 1413 was never true
1414 print("stopping on number-gender")
1415 break
1416 elif "person" in cur_cats and "person" in new_cats:
1417 if celltext == debug_cell_text: 1417 ↛ 1418line 1417 didn't jump to line 1418 because the condition on line 1417 was never true
1418 print("stopping on person-person")
1419 break
1420 else:
1421 # Merge tags and continue to next header up/left in the table.
1422 row_tagsets = and_tagsets(lang, pos, row_tagsets, tagsets)
1423 if celltext == debug_cell_text: 1423 ↛ 1424line 1423 didn't jump to line 1424 because the condition on line 1423 was never true
1424 print("merged: {}".format(coltags))
1425 # Update the row number from which we have last taken headers
1426 last_header_row = hdrspan.rownum
1427 # Merge the final row tagset into coltags
1428 coltags = and_tagsets(lang, pos, coltags, row_tagsets)
1429 # print(
1430 # "HDRSPANS:", list((x.start, x.colspan, x.tagsets) for x in hdrspans)
1431 # )
1432 if celltext == debug_cell_text: 1432 ↛ 1433line 1432 didn't jump to line 1433 because the condition on line 1432 was never true
1433 print("COMPUTE_COLTAGS {} {}: {}".format(start, colspan, coltags))
1434 assert isinstance(coltags, list)
1435 assert all(isinstance(x, tuple) for x in coltags)
1436 return coltags
1439def parse_simple_table(
1440 wxr: WiktextractContext,
1441 tablecontext: "TableContext",
1442 word: str,
1443 lang: str,
1444 pos: str,
1445 rows: list[list[InflCell]],
1446 titles: list[str],
1447 source: str,
1448 after: str,
1449 depth: int,
1450) -> list[FormData]:
1451 """This is the default table parser. Despite its name, it can parse
1452 complex tables. This returns a list of forms to be added to the
1453 part-of-speech, or None if the table could not be parsed."""
1454 assert isinstance(wxr, WiktextractContext)
1455 assert isinstance(tablecontext, TableContext)
1456 assert isinstance(word, str)
1457 assert isinstance(lang, str)
1458 assert isinstance(pos, str)
1459 assert isinstance(rows, list)
1460 assert isinstance(source, str)
1461 assert isinstance(after, str)
1462 assert isinstance(depth, int)
1463 for row in rows:
1464 for cell in row:
1465 assert isinstance(cell, InflCell)
1466 assert isinstance(titles, list)
1467 for x in titles:
1468 assert isinstance(x, str)
1470 # print("PARSE_SIMPLE_TABLE: TITLES:", titles)
1471 if debug_cell_text: 1471 ↛ 1472line 1471 didn't jump to line 1472 because the condition on line 1471 was never true
1472 print("ROWS:")
1473 for row in rows:
1474 print(" ", row)
1476 # Check for forced rowspan kludge. See e.g.
1477 # maorski/Serbo-Croatian. These are essentially multi-row
1478 # cells implemented using <br> rather than separate cell. We fix this
1479 # by identifying rows where this happens, and splitting the current row
1480 # to multiple rows by synthesizing additional cells.
1481 new_rows = []
1482 for row in rows:
1483 split_row = (
1484 any(x.is_title and x.text in ("inanimate\nanimate",) for x in row)
1485 and
1486 # x is an InflCell
1487 all(x.rowspan == 1 for x in row)
1488 )
1489 if not split_row:
1490 new_rows.append(row)
1491 continue
1492 row1 = []
1493 row2 = []
1494 for cell in row:
1495 cell1 = copy.deepcopy(cell)
1496 if "\n" in cell.text:
1497 # Has more than one line - split this cell
1498 parts = cell.text.strip().splitlines()
1499 if len(parts) != 2: 1499 ↛ 1500line 1499 didn't jump to line 1500 because the condition on line 1499 was never true
1500 wxr.wtp.debug(
1501 "forced rowspan kludge got {} parts: {!r}".format(
1502 len(parts), cell.text
1503 ),
1504 sortid="inflection/1234",
1505 )
1506 cell2 = copy.deepcopy(cell)
1507 cell1.text = parts[0]
1508 cell2.text = parts[1]
1509 else:
1510 cell1.rowspan = 2
1511 cell2 = cell1 # ref, not a copy
1512 row1.append(cell1)
1513 row2.append(cell2)
1514 new_rows.append(row1)
1515 new_rows.append(row2)
1516 rows = new_rows
1517 # print("ROWS AFTER FORCED ROWSPAN KLUDGE:")
1518 # for row in rows:
1519 # print(" ", row)
1521 # Parse definitions for references (from table itself and from text
1522 # after it)
1523 def_ht = {}
1525 def add_defs(defs: list[tuple[str, str]]) -> None:
1526 for ref, d in defs:
1527 # print("DEF: ref={} d={}".format(ref, d))
1528 d = d.strip()
1529 d = d.split(". ")[0].strip() # text before ". "
1530 if not d: 1530 ↛ 1531line 1530 didn't jump to line 1531 because the condition on line 1530 was never true
1531 continue
1532 if d.endswith("."): # catc ".."??
1533 d = d[:-1]
1534 tags, topics = decode_tags(d, no_unknown_starts=True)
1535 # print(f"{ref=}, {transformed=}, {tags=}")
1536 if topics or any("error-unknown-tag" in ts for ts in tags):
1537 d = d[0].lower() + d[1:]
1538 tags, topics = decode_tags(d, no_unknown_starts=True)
1539 if topics or any("error-unknown-tag" in ts for ts in tags):
1540 # Failed to parse as tags
1541 # print("Failed: topics={} tags={}"
1542 # .format(topics, tags))
1543 continue
1544 tags1_s: set[str] = set()
1545 for ts in tags:
1546 # Set.update is a union operation: definition tags are flat
1547 tags1_s.update(ts)
1548 tags1 = tuple(sorted(tags1_s))
1549 # print("DEFINED: {} -> {}".format(ref, tags1))
1550 def_ht[ref] = tags1
1552 def generate_tags(
1553 rowtags: list[tuple[str, ...]], table_tags: list[str]
1554 ) -> tuple[
1555 list[tuple[str, ...]], list[tuple[str, ...]], list[tuple[str, ...]]
1556 ]:
1557 new_coltags: list[tuple[str, ...]] = []
1558 all_hdr_tags: list[tuple[str, ...]] = [] # list of tuples
1559 new_rowtags: list[tuple[str, ...]] = []
1560 for rt0 in rowtags:
1561 for ct0 in compute_coltags(
1562 lang,
1563 pos,
1564 hdrspans,
1565 col_idx, # col_idx=>start
1566 colspan,
1567 col, # cell_text
1568 ):
1569 base_tags: set[str] = (
1570 set(rt0) | set(ct0) | set(global_tags) | set(table_tags)
1571 ) # Union.
1572 # print(f"{rt0=}, {ct0=}, {global_tags=},"
1573 # f" {table_tags=}, {base_tags=}")
1574 alt_tags = expand_header(
1575 wxr,
1576 tablecontext,
1577 word,
1578 lang,
1579 pos,
1580 text,
1581 base_tags,
1582 depth=depth,
1583 column_number=col_idx,
1584 )
1585 # base_tags are used in infl_map "if"-conds.
1586 for tt in alt_tags:
1587 if tt not in all_hdr_tags:
1588 all_hdr_tags.append(tt)
1589 tt_s = set(tt)
1590 # Add tags from referenced footnotes
1591 tt_s.update(refs_tags)
1592 # Sort, convert to tuple, and add to set of
1593 # alternatives.
1594 tt = tuple(sorted(tt_s))
1595 if tt not in new_coltags:
1596 new_coltags.append(tt)
1597 # Kludge (saprast/Latvian/Verb): ignore row tags
1598 # if trying to add a non-finite after mood.
1599 if any(valid_tags[t] == "mood" for t in rt0) and any(
1600 valid_tags[t] == "non-finite" for t in tt
1601 ):
1602 tags = tuple(sorted(set(tt) | set(hdr_tags)))
1603 else:
1604 tags = tuple(sorted(set(tt) | set(rt0) | set(hdr_tags)))
1605 if tags not in new_rowtags:
1606 new_rowtags.append(tags)
1607 return new_rowtags, new_coltags, all_hdr_tags
1609 def add_new_hdrspan(
1610 col: str,
1611 hdrspans: list[HdrSpan],
1612 store_new_hdrspan: bool,
1613 col0_followed_by_nonempty: bool,
1614 col0_hdrspan: Optional[HdrSpan],
1615 ) -> tuple[str, bool, Optional[HdrSpan]]:
1616 hdrspan = HdrSpan(
1617 col_idx, colspan, rowspan, rownum, new_coltags, col, all_headers
1618 )
1619 hdrspans.append(hdrspan)
1621 # infl-map tag "dummy-store-hdrspan" causes this new hdrspan
1622 # to be added to a register of stored hdrspans to be used
1623 # later with "dummy-load-stored-hdrspans".
1624 if store_new_hdrspan: 1624 ↛ 1625line 1624 didn't jump to line 1625 because the condition on line 1624 was never true
1625 tablecontext.stored_hdrspans.append(hdrspan)
1627 # Handle headers that are above left-side header
1628 # columns and are followed by personal pronouns in
1629 # remaining columns (basically headers that
1630 # evaluate to no tags). In such cases widen the
1631 # left-side header to the full row.
1632 if previously_seen: # id(cell) in seen_cells previously
1633 col0_followed_by_nonempty = True
1634 return col, col0_followed_by_nonempty, col0_hdrspan
1635 elif col0_hdrspan is None:
1636 col0_hdrspan = hdrspan
1637 elif any(all_hdr_tags): 1637 ↛ 1705line 1637 didn't jump to line 1705 because the condition on line 1637 was always true
1638 col0_cats = tagset_cats(col0_hdrspan.tagsets)
1639 later_cats = tagset_cats(all_hdr_tags)
1640 col0_allowed = get_lang_conf(lang, "hdr_expand_first")
1641 later_allowed = get_lang_conf(lang, "hdr_expand_cont")
1642 later_allowed = later_allowed | set(["dummy"])
1643 # dummy2 has different behavior than plain dummy
1644 # and does not belong here.
1646 # print("col0_cats={} later_cats={} "
1647 # "fol_by_nonempty={} col_idx={} end={} "
1648 # "tagsets={}"
1649 # .format(col0_cats, later_cats,
1650 # col0_followed_by_nonempty, col_idx,
1651 # col0_hdrspan.start +
1652 # col0_hdrspan.colspan,
1653 # col0_hdrspan.tagsets))
1654 # print("col0.rowspan={} rowspan={}"
1655 # .format(col0_hdrspan.rowspan, rowspan))
1656 # Only expand if [col0_cats and later_cats are allowed
1657 # and don't overlap] and [col0 has tags], and there have
1658 # been [no disallowed cells in between].
1659 #
1660 # There are three cases here:
1661 # - col0_hdrspan set, continue with allowed current
1662 # - col0_hdrspan set, expand, start new
1663 # - col0_hdrspan set, no expand, start new
1664 if (
1665 not col0_followed_by_nonempty
1666 and
1667 # XXX Only one cat of tags: kunna/Swedish
1668 # XXX len(col0_cats) == 1 and
1669 col0_hdrspan.rowspan >= rowspan
1670 and
1671 # from hdrspan
1672 not (later_cats - later_allowed)
1673 and not (col0_cats & later_cats)
1674 ):
1675 # First case: col0 set, continue
1676 return col, col0_followed_by_nonempty, col0_hdrspan
1677 # We are going to start new col0_hdrspan. Check if
1678 # we should expand.
1679 if (
1680 not col0_followed_by_nonempty
1681 and not (col0_cats - col0_allowed)
1682 and
1683 # Only "allowed" allowed
1684 # XXX len(col0_cats) == 1 and
1685 col_idx > col0_hdrspan.start + col0_hdrspan.colspan
1686 ):
1687 # col_idx is beyond current colspan
1688 # *Expand* current col0_hdrspan
1689 # print("EXPANDING COL0 MID: {} from {} to {} "
1690 # "cols {}"
1691 # .format(col0_hdrspan.text,
1692 # col0_hdrspan.colspan,
1693 # col_idx - col0_hdrspan.start,
1694 # col0_hdrspan.tagsets))
1695 col0_hdrspan.colspan = col_idx - col0_hdrspan.start
1696 col0_hdrspan.expanded = True
1697 # Clear old col0_hdrspan
1698 if col == debug_cell_text: 1698 ↛ 1699line 1698 didn't jump to line 1699 because the condition on line 1698 was never true
1699 print("START NEW {}".format(hdrspan.tagsets))
1700 col0_hdrspan = None
1701 # Now start new, unless it comes from previous row
1702 if not previously_seen: 1702 ↛ 1705line 1702 didn't jump to line 1705 because the condition on line 1702 was always true
1703 col0_hdrspan = hdrspan
1704 col0_followed_by_nonempty = False
1705 return col, col0_followed_by_nonempty, col0_hdrspan
1707 def split_text_into_alts(col: str) -> tuple[str, list[str], list[str]]:
1708 # Split the cell text into alternatives
1709 split_extra_tags = []
1710 if col and is_superscript(col[0]): 1710 ↛ 1711line 1710 didn't jump to line 1711 because the condition on line 1710 was never true
1711 alts = [col]
1712 else:
1713 separators = [";", "•", r"\n", " or "]
1714 if " + " not in col:
1715 separators.append(",")
1716 if not col.endswith("/"):
1717 separators.append("/")
1718 if col in special_phrase_splits:
1719 # Use language-specific special splits.
1720 # These are phrases and constructions that have
1721 # unique ways of splitting, not specific characters
1722 # to split on like with the default splitting.
1723 alts, tags = special_phrase_splits[col]
1724 split_extra_tags = tags.split()
1725 for x in split_extra_tags:
1726 assert x in valid_tags
1727 assert isinstance(alts, (list, tuple))
1728 assert isinstance(tags, str)
1729 elif ( 1729 ↛ 1749line 1729 didn't jump to line 1749 because the condition on line 1729 was never true
1730 (
1731 m := re.match(
1732 # word1, word2 (romanization1, romanization2)
1733 r"\s*([^(),]+),([^(),]+)\(([^(),]+),([^(),]+)\)",
1734 col,
1735 )
1736 )
1737 # NOT `word, (tag, tag)` with an empty m.group(2)...
1738 # There is a test that fails because of this. It's an
1739 # outdated table, but still, ...Italian_verb1
1740 and all(s.strip() for s in m.groups())
1741 and any(
1742 (
1743 # except for entries like word1, word2 (tag2, tag2)...
1744 classify_desc(s) in ("english", "romanization")
1745 for s in (m.group(3), m.group(4))
1746 )
1747 )
1748 ):
1749 alts = [m.group(1), m.group(2), m.group(3), m.group(4)]
1750 else:
1751 # Use default splitting. However, recognize
1752 # language-specific replacements and change them to magic
1753 # characters before splitting. This way we won't split
1754 # them. This is important for, e.g., recognizing
1755 # alternative pronouns.
1756 # The magic characters are characters out of Unicode scope
1757 # that are given a simple incremental value, int > unicode.
1758 repls = {}
1759 magic_ch = MAGIC_FIRST
1760 trs = get_lang_conf(lang, "form_transformations")
1761 # trs is a list of lists of strings
1762 for _, v, _, _ in trs:
1763 # v is a pattern string, like "^ich"
1764 # form_transformations data is doing double-duty here,
1765 # because the pattern strings are already known to us and
1766 # not meant to be split.
1767 m = re.search(v, col)
1768 if m is not None:
1769 # if pattern found in text
1770 magic = chr(magic_ch)
1771 magic_ch += 1 # next magic character value
1772 col = re.sub(v, magic, col) # replace with magic ch
1773 repls[magic] = m.group(0)
1774 # remember what regex match string each magic char
1775 # replaces. .group(0) is the whole match.
1776 alts0 = split_at_comma_semi(col, separators=separators)
1777 # with magic characters in place, split the text so that
1778 # pre-transformation text is out of the way.
1779 alts = []
1780 for alt in alts0:
1781 # create a new list with the separated items and
1782 # the magic characters replaced with the original texts.
1783 for k, v in repls.items():
1784 alt = re.sub(k, v, alt)
1785 alts.append(alt)
1787 # Remove "*" from beginning of forms, as in non-attested
1788 # or reconstructed forms. Otherwise it might confuse romanization
1789 # detection.
1790 alts = list(re.sub(r"^\*\*?([^ ])", r"\1", x) for x in alts)
1791 alts = list(
1792 x for x in alts if not re.match(r"pronounced with |\(with ", x)
1793 )
1794 alts = list(
1795 re.sub(r"^\((in the sense [^)]*)\)\s+", "", x) for x in alts
1796 )
1797 return col, alts, split_extra_tags
1799 def handle_parens(
1800 form: str, roman: str, clitic: str | None, extra_tags: list[str]
1801 ) -> tuple[str, str, str | None]:
1802 if TYPE_CHECKING:
1803 assert isinstance(paren, str)
1804 assert isinstance(m, re.Match)
1805 if re.match(r"[’'][a-z]([a-z][a-z]?)?$", paren):
1806 # is there a clitic starting with apostrophe?
1807 clitic = paren
1808 # assume the whole paren is a clitic
1809 # then remove paren from form
1810 form = (form[: m.start()] + subst + form[m.end() :]).strip()
1811 elif classify_desc(paren) == "tags":
1812 tagsets1, topics1 = decode_tags(paren)
1813 if not topics1: 1813 ↛ 1834line 1813 didn't jump to line 1834 because the condition on line 1813 was always true
1814 for ts in tagsets1:
1815 ts = tuple(x for x in ts if " " not in x)
1816 # There are some generated tags containing
1817 # spaces; do not let them through here.
1818 extra_tags.extend(ts)
1819 form = (form[: m.start()] + subst + form[m.end() :]).strip()
1820 # brackets contain romanization
1821 elif (
1822 m.start() > 0
1823 and not roman
1824 and classify_desc(form[: m.start()]) == "other"
1825 and
1826 # "other" ~ text
1827 classify_desc(paren) in ("romanization", "english")
1828 and not re.search(r"^with |-form$", paren)
1829 ):
1830 roman = paren
1831 form = (form[: m.start()] + subst + form[m.end() :]).strip()
1832 elif re.search(r"^with |-form", paren): 1832 ↛ 1833line 1832 didn't jump to line 1833 because the condition on line 1832 was never true
1833 form = (form[: m.start()] + subst + form[m.end() :]).strip()
1834 return form, roman, clitic
1836 def merge_row_and_column_tags(
1837 form: str,
1838 some_has_covered_text: bool,
1839 links: list[tuple[str, str]] | None = None,
1840 ) -> tuple[list[FormData], str, bool]:
1841 # Merge column tags and row tags. We give preference
1842 # to moods etc coming from rowtags (cf. austteigen/German/Verb
1843 # imperative forms).
1845 # In certain cases, what a tag means depends on whether
1846 # it is a row or column header. Depending on the language,
1847 # we replace certain tags with others if they're in
1848 # a column or row
1850 ret: list[FormData] = []
1851 # rtagreplacs = get_lang_conf(lang, "rowtag_replacements")
1852 # ctagreplacs = get_lang_conf(lang, "coltag_replacements")
1853 for rt in sorted(rowtags):
1854 if "dummy-use-as-coltags" in rt: 1854 ↛ 1855line 1854 didn't jump to line 1855 because the condition on line 1854 was never true
1855 continue
1856 # if lang was in rowtag_replacements)
1857 # if not rtagreplacs == None:
1858 # rt = replace_directional_tags(rt, rtagreplacs)
1859 for ct in sorted(coltags):
1860 if "dummy-use-as-rowtags" in ct: 1860 ↛ 1861line 1860 didn't jump to line 1861 because the condition on line 1860 was never true
1861 continue
1862 # if lang was in coltag_replacements
1863 # if not ctagreplacs == None:
1864 # ct = replace_directional_tags(ct,
1865 # ctagreplacs)
1866 tags = set(global_tags)
1867 tags.update(extra_tags)
1868 tags.update(rt)
1869 tags.update(refs_tags)
1870 tags.update(tablecontext.section_header)
1871 # Merge tags from column. For certain kinds of tags,
1872 # those coming from row take precedence.
1873 old_tags = set(tags)
1874 for t in ct:
1875 c = valid_tags[t]
1876 if c in ("mood", "case", "number") and any(
1877 valid_tags[tt] == c for tt in old_tags
1878 ):
1879 continue
1880 tags.add(t)
1882 # Extract language-specific tags from the
1883 # form. This may also adjust the form.
1884 form, lang_tags = lang_specific_tags(lang, pos, form)
1885 tags.update(lang_tags)
1887 # For non-finite verb forms, see if they have
1888 # a gender/class suffix
1889 if pos == "verb" and any(
1890 valid_tags[t] == "non-finite" for t in tags
1891 ):
1892 form, tt = parse_head_final_tags(wxr, lang, form)
1893 tags.update(tt)
1895 # Remove "personal" tag if have nth person; these
1896 # come up with e.g. reconhecer/Portuguese/Verb. But
1897 # not if we also have "pronoun"
1898 if (
1899 "personal" in tags
1900 and "pronoun" not in tags
1901 and any(
1902 x in tags
1903 for x in [
1904 "first-person",
1905 "second-person",
1906 "third-person",
1907 ]
1908 )
1909 ):
1910 tags.remove("personal")
1912 # If we have impersonal, remove person and number.
1913 # This happens with e.g. viajar/Portuguese/Verb
1914 if "impersonal" in tags:
1915 tags = tags - set(
1916 [
1917 "first-person",
1918 "second-person",
1919 "third-person",
1920 "singular",
1921 "plural",
1922 ]
1923 )
1925 # Remove unnecessary "positive" tag from verb forms
1926 if pos == "verb" and "positive" in tags:
1927 if "negative" in tags: 1927 ↛ 1928line 1927 didn't jump to line 1928 because the condition on line 1927 was never true
1928 tags.remove("negative")
1929 tags.remove("positive")
1931 # Many Russian (and other Slavic) inflection tables
1932 # have animate/inanimate distinction that generates
1933 # separate entries for neuter/feminine, but the
1934 # distinction only applies to masculine. Remove them
1935 # form neuter/feminine and eliminate duplicates.
1936 if get_lang_conf(lang, "masc_only_animate"):
1937 for t1 in ("animate", "inanimate"):
1938 for t2 in ("neuter", "feminine"):
1939 if (
1940 t1 in tags
1941 and t2 in tags
1942 and "masculine" not in tags
1943 and "plural" not in tags
1944 ):
1945 tags.remove(t1)
1947 # German adjective tables contain "(keiner)" etc
1948 # for mixed declension plural. When the adjective
1949 # disappears and it becomes just one word, remove
1950 # the "includes-article" tag. e.g. eiskalt/German
1951 if "includes-article" in tags and " " not in form:
1952 tags.remove("includes-article")
1954 # Handle ignored forms. We mark that the form was
1955 # provided. This is important information; some words
1956 # just do not have a certain form. However, there also
1957 # many cases where no word in a language has a
1958 # particular form. Post-processing could detect and
1959 # remove such cases.
1960 if form in IGNORED_COLVALUES:
1961 # if cell text seems to be ignorable
1962 if "dummy-ignore-skipped" in tags:
1963 continue
1964 if (
1965 col_idx not in has_covering_hdr
1966 and some_has_covered_text
1967 ):
1968 continue
1969 # don't ignore this cell if there's been a header
1970 # above it
1971 form = "-"
1972 elif col_idx in has_covering_hdr:
1973 some_has_covered_text = True
1975 # Handle ambiguous object concord. If a header
1976 # gives the "dummy-object-concord"-tag to a word,
1977 # replace person, number and gender tags with
1978 # their "object-" counterparts so that the verb
1979 # agrees with the object instead.
1980 # Use only when the verb has ONLY object agreement!
1981 # a پخول/Pashto
1982 if "dummy-object-concord" in tags: 1982 ↛ 1983line 1982 didn't jump to line 1983 because the condition on line 1982 was never true
1983 for subtag, objtag in object_concord_replacements.items():
1984 if subtag in tags:
1985 tags.remove(subtag)
1986 tags.add(objtag)
1988 # Remove the dummy mood tag that we sometimes
1989 # use to block adding other mood and related
1990 # tags
1991 tags = tags - set(
1992 [
1993 "dummy-mood",
1994 "dummy-tense",
1995 "dummy-ignore-skipped",
1996 "dummy-object-concord",
1997 "dummy-reset-headers",
1998 "dummy-use-as-coltags",
1999 "dummy-use-as-rowtags",
2000 "dummy-store-hdrspan",
2001 "dummy-load-stored-hdrspans",
2002 "dummy-reset-stored-hdrspans",
2003 "dummy-section-header",
2004 ]
2005 )
2007 # Perform language-specific tag replacements according
2008 # to rules in a table.
2009 lang_tag_mappings = get_lang_conf(lang, "lang_tag_mappings")
2010 if lang_tag_mappings is not None: 2010 ↛ 2011line 2010 didn't jump to line 2011 because the condition on line 2010 was never true
2011 for pre, post in lang_tag_mappings.items():
2012 if all(t in tags for t in pre):
2013 tags = (tags - set(pre)) | set(post)
2015 # Warn if there are entries with empty tags
2016 if not tags:
2017 wxr.wtp.debug(
2018 "inflection table: empty tags for {}".format(form),
2019 sortid="inflection/1826",
2020 )
2022 # Warn if form looks like IPA
2023 ########## XXX ########
2024 # Because IPA is its own unicode block, we could also
2025 # technically do a Unicode name check to see if a string
2026 # contains IPA. Not all valid IPA characters are in the
2027 # IPA extension block, so you can technically have false
2028 # negatives if it's something like /toki/, but it
2029 # shouldn't give false positives.
2030 # Alternatively, you could make a list of IPA-admissible
2031 # characters and reject non-IPA stuff with that.
2032 if re.match(r"\s*/.*/\s*$", form): 2032 ↛ 2033line 2032 didn't jump to line 2033 because the condition on line 2032 was never true
2033 wxr.wtp.debug(
2034 "inflection table form looks like IPA: "
2035 "form={} tags={}".format(form, tags),
2036 sortid="inflection/1840",
2037 )
2039 # Note that this checks `form`, not `in tags`
2040 if form == "dummy-ignored-text-cell": 2040 ↛ 2041line 2040 didn't jump to line 2041 because the condition on line 2040 was never true
2041 continue
2043 if "dummy-remove-this-cell" in tags: 2043 ↛ 2044line 2043 didn't jump to line 2044 because the condition on line 2043 was never true
2044 continue
2046 # Add the form
2047 tags_list = list(sorted(tags))
2048 dt: FormData = {
2049 "form": form,
2050 "tags": tags_list,
2051 "source": source,
2052 }
2053 if roman:
2054 dt["roman"] = roman
2055 if ipa:
2056 dt["ipa"] = ipa
2057 if cell_links is not None and (
2058 matched_links := match_links_to_form(
2059 wxr, form, cell_links, None
2060 )
2061 ):
2062 dt["links"] = matched_links
2063 ret.append(dt)
2064 # If we got separate clitic form, add it
2065 if clitic:
2066 dt = {
2067 "form": clitic,
2068 "tags": tags_list + ["clitic"],
2069 "source": source,
2070 }
2071 ret.append(dt)
2072 return ret, form, some_has_covered_text
2074 # First extract definitions from cells
2075 # See defs_ht for footnote defs stuff
2076 for row in rows:
2077 for cell in row:
2078 text, refs, defs, hdr_tags = extract_cell_content(
2079 lang, word, cell.text
2080 )
2081 # refs, defs = footnote stuff, defs -> (ref, def)
2082 add_defs(defs)
2083 # Extract definitions from text after table
2084 text, refs, defs, hdr_tags = extract_cell_content(lang, word, after)
2085 add_defs(defs)
2087 # Then extract the actual forms
2088 ret = []
2089 hdrspans: list[HdrSpan] = []
2090 first_col_has_text = False
2091 rownum = 0
2092 title = None
2093 global_tags = []
2094 table_tags = []
2095 special_phrase_splits = get_lang_conf(lang, "special_phrase_splits")
2096 form_replacements = get_lang_conf(lang, "form_replacements")
2097 form_transformations = get_lang_conf(lang, "form_transformations")
2098 possibly_ignored_forms = get_lang_conf(lang, "conditionally_ignored_cells")
2099 cleanup_rules = get_lang_conf(lang, "minor_text_cleanups")
2101 for title in titles:
2102 more_global_tags, more_table_tags, extra_forms = parse_title(
2103 title, source
2104 )
2105 global_tags.extend(more_global_tags)
2106 table_tags.extend(more_table_tags)
2107 ret.extend(extra_forms)
2108 cell_rowcnt: collections.defaultdict[int, int] = collections.defaultdict(
2109 int
2110 )
2111 seen_cells = set()
2112 has_covering_hdr = set()
2113 some_has_covered_text = False
2114 for row in rows:
2115 # print("ROW:", row)
2116 # print("====")
2117 # print(f"Start of PREVIOUS row hdrspans:"
2118 # f"{tuple(sp.tagsets for sp in hdrspans)}")
2119 # print(f"Start of row txt: {tuple(t.text for t in row)}")
2120 if not row: 2120 ↛ 2121line 2120 didn't jump to line 2121 because the condition on line 2120 was never true
2121 continue # Skip empty rows
2122 all_headers = all(x.is_title or not x.text.strip() for x in row)
2123 text = row[0].text
2124 if (
2125 row[0].is_title
2126 and text
2127 and not is_superscript(text[0])
2128 and text not in infl_map # zealous inflation map?
2129 and (
2130 re.match(r"Inflection ", text)
2131 or re.sub(
2132 r"\s+",
2133 " ", # flatten whitespace
2134 re.sub(
2135 r"\s*\([^)]*\)",
2136 "",
2137 # Remove whitespace+parens
2138 text,
2139 ),
2140 ).strip()
2141 not in infl_map
2142 )
2143 and not re.match(infl_start_re, text)
2144 and all(
2145 x.is_title == row[0].is_title and x.text == text
2146 # all InflCells in `row` have the same is_title and text
2147 for x in row
2148 )
2149 ):
2150 if text and title is None:
2151 # Only if there were no titles previously make the first
2152 # text that is found the title
2153 title = text
2154 if re.match(r"(Note:|Notes:)", title): 2154 ↛ 2155line 2154 didn't jump to line 2155 because the condition on line 2154 was never true
2155 continue # not a title
2156 more_global_tags, more_table_tags, extra_forms = parse_title(
2157 title, source
2158 )
2159 global_tags.extend(more_global_tags)
2160 table_tags.extend(more_table_tags)
2161 ret.extend(extra_forms)
2162 continue # Skip title rows without incrementing i
2163 if "dummy-skip-this" in global_tags: 2163 ↛ 2164line 2163 didn't jump to line 2164 because the condition on line 2163 was never true
2164 return []
2165 rowtags: list[tuple[str, ...]] = [()]
2166 # have_hdr = False
2167 # have_hdr never used?
2168 have_text = False
2169 samecell_cnt = 0
2170 col0_hdrspan = None # col0 or later header (despite its name)
2171 col0_followed_by_nonempty = False
2172 row_empty = True
2173 for col_idx, cell in enumerate(row):
2174 colspan = cell.colspan # >= 1
2175 rowspan = cell.rowspan # >= 1
2176 cell_links = cell.links # for weird links
2177 previously_seen = id(cell) in seen_cells
2178 # checks to see if this cell was in the previous ROW
2179 seen_cells.add(id(cell))
2180 if samecell_cnt == 0:
2181 # First column of a (possible multi-column) cell
2182 samecell_cnt = colspan - 1
2183 else:
2184 assert samecell_cnt > 0
2185 samecell_cnt -= 1
2186 continue
2188 # is_first_row_of_cell = cell_rowcnt[id(cell)] == 0
2189 # never used?
2191 # defaultdict(int) around line 1900
2192 cell_rowcnt[id(cell)] += 1
2193 # => how many cols this spans
2194 col: str = cell.text
2195 if not col:
2196 continue
2197 row_empty = False
2198 is_title = cell.is_title
2200 # If the cell has a target, i.e., text after colon, interpret
2201 # it as simply specifying a value for that value and ignore
2202 # it otherwise.
2203 if cell.target:
2204 text, refs, defs, hdr_tags = extract_cell_content(
2205 lang, word, col
2206 )
2207 if not text: 2207 ↛ 2208line 2207 didn't jump to line 2208 because the condition on line 2207 was never true
2208 continue
2209 refs_tags: set[str] = set()
2210 for ref in refs: # gets tags from footnotes 2210 ↛ 2211line 2210 didn't jump to line 2211 because the loop on line 2210 never started
2211 if ref in def_ht:
2212 refs_tags.update(def_ht[ref])
2213 rowtags = expand_header(
2214 wxr,
2215 tablecontext,
2216 word,
2217 lang,
2218 pos,
2219 text,
2220 [],
2221 silent=True,
2222 depth=depth,
2223 column_number=col_idx,
2224 )
2225 rowtags = list(
2226 set(tuple(sorted(set(x) | refs_tags)) for x in rowtags)
2227 )
2228 is_title = False
2229 col = cell.target
2231 # print(rownum, col_idx, col)
2232 # print(f"is_title: {is_title}")
2233 if is_title:
2234 # It is a header cell
2235 text, refs, defs, hdr_tags = extract_cell_content(
2236 lang, word, col
2237 )
2238 if not text:
2239 continue
2240 # Extract tags from referenced footnotes
2241 refs_tags = set()
2242 for ref in refs:
2243 if ref in def_ht:
2244 refs_tags.update(def_ht[ref])
2246 # Expand header to tags
2247 v = expand_header(
2248 wxr,
2249 tablecontext,
2250 word,
2251 lang,
2252 pos,
2253 text,
2254 [],
2255 silent=True,
2256 depth=depth,
2257 column_number=col_idx,
2258 )
2259 # print("EXPANDED {!r} to {}".format(text, v))
2261 if col_idx == 0:
2262 # first_col_has_text is used for a test to ignore
2263 # upper-left cells that are just text without
2264 # header info
2265 first_col_has_text = True
2266 # Check if the header expands to reset hdrspans
2267 if any("dummy-reset-headers" in tt for tt in v):
2268 new_hdrspans = []
2269 for hdrspan in hdrspans:
2270 # if there are HdrSpan objects (abstract headers with
2271 # row- and column-spans) that are to the left or at the
2272 # same row or below, KEEP those; things above and to
2273 # the right of the hdrspan with dummy-reset-headers
2274 # are discarded. Tags from the header together with
2275 # dummy-reset-headers are kept as normal.
2276 if (
2277 hdrspan.start + hdrspan.colspan < col_idx
2278 or hdrspan.rownum > rownum - cell.rowspan
2279 ):
2280 new_hdrspans.append(hdrspan)
2281 hdrspans = new_hdrspans
2283 for tt in v:
2284 if "dummy-section-header" in tt: 2284 ↛ 2285line 2284 didn't jump to line 2285 because the condition on line 2284 was never true
2285 tablecontext.section_header = tt
2286 break
2287 if "dummy-reset-section-header" in tt: 2287 ↛ 2288line 2287 didn't jump to line 2288 because the condition on line 2287 was never true
2288 tablecontext.section_header = tuple()
2289 # Text between headers on a row causes earlier headers to
2290 # be reset
2291 if have_text:
2292 # print(" HAVE_TEXT BEFORE HDR:", col)
2293 # Reset rowtags if new title column after previous
2294 # text cells
2295 # +-----+-----+-----+-----+
2296 # |hdr-a|txt-a|hdr-B|txt-B|
2297 # +-----+-----+-----+-----+
2298 # ^reset rowtags=>
2299 # XXX beware of header "—": "" - must not clear on that if
2300 # it expands to no tags
2301 rowtags = [()]
2302 # have_hdr = True
2303 # have_hdr never used?
2304 # print("HAVE_HDR: {} rowtags={}".format(col, rowtags))
2305 # Update rowtags and coltags
2306 has_covering_hdr.add(col_idx) # col_idx == current column
2307 # has_covering_hdr is a set that has the col_idx-ids of columns
2308 # that have previously had some kind of header. It is never
2309 # resetted inside the col_idx-loops OR the bigger rows-loop, so
2310 # applies to the whole table.
2312 new_coltags: list[tuple[str, ...]]
2313 all_hdr_tags: list[tuple[str, ...]]
2314 rowtags, new_coltags, all_hdr_tags = generate_tags(
2315 rowtags, table_tags
2316 )
2318 if any("dummy-skip-this" in ts for ts in rowtags):
2319 continue # Skip this cell
2321 if any("dummy-load-stored-hdrspans" in ts for ts in v): 2321 ↛ 2322line 2321 didn't jump to line 2322 because the condition on line 2321 was never true
2322 hdrspans.extend(tablecontext.stored_hdrspans)
2324 if any("dummy-reset-stored-hdrspans" in ts for ts in v): 2324 ↛ 2325line 2324 didn't jump to line 2325 because the condition on line 2324 was never true
2325 tablecontext.stored_hdrspans = []
2327 if any("dummy-store-hdrspan" in ts for ts in v): 2327 ↛ 2329line 2327 didn't jump to line 2329 because the condition on line 2327 was never true
2328 # print(f"STORED: {col}")
2329 store_new_hdrspan = True
2330 else:
2331 store_new_hdrspan = False
2333 new_coltags = list(
2334 x
2335 for x in new_coltags
2336 if not any(t in noinherit_tags for t in x)
2337 )
2338 # print("new_coltags={} previously_seen={} all_hdr_tags={}"
2339 # .format(new_coltags, previously_seen, all_hdr_tags))
2340 if any(new_coltags):
2341 (
2342 col,
2343 col0_followed_by_nonempty,
2344 col0_hdrspan,
2345 ) = add_new_hdrspan(
2346 col,
2347 hdrspans,
2348 store_new_hdrspan,
2349 col0_followed_by_nonempty,
2350 col0_hdrspan,
2351 )
2353 continue
2355 # These values are ignored, at least for now
2356 if re.match(r"^(# |\(see )", col): 2356 ↛ 2357line 2356 didn't jump to line 2357 because the condition on line 2356 was never true
2357 continue
2359 if any("dummy-skip-this" in ts for ts in rowtags):
2360 continue # Skip this cell
2362 # If the word has no rowtags and is a multi-row cell, then
2363 # ignore this. This happens with empty separator rows
2364 # within a rowspan>1 cell. cf. wander/English/Conjugation.
2365 if rowtags == [()] and rowspan > 1:
2366 continue
2368 # Minor cleanup. See e.g. είμαι/Greek/Verb present participle.
2369 if cleanup_rules:
2370 for regx, substitution in cleanup_rules.items():
2371 col = re.sub(regx, substitution, col)
2373 if ( 2373 ↛ 2378line 2373 didn't jump to line 2378 because the condition on line 2373 was never true
2374 col_idx == 0
2375 and not first_col_has_text
2376 and get_lang_conf(lang, "ignore_top_left_text_cell") is True
2377 ):
2378 continue # Skip text at top left, as in Icelandic, Faroese
2380 # if col0_hdrspan is not None:
2381 # print("COL0 FOLLOWED NONHDR: {!r} by {!r}"
2382 # .format(col0_hdrspan.text, col))
2383 col0_followed_by_nonempty = True
2384 have_text = True
2386 # Determine column tags for the multi-column cell
2387 combined_coltags = compute_coltags(
2388 lang, pos, hdrspans, col_idx, colspan, col
2389 )
2390 if any("dummy-ignored-text-cell" in ts for ts in combined_coltags): 2390 ↛ 2391line 2390 didn't jump to line 2391 because the condition on line 2390 was never true
2391 continue
2393 # Split the text into separate forms. First simplify spaces except
2394 # newline.
2395 col = re.sub(r"[ \t\r]+", " ", col)
2396 # Split the cell text into alternatives
2398 col, alts, split_extra_tags = split_text_into_alts(col)
2400 # Some cells have mixed form content, like text and romanization,
2401 # or text and IPA. Handle these.
2402 altss = handle_mixed_lines(alts, tablecontext)
2404 altsss = list((x, combined_coltags, cell_links) for x in altss)
2406 # Generate forms from the alternatives
2407 # alts is a list of (tuple of forms, tuple of tags)
2408 coltags: list[tuple[str, ...]]
2409 base_roman: str
2410 ipa: str
2411 for (form, base_roman, ipa), coltags, cell_links in altsss:
2412 form = form.strip()
2413 extra_tags: list[str] = []
2414 extra_tags.extend(split_extra_tags)
2415 # Handle special splits again here, so that we can have custom
2416 # mappings from form to form and tags.
2417 if form in form_replacements:
2418 replacement, tags = form_replacements[form]
2419 for x in tags.split():
2420 assert x in valid_tags
2421 assert isinstance(replacement, str)
2422 assert isinstance(tags, str)
2423 form = replacement
2424 extra_tags.extend(tags.split())
2426 check_romanization_form_transformation = False
2427 # loop over regexes in form_transformation and replace text
2428 # in form using regex patterns
2429 # this does a bit of the same stuff the above does,
2430 # but with regexes and re.sub() instead
2431 subst: str
2432 for (
2433 form_transformations_pos,
2434 vv,
2435 subst,
2436 tags,
2437 ) in form_transformations:
2438 # v is a pattern string, like "^ich"
2439 if (
2440 isinstance(form_transformations_pos, str)
2441 and pos != form_transformations_pos
2442 ) or (
2443 (not isinstance(form_transformations_pos, str))
2444 and pos not in form_transformations_pos
2445 ):
2446 continue
2447 m: re.Match | None = re.search(vv, form)
2448 if m is not None:
2449 if base_roman: 2449 ↛ 2450line 2449 didn't jump to line 2450 because the condition on line 2449 was never true
2450 for _, rom_v, rom_sub, _ in form_transformations:
2451 rom_m = re.search(rom_v, base_roman)
2452 if rom_m is not None:
2453 base_roman = re.sub(
2454 rom_v, rom_sub, base_roman
2455 )
2456 break
2457 form = re.sub(vv, subst, form)
2458 for x in tags.split():
2459 assert x in valid_tags
2460 extra_tags.extend(tags.split())
2461 check_romanization_form_transformation = True
2462 break
2464 # Clean the value, extracting reference symbols
2465 form, refs, defs, hdr_tags = extract_cell_content(
2466 lang, word, form
2467 )
2468 # if refs:
2469 # print("REFS:", refs)
2470 extra_tags.extend(hdr_tags)
2471 # Extract tags from referenced footnotes
2472 refs_tags = set()
2473 for ref in refs:
2474 if ref in def_ht:
2475 refs_tags.update(def_ht[ref])
2477 if base_roman:
2478 if check_romanization_form_transformation: 2478 ↛ 2482line 2478 didn't jump to line 2482 because the condition on line 2478 was never true
2479 # because form_transformations are used to handle things
2480 # where the romanization has the "same" structure, we
2481 # need to handle that here too....
2482 for (
2483 _,
2484 vv,
2485 subst,
2486 _,
2487 ) in form_transformations:
2488 # v is a pattern string, like "^ich"
2489 m = re.search(vv, base_roman)
2490 if m is not None:
2491 base_roman = re.sub(vv, subst, base_roman)
2492 # XXX add tag stuff here if needed
2493 break
2495 base_roman, _, _, hdr_tags = extract_cell_content(
2496 lang, word, base_roman
2497 )
2498 extra_tags.extend(hdr_tags)
2500 # Do some additional cleanup on the cell.
2501 form = re.sub(r"^\s*,\s*", "", form)
2502 form = re.sub(r"\s*,\s*$", "", form)
2503 form = re.sub(r"\s*(,\s*)+", ", ", form)
2504 form = re.sub(r"(?i)^Main:", "", form)
2505 form = re.sub(r"\s+", " ", form)
2506 form = form.strip()
2508 # Look for parentheses that have semantic meaning
2509 form, et = find_semantic_parens(form, lang)
2510 extra_tags.extend(et)
2512 # Handle parentheses in the table element. We parse
2513 # tags anywhere and romanizations anywhere but beginning.
2514 roman: str = base_roman
2515 paren: str | None = None
2516 clitic: str | None = None
2517 m = re.search(r"(\s+|^)\(([^)]*)\)", form)
2518 # start|spaces + (anything)
2519 if m is not None:
2520 subst = m.group(1)
2521 paren = m.group(2)
2522 else:
2523 m = re.search(r"\(([^)]*)\)(\s+|$)", form)
2524 # (anything) + spaces|end
2525 if m is not None: 2525 ↛ 2526line 2525 didn't jump to line 2526 because the condition on line 2525 was never true
2526 paren = m.group(1)
2527 subst = m.group(2)
2528 if paren is not None:
2529 form, roman, clitic = handle_parens(
2530 form, roman, clitic, extra_tags
2531 )
2533 # Ignore certain forms that are not really forms,
2534 # unless they're really, really close to the article title
2535 if form in ( 2535 ↛ 2540line 2535 didn't jump to line 2540 because the condition on line 2535 was never true
2536 "",
2537 "unchanged",
2538 "after an", # in sona/Irish/Adj/Mutation
2539 ):
2540 Lev = distw([form], word)
2541 if form and Lev < 0.1:
2542 wxr.wtp.debug(
2543 "accepted possible false positive '{}' with"
2544 "> 0.1 Levenshtein distance in {}/{}".format(
2545 form, word, lang
2546 ),
2547 sortid="inflection/2213",
2548 )
2549 elif form and Lev < 0.3:
2550 wxr.wtp.debug(
2551 "skipped possible match '{}' with > 0.3"
2552 "Levenshtein distance in {}/{}".format(
2553 form, word, lang
2554 ),
2555 sortid="inflection/2218",
2556 )
2557 continue
2558 else:
2559 continue
2560 # print("ROWTAGS={} COLTAGS={} REFS_TAGS={} "
2561 # "FORM={!r} ROMAN={!r}"
2562 # .format(rowtags, coltags, refs_tags,
2563 # form, roman))
2565 # Merge tags from row and column and do miscellaneous
2566 # tag-related handling.
2567 (
2568 merge_ret,
2569 form,
2570 some_has_covered_text,
2571 ) = merge_row_and_column_tags(
2572 form, some_has_covered_text, cell_links
2573 )
2574 ret.extend(merge_ret)
2576 # End of row.
2577 rownum += 1
2578 # For certain languages, if the row was empty, reset
2579 # hdrspans (saprast/Latvian/Verb, but not aussteigen/German/Verb).
2580 if row_empty and get_lang_conf(lang, "empty_row_resets"):
2581 hdrspans = []
2582 # Check if we should expand col0_hdrspan.
2583 if col0_hdrspan is not None:
2584 col0_allowed = get_lang_conf(lang, "hdr_expand_first")
2585 col0_cats = tagset_cats(col0_hdrspan.tagsets)
2586 # Only expand if col0_cats and later_cats are allowed
2587 # and don't overlap and col0 has tags, and there have
2588 # been no disallowed cells in between.
2589 if (
2590 not col0_followed_by_nonempty
2591 and not (col0_cats - col0_allowed)
2592 and
2593 # len(col0_cats) == 1 and
2594 col_idx > col0_hdrspan.start + col0_hdrspan.colspan
2595 ):
2596 # If an earlier header is only followed by headers that yield
2597 # no tags, expand it to entire row
2598 # print("EXPANDING COL0: {} from {} to {} cols {}"
2599 # .format(col0_hdrspan.text, col0_hdrspan.colspan,
2600 # len(row) - col0_hdrspan.start,
2601 # col0_hdrspan.tagsets))
2602 col0_hdrspan.colspan = len(row) - col0_hdrspan.start
2603 col0_hdrspan.expanded = True
2604 # XXX handle refs and defs
2605 # for x in hdrspans:
2606 # print(" HDRSPAN {} {} {} {!r}"
2607 # .format(x.start, x.colspan, x.tagsets, x.text))
2609 # Post-process German nouns with articles in separate columns. We move the
2610 # definite/indefinite/usually-without-article markers into the noun and
2611 # remove the article entries.
2612 if get_lang_conf(lang, "articles_in_separate_columns") and any(
2613 "noun" in x["tags"] for x in ret
2614 ):
2615 new_ret = []
2616 saved_tags: set[str] = set()
2617 had_noun = False
2618 for dt in ret:
2619 tags = dt["tags"]
2620 # print(tags)
2621 if "noun" in tags:
2622 tags = list(
2623 sorted(set(t for t in tags if t != "noun") | saved_tags)
2624 )
2625 had_noun = True
2626 elif ( 2626 ↛ 2653line 2626 didn't jump to line 2653 because the condition on line 2626 was always true
2627 "indefinite" in tags
2628 or "definite" in tags
2629 or "usually-without-article" in tags
2630 or "without-article" in tags
2631 ):
2632 if had_noun:
2633 saved_tags = set(tags)
2634 else:
2635 saved_tags = saved_tags | set(tags) # E.g. Haus/German
2636 remove_useless_tags(lang, pos, saved_tags)
2637 saved_tags = saved_tags & set(
2638 [
2639 "masculine",
2640 "feminine",
2641 "neuter",
2642 "singular",
2643 "plural",
2644 "indefinite",
2645 "definite",
2646 "usually-without-article",
2647 "without-article",
2648 ]
2649 )
2650 had_noun = False
2651 continue # Skip the articles
2653 dt = dt.copy()
2654 dt["tags"] = tags
2655 new_ret.append(dt)
2656 ret = new_ret
2658 elif possibly_ignored_forms:
2659 # Some languages have tables with cells that are kind of separated
2660 # and difficult to handle, like eulersche Formel/German where
2661 # the definite and indefinite articles are just floating.
2662 # If a language has a dict of conditionally_ignored_cells,
2663 # and if the contents of a cell is found in one of the rules
2664 # there, ignore that cell if it
2665 # 1. Does not have the appropriate tag (like "definite" for "die")
2666 # and
2667 # 2. The title of the article is not one of the other co-words
2668 # (ie. it's an article for the definite articles in german etc.)
2669 # pass
2670 new_ret = []
2671 for cell_data in ret:
2672 tags = cell_data["tags"]
2673 text = cell_data["form"]
2674 skip_this = False
2675 for key_tag, ignored_forms in possibly_ignored_forms.items():
2676 if text not in ignored_forms: 2676 ↛ 2678line 2676 didn't jump to line 2678 because the condition on line 2676 was always true
2677 continue
2678 if word in ignored_forms:
2679 continue
2680 if key_tag not in tags:
2681 skip_this = True
2683 if skip_this: 2683 ↛ 2684line 2683 didn't jump to line 2684 because the condition on line 2683 was never true
2684 continue
2685 new_ret.append(cell_data)
2687 ret = new_ret
2689 # Post-process English inflection tables, addding "multiword-construction"
2690 # when the number of words has increased.
2691 if lang == "English" and pos == "verb":
2692 word_words = len(word.split())
2693 new_ret = []
2694 for dt in ret:
2695 form = dt.get("form", "")
2696 if len(form.split()) > word_words:
2697 dt = dt.copy()
2698 dt["tags"] = list(dt.get("tags", []))
2699 # This strange copy-assigning shuffle is preventative black
2700 # magic; do not touch lest you invoke deep bugs.
2701 data_append(dt, "tags", "multiword-construction")
2702 new_ret.append(dt)
2703 ret = new_ret
2705 # Always insert "table-tags" detail as the first entry in any inflection
2706 # table. This way we can reliably detect where a new table starts.
2707 # Table-tags applies until the next table-tags entry.
2708 if ret or table_tags:
2709 table_tags = sorted(set(table_tags))
2710 dt = {
2711 "form": " ".join(table_tags),
2712 "source": source,
2713 "tags": ["table-tags"],
2714 }
2715 if dt["form"] == "":
2716 dt["form"] = "no-table-tags"
2717 if tablecontext.template_name:
2718 tn: FormData = {
2719 "form": tablecontext.template_name,
2720 "source": source,
2721 "tags": ["inflection-template"],
2722 }
2723 ret = [dt] + [tn] + ret
2724 else:
2725 ret = [dt] + ret
2727 return ret
2730def find_semantic_parens(form: str, lang: str) -> tuple[str, list[str]]:
2731 # "Some languages" (=Greek) use brackets to mark things that
2732 # require tags, like (informality), [rarity] and {archaicity}.
2733 extra_tags = []
2734 if re.match(r"\([^][(){}]*\)$", form):
2735 if get_lang_conf(lang, "parentheses_for_informal"):
2736 form = form[1:-1]
2737 extra_tags.append("informal")
2738 else:
2739 form = form[1:-1]
2740 elif re.match(r"\{\[[^][(){}]*\]\}$", form):
2741 if get_lang_conf(lang, "square_brackets_for_rare") and get_lang_conf( 2741 ↛ 2748line 2741 didn't jump to line 2748 because the condition on line 2741 was always true
2742 lang, "curly_brackets_for_archaic"
2743 ):
2744 # είμαι/Greek/Verb
2745 form = form[2:-2]
2746 extra_tags.extend(["rare", "archaic"])
2747 else:
2748 form = form[2:-2]
2749 elif re.match(r"\{[^][(){}]*\}$", form):
2750 if get_lang_conf(lang, "curly_brackets_for_archaic"): 2750 ↛ 2755line 2750 didn't jump to line 2755 because the condition on line 2750 was always true
2751 # είμαι/Greek/Verb
2752 form = form[1:-1]
2753 extra_tags.extend(["archaic"])
2754 else:
2755 form = form[1:-1]
2756 elif re.match(r"\[[^][(){}]*\]$", form):
2757 if get_lang_conf(lang, "square_brackets_for_rare"): 2757 ↛ 2762line 2757 didn't jump to line 2762 because the condition on line 2757 was always true
2758 # είμαι/Greek/Verb
2759 form = form[1:-1]
2760 extra_tags.append("rare")
2761 else:
2762 form = form[1:-1]
2763 return form, extra_tags
2766def handle_mixed_lines(
2767 alts: list[str], tablecontext: "TableContext"
2768) -> list[tuple[str, str, str]]:
2769 # Handle the special case where romanization is given under
2770 # normal form, e.g. in Russian. There can be multiple
2771 # comma-separated forms in each case. We also handle the case
2772 # where instead of romanization we have IPA pronunciation
2773 # (e.g., avoir/French/verb).
2774 len2 = len(alts) // 2
2776 if len(alts) == 1 and "(" not in alts[0]:
2777 return [(alts[0], "", "")]
2779 # Check for IPAs (forms first, IPAs under)
2780 # base, base, IPA, IPA
2781 if (
2782 len(alts) % 2 == 0 # Divisibly by two
2783 and all(
2784 re.match(r"^\s*/.*/\s*$", x) # Inside slashes = IPA
2785 for x in alts[len2:]
2786 )
2787 and not any(
2788 re.match(r"^\s*/.*/\s*$", x) # first half without slashes
2789 for x in alts[:len2]
2790 )
2791 ): # In the second half of alts
2792 return list(
2793 (alts[i], "", alts[i + len2])
2794 # List of tuples: (base, "", ipa)
2795 for i in range(len2)
2796 )
2797 # base, base, base, IPA
2798 elif (
2799 len(alts) > 2
2800 and re.match(r"^\s*/.*/\s*$", alts[-1])
2801 and all(not x.startswith("/") for x in alts[:-1])
2802 ):
2803 # Only if the last alt is IPA
2804 return list((alts[i], "", alts[-1]) for i in range(len(alts) - 1))
2806 # base, IPA, IPA, IPA
2807 elif (
2808 len(alts) > 2
2809 and not alts[0].startswith("/")
2810 and all(re.match(r"^\s*/.*/\s*$", x) for x in alts[1:])
2811 ):
2812 # First is base and the rest is IPA alternatives
2813 return list((alts[0], "", x) for x in alts[1:])
2815 alt_classifications = list(
2816 classify_desc(
2817 re.sub(
2818 r"\^.*$",
2819 "",
2820 # Remove ends of strings starting from ^.
2821 # Supescripts have been already removed
2822 # from the string, while ^xyz needs to be
2823 # removed separately, though it's usually
2824 # something with a single letter?
2825 "".join(xx for xx in x if not is_superscript(xx))
2826 # Remove trailing footnote asterisks that mess with
2827 # classification
2828 .strip("* "),
2829 )
2830 )
2831 for x in alts
2832 )
2834 # Check for romanizations, forms first, romanizations under
2835 if (
2836 len(alts) % 2 == 0
2837 and not any("(" in x for x in alts)
2838 and all(x == "other" for x in alt_classifications[:len2])
2839 and all(
2840 x in ("romanization", "english") for x in alt_classifications[len2:]
2841 )
2842 ):
2843 return list((alts[i], alts[i + len2], "") for i in range(len2))
2844 # Check for romanizations, forms and romanizations alternating
2845 elif (
2846 len(alts) % 2 == 0
2847 and not any("(" in x for x in alts)
2848 and all(
2849 alt_classifications[i] == "other" for i in range(0, len(alts), 2)
2850 )
2851 and all(
2852 alt_classifications[i] in ("romanization", "english")
2853 for i in range(1, len(alts), 2)
2854 )
2855 ):
2856 # odds
2857 return list((alts[i], alts[i + 1], "") for i in range(0, len(alts), 2))
2858 # evens
2859 # Handle complex Georgian entries with alternative forms and*
2860 # *romanizations. It's a bit of a mess. Remove this kludge if not
2861 # needed anymore. NOTE THAT THE PARENTHESES ON THE WEBSITE ARE NOT
2862 # DISPLAYED. They are put inside their own span elements that are
2863 # then hidden with some CSS.
2864 # https://en.wiktionary.org/wiki/%E1%83%90%E1%83%9B%E1%83%94%E1%83%A0%E1%83%98%E1%83%99%E1%83%98%E1%83%A1_%E1%83%A8%E1%83%94%E1%83%94%E1%83%A0%E1%83%97%E1%83%94%E1%83%91%E1%83%A3%E1%83%9A%E1%83%98_%E1%83%A8%E1%83%A2%E1%83%90%E1%83%A2%E1%83%94%E1%83%91%E1%83%98
2865 # ამერიკის შეერთებულ შტატებს(ა) (ameriḳis šeertebul šṭaṭebs(a))
2866 # The above should generate two alts entries, with two different
2867 # parallel versions, one without (a) and with (a) at the end,
2868 # for both the Georgian original and the romanization.
2869 elif (
2870 tablecontext.template_name == "ka-decl-noun"
2871 and len(alts) >= 1
2872 and any(" (" in alt_ for alt_ in alts)
2873 ):
2874 return ka_decl_noun_template_cell(alts)
2875 elif (
2876 len(alts) > 2
2877 and alt_classifications[0] == "other"
2878 and all(
2879 x in ("romanization", "english") for x in alt_classifications[1:]
2880 )
2881 ):
2882 return list((alts[0], x, "") for x in alts[1:])
2883 else:
2884 new_alts = []
2885 for alt in alts:
2886 lst = [""]
2887 idx = 0
2888 for m in re.finditer(
2889 r"(^|\w|\*)\((\w+(/\w+)*)\)",
2890 # start OR letter OR asterisk (word/word*)
2891 # \\___________group 1_______/ \ \_g3_///
2892 # \ \__gr. 2_//
2893 # \_____________group 0________________/
2894 alt,
2895 ):
2896 v = m.group(2) # (word/word/word...)
2897 if (
2898 classify_desc(v) == "tags" # Tags inside parens
2899 or m.group(0) == alt
2900 ): # All in parens
2901 continue
2902 new_lst = []
2903 for x in lst:
2904 x += alt[idx : m.start()] + m.group(1)
2905 # alt until letter or asterisk
2906 idx = m.end()
2907 vparts = v.split("/")
2908 # group(2) = ["word", "wörd"...]
2909 if len(vparts) == 1:
2910 new_lst.append(x)
2911 new_lst.append(x + v)
2912 # "kind(er)" -> ["kind", "kinder"]
2913 else:
2914 for vv in vparts:
2915 new_lst.append(x + vv)
2916 # "lampai(tten/den)" ->
2917 # ["lampaitten", "lampaiden"]
2918 lst = new_lst
2919 for x in lst:
2920 new_alts.append(x + alt[idx:])
2921 # add the end of alt
2922 return list((x, "", "") for x in new_alts)
2923 # [form, no romz, no ipa]
2924 return []
2927def handle_generic_table(
2928 wxr: WiktextractContext,
2929 tablecontext: "TableContext",
2930 data: WordData,
2931 word: str,
2932 lang: str,
2933 pos: str,
2934 rows: list[list[InflCell]],
2935 titles: list[str],
2936 source: str,
2937 after: str,
2938 depth: int,
2939) -> None:
2940 assert isinstance(wxr, WiktextractContext)
2941 assert isinstance(data, dict)
2942 assert isinstance(word, str)
2943 assert isinstance(lang, str)
2944 assert isinstance(pos, str)
2945 assert isinstance(rows, list)
2946 assert isinstance(source, str)
2947 assert isinstance(after, str)
2948 assert isinstance(depth, int)
2949 for row in rows:
2950 assert isinstance(row, list)
2951 for x in row:
2952 assert isinstance(x, InflCell)
2953 assert isinstance(titles, list)
2954 for s in titles:
2955 assert isinstance(s, str)
2957 # Try to parse the table as a simple table
2958 ret = parse_simple_table(
2959 wxr, tablecontext, word, lang, pos, rows, titles, source, after, depth
2960 )
2961 if ret is None: 2961 ↛ 2964line 2961 didn't jump to line 2964 because the condition on line 2961 was never true
2962 # XXX handle other table formats
2963 # We were not able to handle the table
2964 wxr.wtp.debug(
2965 "unhandled inflection table format, {}/{}".format(word, lang),
2966 sortid="inflection/2370",
2967 )
2968 return
2970 # Add the returned forms but eliminate duplicates.
2971 have_forms = set()
2972 for dt in ret:
2973 fdt = freeze(dt)
2974 if fdt in have_forms:
2975 continue # Don't add duplicates
2976 # Some Russian words have Declension and Pre-reform declension partially
2977 # duplicating same data. Don't add "dated" tags variant if already have
2978 # the same without "dated" from the modern declension table
2980 tags = dt.get("tags", [])
2981 for dated_tag in ("dated",):
2982 if dated_tag in tags:
2983 dt2 = dt.copy()
2984 tags2 = list(x for x in tags if x != dated_tag)
2985 dt2["tags"] = tags2
2986 if tags2 and freeze(dt2) in have_forms: 2986 ↛ 2987line 2986 didn't jump to line 2987 because the condition on line 2986 was never true
2987 break # Already have without archaic
2988 else:
2989 if "table-tags" not in tags:
2990 have_forms.add(fdt)
2991 data_append(data, "forms", dt)
2994def determine_header(
2995 wxr: WiktextractContext,
2996 tablecontext,
2997 lang: str,
2998 word: str,
2999 pos: str,
3000 table_kind: NodeKind,
3001 kind: NodeKind | str,
3002 style: str | None,
3003 row: list[InflCell],
3004 col: WikiNode,
3005 celltext: str,
3006 titletext: str,
3007 cols_headered: list[bool],
3008 target: str | None,
3009 cellstyle: str,
3010 # is_title,
3011 # hdr_expansion,
3012 # target,
3013 # celltext,
3014) -> tuple[bool, list[tuple[str, ...]], str | None, str]:
3015 assert isinstance(table_kind, NodeKind)
3016 assert isinstance(kind, (NodeKind, str))
3017 assert style is None or isinstance(style, str)
3018 assert cellstyle is None or isinstance(cellstyle, str)
3020 header_kind: NodeKind | str
3021 if table_kind == NodeKind.TABLE:
3022 header_kind = NodeKind.TABLE_HEADER_CELL
3023 elif table_kind == NodeKind.HTML: 3023 ↛ 3025line 3023 didn't jump to line 3025 because the condition on line 3023 was always true
3024 header_kind = "th"
3025 idx = celltext.find(": ")
3026 is_title = False
3027 # remove anything in parentheses, compress whitespace, .strip()
3028 cleaned_titletext = re.sub(
3029 r"\s+", " ", re.sub(r"\s*\([^)]*\)", "", titletext)
3030 ).strip()
3031 cleaned, _, _, _ = extract_cell_content(lang, word, celltext)
3032 cleaned = re.sub(r"\s+", " ", cleaned)
3033 hdr_expansion = expand_header(
3034 wxr,
3035 tablecontext,
3036 word,
3037 lang,
3038 pos,
3039 cleaned,
3040 [],
3041 silent=True,
3042 ignore_tags=True,
3043 )
3044 candidate_hdr = not any(
3045 any(t.startswith("error-") for t in ts) for ts in hdr_expansion
3046 )
3047 # KJ candidate_hdr says that a specific cell is a candidate
3048 # for being a header because it passed through expand_header
3049 # without getting any "error-" tags; that is, the contents
3050 # is "valid" for being a header; these are the false positives
3051 # we want to catch
3052 ignored_cell = any(
3053 any(t.startswith("dummy-") for t in ts) for ts in hdr_expansion
3054 )
3055 # ignored_cell should NOT be used to filter for headers, like
3056 # candidate_hdr is used, but only to filter for related *debug
3057 # messages*: some dummy-tags are actually half-way to headers,
3058 # like ones with "Notes", so they MUST be headers, but later
3059 # on they're ignored *as* headers so they don't need to print
3060 # out any cells-as-headers debug messages.
3061 if (
3062 candidate_hdr
3063 and kind != header_kind
3064 and cleaned != ""
3065 and cleaned != "dummy-ignored-text-cell"
3066 and cleaned not in IGNORED_COLVALUES
3067 ):
3068 # print("col: {}".format(col))
3069 if not ignored_cell and lang not in LANGUAGES_WITH_CELLS_AS_HEADERS:
3070 wxr.wtp.debug(
3071 "rejected heuristic header: "
3072 "table cell identified as header and given "
3073 "candidate status, BUT {} is not in "
3074 "LANGUAGES_WITH_CELLS_AS_HEADERS; "
3075 "cleaned text: {}".format(lang, cleaned),
3076 sortid="inflection/2447",
3077 )
3078 candidate_hdr = False
3079 elif cleaned not in LANGUAGES_WITH_CELLS_AS_HEADERS.get(lang, ""):
3080 wxr.wtp.debug(
3081 "rejected heuristic header: "
3082 "table cell identified as header and given "
3083 "candidate status, BUT the cleaned text is "
3084 "not in LANGUAGES_WITH_CELLS_AS_HEADERS[{}]; "
3085 "cleaned text: {}".format(lang, cleaned),
3086 sortid="inflection/2457",
3087 )
3088 candidate_hdr = False
3089 else:
3090 wxr.wtp.debug(
3091 "accepted heuristic header: "
3092 "table cell identified as header and given "
3093 "candidate status, AND the cleaned text is "
3094 "in LANGUAGES_WITH_CELLS_AS_HEADERS[{}]; "
3095 "cleaned text: {}".format(lang, cleaned),
3096 sortid="inflection/2466",
3097 )
3099 # If the cell starts with something that could start a
3100 # definition (typically a reference symbol), make it a candidate
3101 # regardless of whether the language is listed.
3102 if re.match(def_re, cleaned) and not re.match(nondef_re, cleaned): 3102 ↛ 3103line 3102 didn't jump to line 3103 because the condition on line 3102 was never true
3103 candidate_hdr = True
3105 # print("titletext={!r} hdr_expansion={!r} candidate_hdr={!r} "
3106 # "lang={} pos={}"
3107 # .format(titletext, hdr_expansion, candidate_hdr,
3108 # lang, pos))
3109 if idx >= 0 and titletext[:idx] in infl_map:
3110 target = titletext[idx + 2 :].strip()
3111 celltext = celltext[:idx]
3112 is_title = True
3113 elif (
3114 kind == header_kind
3115 and " + " not in titletext # For "avoir + blah blah"?
3116 and not any(
3117 isinstance(x, WikiNode)
3118 and x.kind == NodeKind.HTML
3119 and x.sarg == "span"
3120 and x.attrs.get("lang") in ("az",)
3121 for x in col.children
3122 )
3123 ):
3124 is_title = True
3125 elif (
3126 candidate_hdr
3127 and cleaned_titletext not in IGNORED_COLVALUES
3128 and distw([cleaned_titletext], word) > 0.3
3129 and cleaned_titletext not in ("I", "es")
3130 ):
3131 is_title = True
3132 # if first column or same style as first column
3133 elif (
3134 style == cellstyle
3135 and
3136 # and title is not identical to word name
3137 titletext != word
3138 and cleaned not in IGNORED_COLVALUES
3139 and cleaned != "dummy-ignored-text-cell"
3140 and
3141 # the style composite string is not broken
3142 not style.startswith("////")
3143 and " + " not in titletext
3144 ):
3145 if not ignored_cell and lang not in LANGUAGES_WITH_CELLS_AS_HEADERS: 3145 ↛ 3146line 3145 didn't jump to line 3146 because the condition on line 3145 was never true
3146 wxr.wtp.debug(
3147 "rejected heuristic header: "
3148 "table cell identified as header based "
3149 "on style, BUT {} is not in "
3150 "LANGUAGES_WITH_CELLS_AS_HEADERS; "
3151 "cleaned text: {}, style: {}".format(lang, cleaned, style),
3152 sortid="inflection/2512",
3153 )
3154 elif ( 3154 ↛ 3158line 3154 didn't jump to line 3158 because the condition on line 3154 was never true
3155 not ignored_cell
3156 and cleaned not in LANGUAGES_WITH_CELLS_AS_HEADERS.get(lang, "")
3157 ):
3158 wxr.wtp.debug(
3159 "rejected heuristic header: "
3160 "table cell identified as header based "
3161 "on style, BUT the cleaned text is "
3162 "not in LANGUAGES_WITH_CELLS_AS_HEADERS[{}]; "
3163 "cleaned text: {}, style: {}".format(lang, cleaned, style),
3164 sortid="inflection/2522",
3165 )
3166 else:
3167 wxr.wtp.debug(
3168 "accepted heuristic header: "
3169 "table cell identified as header based "
3170 "on style, AND the cleaned text is "
3171 "in LANGUAGES_WITH_CELLS_AS_HEADERS[{}]; "
3172 "cleaned text: {}, style: {}".format(lang, cleaned, style),
3173 sortid="inflection/2530",
3174 )
3175 is_title = True
3176 if ( 3176 ↛ 3183line 3176 didn't jump to line 3183 because the condition on line 3176 was never true
3177 not is_title
3178 and len(row) < len(cols_headered)
3179 and cols_headered[len(row)]
3180 ):
3181 # Whole column has title suggesting they are headers
3182 # (e.g. "Case")
3183 is_title = True
3184 if re.match(
3185 r"Conjugation of |Declension of |Inflection of |"
3186 r"Mutation of |Notes\b", # \b is word-boundary
3187 titletext,
3188 ):
3189 is_title = True
3190 return is_title, hdr_expansion, target, celltext
3193class TableContext:
3194 """Saved context used when parsing a table and its subtables."""
3196 __slot__ = (
3197 "stored_hdrspans",
3198 "section_header",
3199 "template_name",
3200 )
3202 def __init__(self, template_name: str | None = None) -> None:
3203 self.stored_hdrspans: list[HdrSpan] = []
3204 self.section_header: tuple[str, ...] = tuple()
3205 if template_name is None:
3206 self.template_name = ""
3207 else:
3208 self.template_name = template_name
3211def handle_wikitext_or_html_table(
3212 wxr: WiktextractContext,
3213 word: str,
3214 lang: str,
3215 pos: str,
3216 data: WordData,
3217 tree: WikiNode,
3218 titles: list[str],
3219 source: str,
3220 after: str,
3221 tablecontext: TableContext | None = None,
3222):
3223 """Parses a table from parsed Wikitext format into rows and columns of
3224 InflCell objects and then calls handle_generic_table() to parse it into
3225 forms. This adds the forms into ``data``."""
3226 assert isinstance(wxr, WiktextractContext)
3227 assert isinstance(word, str)
3228 assert isinstance(lang, str)
3229 assert isinstance(pos, str)
3230 assert isinstance(data, dict)
3231 assert isinstance(tree, WikiNode)
3232 assert tree.kind == NodeKind.TABLE or (
3233 tree.kind == NodeKind.HTML and tree.sarg == "table"
3234 )
3235 assert isinstance(titles, list)
3236 assert isinstance(source, str)
3237 for x in titles:
3238 assert isinstance(x, str)
3239 assert isinstance(after, str)
3240 assert tablecontext is None or isinstance(tablecontext, TableContext)
3241 # Imported here to avoid a circular import
3242 from wiktextract.page import clean_node, recursively_extract
3244 # from wikitextprocessor.parser import print_tree
3245 # print_tree(tree)
3246 # print("-------==========-------")
3248 if not tablecontext:
3249 tablecontext = TableContext()
3251 # Get language specific text removal patterns
3252 remove_text_patterns: tuple[str | re.Pattern, ...] | None = None
3253 if rem := get_lang_conf(lang, "remove_text_patterns"):
3254 for poses in rem.keys():
3255 if pos in poses:
3256 remove_text_patterns = rem[poses]
3257 break
3259 def handle_table1(
3260 wxr: WiktextractContext,
3261 tablecontext: TableContext,
3262 word: str,
3263 lang: str,
3264 pos: str,
3265 data: WordData,
3266 tree: WikiNode,
3267 titles: list[str],
3268 source: str,
3269 after: str,
3270 depth: int,
3271 ) -> list[tuple[list[list[InflCell]], list[str], str, int]]:
3272 # rows, titles, after, depth
3273 """Helper function allowing the 'flattening' out of the table
3274 recursion: instead of handling the tables in the wrong order
3275 (recursively), this function adds to new_row that is then
3276 iterated through in the main function at the end, creating
3277 a longer table (still in pieces) in the correct order."""
3279 assert isinstance(data, dict)
3280 assert isinstance(titles, list)
3281 assert isinstance(source, str)
3282 for x in titles:
3283 assert isinstance(x, str)
3284 assert isinstance(after, str)
3285 assert isinstance(depth, int)
3286 # print("HANDLE_WIKITEXT_TABLE", titles)
3287 # if len(titles) > 0:
3288 # wxr.wtp.debug(f"HANDLE_WIKITEXT_TABLE {titles=}")
3290 # Filling for columns with rowspan > 1
3291 col_gap_data: list[InflCell | None] = []
3292 # Number of remaining rows for which to fill the column
3293 vertical_still_left: list[int] = []
3294 cols_headered: list[bool] = [] # [F, T, F, F...]
3295 # True when the whole column contains headers, even
3296 # when the cell is not considered a header; triggered
3297 # by the "*" inflmap meta-tag.
3298 rows: list[list[InflCell]] = []
3300 sub_ret = []
3302 # from wikitextprocessor.parser import print_tree
3303 # print_tree(tree)
3304 for node in tree.children:
3305 if not isinstance(node, WikiNode):
3306 continue
3307 kind: NodeKind | str
3308 if node.kind == NodeKind.HTML:
3309 kind = node.sarg
3310 else:
3311 kind = node.kind
3313 # print(" {}".format(node))
3314 if kind in (NodeKind.TABLE_CAPTION, "caption"):
3315 # print(" CAPTION:", node)
3316 if "inflection-table-title" in node.attrs.get("class", ""): 3316 ↛ 3317line 3316 didn't jump to line 3317 because the condition on line 3316 was never true
3317 titles = [clean_node(wxr, None, node.children)]
3318 elif kind in (NodeKind.TABLE_ROW, "tr"):
3319 if "vsShow" in node.attrs.get("class", "").split():
3320 # vsShow rows are those that are intially shown in tables
3321 # that have more data. The hidden data duplicates these
3322 # rows, so we skip it and just process the hidden data.
3323 continue
3325 # if (
3326 # len(node.children) == 1
3327 # and node.children[0].attrs.get("class") == "separator"
3328 # ):
3329 # print("------------------ skip separator")
3330 # continue
3332 # Parse a table row.
3333 row: list[InflCell] = []
3334 style = None
3335 row_has_nonempty_cells = False
3336 # Have nonempty cell not from rowspan
3337 for col in get_table_cells(node):
3338 # loop through each cell in the ROW
3340 # The below skip is not needed anymore, because we "skip" in
3341 # get_table_cells, but left here as a comment
3342 # if not isinstance(col, WikiNode):
3343 # # This skip is not used for counting,
3344 # # "None" is not used in
3345 # # indexing or counting or looping.
3346 # continue
3347 if col.kind == NodeKind.HTML:
3348 kind = col.sarg
3349 else:
3350 kind = col.kind
3351 if kind not in ( 3351 ↛ 3357line 3351 didn't jump to line 3357 because the condition on line 3351 was never true
3352 NodeKind.TABLE_HEADER_CELL,
3353 NodeKind.TABLE_CELL,
3354 "th",
3355 "td",
3356 ):
3357 print(" UNEXPECTED ROW CONTENT: {}".format(col))
3358 continue
3360 while (
3361 len(row) < len(vertical_still_left)
3362 and vertical_still_left[len(row)] > 0
3363 ):
3364 # vertical_still_left is [...0, 0, 2...] for each
3365 # column. It is populated at the end of the loop, at the
3366 # same time as col_gap_data. This needs to be looped and
3367 # filled this way because each `for col`-looping jumps
3368 # straight to the next meaningful cell; there is no
3369 # "None" cells, only emptiness between, and rowspan and
3370 # colspan are just to generate the "fill-
3371 vertical_still_left[len(row)] -= 1
3373 # KJ Apr 2026
3374 # type checking is ignored; I am pretty sure that
3375 # row will never contain None, even if col_gap_data
3376 # is `InflCell | None`, but this code is such
3377 # spaghetti that it's hard to figure out, except
3378 # by the process of elimination: this has never
3379 # caused trouble before, ergo, it works.
3380 row.append(col_gap_data[len(row)]) # type: ignore
3382 # appending row is how "indexing" is
3383 # done here; something is appended,
3384 # like a filler-cell here or a "start"
3385 # cell at the end of the row-loop,
3386 # which increased len(row) which is
3387 # then used as the target-index to check
3388 # for gaps. vertical_still_left is
3389 # the countdown to when to stop
3390 # filling in gaps, and goes down to 0,
3391 # and col_gap_data is not touched
3392 # except when a new rowspan is needed,
3393 # at the same time that
3394 # vertical_still_left gets reassigned.
3396 try:
3397 rowspan = int(col.attrs.get("rowspan", "1")) # 🡙
3398 colspan = int(col.attrs.get("colspan", "1")) # 🡘
3399 except ValueError:
3400 rowspan = 1
3401 colspan = 1
3402 # print("COL:", col)
3404 # Too many of these errors
3405 if colspan > 100:
3406 # wxr.wtp.error(
3407 # f"Colspan {colspan} over 30, set to 1",
3408 # sortid="inflection/20250113a",
3409 # )
3410 colspan = 100
3411 if rowspan > 100: 3411 ↛ 3416line 3411 didn't jump to line 3416 because the condition on line 3411 was never true
3412 # wxr.wtp.error(
3413 # f"Rowspan {rowspan} over 30, set to 1",
3414 # sortid="inflection/20250113b",
3415 # )
3416 rowspan = 100
3418 # Process any nested tables recursively.
3419 tables, rest = recursively_extract(
3420 col,
3421 lambda x: (
3422 isinstance(x, WikiNode)
3423 and (x.kind == NodeKind.TABLE or x.sarg == "table")
3424 ),
3425 )
3427 # Clean the rest of the cell.
3428 link_capture_dict: dict = {}
3429 celltext = clean_node(
3430 wxr, link_capture_dict, rest, collect_links=True
3431 )
3432 cell_links: list[tuple[str, str]] | None = (
3433 link_capture_dict.get("links", None)
3434 )
3435 # print(f"CLEANED: {celltext=}")
3436 # print(f"SUBTABLES: {tables}")
3437 # print(f"{link_capture_dict=}")
3439 # Remove regexed patterns from text
3440 if remove_text_patterns is not None:
3441 for pat in remove_text_patterns:
3442 celltext = re.sub(pat, "", celltext)
3443 # print(f"AFTER: {celltext=} <<")
3445 # Handle nested tables.
3446 for tbl in tables:
3447 # Some nested tables (e.g., croí/Irish) have subtitles
3448 # as normal paragraphs in the same cell under a descrip-
3449 # tive text that should be treated as a title (e.g.,
3450 # "Forms with the definite article", with "definite" not
3451 # mentioned elsewhere).
3452 new_titles = list(titles)
3453 if celltext:
3454 new_titles.append(celltext)
3455 subtbl = handle_table1(
3456 wxr,
3457 tablecontext,
3458 word,
3459 lang,
3460 pos,
3461 data,
3462 tbl, # type: ignore
3463 new_titles,
3464 source,
3465 "",
3466 depth + 1,
3467 )
3468 if subtbl: 3468 ↛ 3446line 3468 didn't jump to line 3446 because the condition on line 3468 was always true
3469 sub_ret.append((rows, titles, after, depth))
3470 rows = []
3471 titles = []
3472 after = ""
3473 sub_ret.extend(subtbl)
3475 # This magic value is used as part of header detection
3476 cellstyle = (
3477 col.attrs.get("style", "")
3478 + "//"
3479 + col.attrs.get("class", "")
3480 + "//"
3481 + str(kind)
3482 )
3484 if not row: # if first column in row
3485 style = cellstyle
3486 target = None
3487 titletext = celltext.strip()
3488 while titletext and is_superscript(titletext[-1]):
3489 titletext = titletext[:-1]
3491 (
3492 is_title,
3493 hdr_expansion,
3494 target,
3495 celltext,
3496 ) = determine_header(
3497 wxr,
3498 tablecontext,
3499 lang,
3500 word,
3501 pos,
3502 tree.kind,
3503 kind,
3504 style,
3505 row,
3506 col,
3507 celltext,
3508 titletext,
3509 cols_headered,
3510 None,
3511 cellstyle,
3512 )
3514 if is_title:
3515 # If this cell gets a "*" tag, make the whole column
3516 # below it (toggling it in cols_headered = [F, F, T...])
3517 # into headers.
3518 while len(cols_headered) <= len(row):
3519 cols_headered.append(False)
3520 if any("*" in tt for tt in hdr_expansion):
3521 cols_headered[len(row)] = True
3522 celltext = ""
3523 # if row_has_nonempty_cells has been True at some point, it
3524 # keeps on being True.
3525 # if row_has_nonempty_cells or is_title or celltext != "":
3526 # row_has_nonempty_cells = True
3527 # ⇓⇓⇓⇓⇓⇓⇓⇓⇓⇓⇓⇓⇓⇓⇓⇓⇓⇓⇓⇓
3528 row_has_nonempty_cells |= is_title or celltext != ""
3529 cell = InflCell(
3530 celltext, is_title, colspan, rowspan, target, cell_links
3531 )
3532 for _ in range(0, colspan):
3533 # colspan🡘 current loop (col) or 1
3534 # All the data-filling for colspan
3535 # is done simply in this loop,
3536 # while rowspan needs to use
3537 # vertical_still_left to count gaps
3538 # and col_gap_data to fill in
3539 # those gaps with InflCell data.
3540 if rowspan > 1: # rowspan🡙 current loop (col) or 1
3541 while len(col_gap_data) <= len(row):
3542 # Initialize col_gap_data/ed if
3543 # it is lacking slots
3544 # for each column; col_gap_data and
3545 # vertical_still_left are never
3546 # reset to [], during
3547 # the whole table function.
3548 col_gap_data.append(None)
3549 vertical_still_left.append(0)
3550 # Below is where the "rectangle" block of rowspan
3551 # and colspan is filled for the future.
3552 col_gap_data[len(row)] = cell
3553 # col_gap_data contains cells that
3554 # will be used in the
3555 # future, or None
3556 vertical_still_left[len(row)] = rowspan - 1
3557 # A counter for how many gaps🡙 are still left to be
3558 # filled (row.append or
3559 # row[col_gap_data[len(row)] =>
3560 # rows), it is not reset to [], but decremented to 0
3561 # each time a row gets something from col_gap_data.
3562 # Append this cell 1+ times for colspan🡘
3563 row.append(cell)
3564 if not row:
3565 continue
3566 # After looping the original row-nodes above, fill
3567 # in the rest of the row if the final cell has colspan
3568 # (inherited from above, so a cell with rowspan and colspan)
3569 for i in range(len(row), len(vertical_still_left)):
3570 if vertical_still_left[i] <= 0:
3571 continue
3572 vertical_still_left[i] -= 1
3573 while len(row) < i:
3574 row.append(InflCell("", False, 1, 1, None))
3575 row.append(col_gap_data[i]) # type: ignore
3576 # print(" ROW {!r}".format(row))
3577 if row_has_nonempty_cells: 3577 ↛ 3304line 3577 didn't jump to line 3304 because the condition on line 3577 was always true
3578 rows.append(row)
3579 elif kind in ( 3579 ↛ 3304line 3579 didn't jump to line 3304 because the condition on line 3579 was always true
3580 NodeKind.TABLE_HEADER_CELL,
3581 NodeKind.TABLE_CELL,
3582 "th",
3583 "td",
3584 "span",
3585 ):
3586 # print(" TOP-LEVEL CELL", node)
3587 pass
3589 if sub_ret:
3590 main_ret = sub_ret
3591 main_ret.append((rows, titles, after, depth))
3592 else:
3593 main_ret = [(rows, titles, after, depth)]
3594 return main_ret
3596 new_rows = handle_table1(
3597 wxr, tablecontext, word, lang, pos, data, tree, titles, source, after, 0
3598 )
3600 # Now we have a table that has been parsed into rows and columns of
3601 # InflCell objects. Parse the inflection table from that format.
3602 if new_rows: 3602 ↛ exitline 3602 didn't return from function 'handle_wikitext_or_html_table' because the condition on line 3602 was always true
3603 for rows, titles, after, depth in new_rows:
3604 handle_generic_table(
3605 wxr,
3606 tablecontext,
3607 data,
3608 word,
3609 lang,
3610 pos,
3611 rows,
3612 titles,
3613 source,
3614 after,
3615 depth,
3616 )
3619def get_table_cells(node: WikiNode) -> Generator[WikiNode, None, None]:
3620 """If a wikitext table cell contains HTML cells `<td>`, as they sometimes
3621 do because it is easier to write wikitext conditionals that way,
3622 those td-elements are parsed as child elements of the Wikitext cell.
3623 This generator will yield wikitext and HTML direct children of
3624 `node` and if a Wikitext TABLE_CELL has direct td-element children,
3625 those are also yielded."""
3626 for col in node.children:
3627 if not isinstance(col, WikiNode):
3628 continue
3629 if any(
3630 isinstance(c, HTMLNode) and c.sarg in ("th", "td")
3631 for c in col.children
3632 ):
3633 html_cells = []
3634 content = []
3635 for c in col.children:
3636 if isinstance(c, HTMLNode) and c.sarg in ("th", "td"):
3637 html_cells.append(c)
3638 else:
3639 content.append(c)
3640 # Remove td-elements from col so they are not returned twice
3641 col.children = content
3642 yield col
3643 for c in html_cells:
3644 yield c
3645 else:
3646 yield col
3649def handle_html_table(
3650 wxr: WiktextractContext,
3651 word: str,
3652 lang: str,
3653 pos: str,
3654 data: WordData,
3655 tree: WikiNode,
3656 titles: list[str],
3657 source: str,
3658 after: str,
3659 tablecontext: TableContext | None = None,
3660) -> None:
3661 """A passer-on function for html-tables, XXX, remove these?"""
3662 handle_wikitext_or_html_table(
3663 wxr, word, lang, pos, data, tree, titles, source, after, tablecontext
3664 )
3667def handle_wikitext_table(
3668 wxr: WiktextractContext,
3669 word: str,
3670 lang: str,
3671 pos: str,
3672 data: WordData,
3673 tree: WikiNode,
3674 titles: list[str],
3675 source: str,
3676 after: str,
3677 tablecontext: TableContext | None = None,
3678) -> None:
3679 """A passer-on function for html-tables, XXX, remove these?"""
3680 handle_wikitext_or_html_table(
3681 wxr, word, lang, pos, data, tree, titles, source, after, tablecontext
3682 )
3685def parse_inflection_section(
3686 wxr: WiktextractContext,
3687 data: WordData,
3688 word: str,
3689 lang: str,
3690 pos: str,
3691 section: str,
3692 tree: WikiNode,
3693 tablecontext: TableContext | None = None,
3694) -> None:
3695 """Parses an inflection section on a page. ``data`` should be the
3696 data for a part-of-speech, and inflections will be added to it."""
3698 # print("PARSE_INFLECTION_SECTION {}/{}/{}/{}"
3699 # .format(word, lang, pos, section))
3700 assert isinstance(wxr, WiktextractContext)
3701 assert isinstance(data, dict)
3702 assert isinstance(word, str)
3703 assert isinstance(lang, str)
3704 assert isinstance(section, str)
3705 assert isinstance(tree, WikiNode)
3706 assert tablecontext is None or isinstance(tablecontext, TableContext)
3707 source = section
3708 tables: list[
3709 tuple[Literal["html", "wikitext"], WikiNode, list[str], list[str]]
3710 ] = []
3711 titleparts: list[str] = []
3712 preceding_bolded_title = ""
3714 # from wikitextprocessor.parser import print_tree
3715 # print_tree(tree)
3716 # print("--------------******************----------------")
3718 def process_tables() -> None:
3719 for kind, node, titles, after_l in tables:
3720 after = "".join(after_l).strip()
3721 after = clean_value(wxr, after)
3722 if kind == "wikitext":
3723 handle_wikitext_table(
3724 wxr,
3725 word,
3726 lang,
3727 pos,
3728 data,
3729 node,
3730 titles,
3731 source,
3732 after,
3733 tablecontext=tablecontext,
3734 )
3735 elif kind == "html": 3735 ↛ 3749line 3735 didn't jump to line 3749 because the condition on line 3735 was always true
3736 handle_html_table(
3737 wxr,
3738 word,
3739 lang,
3740 pos,
3741 data,
3742 node,
3743 titles,
3744 source,
3745 after,
3746 tablecontext=tablecontext,
3747 )
3748 else:
3749 raise RuntimeError(
3750 "{}: unimplemented table kind {}".format(word, kind)
3751 )
3753 def recurse_navframe(node: WikiNode | str, titles: list[str]) -> None:
3754 nonlocal tables
3755 nonlocal titleparts
3756 titleparts = []
3757 old_tables = tables
3758 tables = []
3760 recurse(node, [], navframe=True)
3762 process_tables()
3763 tables = old_tables
3765 def recurse(
3766 node: WikiNode
3767 | str
3768 | list[WikiNode | str]
3769 | list[list[WikiNode | str]],
3770 titles: list[str],
3771 navframe=False,
3772 ) -> None:
3773 nonlocal tables
3774 if isinstance(node, (list, tuple)):
3775 for x in node:
3776 recurse(x, titles, navframe)
3777 return
3778 if isinstance(node, str):
3779 if tables:
3780 tables[-1][-1].append(node)
3781 elif navframe:
3782 titleparts.append(node)
3783 return
3784 if not isinstance(node, WikiNode): 3784 ↛ 3785line 3784 didn't jump to line 3785 because the condition on line 3784 was never true
3785 if navframe:
3786 wxr.wtp.debug(
3787 "inflection table: unhandled in NavFrame: {}".format(node),
3788 sortid="inflection/2907",
3789 )
3790 return
3791 kind = node.kind
3792 if navframe:
3793 if kind == NodeKind.HTML:
3794 classes = node.attrs.get("class", "").split()
3795 if "NavToggle" in classes: 3795 ↛ 3796line 3795 didn't jump to line 3796 because the condition on line 3795 was never true
3796 return
3797 if "NavHead" in classes:
3798 # print("NAVHEAD:", node)
3799 recurse(node.children, titles, navframe)
3800 return
3801 if "NavContent" in classes:
3802 # print("NAVCONTENT:", node)
3803 title = "".join(titleparts).strip()
3804 title = html.unescape(title)
3805 title = title.strip()
3806 new_titles = list(titles)
3807 if not re.match(r"(Note:|Notes:)", title): 3807 ↛ 3809line 3807 didn't jump to line 3809 because the condition on line 3807 was always true
3808 new_titles.append(title)
3809 recurse(node, new_titles, navframe=False)
3810 return
3811 else:
3812 if kind == NodeKind.TABLE:
3813 tables.append(("wikitext", node, titles, []))
3814 return
3815 elif kind == NodeKind.HTML and node.sarg == "table":
3816 htmlclasses = node.attrs.get("class", ())
3817 if "audiotable" in htmlclasses:
3818 return
3819 tables.append(("html", node, titles, []))
3820 return
3821 elif kind in ( 3821 ↛ 3828line 3821 didn't jump to line 3828 because the condition on line 3821 was never true
3822 NodeKind.LEVEL2,
3823 NodeKind.LEVEL3,
3824 NodeKind.LEVEL4,
3825 NodeKind.LEVEL5,
3826 NodeKind.LEVEL6,
3827 ):
3828 return # Skip subsections
3829 if (
3830 kind == NodeKind.HTML
3831 and node.sarg == "div"
3832 and "NavFrame" in node.attrs.get("class", "").split()
3833 ):
3834 recurse_navframe(node, titles)
3835 return
3836 if kind == NodeKind.LINK:
3837 if len(node.largs) > 1:
3838 recurse(node.largs[1:], titles, navframe)
3839 else:
3840 recurse(node.largs[0], titles, navframe)
3841 return
3842 if kind == NodeKind.HTML and node.sarg == "ref":
3843 return
3844 if kind == NodeKind.LIST and node.sarg == ";":
3845 nonlocal preceding_bolded_title
3846 from wiktextract.page import clean_node
3848 preceding_bolded_title = clean_node(wxr, None, node).strip("; ")
3849 for x in node.children:
3850 recurse(x, titles, navframe)
3852 assert tree.kind == NodeKind.ROOT
3853 for x in tree.children:
3854 if preceding_bolded_title != "":
3855 recurse(x, [preceding_bolded_title])
3856 else:
3857 recurse(x, [])
3859 # Process the tables we found
3860 process_tables()
3862 # XXX this code is used for extracting tables for inflection tests
3863 if wxr.config.expand_tables: 3863 ↛ 3864line 3863 didn't jump to line 3864 because the condition on line 3863 was never true
3864 if section != "Mutation":
3865 with open(wxr.config.expand_tables, "w") as f:
3866 f.write(word + "\n")
3867 f.write(lang + "\n")
3868 f.write(pos + "\n")
3869 f.write(section + "\n")
3870 text = wxr.wtp.node_to_wikitext(tree)
3871 f.write(text + "\n")