Coverage for src/wiktextract/extractor/en/form_descriptions.py: 79%
1442 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 07:03 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 07:03 +0000
1# Code for parsing linguistic form descriptions and tags for word senses
2# (both the word entry head - initial part and parenthesized parts -
3# and tags at the beginning of word senses)
4#
5# Copyright (c) 2020-2022 Tatu Ylonen. See file LICENSE and https://ylonen.org
7import functools
8import re
9import unicodedata
10from typing import (
11 Any,
12 Literal,
13 Optional,
14 Sequence,
15 Union,
16)
18import Levenshtein
19from nltk import TweetTokenizer # type:ignore[import-untyped]
20from wikitextprocessor.parser import WikiNode
22from ...datautils import data_append, data_extend, split_at_comma_semi
23from ...page import extract_links_from_node
24from ...tags import (
25 alt_of_tags,
26 form_of_tags,
27 head_final_bantu_langs,
28 head_final_bantu_map,
29 head_final_numeric_langs,
30 head_final_other_langs,
31 head_final_other_map,
32 head_final_semitic_langs,
33 head_final_semitic_map,
34 uppercase_tags,
35 valid_tags,
36 xlat_descs_map,
37 xlat_head_map,
38 xlat_tags_map,
39)
40from ...topics import topic_generalize_map, valid_topics
41from ...wxr_context import WiktextractContext
42from .english_words import (
43 english_words,
44 not_english_words,
45 potentially_english_words,
46)
47from .form_descriptions_known_firsts import known_firsts
48from .taxondata import known_species
49from .type_utils import (
50 AltOf,
51 FormData,
52 LinkageData,
53 SenseData,
54 SoundData,
55 TemplateData,
56 TranslationData,
57 WordData,
58)
60# Tokenizer for classify_desc()
61tokenizer = TweetTokenizer()
63# These are ignored as the value of a related form in form head.
64IGNORED_RELATED: set[str] = set(
65 [
66 "-",
67 "־",
68 "᠆",
69 "‐",
70 "‑",
71 "‒",
72 "–",
73 "—",
74 "―",
75 "−",
76 "⸺",
77 "⸻",
78 "﹘",
79 "﹣",
80 "-",
81 "?",
82 "(none)",
83 ]
84)
87# First words of unicodedata.name() that indicate scripts that cannot be
88# accepted in romanizations or english (i.e., should be considered "other"
89# in classify_desc()).
90non_latin_scripts: list[str] = [
91 "ADLAM",
92 "ARABIC",
93 "ARABIC-INDIC",
94 "ARMENIAN",
95 "BALINESE",
96 "BENGALI",
97 "BRAHMI",
98 "BRAILLE",
99 "CANADIAN",
100 "CHAKMA",
101 "CHAM",
102 "CHEROKEE",
103 "CJK",
104 "COPTIC",
105 "COUNTING ROD",
106 "CUNEIFORM",
107 "CYRILLIC",
108 "DOUBLE-STRUCK",
109 "EGYPTIAN",
110 "ETHIOPIC",
111 "EXTENDED ARABIC-INDIC",
112 "GEORGIAN",
113 "GLAGOLITIC",
114 "GOTHIC",
115 "GREEK",
116 "GUJARATI",
117 "GURMUKHI",
118 "HANGUL",
119 "HANIFI ROHINGYA",
120 "HEBREW",
121 "HIRAGANA",
122 "JAVANESE",
123 "KANNADA",
124 "KATAKANA",
125 "KAYAH LI",
126 "KHMER",
127 "KHUDAWADI",
128 "LAO",
129 "LEPCHA",
130 "LIMBU",
131 "MALAYALAM",
132 "MEETEI",
133 "MYANMAR",
134 "NEW TAI LUE",
135 "NKO",
136 "OL CHIKI",
137 "OLD PERSIAN",
138 "OLD SOUTH ARABIAN",
139 "ORIYA",
140 "OSMANYA",
141 "PHOENICIAN",
142 "SAURASHTRA",
143 "SHARADA",
144 "SINHALA",
145 "SUNDANESE",
146 "SYLOTI",
147 "TAI THAM",
148 "TAKRI",
149 "TAMIL",
150 "TELUGU",
151 "THAANA",
152 "THAI",
153 "TIBETAN",
154 "TIFINAGH",
155 "TIRHUTA",
156 "UGARITIC",
157 "WARANG CITI",
158 "YI",
159]
160non_latin_scripts_re = re.compile(
161 r"(" + r"|".join(re.escape(x) for x in non_latin_scripts) + r")\b"
162)
164# Sanity check xlat_head_map values
165for k, v in xlat_head_map.items():
166 if v.startswith("?"):
167 v = v[1:]
168 for tag in v.split():
169 if tag not in valid_tags: 169 ↛ 170line 169 didn't jump to line 170 because the condition on line 169 was never true
170 print(
171 "WARNING: xlat_head_map[{}] contains"
172 " unrecognized tag {}".format(k, tag)
173 )
175# Regexp for finding nested translations from translation items (these are
176# used in, e.g., year/English/Translations/Arabic). This is actually used
177# in page.py.
178nested_translations_re = re.compile(
179 r"\s+\((({}): ([^()]|\([^()]+\))+)\)".format(
180 "|".join(
181 re.escape(x.removeprefix("?"))
182 for x in sorted(xlat_head_map.values(), key=len, reverse=True)
183 if x and not x.startswith("class-")
184 )
185 )
186)
188# Regexp that matches head tag specifiers. Used to match tags from end of
189# translations and linkages
190head_final_re_text = r"( -)?( ({}))+".format(
191 "|".join(
192 re.escape(x)
193 for x in
194 # The sort is to put longer ones first, preferring them in
195 # the regexp match
196 sorted(xlat_head_map.keys(), key=len, reverse=True)
197 )
198)
199head_final_re = re.compile(head_final_re_text + r"$")
201# Regexp used to match head tag specifiers at end of a form for certain
202# Bantu languages (particularly Swahili and similar languages).
203head_final_bantu_re_text = r" ({})".format(
204 "|".join(re.escape(x) for x in head_final_bantu_map.keys())
205)
206head_final_bantu_re = re.compile(head_final_bantu_re_text + "$")
208# Regexp used to match head tag specifiers at end of a form for certain
209# Semitic languages (particularly Arabic and similar languages).
210head_final_semitic_re_text = r" ({})".format(
211 "|".join(re.escape(x) for x in head_final_semitic_map.keys())
212)
213head_final_semitic_re = re.compile(head_final_semitic_re_text + "$")
215# Regexp used to match head tag specifiers at end of a form for certain
216# other languages (e.g., Lithuanian, Finnish, French).
217head_final_other_re_text = r" ({})".format(
218 "|".join(re.escape(x) for x in head_final_other_map.keys())
219)
220head_final_other_re = re.compile(head_final_other_re_text + "$")
222# Regexp for splitting heads. See parse_word_head().
223head_split_re_text_part_1 = (
224 "("
225 + head_final_re_text
226 + "|"
227 + head_final_bantu_re_text
228 + "|"
229 + head_final_semitic_re_text
230 + "|"
231 + head_final_other_re_text
232)
234head_split_re_text = head_split_re_text_part_1 + ")?( or |[,;]+| *$)"
236head_split_re_text_no_semicolon = head_split_re_text_part_1 + ")?( or |,+| *$)"
238head_split_re = re.compile(head_split_re_text)
239head_split_no_semicolon_re = re.compile(head_split_re_text_no_semicolon)
241head_split_re_parens = 0
242for m in re.finditer(r"(^|[^\\])[(]+", head_split_re_text):
243 head_split_re_parens += m.group(0).count("(")
245# Parenthesized parts that are ignored in translations
246tr_ignored_parens: set[str] = set(
247 [
248 "please verify",
249 "(please verify)",
250 "transliteration needed",
251 "(transliteration needed)",
252 "in words with back vowel harmony",
253 "(in words with back vowel harmony)",
254 "in words with front vowel harmony",
255 "(in words with front vowel harmony)",
256 "see below",
257 "see usage notes below",
258 ]
259)
260tr_ignored_parens_re = re.compile(
261 r"^("
262 + "|".join(re.escape(x) for x in tr_ignored_parens)
263 + ")$"
264 + r"|^(Can we clean up|Can we verify|for other meanings see "
265 r"lit\. )"
266)
268# Translations that are ignored
269ignored_translations: set[str] = set(
270 [
271 "[script needed]",
272 "please add this translation if you can",
273 ]
274)
276# Put english text into the "note" field in a translation if it contains one
277# of these words
278tr_note_re = re.compile(
279 r"(\b(article|definite|indefinite|superlative|comparative|pattern|"
280 r"adjective|adjectives|clause|clauses|pronoun|pronouns|preposition|prep|"
281 r"postposition|postp|action|actions|articles|"
282 r"adverb|adverbs|noun|nouns|verb|verbs|before|"
283 r"after|placed|prefix|suffix|used with|translated|"
284 r"nominative|genitive|dative|infinitive|participle|past|perfect|imperfect|"
285 r"perfective|imperfective|auxiliary|negative|future|present|tense|aspect|"
286 r"conjugation|declension|class|category|plural|singular|positive|"
287 r"seldom used|formal|informal|familiar|unspoken|spoken|written|"
288 r"indicative|progressive|conditional|potential|"
289 r"accusative|adessive|inessive|superessive|elative|allative|"
290 r"dialect|dialects|object|subject|predicate|movies|recommended|language|"
291 r"locative|continuous|simple|continuousness|gerund|subjunctive|"
292 r"periphrastically|no equivalent|not used|not always used|"
293 r"used only with|not applicable|use the|signifying|wordplay|pronounced|"
294 r"preconsonantal|spelled|spelling|respelling|respellings|phonetic|"
295 r"may be replaced|stricter sense|for nonhumans|"
296 r"sense:|used:|in full:|informally used|followed by|"
297 r"not restricted to|pertaining to|or optionally with|are optional|"
298 r"in conjunction with|in compounds|depending on the relationship|"
299 r"person addressed|one person|multiple persons|may be replaced with|"
300 r"optionally completed with|in the phrase|in response to|"
301 r"before a|before an|preceded by|verbs ending|very common|after a verb|"
302 r"with verb|with uncountable|with the objects|with stative|"
303 r"can be replaced by|often after|used before|used after|"
304 r"used in|clipping of|spoken|somewhat|capitalized|"
305 r"short form|shortening of|shortened form|initialism of|"
306 r"said to|rare:|rarer also|is rarer|negatively connoted|"
307 r"previously mentioned|uncountable noun|countable noun|"
308 r"countable nouns|uncountable nouns|"
309 r"with predicative|with -|with imperfect|with a negated|"
310 r"colloquial|misspelling|holophrastic|frequently|esp\.|especially|"
311 r'"|'
312 r"general term|after a vowel|before a vowel|"
313 r"form|regular|irregular|alternative)"
314 r")($|[) ])|^("
315 # Following are only matched at the beginning of the string
316 r"pl|pl\.|see:|pl:|sg:|plurals:|e\.g\.|e\.g\.:"
317 r"|e\.g\.,|cf\.|compare|such as|"
318 r"see|only|often|usually|used|usage:|of|not|in|compare|usu\.|"
319 r"as|about|abbrv\.|abbreviation|abbr\.|that:|optionally|"
320 r"mainly|from|for|also|also:|acronym|"
321 r"\+|with) "
322)
323# \b does not work at the end???
325# Related forms matching this regexp will be considered suspicious if the
326# page title does not also match one of these.
327suspicious_related_re = re.compile(
328 r"(^| )(f|m|n|c|or|pl|sg|inan|anim|pers|anml|impf|pf|vir|nvir)( |$)"
329 r"|[][:=<>&#*|]"
330 r"| \d+$"
331)
333# Word forms (head forms, translations, etc) that will be considered ok and
334# silently accepted even if they would otherwise trigger a suspicious
335# form warning.
336ok_suspicious_forms: set[str] = set(
337 [
338 "but en or", # "golden goal"/English/Tr/French
339 "cœur en or", # "heart of gold"/Eng/Tr/French
340 "en or", # golden/Eng/Tr/French
341 "men du", # jet/Etym2/Noun/Tr/Cornish
342 "parachute en or", # "golden parachute"/Eng/Tr/French
343 "vieil or", # "old gold"/Eng/Tr/French
344 # "all that glitters is not gold"/Eng/Tr/French
345 "tout ce qui brille n’est pas or",
346 "μη αποκλειστικό or", # inclusive or/Eng/Tr/Greek
347 "period or full stop",
348 ]
349)
352# Replacements to be done in classify_desc before tokenizing. This is a
353# workaround for shortcomings in TweetTokenizer.
354tokenizer_fixup_map = {
355 r"a.m.": "AM",
356 r"p.m.": "PM",
357}
358tokenizer_fixup_re = re.compile(
359 r"\b("
360 + "|".join(
361 re.escape(x)
362 for x in sorted(
363 tokenizer_fixup_map.keys(), key=lambda x: len(x), reverse=True
364 )
365 )
366 + r")"
367)
369# Unknown tags starting with these words will be silently ignored.
370ignored_unknown_starts: set[str] = set(
371 [
372 "originally",
373 "e.g.",
374 "c.f.",
375 "supplanted by",
376 "supplied by",
377 ]
378)
380ignored_unknown_starts_re = re.compile(
381 r"^("
382 + "|".join(
383 re.escape(x)
384 for x in sorted(ignored_unknown_starts, key=lambda x: -len(x))
385 )
386 + ") "
387)
389# If an unknown sequence starts with one of these, it will continue as an
390# unknown sequence until the end, unless it turns out to have a replacement.
391allowed_unknown_starts: set[str] = set(
392 [
393 "Relating",
394 "accompanied",
395 "added",
396 "after",
397 "answering",
398 "as",
399 "based",
400 "before",
401 "conjugated",
402 "conjunction",
403 "construed",
404 "especially",
405 "expression:",
406 "figurative:",
407 "followed",
408 "for",
409 "forms",
410 "from",
411 "governs",
412 "in",
413 "indicating",
414 "modifying",
415 "normally",
416 "not",
417 "of",
418 "preceding",
419 "prefixed",
420 "referring",
421 "relating",
422 "revived",
423 "said",
424 "since",
425 "takes",
426 "used",
427 "with",
428 "With",
429 "without",
430 ]
431)
432# Allow the ignored unknown starts without complaining
433allowed_unknown_starts.update(ignored_unknown_starts)
435# Full unknown tags that will be ignored in decode_tags()
436# XXX this is unused, ask Tatu where the contents is now
437ignored_unknown_tags: set[str] = set([])
439# Head endings that are mapped to tags
440head_end_map = {
441 " 1st conj.": "conjugation-1",
442 " 2nd conj.": "conjugation-2",
443 " 3rd conj.": "conjugation-3",
444 " 4th conj.": "conjugation-4",
445 " 5th conj.": "conjugation-5",
446 " 6th conj.": "conjugation-6",
447 " 7th conj.": "conjugation-7",
448}
449head_end_re = re.compile(
450 r"(" + "|".join(re.escape(x) for x in head_end_map.keys()) + r")$"
451)
453# Transitivity and inflection class labels at the end of Japanese heads,
454# e.g. 走る "intransitive godan", 給ふ "transitive or intransitive ^†yodan"
455ja_head_end_re = re.compile(
456 r" (?:\^†)?(transitive|intransitive|godan|ichidan|suru|kuru|yodan|nidan"
457 r"|irregular|uninflectable"
458 r"|-i|-na|-nari|-tari|-ku|-shiku|-ka|-ri|-se|-ko|-n)(?: or)?$"
459)
462# Dictionary of language-specific parenthesized head part starts that
463# either introduce new tags or modify previous tags. The value for each
464# language is a dictionary that maps the first word of the head part to
465# (rem_tags, add_tags), where ``rem_tags`` can be True to remove all previous
466# tags or a space-separated string of tags to remove, and ``add_tags`` should
467# be a string of tags to add.
468lang_specific_head_map: dict[
469 str, dict[str, Union[tuple[str, str], tuple[Literal[True], str]]]
470] = {
471 "Danish": {
472 # prefix: (rem_tags space separate string/True, add_tags s-sep str)
473 "c": ("neuter", "common-gender"),
474 "n": ("common-gender", "neuter"),
475 "pl": ("singular neuter common-gender", "plural"),
476 "sg": ("plural neuter common-gender", "singular"),
477 },
478}
481# Regular expression used to strip additional stuff from the end of alt_of and
482# form_of.
483alt_of_form_of_clean_re = re.compile(
484 r"(?s)("
485 + "|".join(
486 [
487 r":",
488 r'[“"]',
489 r";",
490 r" \(",
491 r" - ",
492 r" ־ ",
493 r" ᠆ ",
494 r" ‐ ",
495 r" ‑ ",
496 r" ‒ ",
497 r" – ",
498 r" — ",
499 r" ― ",
500 r" − ",
501 r" ⸺ ",
502 r" ⸻ ",
503 r" ﹘ ",
504 r" ﹣ ",
505 r" - ",
506 r" \+ ",
507 r" \(with ",
508 r" with -ra/-re",
509 r"\. Used ",
510 r"\. Also ",
511 r"\. Since ",
512 r"\. A ",
513 r"\.\. A ",
514 r"\. An ",
515 r"\.\. An ",
516 r"\. an ",
517 r"\. The ",
518 r"\. Spanish ",
519 r"\. Language ",
520 r"\. former name of ",
521 r"\. AIM",
522 r"\. OT",
523 r"\. Not ",
524 r"\. Now ",
525 r"\. Nowadays ",
526 r"\. Early ",
527 r"\. ASEAN",
528 r"\. UN",
529 r"\. IMF",
530 r"\. WHO",
531 r"\. WIPO",
532 r"\. AC",
533 r"\. DC",
534 r"\. DNA",
535 r"\. RNA",
536 r"\. SOB",
537 r"\. IMO",
538 r"\. Behavior",
539 r"\. Income ",
540 r"\. More ",
541 r"\. Most ",
542 r"\. Only ",
543 r"\. Also ",
544 r"\. From ",
545 r"\. Of ",
546 r"\.\. Of ",
547 r"\. To ",
548 r"\. For ",
549 r"\. If ",
550 r"\. Praenominal ",
551 r"\. This ",
552 r"\. Replaced ",
553 r"\. CHCS is the ",
554 r"\. Equivalent ",
555 r"\. Initialism ",
556 r"\. Note ",
557 r"\. Alternative ",
558 r"\. Compare ",
559 r"\. Cf\. ",
560 r"\. Comparable ",
561 r"\. Involves ",
562 r"\. Sometimes ",
563 r"\. Commonly ",
564 r"\. Often ",
565 r"\. Typically ",
566 r"\. Possibly ",
567 r"\. Although ",
568 r"\. Rare ",
569 r"\. Instead ",
570 r"\. Integrated ",
571 r"\. Distinguished ",
572 r"\. Given ",
573 r"\. Found ",
574 r"\. Was ",
575 r"\. In ",
576 r"\. It ",
577 r"\.\. It ",
578 r"\. One ",
579 r"\. Any ",
580 r"\. They ",
581 r"\. Members ",
582 r"\. Each ",
583 r"\. Original ",
584 r"\. Especially ",
585 r"\. Usually ",
586 r"\. Known ",
587 r"\.\. Known ",
588 r"\. See ",
589 r"\. see ",
590 r"\. target was not ",
591 r"\. Popular ",
592 r"\. Pedantic ",
593 r"\. Positive ",
594 r"\. Society ",
595 r"\. Plan ",
596 r"\. Environmentally ",
597 r"\. Affording ",
598 r"\. Encompasses ",
599 r"\. Expresses ",
600 r"\. Indicates ",
601 r"\. Text ",
602 r"\. Large ",
603 r"\. Sub-sorting ",
604 r"\. Sax",
605 r"\. First-person ",
606 r"\. Second-person ",
607 r"\. Third-person ",
608 r"\. 1st ",
609 r"\. 2nd ",
610 r"\. 3rd ",
611 r"\. Term ",
612 r"\. Northeastern ",
613 r"\. Northwestern ",
614 r"\. Southeast ",
615 r"\. Egyptian ",
616 r"\. English ",
617 r"\. Cape Province was split into ",
618 r"\. Pañcat",
619 r"\. of the ",
620 r"\. is ",
621 r"\. after ",
622 r"\. or ",
623 r"\. chromed",
624 r"\. percussion",
625 r"\. with his ",
626 r"\. a\.k\.a\. ",
627 r"\. comparative form ",
628 r"\. singular ",
629 r"\. plural ",
630 r"\. present ",
631 r"\. his ",
632 r"\. her ",
633 r"\. equivalent ",
634 r"\. measuring ",
635 r"\. used in ",
636 r"\. cutely ",
637 r"\. Protects",
638 r'\. "',
639 r"\.^",
640 r"\. \+ ",
641 r"\., ",
642 r". — ",
643 r", a ",
644 r", an ",
645 r", the ",
646 r", obsolete ",
647 r", possessed", # 'd/English
648 r", imitating", # 1/English
649 r", derived from",
650 r", called ",
651 r", especially ",
652 r", slang for ",
653 r", used to", # c/o /English
654 r", commonly", # b/w /English
655 r" corresponding to ",
656 r" equivalent to ",
657 r" popularized by ",
658 r" denoting ",
659 r" in its various senses\.",
660 r" used by ",
661 r" but not for ",
662 r" since ",
663 r" i\.e\. ",
664 r" i\. e\. ",
665 r" e\.g\. ",
666 r" eg\. ",
667 r" etc\. ",
668 r"\[http",
669 r" — used as ",
670 r" by K\. Forsyth ",
671 r" by J\. R\. Allen ",
672 r" by S\. Ferguson ",
673 r" by G\. Donaldson ",
674 r" May refer to ",
675 r" An area or region ",
676 ]
677 )
678 + r").*$"
679)
682class ValidNode:
683 """Node in the valid_sequences tree. Each node is part of a chain
684 or chains that form sequences built out of keys in key->tags
685 maps like xlat_tags, etc. The ValidNode's 'word' is the key
686 by which it is refered to in the root dict or a `children` dict,
687 `end` marks that the node is the end-terminus of a sequence (but
688 it can still continue if the sequence is shared by the start of
689 other sequences: "nominative$" and "nominative plural$" for example),
690 `tags` and `topics` are the dicts containing tag and topic strings
691 for terminal nodes (end==True)."""
693 __slots__ = (
694 "end",
695 "tags",
696 "topics",
697 "children",
698 )
700 def __init__(
701 self,
702 end=False,
703 tags: Optional[list[str]] = None,
704 topics: Optional[list[str]] = None,
705 children: Optional[dict[str, "ValidNode"]] = None,
706 ) -> None:
707 self.end = end
708 self.tags: list[str] = tags or []
709 self.topics: list[str] = topics or []
710 self.children: dict[str, "ValidNode"] = children or {}
713def add_to_valid_tree(tree: ValidNode, desc: str, v: Optional[str]) -> None:
714 """Helper function for building trees of valid tags/sequences during
715 initialization."""
716 assert isinstance(tree, ValidNode)
717 assert isinstance(desc, str)
718 assert v is None or isinstance(v, str)
719 node = tree
721 # Build the tree structure: each node has children nodes
722 # whose names are denoted by their dict key.
723 for w in desc.split(" "):
724 if w in node.children:
725 node = node.children[w]
726 else:
727 new_node = ValidNode()
728 node.children[w] = new_node
729 node = new_node
730 if not node.end:
731 node.end = True
732 if not v:
733 return None # Terminate early because there are no tags
735 tagslist = []
736 topicslist = []
737 for vv in v.split():
738 if vv in valid_tags:
739 tagslist.append(vv)
740 elif vv in valid_topics: 740 ↛ 743line 740 didn't jump to line 743 because the condition on line 740 was always true
741 topicslist.append(vv)
742 else:
743 print(
744 "WARNING: tag/topic {!r} maps to unknown {!r}".format(desc, vv)
745 )
746 topics = " ".join(topicslist)
747 tags = " ".join(tagslist)
748 # Changed to "_tags" and "_topics" to avoid possible key-collisions.
749 if topics:
750 node.topics.extend([topics])
751 if tags:
752 node.tags.extend([tags])
755def add_to_valid_tree1(
756 tree: ValidNode,
757 k: str,
758 v: Union[list[str], tuple[str, ...], str],
759 valid_values: Union[set[str], dict[str, Any]],
760) -> list[str]:
761 assert isinstance(tree, ValidNode)
762 assert isinstance(k, str)
763 assert v is None or isinstance(v, (list, tuple, str))
764 assert isinstance(valid_values, (set, dict))
765 if not v: 765 ↛ 766line 765 didn't jump to line 766 because the condition on line 765 was never true
766 add_to_valid_tree(valid_sequences, k, None)
767 return []
768 elif isinstance(v, str):
769 v = [v]
770 q = []
771 for vv in v:
772 assert isinstance(vv, str)
773 add_to_valid_tree(valid_sequences, k, vv)
774 vvs = vv.split()
775 for x in vvs:
776 q.append(x)
777 # return each individual tag
778 return q
781def add_to_valid_tree_mapping(
782 tree: ValidNode,
783 mapping: Union[dict[str, Union[list[str], str]], dict[str, str]],
784 valid_values: Union[set[str], dict[str, Any]],
785 recurse: bool,
786) -> None:
787 assert isinstance(tree, ValidNode)
788 assert isinstance(mapping, dict)
789 assert isinstance(valid_values, (set, dict))
790 assert recurse in (True, False)
791 for k, v in mapping.items():
792 assert isinstance(k, str)
793 assert isinstance(v, (list, str))
794 if isinstance(v, str):
795 q = add_to_valid_tree1(tree, k, [v], valid_values)
796 else:
797 q = add_to_valid_tree1(tree, k, v, valid_values)
798 if recurse:
799 visited = set()
800 while q:
801 v = q.pop()
802 if v in visited:
803 continue
804 visited.add(v)
805 if v not in mapping:
806 continue
807 vv = mapping[v]
808 qq = add_to_valid_tree1(tree, k, vv, valid_values)
809 q.extend(qq)
812# Tree of sequences considered to be tags (includes sequences that are
813# mapped to something that becomes one or more valid tags)
814valid_sequences = ValidNode()
815sequences_with_slashes: set[str] = set()
816for tag in valid_tags:
817 # The basic tags used in our tag system; some are a bit weird, but easier
818 # to implement this with 'false' positives than filter out stuff no one else
819 # uses.
820 if "/" in tag:
821 sequences_with_slashes.add(tag)
822 add_to_valid_tree(valid_sequences, tag, tag)
823for tag in uppercase_tags:
824 hyphenated = re.sub(r"\s+", "-", tag)
825 if "/" in tag:
826 sequences_with_slashes.add(tag)
827 add_to_valid_tree(valid_sequences, tag, hyphenated)
829# xlat_tags_map!
830add_to_valid_tree_mapping(valid_sequences, xlat_tags_map, valid_tags, False)
831for k in xlat_tags_map:
832 if "/" in k:
833 sequences_with_slashes.add(k)
834# Add topics to the same table, with all generalized topics also added
835for topic in valid_topics:
836 assert " " not in topic
837 if "/" in topic: 837 ↛ 838line 837 didn't jump to line 838 because the condition on line 837 was never true
838 sequences_with_slashes.add(topic)
839 add_to_valid_tree(valid_sequences, topic, topic)
840# Let each original topic value stand alone. These are not generally on
841# valid_topics. We add the original topics with spaces replaced by hyphens.
842for topic in topic_generalize_map.keys():
843 hyphenated = re.sub(r"\s+", "-", topic)
844 if "/" in topic: 844 ↛ 845line 844 didn't jump to line 845 because the condition on line 844 was never true
845 sequences_with_slashes.add(topic)
846 add_to_valid_tree(valid_sequences, topic, hyphenated)
847# Add canonicalized/generalized topic values
848add_to_valid_tree_mapping(
849 valid_sequences, topic_generalize_map, valid_topics, True
850)
852# Regex used to divide a decode candidate into parts that shouldn't
853# have their slashes turned into spaces
854slashes_re = re.compile(
855 r"(" + "|".join((re.escape(s) for s in sequences_with_slashes)) + r")"
856)
858# Regexp used to find "words" from word heads and linguistic descriptions
859word_pattern = (
860 r"[^ ,;()\u200e]+|"
861 r"\([^ ,;()\u200e]+\)[^ ,;()\u200e]+|"
862 r"[\u2800-\u28ff]|" # Braille characters
863 r"\(([^()]|\([^()]*\))*\)"
864)
866word_re_global = re.compile(word_pattern)
869def distw(titleparts: Sequence[str], word: str) -> float:
870 """Computes how distinct ``word`` is from the most similar word in
871 ``titleparts``. Returns 1 if words completely distinct, 0 if
872 identical, or otherwise something in between."""
873 assert isinstance(titleparts, (list, tuple))
874 assert isinstance(word, str)
875 w = min(
876 Levenshtein.distance(word, tw) / max(len(tw), len(word))
877 for tw in titleparts
878 )
879 return w
882def map_with(
883 ht: dict[str, str | list[str]] | dict[str, str],
884 lst: Sequence[str],
885) -> list[str]:
886 """Takes alternatives from ``lst``, maps them using ``ht`` to zero or
887 more alternatives each, and returns a combined list of alternatives."""
888 assert isinstance(ht, dict)
889 assert isinstance(lst, (list, tuple))
890 ret = []
891 for x in lst:
892 assert isinstance(x, str)
893 x = x.strip()
894 x = ht.get(x, x)
895 if isinstance(x, str): 895 ↛ 898line 895 didn't jump to line 898 because the condition on line 895 was always true
896 if x: 896 ↛ 891line 896 didn't jump to line 891 because the condition on line 896 was always true
897 ret.append(x)
898 elif isinstance(x, (list, tuple)):
899 ret.extend(x)
900 else:
901 raise RuntimeError("map_with unexpected value: {!r}".format(x))
902 return ret
905TagList = list[str]
906PosPathStep = tuple[int, TagList, TagList]
909def check_unknown(
910 from_i: int,
911 to_i: int,
912 i: int,
913 wordlst: Sequence[str],
914 allow_any: bool,
915 no_unknown_starts: bool,
916) -> list[PosPathStep]:
917 """Check if the current section from_i->to_i is actually unknown
918 or if it needs some special handling. We already presupposed that
919 this is UNKNOWN; this is just called to see what *kind* of UNKNOWN."""
920 assert isinstance(to_i, int)
921 assert isinstance(from_i, int)
922 assert isinstance(i, int)
923 # Adds unknown tag if needed. Returns new last_i
924 # print("check_unknown to_i={} from_i={} i={}"
925 # .format(to_i, from_i, i))
926 if from_i >= to_i:
927 return []
928 words = wordlst[from_i:to_i]
929 tag = " ".join(words)
930 assert tag
931 # print(f"{tag=}")
932 if re.match(ignored_unknown_starts_re, tag):
933 # Tags with this start are to be ignored
934 return [(from_i, ["UNKNOWN"], [])]
935 if tag in ignored_unknown_tags: 935 ↛ 936line 935 didn't jump to line 936 because the condition on line 935 was never true
936 return [] # One of the tags listed as to be ignored
937 if tag in ("and", "or"):
938 return []
939 if (
940 not allow_any
941 and not words[0].startswith("~")
942 and (
943 no_unknown_starts
944 or words[0] not in allowed_unknown_starts
945 or len(words) <= 1
946 )
947 ):
948 # print("ERR allow_any={} words={}"
949 # .format(allow_any, words))
950 return [
951 (from_i, ["UNKNOWN"], ["error-unknown-tag"])
952 ] # Add ``tag`` here to include
953 else:
954 return [(from_i, ["UNKNOWN"], [tag])]
957def add_new1(
958 node: ValidNode,
959 i: int,
960 start_i: int,
961 last_i: int,
962 new_paths: list[list[PosPathStep]],
963 new_nodes: list[tuple[ValidNode, int, int]],
964 pos_paths: list[list[list[PosPathStep]]],
965 wordlst: list[str],
966 allow_any: bool,
967 no_unknown_starts: bool,
968 max_last_i: int,
969) -> int:
970 assert isinstance(new_paths, list)
971 # print("add_new: start_i={} last_i={}".format(start_i, last_i))
972 # print("$ {} last_i={} start_i={}"
973 # .format(w, last_i, start_i))
974 max_last_i = max(max_last_i, last_i) # if last_i has grown
975 if (node, start_i, last_i) not in new_nodes:
976 new_nodes.append((node, start_i, last_i))
977 if node.end:
978 # We can see a terminal point in the search tree.
979 u = check_unknown(
980 last_i, start_i, i, wordlst, allow_any, no_unknown_starts
981 )
982 # Create new paths candidates based on different past possible
983 # paths; pos_path[last_i] contains possible paths, so add this
984 # new one at the beginning(?)
985 # The list comprehension inside the parens generates an iterable
986 # of lists, so this is .extend( [(last_i...)], [(last_i...)], ... )
987 # XXX: this is becoming impossible to annotate, nodes might
988 # need to become classed objects and not just dicts, or at least
989 # a TypedDict with a "children" node
990 new_paths.extend(
991 [(last_i, node.tags, node.topics)] + u + x
992 for x in pos_paths[last_i]
993 )
994 max_last_i = i + 1
995 return max_last_i
998@functools.lru_cache(maxsize=65536)
999def decode_tags(
1000 src: str,
1001 allow_any=False,
1002 no_unknown_starts=False,
1003) -> tuple[list[tuple[str, ...]], list[str]]:
1004 tagsets, topics = decode_tags1(src, allow_any, no_unknown_starts)
1005 # print(f"decode_tags: {src=}, {tagsets=}")
1007 # Insert retry-code here that modifies the text source
1008 if (
1009 any(s.startswith("error-") for tagset in tagsets for s in tagset)
1010 # I hate Python's *nested* list comprehension syntax ^
1011 or any(s.startswith("error-") for s in topics)
1012 ):
1013 new_tagsets: list[tuple[str, ...]] = []
1014 new_topics: list[str] = []
1016 if "/" in src:
1017 # slashes_re contains valid key entries with slashes; we're going
1018 # to skip them by splitting the string and skipping handling every
1019 # second entry, which contains the splitting group like "masculine/
1020 # feminine" style keys.
1021 split_parts = re.split(slashes_re, src)
1022 new_parts: list[str] = []
1023 if len(split_parts) > 1:
1024 for i, s in enumerate(split_parts):
1025 if i % 2 == 0:
1026 new_parts.append(s.replace("/", " "))
1027 else:
1028 new_parts.append(s)
1029 new_src = "".join(new_parts)
1030 else:
1031 new_src = src
1032 new_tagsets, new_topics = decode_tags1(
1033 new_src, allow_any, no_unknown_starts
1034 )
1035 elif " or " in src or " and " in src:
1036 # Annoying kludge.
1037 new_src = src.replace(" and ", " ")
1038 new_src = new_src.replace(" or ", " ")
1039 new_tagsets, new_topics = decode_tags1(
1040 new_src, allow_any, no_unknown_starts
1041 )
1042 # print(f"{new_tagsets=}")
1044 if new_tagsets or new_topics:
1045 old_errors = sum(
1046 1 for tagset in tagsets for s in tagset if s.startswith("error")
1047 )
1048 old_errors += sum(1 for s in topics if s.startswith("error"))
1049 new_errors = sum(
1050 1
1051 for new_tagset in new_tagsets
1052 for s in new_tagset
1053 if s.startswith("error")
1054 )
1055 new_errors += sum(1 for s in new_topics if s.startswith("error"))
1057 if new_errors <= old_errors: 1057 ↛ 1060line 1057 didn't jump to line 1060 because the condition on line 1057 was always true
1058 return new_tagsets, new_topics
1060 return tagsets, topics
1063def decode_tags1(
1064 src: str,
1065 allow_any=False,
1066 no_unknown_starts=False,
1067) -> tuple[list[tuple[str, ...]], list[str]]:
1068 """Decodes tags, doing some canonicalizations. This returns a list of
1069 lists of tags and a list of topics."""
1070 assert isinstance(src, str)
1072 # print("decode_tags: src={!r}".format(src))
1074 pos_paths: list[list[list[PosPathStep]]] = [[[]]]
1075 wordlst: list[str] = []
1076 max_last_i = 0 # pre-initialized here so that it can be used as a ref
1078 add_new = functools.partial(
1079 add_new1, # pre-set parameters and references for function
1080 pos_paths=pos_paths,
1081 wordlst=wordlst,
1082 allow_any=allow_any,
1083 no_unknown_starts=no_unknown_starts,
1084 max_last_i=max_last_i,
1085 )
1086 # First split the tags at commas and semicolons. Their significance is that
1087 # a multi-word sequence cannot continue across them.
1088 parts = split_at_comma_semi(src, extra=[";", ":"])
1090 for part in parts:
1091 max_last_i = len(wordlst) # "how far have we gone?"
1092 lst1 = part.split()
1093 if not lst1:
1094 continue
1095 wordlst.extend(lst1)
1096 cur_nodes: list[tuple[ValidNode, int, int]] = [] # Currently seen
1097 for w in lst1:
1098 i = len(pos_paths) - 1
1099 new_nodes: list[tuple[ValidNode, int, int]] = []
1100 # replacement nodes for next loop
1101 new_paths: list[list[PosPathStep]] = []
1102 # print("ITER i={} w={} max_last_i={} wordlst={}"
1103 # .format(i, w, max_last_i, wordlst))
1104 node: ValidNode
1105 start_i: int
1106 last_i: int
1107 for node, start_i, last_i in cur_nodes:
1108 # ValidNodes are part of a search tree that checks if a
1109 # phrase is found in xlat_tags_map and other text->tags dicts.
1110 if w in node.children:
1111 # the phrase continues down the tree
1112 # print("INC", w)
1113 max_last_i = add_new(
1114 node.children[w],
1115 i,
1116 start_i,
1117 last_i,
1118 new_paths,
1119 new_nodes,
1120 )
1121 if node.end:
1122 # we've hit an end point, the tags and topics have already
1123 # been gathered at some point, don't do anything with the
1124 # old stuff
1125 if w in valid_sequences.children:
1126 # This starts a *new* possible section
1127 max_last_i = add_new(
1128 valid_sequences.children[w], # root->
1129 i,
1130 i,
1131 i,
1132 new_paths,
1133 new_nodes,
1134 )
1135 if w not in node.children and not node.end:
1136 # print("w not in node and $: i={} last_i={} wordlst={}"
1137 # .format(i, last_i, wordlst))
1138 # If i == last_i == 0, for example (beginning)
1139 if (
1140 i == last_i
1141 or no_unknown_starts
1142 or wordlst[last_i] not in allowed_unknown_starts
1143 ):
1144 # print("NEW", w)
1145 if w in valid_sequences.children:
1146 # Start new sequences here
1147 max_last_i = add_new(
1148 valid_sequences.children[w],
1149 i,
1150 i,
1151 last_i,
1152 new_paths,
1153 new_nodes,
1154 )
1155 if not new_nodes:
1156 # This is run at the start when i == max_last_i == 0,
1157 # which is what populates the first node in new_nodes.
1158 # Some initial words cause the rest to be interpreted as unknown
1159 # print("not new nodes: i={} last_i={} wordlst={}"
1160 # .format(i, max_last_i, wordlst))
1161 if (
1162 i == max_last_i
1163 or no_unknown_starts
1164 or wordlst[max_last_i] not in allowed_unknown_starts
1165 ):
1166 # print("RECOVER w={} i={} max_last_i={} wordlst={}"
1167 # .format(w, i, max_last_i, wordlst))
1168 if w in valid_sequences.children:
1169 max_last_i = add_new(
1170 # new sequence from root
1171 valid_sequences.children[w],
1172 i,
1173 i,
1174 max_last_i,
1175 new_paths,
1176 new_nodes,
1177 )
1178 cur_nodes = new_nodes # Completely replace nodes!
1179 # 2023-08-18, fix to improve performance
1180 # Decode tags does a big search of the best-shortest matching
1181 # sequences of tags, but the original algorithm didn't have
1182 # any culling happen during operation, so in a case with
1183 # a lot of tags (for example, big blocks of text inserted
1184 # somewhere by mistake that is processed by decode_tags),
1185 # it would lead to exponential growth of new_paths contents.
1186 # This culling, using the same weighting algorithm code as
1187 # in the original is just applied to new_paths before it is
1188 # added to pos_paths. Basically it's "take the 10 best paths".
1189 # This *can* cause bugs if it gets stuck in a local minimum
1190 # or something, but this whole process is one-dimensional
1191 # and not that complex, so hopefully it works out...
1192 pw = []
1193 path: list[PosPathStep]
1194 for path in new_paths:
1195 weight = len(path)
1196 if any(x[1] == ["UNKNOWN"] for x in path):
1197 weight += 100 # Penalize unknown paths
1198 pw.append((weight, path))
1199 new_paths = [weightpath[1] for weightpath in sorted(pw)[:10]]
1200 pos_paths.append(new_paths)
1202 # print("END max_last_i={} len(wordlst)={} len(pos_paths)={}"
1203 # .format(max_last_i, len(wordlst), len(pos_paths)))
1205 if cur_nodes:
1206 # print("END HAVE_NODES")
1207 for node, start_i, last_i in cur_nodes:
1208 if node.end:
1209 # print("$ END start_i={} last_i={}"
1210 # .format(start_i, last_i))
1211 for path in pos_paths[start_i]:
1212 pos_paths[-1].append(
1213 [(last_i, node.tags, node.topics)] + path
1214 )
1215 else:
1216 # print("UNK END start_i={} last_i={} wordlst={}"
1217 # .format(start_i, last_i, wordlst))
1218 u = check_unknown(
1219 last_i,
1220 len(wordlst),
1221 len(wordlst),
1222 wordlst,
1223 allow_any,
1224 no_unknown_starts,
1225 )
1226 if pos_paths[start_i]:
1227 for path in pos_paths[start_i]:
1228 pos_paths[-1].append(u + path)
1229 else:
1230 pos_paths[-1].append(u)
1231 else:
1232 # Check for a final unknown tag
1233 # print("NO END NODES max_last_i={}".format(max_last_i))
1234 paths = pos_paths[max_last_i] or [[]]
1235 u = check_unknown(
1236 max_last_i,
1237 len(wordlst),
1238 len(wordlst),
1239 wordlst,
1240 allow_any,
1241 no_unknown_starts,
1242 )
1243 if u:
1244 # print("end max_last_i={}".format(max_last_i))
1245 for path in list(paths): # Copy in case it is the last pos
1246 pos_paths[-1].append(u + path)
1248 # import json
1249 # print("POS_PATHS:", json.dumps(pos_paths, indent=2, sort_keys=True))
1251 if not pos_paths[-1]:
1252 # print("decode_tags: {}: EMPTY POS_PATHS[-1]".format(src))
1253 return [], []
1255 # Find the best path
1256 pw = []
1257 for path in pos_paths[-1]:
1258 weight = len(path)
1259 if any(x[1] == ["UNKNOWN"] for x in path):
1260 weight += 100 # Penalize unknown paths
1261 pw.append((weight, path))
1262 path = min(pw)[1]
1264 # Convert the best path to tagsets and topics
1265 tagsets: list[list[str]] = [[]]
1266 topics: list[str] = []
1267 for i, tagspec, topicspec in path:
1268 if len(tagsets or "") > 16:
1269 # ctx.error("Too many tagsets! This is probably exponential",
1270 # sortid="form_descriptions/20230818")
1271 return [("error-unknown-tag", "error-exponential-tagsets")], []
1272 if tagspec == ["UNKNOWN"]:
1273 new_tagsets = []
1274 for x in tagsets:
1275 new_tagsets.append(x + topicspec)
1276 tagsets = new_tagsets
1277 continue
1278 if tagspec:
1279 new_tagsets = []
1280 for x in tagsets:
1281 for t in tagspec:
1282 if t: 1282 ↛ 1289line 1282 didn't jump to line 1289 because the condition on line 1282 was always true
1283 new_tags = list(x)
1284 for tag in t.split():
1285 if tag not in new_tags:
1286 new_tags.append(tag)
1287 new_tagsets.append(new_tags)
1288 else:
1289 new_tagsets.append(x)
1290 tagsets = new_tagsets
1291 if topicspec:
1292 for t in topicspec:
1293 for topic in t.split():
1294 if topic not in topics:
1295 topics.append(topic)
1297 # print("unsorted tagsets:", tagsets)
1298 ret_tagsets = sorted(set(tuple(sorted(set(tags))) for tags in tagsets))
1299 # topics = list(sorted(set(topics))) XXX tests expect not sorted
1300 # print("decode_tags: {} -> {} topics {}".format(src, tagsets, topics))
1301 # Yes, ret_tagsets is a list of tags in tuples, while topics is a LIST
1302 # of tags. Turning topics into a tuple breaks tests, turning the tuples
1303 # inside tagsets into lists breaks tests, I'm leaving them mismatched
1304 # for now. XXX
1305 return ret_tagsets, topics
1308def parse_head_final_tags(
1309 wxr: WiktextractContext, lang: str, form: str
1310) -> tuple[str, list[str]]:
1311 """Parses tags that are allowed at the end of a form head from the end
1312 of the form. This can also be used for parsing the final gender etc tags
1313 from translations and linkages."""
1314 assert isinstance(wxr, WiktextractContext)
1315 assert isinstance(lang, str) # Should be language that "form" is for
1316 assert isinstance(form, str)
1318 # print("parse_head_final_tags: lang={} form={!r}".format(lang, form))
1320 # Make sure there are no double spaces in the form as this code does not
1321 # handle them otherwise.
1322 form = re.sub(r"\s+", " ", form.strip())
1323 if not form:
1324 return form, []
1326 origform = form
1328 tags = []
1330 # If parsing for certain Bantu languages (e.g., Swahili), handle
1331 # some extra head-final tags first
1332 if lang in head_final_bantu_langs:
1333 m = re.search(head_final_bantu_re, form)
1334 if m is not None:
1335 tagkeys = m.group(1)
1336 if not wxr.wtp.title.endswith(tagkeys): # type:ignore[union-attr] 1336 ↛ 1351line 1336 didn't jump to line 1351 because the condition on line 1336 was always true
1337 form = form[: m.start()]
1338 v = head_final_bantu_map[tagkeys]
1339 if v.startswith("?"): 1339 ↛ 1340line 1339 didn't jump to line 1340 because the condition on line 1339 was never true
1340 v = v[1:]
1341 wxr.wtp.debug(
1342 "suspicious suffix {!r} in language {}: {}".format(
1343 tagkeys, lang, origform
1344 ),
1345 sortid="form_descriptions/1028",
1346 )
1347 tags.extend(v.split())
1349 # If parsing for certain Semitic languages (e.g., Arabic), handle
1350 # some extra head-final tags first
1351 if lang in head_final_semitic_langs:
1352 m = re.search(head_final_semitic_re, form)
1353 if m is not None:
1354 tagkeys = m.group(1)
1355 if not wxr.wtp.title.endswith(tagkeys): # type:ignore[union-attr] 1355 ↛ 1370line 1355 didn't jump to line 1370 because the condition on line 1355 was always true
1356 form = form[: m.start()]
1357 v = head_final_semitic_map[tagkeys]
1358 if v.startswith("?"): 1358 ↛ 1359line 1358 didn't jump to line 1359 because the condition on line 1358 was never true
1359 v = v[1:]
1360 wxr.wtp.debug(
1361 "suspicious suffix {!r} in language {}: {}".format(
1362 tagkeys, lang, origform
1363 ),
1364 sortid="form_descriptions/1043",
1365 )
1366 tags.extend(v.split())
1368 # If parsing for certain other languages (e.g., Lithuanian,
1369 # French, Finnish), handle some extra head-final tags first
1370 if lang in head_final_other_langs:
1371 m = re.search(head_final_other_re, form)
1372 if m is not None:
1373 tagkeys = m.group(1)
1374 if not wxr.wtp.title.endswith(tagkeys): # type:ignore[union-attr] 1374 ↛ 1380line 1374 didn't jump to line 1380 because the condition on line 1374 was always true
1375 form = form[: m.start()]
1376 tags.extend(head_final_other_map[tagkeys].split(" "))
1378 # Handle normal head-final tags
1379 # Loop this until nothing is found
1380 while True:
1381 prev_form = form
1382 m = re.search(head_final_re, form)
1383 if m is not None:
1384 # print(f"{m=}, {m.groups()=}")
1385 tagkeys = m.group(3)
1386 # Only replace tags ending with numbers in languages that have
1387 # head-final numeric tags (e.g., Bantu classes); also, don't replace
1388 # tags if the main title ends with them (then presume they are part
1389 # of the word)
1390 # print("head_final_tags form={!r} tagkeys={!r} lang={}"
1391 # .format(form, tagkeys, lang))
1392 tagkeys_contains_digit = re.search(r"\d", tagkeys)
1393 if (
1394 (not tagkeys_contains_digit or lang in head_final_numeric_langs)
1395 and not wxr.wtp.title.endswith(" " + tagkeys) # type:ignore[union-attr]
1396 and
1397 # XXX the above test does not capture when the whole word is a
1398 # xlat_head_map key, so I added the below test to complement
1399 # it; does this break anything?
1400 not wxr.wtp.title == tagkeys
1401 ): # defunct/English,
1402 # "more defunct" -> "more" ["archaic"]
1403 if ( 1403 ↛ 1421line 1403 didn't jump to line 1421 because the condition on line 1403 was always true
1404 not tagkeys_contains_digit
1405 or lang in head_final_numeric_langs
1406 ):
1407 # m.start(3) gets the start of what is in m.group(3), handy
1408 form = form[: m.start(3)].strip()
1409 v = xlat_head_map[tagkeys]
1410 if v.startswith("?"): 1410 ↛ 1411line 1410 didn't jump to line 1411 because the condition on line 1410 was never true
1411 v = v[1:]
1412 wxr.wtp.debug(
1413 "suspicious suffix {!r} in language {}: {}".format(
1414 tagkeys, lang, origform
1415 ),
1416 sortid="form_descriptions/1077",
1417 )
1418 tags.extend(v.split())
1419 else:
1420 break
1421 if prev_form == form:
1422 break
1424 # Generate warnings about words ending in " or" after processing
1425 if (
1426 (form.endswith(" or") and not origform.endswith(" or"))
1427 or re.search(
1428 r" (1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|"
1429 r"1a|2a|9a|10a|m1|f1|f2|m2|f3|m3|f4|m4|f5|m5|or|\?)"
1430 r"($|/| (f|m|sg|pl|anim|inan))",
1431 form,
1432 )
1433 or form.endswith(" du")
1434 ):
1435 if form not in ok_suspicious_forms:
1436 wxr.wtp.debug(
1437 "suspicious unhandled suffix in {}:"
1438 " {!r}, originally {!r}".format(lang, form, origform),
1439 sortid="form_descriptions/1089",
1440 )
1442 # print("parse_head_final_tags: form={!r} tags={}".format(form, tags))
1443 return form, tags
1446def quote_kept_parens(s: str) -> str:
1447 """Changes certain parenthesized expressions so that they won't be
1448 interpreted as parentheses. This is used for parts that are kept as
1449 part of the word, such as "read admiral (upper half)"."""
1450 return re.sub(
1451 r"\((lower half|upper half|k|s|n|II|III|A|C|G|U|Y|"
1452 r"vinyl|p-phenylene vinylene|\(\(\s*\)\))\)",
1453 r"__lpar__\1__rpar__",
1454 s,
1455 )
1458def quote_kept_ruby(
1459 wxr: WiktextractContext,
1460 ruby_tuples: list[
1461 tuple[
1462 str,
1463 str,
1464 ]
1465 ],
1466 s: str,
1467) -> str:
1468 if len(ruby_tuples) < 1: 1468 ↛ 1469line 1468 didn't jump to line 1469 because the condition on line 1468 was never true
1469 wxr.wtp.debug(
1470 "quote_kept_ruby called with no ruby",
1471 sortid="form_description/1114/20230517",
1472 )
1473 return s
1474 ks = []
1475 rs = []
1476 for k, r in ruby_tuples:
1477 ks.append(re.escape(k))
1478 rs.append(re.escape(r))
1479 if not (ks and rs): 1479 ↛ 1480line 1479 didn't jump to line 1480 because the condition on line 1479 was never true
1480 wxr.wtp.debug(
1481 f"empty column in ruby_tuples: {ruby_tuples}",
1482 sortid="form_description/1124/20230606",
1483 )
1484 return s
1485 newm = re.compile(
1486 r"({})\s*\(\s*({})\s*\)".format("|".join(ks), "|".join(rs))
1487 )
1488 rub_re = re.compile(
1489 r"({})".format(
1490 r"|".join(
1491 r"{}\(*{}\)*".format(
1492 re.escape(k),
1493 re.escape(r),
1494 )
1495 for k, r in ruby_tuples
1496 )
1497 )
1498 )
1500 def paren_replace(m: re.Match) -> str:
1501 return re.sub(newm, r"\1__lrub__\2__rrub__", m.group(0))
1503 return re.sub(rub_re, paren_replace, s)
1506def unquote_kept_parens(s: str) -> str:
1507 """Converts the quoted parentheses back to normal parentheses."""
1508 return re.sub(r"__lpar__(.*?)__rpar__", r"(\1)", s)
1511def add_romanization(
1512 wxr: WiktextractContext,
1513 data: WordData,
1514 roman: str,
1515 text: str,
1516 is_reconstruction: bool,
1517 head_group: Optional[int],
1518 ruby: Sequence[tuple[str, str]],
1519) -> None:
1520 tags_lst = ["romanization"]
1521 m = re.match(r"([^:]+):(.+)", roman)
1522 # This function's purpose is to intercept broken romanizations,
1523 # like "Yale: hēnpyeng" style tags. Most romanization styles
1524 # are already present as tags, so we can use decode_tags to find
1525 # them.
1526 if m: 1526 ↛ 1527line 1526 didn't jump to line 1527 because the condition on line 1526 was never true
1527 tagsets, topics = decode_tags(m.group(1))
1528 if tagsets:
1529 for tags in tagsets:
1530 tags_lst.extend(tags)
1531 roman = m.group(2)
1532 add_related(
1533 wxr,
1534 data,
1535 tags_lst,
1536 [roman],
1537 text,
1538 True,
1539 is_reconstruction,
1540 head_group,
1541 ruby,
1542 )
1545def add_related(
1546 wxr: WiktextractContext,
1547 data: WordData,
1548 tags_lst: Union[list[str], tuple[str, ...]],
1549 related_list: list[str],
1550 origtext: str,
1551 add_all_canonicals: bool,
1552 is_reconstruction: bool,
1553 head_group: Optional[int],
1554 ruby_data: Optional[Sequence[tuple[str, str]]] = None,
1555 links: list[tuple[str, str]] | None = None,
1556 link_dict: dict[str, list[str]] | None = None,
1557) -> Optional[list[tuple[str, ...]]]:
1558 """Internal helper function for some post-processing entries for related
1559 forms (e.g., in word head). This returns a list of list of tags to be
1560 added to following related forms or None (cf. walrus/English word head,
1561 parenthesized part starting with "both")."""
1562 assert isinstance(wxr, WiktextractContext)
1563 assert isinstance(tags_lst, (list, tuple))
1564 for x in tags_lst:
1565 assert isinstance(x, str)
1566 assert isinstance(related_list, (list, tuple))
1567 assert isinstance(origtext, str)
1568 assert add_all_canonicals in (True, False)
1569 assert isinstance(ruby_data, (list, tuple)) or ruby_data is None
1570 if ruby_data is None: 1570 ↛ 1571line 1570 didn't jump to line 1571 because the condition on line 1570 was never true
1571 ruby_data = []
1572 related = " ".join(related_list)
1573 # print("add_related: tags_lst={} related={}".format(tags_lst, related))
1574 if related == "[please provide]": 1574 ↛ 1575line 1574 didn't jump to line 1575 because the condition on line 1574 was never true
1575 return None
1576 if related in IGNORED_RELATED: 1576 ↛ 1577line 1576 didn't jump to line 1577 because the condition on line 1576 was never true
1577 return None
1578 if is_reconstruction and related.startswith("*") and len(related) > 1:
1579 related = related[1:]
1581 # print(f"{links=}, {link_dict=}")
1582 # Get title word, with any reconstruction prefix removed
1583 titleword = re.sub(r"^Reconstruction:[^/]*/", "", wxr.wtp.title) # type:ignore[arg-type]
1585 def check_related(related: str) -> None:
1586 # Warn about some suspicious related forms
1587 m = re.search(suspicious_related_re, related)
1588 if (m and m.group(0) not in titleword) or (
1589 related in ("f", "m", "n", "c") and len(titleword) >= 3
1590 ):
1591 if "eumhun" in tags_lst: 1591 ↛ 1592line 1591 didn't jump to line 1592 because the condition on line 1591 was never true
1592 return
1593 if "cangjie-input" in tags_lst: 1593 ↛ 1594line 1593 didn't jump to line 1594 because the condition on line 1593 was never true
1594 return
1595 if "class" in tags_lst: 1595 ↛ 1596line 1595 didn't jump to line 1596 because the condition on line 1595 was never true
1596 return
1597 if wxr.wtp.section == "Korean" and re.search( 1597 ↛ 1601line 1597 didn't jump to line 1601 because the condition on line 1597 was never true
1598 r"^\s*\w*>\w*\s*$", related
1599 ):
1600 # ignore Korean "i>ni" / "라>나" values
1601 return
1602 if ( 1602 ↛ 1609line 1602 didn't jump to line 1609 because the condition on line 1602 was never true
1603 wxr.wtp.section == "Burmese"
1604 and "romanization" in tags_lst
1605 and re.search(r":", related)
1606 ):
1607 # ignore Burmese with ":", that is used in Burmese
1608 # translitteration of "း", the high-tone visarga.
1609 return
1610 wxr.wtp.debug(
1611 "suspicious related form tags {}: {!r} in {!r}".format(
1612 tags_lst, related, origtext
1613 ),
1614 sortid="form_descriptions/1147",
1615 )
1617 following_tagsets = None # Tagsets to add to following related forms
1618 roman = None
1619 tagsets1: list[tuple[str, ...]] = [tuple()]
1620 topics1: list[str] = []
1622 m = re.match(r"\((([^()]|\([^()]*\))*)\)\s+", related)
1623 if m:
1624 paren = m.group(1)
1625 related = related[m.end() :]
1626 m = re.match(r"^(all|both) (.*)", paren)
1627 if m: 1627 ↛ 1628line 1627 didn't jump to line 1628 because the condition on line 1627 was never true
1628 tagsets1, topics1 = decode_tags(m.group(2))
1629 following_tagsets = tagsets1
1630 else:
1631 tagsets1, topics1 = decode_tags(paren)
1632 else:
1633 m = re.search(r"\s+\((([^()]|\([^()]*\))*)\)$", related)
1634 if m:
1635 paren = m.group(1)
1636 if paren.startswith("U+"): 1636 ↛ 1637line 1636 didn't jump to line 1637 because the condition on line 1636 was never true
1637 related = related[: m.start()]
1638 else:
1639 cls = classify_desc(paren)
1640 if ( 1640 ↛ 1647line 1640 didn't jump to line 1647 because the condition on line 1640 was always true
1641 cls in ("romanization", "english")
1642 and classify_desc(related[: m.start()]) == "other"
1643 ):
1644 roman = paren
1645 related = related[: m.start()]
1646 else:
1647 related = related[: m.start()]
1648 tagsets1, topics1 = decode_tags(paren)
1649 if related and related.startswith("{{"): 1649 ↛ 1650line 1649 didn't jump to line 1650 because the condition on line 1649 was never true
1650 wxr.wtp.debug(
1651 "`{{` in word head form - possible Wiktionary error: {!r}".format(
1652 related
1653 ),
1654 sortid="form_descriptions/1177",
1655 )
1656 return None # Likely Wiktionary coding error
1657 related = unquote_kept_parens(related)
1658 # Split related by "/" (e.g., grande/Spanish) superlative in head
1659 # Do not split if / in word title, see π//Japanese
1660 if len(related) > 5 and "/" not in wxr.wtp.title: # type:ignore[operator]
1661 alts = split_at_comma_semi(related, separators=["/"])
1662 else:
1663 alts = [related]
1664 if ruby_data:
1665 # prepare some regex stuff in advance
1666 ks, rs = [], []
1667 for k, r in ruby_data:
1668 ks.append(re.escape(k))
1669 rs.append(re.escape(r))
1670 splitter = r"((?:{})__lrub__(?:{})__rrub__)".format(
1671 "|".join(ks), "|".join(rs)
1672 )
1673 for related in alts:
1674 ruby: list[tuple[str, str]] = []
1675 if ruby_data:
1676 new_related = []
1677 rub_split = re.split(splitter, related)
1678 for s in rub_split:
1679 m = re.match(r"(.+)__lrub__(.+)__rrub__", s)
1680 if m:
1681 # add ruby with (\1, \2)
1682 ruby.append((m.group(1), m.group(2)))
1683 new_related.append(m.group(1))
1684 else:
1685 new_related.append(s)
1686 related = "".join(new_related)
1687 tagsets2, topics2 = decode_tags(" ".join(tags_lst))
1688 for tags1 in tagsets1:
1689 assert isinstance(tags1, (list, tuple))
1690 for tags2 in tagsets2:
1691 assert isinstance(tags1, (list, tuple))
1692 dt: LinkageData = {"word": related}
1693 if roman:
1694 dt["roman"] = roman
1695 if ruby:
1696 dt["ruby"] = ruby
1697 if "alt-of" in tags2: 1697 ↛ 1698line 1697 didn't jump to line 1698 because the condition on line 1697 was never true
1698 check_related(related)
1699 data_extend(data, "tags", tags1)
1700 data_extend(data, "tags", tags2)
1701 data_extend(data, "topics", topics1)
1702 data_extend(data, "topics", topics2)
1703 data_append(data, "alt_of", dt)
1704 elif "form-of" in tags2: 1704 ↛ 1705line 1704 didn't jump to line 1705 because the condition on line 1704 was never true
1705 check_related(related)
1706 data_extend(data, "tags", tags1)
1707 data_extend(data, "tags", tags2)
1708 data_extend(data, "topics", topics1)
1709 data_extend(data, "topics", topics2)
1710 data_append(data, "form_of", dt)
1711 elif "compound-of" in tags2: 1711 ↛ 1712line 1711 didn't jump to line 1712 because the condition on line 1711 was never true
1712 check_related(related)
1713 data_extend(data, "tags", tags1)
1714 data_extend(data, "tags", tags2)
1715 data_extend(data, "topics", topics1)
1716 data_extend(data, "topics", topics2)
1717 data_append(data, "compound", related)
1718 else:
1719 lang = wxr.wtp.section or "LANG_MISSING"
1720 related, final_tags = parse_head_final_tags(
1721 wxr, lang, related
1722 )
1723 # print("add_related: related={!r} tags1={!r} tags2={!r} "
1724 # "final_tags={!r}"
1725 # .format(related, tags1, tags2, final_tags))
1726 tags = list(tags1) + list(tags2) + list(final_tags)
1727 check_related(related)
1728 form: FormData = {"form": related}
1729 if (
1730 links
1731 and link_dict
1732 and (
1733 form_links := match_links_to_form(
1734 wxr, related, links, link_dict
1735 )
1736 )
1737 ):
1738 form["links"] = form_links
1739 if head_group:
1740 form["head_nr"] = head_group
1741 if roman:
1742 form["roman"] = roman
1743 if ruby:
1744 form["ruby"] = ruby
1745 data_extend(form, "topics", topics1)
1746 data_extend(form, "topics", topics2)
1747 if topics1 or topics2: 1747 ↛ 1748line 1747 didn't jump to line 1748 because the condition on line 1747 was never true
1748 wxr.wtp.debug(
1749 "word head form has topics: {}".format(form),
1750 sortid="form_descriptions/1233",
1751 )
1752 # Add tags from canonical form into the main entry
1753 if "canonical" in tags:
1754 if related in ("m", "f") and len(titleword) > 1: 1754 ↛ 1755line 1754 didn't jump to line 1755 because the condition on line 1754 was never true
1755 wxr.wtp.debug(
1756 "probably incorrect canonical form "
1757 "{!r} ignored (probably tag combination "
1758 "missing from xlat_head_map)".format(related),
1759 sortid="form_descriptions/1241",
1760 )
1761 continue
1762 if (
1763 related != titleword
1764 or add_all_canonicals
1765 or topics1
1766 or topics2
1767 or ruby
1768 ):
1769 data_extend(form, "tags", sorted(set(tags)))
1770 else:
1771 # We won't add canonical form here
1772 filtered_tags = list(
1773 x for x in tags if x != "canonical"
1774 )
1775 data_extend(data, "tags", filtered_tags)
1776 continue
1777 else:
1778 data_extend(form, "tags", sorted(set(tags)))
1779 # Only insert if the form is not already there
1780 for old in data.get("forms", ()):
1781 if form == old: 1781 ↛ 1782line 1781 didn't jump to line 1782 because the condition on line 1781 was never true
1782 break
1783 else:
1784 data_append(data, "forms", form)
1786 # If this form had pre-tags that started with "both" or "all", add those
1787 # tags also to following related forms that don't have their own tags
1788 # specified.
1789 return following_tagsets
1792def match_links_to_form(
1793 wxr: WiktextractContext,
1794 form: str,
1795 links: list[tuple[str, str]],
1796 link_dict: dict[str, list[str]] | None,
1797) -> list[tuple[str, str]] | None:
1798 if not links: 1798 ↛ 1799line 1798 didn't jump to line 1799 because the condition on line 1798 was never true
1799 return None
1800 if link_dict is None:
1801 link_dict = {}
1802 for ltxt, ltrg in links:
1803 if ltxt not in link_dict:
1804 link_dict[ltxt] = [
1805 ltrg,
1806 ]
1807 else:
1808 link_dict[ltxt].append(ltrg)
1809 ret: list[tuple[str, str]] = []
1810 if form in link_dict:
1811 if len(link_dict[form]) > 1 and any( 1811 ↛ 1814line 1811 didn't jump to line 1814 because the condition on line 1811 was never true
1812 x != link_dict[form][0] for x in link_dict[form]
1813 ):
1814 wxr.wtp.warning(
1815 f"{form=} has many different "
1816 f"link candidates `{link_dict[form]}`, "
1817 f"which can't be disambiguated.",
1818 sortid="form_descriptions/match_links_to_form",
1819 )
1820 for ltarg in link_dict[form]:
1821 ret.append((form, ltarg))
1822 elif " " in form:
1823 # split and search for a sequence of links...
1824 split_forms = form.split()
1825 found = False
1826 for i, (ltext, ltarg) in enumerate(links):
1827 if ltext == split_forms[0]:
1828 for j, f in enumerate(split_forms):
1829 if i + j >= len(links):
1830 break
1831 if f.strip(",;() ") != links[i + j][0].strip(",;() "):
1832 break
1833 if i + j == len(links): 1833 ↛ 1834line 1833 didn't jump to line 1834 because the condition on line 1833 was never true
1834 break
1835 else:
1836 found = True
1837 if found:
1838 ret = links[i : i + len(split_forms)]
1839 break
1840 # We only care about weird links
1841 # print(f"{len(ret)=}, {ret}")
1842 for txt, tar in ret:
1843 if txt != tar and txt != tar[: tar.find("#")]:
1844 break
1845 else:
1846 return None
1847 return ret or None
1850# Issue #967, in English word forms sometimes forms are skipped because
1851# they are taggable words and their distw() is too big, like clipping from clip
1852WORDS_WITH_FALSE_POSITIVE_TAGS: dict[str, list[str]] = {
1853 "clip": ["clipping"], # XXX remember to change me back to clipping after
1854 "English": ["English", "Englishes"],
1855 "common": ["common", "commoner"],
1856 # tests.
1857}
1859WORDS_WITH_FALSE_POSITIVE_FORMS: dict[str, list[str]] = {
1860 "unaccountability": ["countable", "uncountable"],
1861 "uncountability": ["countable", "uncountable"],
1862}
1864FALSE_POSITIVE_MISSING_FORMS: dict[str, list[str]] = {}
1866FORM_ASSOCIATED_TAG_WORDS: set[str] = {
1867 "participle",
1868 "past",
1869 "present",
1870 "singular",
1871 "plural",
1872 "first-person",
1873 "second-person",
1874 "third-person",
1875 "gerund",
1876}
1878SEMICOLON_REPLACEMENT = "__SEMICOLON__"
1881def parse_word_head(
1882 wxr: WiktextractContext,
1883 word: str,
1884 pos: str,
1885 text: str,
1886 data: WordData,
1887 is_reconstruction: bool,
1888 head_group: Optional[int],
1889 original_header_nodes: list[WikiNode | str] | None = None,
1890 ruby=None,
1891 links: list[
1892 tuple[
1893 str,
1894 str,
1895 ]
1896 ]
1897 | None = None,
1898 label_templates: list[TemplateData] | None = None,
1899) -> None:
1900 """Parses the head line for a word for in a particular language and
1901 part-of-speech, extracting tags and related forms."""
1902 assert isinstance(wxr, WiktextractContext)
1903 assert isinstance(pos, str)
1904 assert isinstance(text, str)
1905 assert isinstance(data, dict)
1906 assert isinstance(ruby, (list, tuple)) or ruby is None
1907 if ruby is None:
1908 ruby = []
1909 assert is_reconstruction in (True, False)
1910 # print("PARSE_WORD_HEAD: {}: {!r}".format(wxr.wtp.section, text))
1911 # print(f"PARSE_WORD_HEAD: {data=}")
1912 # print(f"PARSE_WORD_HEAD: {links=}")
1914 # Save original text for if we want to look for mismatched form-links
1916 link_dict: dict[str, list[str]] | None
1917 if links is not None:
1918 link_dict = {}
1919 for ltxt, ltrg in links:
1920 if ltxt not in link_dict:
1921 link_dict[ltxt] = [
1922 ltrg,
1923 ]
1924 else:
1925 link_dict[ltxt].append(ltrg)
1926 else:
1927 link_dict = None
1929 # print(f"MAIN: {links=}")
1930 link_words_not_alnum = []
1931 if not word.isalnum() and not word.replace("-", "").isalnum():
1932 # `-` is kosher, add more of these if needed.
1933 # if the word contains non-letter or -number characters, it
1934 # might have something that messes with split-at-semi-comma; we
1935 # collect links so that we can skip splitting them.
1936 if links is None and original_header_nodes is not None:
1937 links, _ = extract_links_from_node(
1938 wxr,
1939 original_header_nodes,
1940 remove_anchor_tags=True,
1941 expand_nodes=True,
1942 )
1943 if links is not None: 1943 ↛ 1947line 1943 didn't jump to line 1947 because the condition on line 1943 was always true
1944 for ltext, ltar in links:
1945 if not ltext.isalnum():
1946 link_words_not_alnum.append(ltext)
1947 if word not in link_words_not_alnum: 1947 ↛ 1950line 1947 didn't jump to line 1950 because the condition on line 1947 was always true
1948 link_words_not_alnum.append(word)
1950 if link_words_not_alnum is None: 1950 ↛ 1951line 1950 didn't jump to line 1951 because the condition on line 1950 was never true
1951 link_words_not_alnum = []
1953 if len(link_words_not_alnum) > 0:
1954 # if we have link data (that is, links with stuff like commas and
1955 # spaces, replace word_re with a modified local scope pattern
1956 # print(f"links {list((c, ord(c)) for link in links for c in link)=}")
1957 word_re = re.compile(
1958 r"\b" # In case we have forms that are longer and contain links
1959 +
1960 # or words as a substring...
1961 r"\b|\b".join(
1962 sorted(
1963 (re.escape(s) for s in link_words_not_alnum),
1964 key=lambda x: -len(x),
1965 )
1966 )
1967 + r"\b|"
1968 + word_pattern
1969 )
1970 else:
1971 word_re = word_re_global
1973 if "Lua execution error" in text or "Lua timeout error" in text: 1973 ↛ 1974line 1973 didn't jump to line 1974 because the condition on line 1973 was never true
1974 return
1976 # Fix words with "superlative:" or "comparative:" at end of head
1977 # e.g. grande/Spanish/Adj
1978 text = re.sub(r" (superlative|comparative): (.*)", r" (\1 \2)", text)
1980 # Parse Arabic non-past forms, e.g. أبلع/Arabic/Verb
1981 m = re.search(r", non-past ([^)]+ \([^)]+\))", text)
1982 if m:
1983 add_related(
1984 wxr,
1985 data,
1986 ["non-past"],
1987 [m.group(1)],
1988 text,
1989 True,
1990 is_reconstruction,
1991 head_group,
1992 ruby,
1993 links,
1994 link_dict,
1995 )
1996 text = text[: m.start()] + text[m.end() :]
1998 language = wxr.wtp.section
1999 titleword = re.sub(
2000 r"^Reconstruction:[^/]*/", "", wxr.wtp.title or "MISSING_TITLE"
2001 )
2002 titleparts = list(
2003 m.group(0)
2004 for m in re.finditer(word_re, wxr.wtp.title or "MISSING_TITLE")
2005 )
2006 if not titleparts: 2006 ↛ 2007line 2006 didn't jump to line 2007 because the condition on line 2006 was never true
2007 return
2009 # Remove " or" from the end to prevent weird canonical forms
2010 if text.endswith(" or"):
2011 for tp in titleparts:
2012 if text.endswith(tp): 2012 ↛ 2013line 2012 didn't jump to line 2013 because the condition on line 2012 was never true
2013 break
2014 else:
2015 text = text.removesuffix(" or").rstrip()
2017 # Handle the part of the head that is not in parentheses. However, certain
2018 # parenthesized parts are part of word, and those must be handled
2019 # specially here.
2020 if ruby:
2021 text = quote_kept_ruby(wxr, ruby, text)
2022 base = text
2023 base = quote_kept_parens(base)
2024 base = remove_text_in_parentheses(base)
2025 base = base.replace("?", "") # Removes uncertain articles etc
2026 base = re.sub(r"\s+", " ", base)
2027 base = re.sub(r" ([,;])", r"\1", base)
2028 base = re.sub(r" • ", r" ", base)
2029 # Many languages use • as a punctuation mark separating the base
2030 # from the rest of the head. στάδιος/Ancient Greek, issue #176
2031 base = base.strip()
2032 # print(f"{base=}, {text=}")
2034 # Check for certain endings in head (mostly for compatibility with weird
2035 # heads, e.g. rata/Romanian "1st conj." at end)
2036 m = re.search(head_end_re, base)
2037 tags: Union[tuple[str, ...], list[str]] = []
2038 if m: 2038 ↛ 2039line 2038 didn't jump to line 2039 because the condition on line 2038 was never true
2039 tags = head_end_map[m.group(1).lower()].split()
2040 data_extend(data, "tags", tags)
2041 base = base[: m.start()]
2043 if language == "Japanese":
2044 # Remove the leftover "^" of the historical kana superscript,
2045 # e.g. 静か "^(←しづか)-na"
2046 base = re.sub(r" \^(?!†)", " ", base).strip()
2047 while m := ja_head_end_re.search(base):
2048 tag = m.group(1)
2049 if tag == "uninflectable": 2049 ↛ 2050line 2049 didn't jump to line 2050 because the condition on line 2049 was never true
2050 tag = "uninflected"
2051 data_extend(data, "tags", [tag])
2052 base = base[: m.start()]
2054 # Special case: handle Hán Nôm readings for Vietnamese characters
2055 m = re.match(
2056 r"{}: (Hán Nôm) readings: (.*)".format(re.escape(titleword)), base
2057 )
2058 if m: 2058 ↛ 2059line 2058 didn't jump to line 2059 because the condition on line 2058 was never true
2059 tag, readings = m.groups()
2060 tag = re.sub(r"\s+", "-", tag)
2061 for reading in split_at_comma_semi(
2062 readings, skipped=link_words_not_alnum
2063 ):
2064 add_related(
2065 wxr,
2066 data,
2067 [tag],
2068 [reading],
2069 text,
2070 True,
2071 is_reconstruction,
2072 head_group,
2073 ruby,
2074 links,
2075 link_dict,
2076 )
2077 return
2079 # Special case: Hebrew " [pattern: nnn]" ending
2080 m = re.search(r"\s+\[pattern: ([^]]+)\]", base)
2081 if m: 2081 ↛ 2082line 2081 didn't jump to line 2082 because the condition on line 2081 was never true
2082 add_related(
2083 wxr,
2084 data,
2085 ["class"],
2086 [m.group(1)],
2087 text,
2088 True,
2089 is_reconstruction,
2090 head_group,
2091 ruby,
2092 links,
2093 link_dict,
2094 )
2095 base = base[: m.start()] + base[m.end() :]
2097 # Clean away some messy "Upload an image" template text used in
2098 # American Sign Language:
2099 # S@NearBaseForearm-PalmUp Frontandback S@BaseForearm-PalmUp
2100 m = re.search(r"Upload .+ gif image.", base)
2101 if m: 2101 ↛ 2102line 2101 didn't jump to line 2102 because the condition on line 2101 was never true
2102 base = base[: m.start()] + base[m.end() :]
2104 semicolon_present = False
2105 # Split the head into alternatives. This is a complicated task, as
2106 # we do not want so split on "or" or "," when immediately followed by more
2107 # head-final tags, but otherwise do want to split by them.
2108 # 20230907 added "or" to this to handle 'true or false', titles with 'or'
2109 if wxr.wtp.title and (
2110 "," in wxr.wtp.title or ";" in wxr.wtp.title or " or " in wxr.wtp.title
2111 ):
2112 # If the title has ";", we don't want to split on that and can remove
2113 # the ; from the splitting regex pretty easily because it's uncommon.
2114 # However, commas are so common that not splitting on them is just
2115 # not feasible, and we have to just deal with that if there are
2116 # alternative forms or variations with stray commas that shouldn't
2117 # be split.
2118 if ";" in wxr.wtp.title:
2119 semicolon_present = True
2120 base = base.replace(";", SEMICOLON_REPLACEMENT)
2121 default_splitter = head_split_no_semicolon_re
2122 else:
2123 default_splitter = head_split_re
2124 # A kludge to handle article titles/phrases with commas.
2125 # Preprocess splits to first capture the title, then handle
2126 # all the others as usual.
2127 presplits = re.split(r"({})".format(wxr.wtp.title), base)
2128 splits = []
2129 for psplit in presplits:
2130 if psplit == wxr.wtp.title:
2131 splits.append(psplit)
2132 else:
2133 splits.extend(re.split(default_splitter, psplit))
2134 else:
2135 # Do the normal split; previous behavior.
2136 splits = re.split(head_split_re, base)
2137 # print("BASE: ", repr(base))
2138 # print("SPLITS:", splits)
2139 alts: list[str] = []
2140 # print("parse_word_head: splits:", splits,
2141 # "head_split_re_parens:", head_split_re_parens)
2142 for i in range(
2143 0, len(splits) - head_split_re_parens, head_split_re_parens + 1
2144 ):
2145 v = splits[i]
2146 ending = splits[i + 1] or "" # XXX is this correct???
2147 # print("parse_word_head alts v={!r} ending={!r} alts={}"
2148 # .format(v, ending, alts))
2149 if alts and (v == "" and ending):
2150 assert ending[0] == " "
2151 alts[-1] += " or" + ending # endings starts with space
2152 elif v or ending:
2153 alts.append((v or "") + (ending or ""))
2154 last = splits[-1].strip()
2155 conn = "" if len(splits) < 3 else splits[-2]
2156 # print("parse_word_head alts last={!r} conn={!r} alts={}"
2157 # .format(last, conn, alts))
2158 if ( 2158 ↛ 2169line 2158 didn't jump to line 2169 because the condition on line 2158 was never true
2159 alts
2160 and last
2161 and (
2162 last.split()[0] in xlat_head_map
2163 or (
2164 conn == " or "
2165 and (alts[-1] + " or " + last).strip() in xlat_head_map
2166 )
2167 )
2168 ):
2169 alts[-1] += " or " + last
2170 elif last: 2170 ↛ 2171line 2170 didn't jump to line 2171 because the condition on line 2170 was never true
2171 alts.append(last)
2173 # print("parse_word_head alts: {}".format(alts))
2174 # print(f"{base=}")
2176 # Process the head alternatives
2177 canonicals: list[tuple[list[str], list[str]]] = []
2178 mode: Optional[str] = None
2179 for alt_i, alt in enumerate(alts):
2180 alt = alt.strip()
2181 if alt.startswith("compound form:"): 2181 ↛ 2182line 2181 didn't jump to line 2182 because the condition on line 2181 was never true
2182 mode = "compound-form"
2183 alt = alt[14:].strip()
2184 if ((dash_i := alt.find(" -")) > 0) and ( 2184 ↛ 2190line 2184 didn't jump to line 2190 because the condition on line 2184 was never true
2185 dash_i > (wxr.wtp.title or "").find(" -")
2186 ):
2187 # test_en_head / test_suffixes_at_end_of_form1
2188 # Some heads have suffixes that end up attached to the form
2189 # like in https://en.wiktionary.org/wiki/%E6%A5%BD%E3%81%97%E3%81%84
2190 alt = alt[:dash_i]
2191 if mode == "compound-form": 2191 ↛ 2192line 2191 didn't jump to line 2192 because the condition on line 2191 was never true
2192 add_related(
2193 wxr,
2194 data,
2195 ["in-compounds"],
2196 [alt],
2197 text,
2198 True,
2199 is_reconstruction,
2200 head_group,
2201 ruby,
2202 links,
2203 link_dict,
2204 )
2205 continue
2206 # For non-first parts, see if it can be treated as tags-only
2207 if alt_i == 0:
2208 expanded_alts = [alt]
2209 else:
2210 expanded_alts = map_with(xlat_descs_map, [alt])
2211 # print("EXPANDED_ALTS:", expanded_alts)
2212 tagsets: Optional[list[tuple[str, ...]]]
2213 for alt in expanded_alts:
2214 baseparts = list(m.group(0) for m in word_re.finditer(alt))
2215 if alt_i > 0:
2216 tagsets, topics = decode_tags(" ".join(baseparts))
2217 if not any("error-unknown-tag" in x for x in tagsets):
2218 data_extend(data, "topics", topics)
2219 for tags1 in tagsets:
2220 data_extend(data, "tags", tags1)
2221 continue
2223 alt, tags = parse_head_final_tags(
2224 wxr, language or "MISSING_LANG", alt
2225 )
2226 tags = list(tags) # Make sure we don't modify anything cached
2227 tags.append("canonical")
2228 if alt_i == 0 and "," in wxr.wtp.title or ";" in wxr.wtp.title: # type:ignore[operator]
2229 # Kludge to handle article titles/phrases with commas.
2230 # basepart's regex strips commas, which leads to a
2231 # canonical form that is the title phrase without a comma.
2232 # basepart in add_related is almost immediately joined with
2233 # spaces anyhow. XXX not exactly sure why it's
2234 # canonicals.append((tags, baseparts)) and not (tags, [alt])
2235 baseparts = [alt]
2236 canonicals.append((tags, baseparts))
2238 # If more of this kind of replace-and-return-original kind of stuff is
2239 # needed, make semicolon_present into a flag enum, something like `modified`
2240 if semicolon_present:
2241 new_cans = []
2242 for tags, baseparts in canonicals:
2243 new_cans.append(
2244 (
2245 tags,
2246 [s.replace(SEMICOLON_REPLACEMENT, ";") for s in baseparts],
2247 )
2248 )
2249 canonicals = new_cans
2250 for tags, baseparts in canonicals:
2251 add_related(
2252 wxr,
2253 data,
2254 tags,
2255 baseparts,
2256 text,
2257 len(canonicals) > 1,
2258 is_reconstruction,
2259 head_group,
2260 ruby,
2261 links,
2262 link_dict,
2263 )
2265 # Handle parenthesized descriptors for the word form and links to
2266 # related words
2267 text = quote_kept_parens(text)
2268 parens = list(
2269 m.group(2)
2270 for m in re.finditer(r"(^|\s)\((([^()]|\([^()]*\))*)\)", text)
2271 )
2272 parens.extend(
2273 m.group(1)
2274 for m in re.finditer(r"[^\s]\((([^()]|\([^()]*\))*)\)($|\s)", text)
2275 )
2276 have_romanization = False
2277 have_ruby = False
2278 hiragana = ""
2279 katakana = ""
2280 for paren in parens:
2281 paren = paren.strip()
2282 if not paren: 2282 ↛ 2283line 2282 didn't jump to line 2283 because the condition on line 2282 was never true
2283 continue
2284 can_be_form = True
2285 if label_templates is not None and paren.startswith(
2286 "__LABEL_TEMPLATE_"
2287 ):
2288 # wxr.wtp.warning("Found label template in head")
2289 # continue
2290 can_be_form = False
2291 m = re.match(r"__LABEL_TEMPLATE_(\d+)__", paren)
2292 if m is None: 2292 ↛ 2293line 2292 didn't jump to line 2293 because the condition on line 2292 was never true
2293 wxr.wtp.warning(
2294 f"Label template list magic phrase is broken: `{paren}`",
2295 sortid="20260508/label list index broken",
2296 )
2297 continue
2298 ht = label_templates[int(m.group(1))]
2299 desc = ht.get("expansion", "").strip()
2300 if desc: 2300 ↛ 2303line 2300 didn't jump to line 2303 because the condition on line 2300 was always true
2301 paren = desc
2302 else:
2303 wxr.wtp.warning(
2304 f"Label template seems to have no text contents: {ht=}",
2305 sortid="20260508/label_templates",
2306 )
2307 continue
2308 if paren.startswith("see "):
2309 continue
2310 if paren.startswith("U+"): 2310 ↛ 2311line 2310 didn't jump to line 2311 because the condition on line 2310 was never true
2311 continue
2312 # In some rare cases, strip word that inflects form the form
2313 # description, e.g. "look through rose-tinted glasses"/English.
2314 # `([looks])`
2315 paren = re.sub(r"\s*\(\[[^])]*\]\)", "", paren)
2317 # If it starts with hiragana or katakana, treat as such form. Note
2318 # that each hiragana/katakana character is in separate parentheses,
2319 # so we must concatenate them.
2320 try:
2321 un = unicodedata.name(paren[0]).split()[0]
2322 except ValueError:
2323 un = "INVALID"
2324 if un == "KATAKANA": 2324 ↛ 2325line 2324 didn't jump to line 2325 because the condition on line 2324 was never true
2325 katakana += paren
2326 have_ruby = True
2327 continue
2328 if un == "HIRAGANA": 2328 ↛ 2329line 2328 didn't jump to line 2329 because the condition on line 2328 was never true
2329 hiragana += paren
2330 have_ruby = True
2331 continue
2333 # Parse format ", 16 (Japan, Mainland), 17 (Hong Kong, Taiwan) strokes,"
2334 # in the middle of the parenthesized expression, e.g. 薄
2335 def strokes_repl(m: re.Match) -> str:
2336 strokes1, tags1, strokes2, tags2 = m.groups()
2337 for strokes, tags in [[strokes1, tags1], [strokes2, tags2]]:
2338 tags = tags.split(", ")
2339 tags = list(
2340 "Mainland China" if t == "Mainland" else t for t in tags
2341 )
2342 tags.append("strokes")
2343 add_related(
2344 wxr,
2345 data,
2346 tags,
2347 [strokes],
2348 text,
2349 True,
2350 is_reconstruction,
2351 head_group,
2352 ruby,
2353 links,
2354 link_dict,
2355 )
2356 return ", "
2358 if can_be_form is True:
2359 paren = re.sub(
2360 r", (\d+) \(([^()]+)\), (\d+) \(([^()]+)\) strokes, ",
2361 strokes_repl,
2362 paren,
2363 )
2365 descriptors = map_with(xlat_descs_map, [paren])
2366 new_desc = []
2367 for desc in descriptors:
2368 new_desc.extend(
2369 map_with(
2370 xlat_tags_map,
2371 split_at_comma_semi(
2372 desc, extra=[", or "], skipped=link_words_not_alnum
2373 ),
2374 )
2375 )
2376 prev_tags: Union[list[list[str]], list[tuple[str, ...]], None] = None
2377 following_tags = None # Added to prev_tags from previous parenthesized
2378 # part, e.g. walrus/English
2379 # "(both nonstandard, proscribed, uncommon)"
2380 for desc_i, desc in enumerate(new_desc):
2381 # print("HEAD DESC: {!r}".format(desc))
2383 # Abort on certain descriptors (assume remaining values are
2384 # examples or uninteresting, cf. gaan/Navajo, horior/Latin)
2385 if re.match(r"^(per |e\.g\.$)", desc): 2385 ↛ 2386line 2385 didn't jump to line 2386 because the condition on line 2385 was never true
2386 break
2388 # If it all consists of CJK characters, add it with the
2389 # CJK tag. This is used at least for some Vietnamese
2390 # words (e.g., ba/Vietnamese)
2391 try:
2392 if ( 2392 ↛ 2396line 2392 didn't jump to line 2396 because the condition on line 2392 was never true
2393 all(unicodedata.name(x).startswith("CJK ") for x in desc)
2394 and can_be_form
2395 ):
2396 add_related(
2397 wxr,
2398 data,
2399 ["CJK"],
2400 [desc],
2401 text,
2402 True,
2403 is_reconstruction,
2404 head_group,
2405 ruby,
2406 links,
2407 link_dict,
2408 )
2409 continue
2410 except ValueError:
2411 pass
2413 # Handle some special cases
2414 splitdesc = desc.split()
2415 if ( 2415 ↛ 2425line 2415 didn't jump to line 2425 because the condition on line 2415 was never true
2416 len(splitdesc) >= 3
2417 and splitdesc[1] == "superlative"
2418 and classify_desc(splitdesc[0]) != "tags"
2419 and prev_tags
2420 and can_be_form
2421 ):
2422 # Handle the special case of second comparative after comma,
2423 # followed by superlative without comma. E.g.
2424 # mal/Portuguese/Adv
2425 for ts in prev_tags:
2426 add_related(
2427 wxr,
2428 data,
2429 ts,
2430 [splitdesc[0]],
2431 text,
2432 True,
2433 is_reconstruction,
2434 head_group,
2435 ruby,
2436 links,
2437 link_dict,
2438 )
2439 desc = " ".join(splitdesc[1:])
2440 elif ( 2440 ↛ 2449line 2440 didn't jump to line 2449 because the condition on line 2440 was never true
2441 len(splitdesc) == 2
2442 and splitdesc[0] in ("also", "and")
2443 and prev_tags
2444 and classify_desc(splitdesc[1]) != "tags"
2445 and can_be_form
2446 ):
2447 # Sometimes alternative forms are prefixed with "also" or
2448 # "and"
2449 for ts in prev_tags:
2450 add_related(
2451 wxr,
2452 data,
2453 ts,
2454 [splitdesc[1]],
2455 text,
2456 True,
2457 is_reconstruction,
2458 head_group,
2459 ruby,
2460 links,
2461 link_dict,
2462 )
2463 continue
2464 elif len(splitdesc) >= 2 and splitdesc[0] in ("including",): 2464 ↛ 2465line 2464 didn't jump to line 2465 because the condition on line 2464 was never true
2465 continue
2467 # If only one word, assume it is comma-separated alternative
2468 # to the previous one
2469 if len(splitdesc) == 1:
2470 cls = classify_desc(desc)
2471 if cls != "tags" and can_be_form:
2472 if prev_tags: 2472 ↛ 2474line 2472 didn't jump to line 2474 because the condition on line 2472 was never true
2473 # Assume comma-separated alternative to previous one
2474 for ts in prev_tags:
2475 add_related(
2476 wxr,
2477 data,
2478 ts,
2479 [desc],
2480 text,
2481 True,
2482 is_reconstruction,
2483 head_group,
2484 ruby,
2485 links,
2486 link_dict,
2487 )
2488 continue
2489 elif distw(titleparts, desc) <= 0.5: 2489 ↛ 2492line 2489 didn't jump to line 2492 because the condition on line 2489 was never true
2490 # Similar to head word, assume a dialectal variation to
2491 # the base form. Cf. go/Alemannic German/Verb
2492 add_related(
2493 wxr,
2494 data,
2495 ["alternative"],
2496 [desc],
2497 text,
2498 True,
2499 is_reconstruction,
2500 head_group,
2501 ruby,
2502 links,
2503 link_dict,
2504 )
2505 continue
2506 elif (
2507 cls in ("romanization", "english")
2508 and not have_romanization
2509 and classify_desc(titleword) == "other"
2510 and not (
2511 "categories" in data and desc in data["categories"]
2512 )
2513 ):
2514 # Assume it to be a romanization
2515 add_romanization(
2516 wxr,
2517 data,
2518 desc,
2519 text,
2520 is_reconstruction,
2521 head_group,
2522 ruby,
2523 )
2524 have_romanization = True
2525 continue
2527 m = re.match(r"^(\d+) strokes?$", desc)
2528 if m and can_be_form:
2529 # Special case, used to give #strokes for Han characters
2530 add_related(
2531 wxr,
2532 data,
2533 ["strokes"],
2534 [m.group(1)],
2535 text,
2536 True,
2537 is_reconstruction,
2538 head_group,
2539 ruby,
2540 links,
2541 link_dict,
2542 )
2543 continue
2545 # See if it is radical+strokes
2546 m = re.match(
2547 r"^([\u2F00-\u2FDF\u2E80-\u2EFF\U00018800-\U00018AFF"
2548 r"\uA490-\uA4CF\u4E00-\u9FFF]\+\d+)"
2549 r"( in (Japanese|Chinese|traditional Chinese|"
2550 r"simplified Chinese))?$",
2551 desc,
2552 )
2553 if m and can_be_form: 2553 ↛ 2556line 2553 didn't jump to line 2556 because the condition on line 2553 was never true
2554 # Special case, used to give radical + strokes for Han
2555 # characters
2556 radical_strokes = m.group(1)
2557 lang = m.group(3)
2558 t = ["radical+strokes"]
2559 if lang:
2560 t.extend(lang.split())
2561 add_related(
2562 wxr,
2563 data,
2564 t,
2565 [radical_strokes],
2566 text,
2567 True,
2568 is_reconstruction,
2569 head_group,
2570 ruby,
2571 links,
2572 link_dict,
2573 )
2574 prev_tags = None
2575 following_tags = None
2576 continue
2578 # See if it indicates historical Katakana ortography (←) or
2579 # just otherwise katakana/hiragana form
2580 m = re.match(r"←\s*|kana\s+", desc)
2581 if m: 2581 ↛ 2582line 2581 didn't jump to line 2582 because the condition on line 2581 was never true
2582 if desc.startswith("←"):
2583 t1 = "historical "
2584 else:
2585 t1 = ""
2586 x = desc[m.end() :]
2587 if x.endswith("?"):
2588 x = x[:-1]
2589 # XXX should we add a tag indicating uncertainty?
2590 if x:
2591 name = unicodedata.name(x[0])
2592 if name.startswith("HIRAGANA "):
2593 desc = t1 + "hiragana " + x
2594 elif name.startswith("KATAKANA "):
2595 desc = t1 + "katakana " + x
2597 # See if it is "n strokes in Chinese" or similar
2598 m = re.match(
2599 r"(\d+) strokes in (Chinese|Japanese|"
2600 r"traditional Chinese|simplified Chinese)$",
2601 desc,
2602 )
2603 if m and can_be_form: 2603 ↛ 2605line 2603 didn't jump to line 2605 because the condition on line 2603 was never true
2604 # Special case, used to give just strokes for some Han chars
2605 strokes = m.group(1)
2606 lang = m.group(2)
2607 t = ["strokes"]
2608 t.extend(lang.split())
2609 add_related(
2610 wxr,
2611 data,
2612 t,
2613 [strokes],
2614 text,
2615 True,
2616 is_reconstruction,
2617 head_group,
2618 ruby,
2619 links,
2620 link_dict,
2621 )
2622 prev_tags = None
2623 following_tags = None
2624 continue
2626 # American Sign Language has images (or requests for image)
2627 # as heads, + this ASL gloss after.
2628 m2 = re.search(r"\(ASL gloss:\s+(.*)\)", text)
2629 if m2 and can_be_form: 2629 ↛ 2630line 2629 didn't jump to line 2630 because the condition on line 2629 was never true
2630 add_related(
2631 wxr,
2632 data,
2633 ["ASL-gloss"],
2634 [m2.group(1)],
2635 text,
2636 True,
2637 is_reconstruction,
2638 head_group,
2639 ruby,
2640 links,
2641 link_dict,
2642 )
2643 continue
2645 parts = list(m.group(0) for m in re.finditer(word_re, desc))
2646 if not parts: 2646 ↛ 2647line 2646 didn't jump to line 2647 because the condition on line 2646 was never true
2647 prev_tags = None
2648 following_tags = None
2649 continue
2651 # Check for certain language-specific header part starts that
2652 # modify
2653 if ( 2653 ↛ 2658line 2653 didn't jump to line 2658 because the condition on line 2653 was never true
2654 len(parts) == 2
2655 and language in lang_specific_head_map
2656 and can_be_form
2657 ):
2658 ht2 = lang_specific_head_map[language]
2659 if parts[0] in ht2:
2660 rem_tags, add_tags = ht2[parts[0]]
2661 new_prev_tags1: list[list[str]] = []
2662 tags2: Union[tuple[str, ...], list[str]]
2663 for tags2 in prev_tags or [()]:
2664 if rem_tags is True: # Remove all old tags
2665 tsets = set()
2666 else:
2667 tsets = set(tags2) - set(rem_tags.split())
2668 tsets = tsets | set(add_tags.split())
2669 tags = list(sorted(tsets))
2670 add_related(
2671 wxr,
2672 data,
2673 tags,
2674 [parts[1]],
2675 text,
2676 True,
2677 is_reconstruction,
2678 head_group,
2679 ruby,
2680 links,
2681 link_dict,
2682 )
2683 new_prev_tags1.append(tags)
2684 prev_tags = new_prev_tags1
2685 following_tags = None
2686 continue
2688 # Handle the special case of descriptors that are parenthesized,
2689 # e.g., (archaic or Scotland)
2690 m = re.match(r"\(([^)]+)\)\s+(.*)$", desc)
2691 if m is not None and classify_desc(m.group(1)) == "tags": 2691 ↛ 2692line 2691 didn't jump to line 2692 because the condition on line 2691 was never true
2692 tagpart = m.group(1)
2693 related = [m.group(2)]
2694 tagsets, topics = decode_tags(tagpart, no_unknown_starts=True)
2695 if topics:
2696 wxr.wtp.debug(
2697 "parenthized head part {!r} contains topics: {}".format(
2698 tagpart, topics
2699 ),
2700 sortid="form_descriptions/1647",
2701 )
2702 elif m is not None and re.match(r"in the sense ", m.group(1)): 2702 ↛ 2705line 2702 didn't jump to line 2705 because the condition on line 2702 was never true
2703 # Handle certain ignored cases
2704 # e.g. bord/Danish: in the sense "plank"
2705 related = [m.group(2)]
2706 tagsets = [()]
2707 else:
2708 # Normal parsing of the descriptor
2709 alt_related = None
2710 alt_tagsets = None
2711 tagsets = None
2712 for i in range(len(parts), 0, -1):
2713 related = parts[i:]
2714 tagparts = parts[:i]
2715 # print(" i={} related={} tagparts={}"
2716 # .format(i, related, tagparts))
2717 tagsets, topics = decode_tags(
2718 " ".join(tagparts), no_unknown_starts=True
2719 )
2720 # print("tagparts={!r} tagsets={} topics={} related={} "
2721 # "alt_related={} distw={:.2f}"
2722 # .format(tagparts, tagsets, topics, related,
2723 # alt_related,
2724 # distw(titleparts, parts[i - 1])))
2725 if (
2726 topics
2727 or not tagsets
2728 or any("error-unknown-tag" in x for x in tagsets)
2729 ):
2730 if alt_related is not None: 2730 ↛ 2732line 2730 didn't jump to line 2732 because the condition on line 2730 was never true
2731 # We already had a good division, so let's stop.
2732 break
2733 # Bad division, try deeper
2734 continue
2735 # print(f"{parts[i-1]=}, {parts=}")
2736 if (
2737 i > 1
2738 and len(parts[i - 1]) >= 4
2739 and (
2740 distw(titleparts, parts[i - 1]) <= 0.4
2741 or (
2742 wxr.wtp.section == "English"
2743 and wxr.wtp.title
2744 in WORDS_WITH_FALSE_POSITIVE_TAGS
2745 and parts[i - 1]
2746 in WORDS_WITH_FALSE_POSITIVE_TAGS[wxr.wtp.title]
2747 )
2748 )
2749 # Fixes 'unaccountability' wiktext #1196
2750 and not (
2751 wxr.wtp.section == "English"
2752 and wxr.wtp.title in WORDS_WITH_FALSE_POSITIVE_FORMS
2753 and parts[i - 1]
2754 in WORDS_WITH_FALSE_POSITIVE_FORMS[wxr.wtp.title]
2755 )
2756 # Fixes wiktextract #983, where "participle"
2757 # was too close to "Martinize" and so this accepted
2758 # ["participle", "Martinize"] as matching; this
2759 # kludge prevents this from happening if titleparts
2760 # is shorter than what would be 'related'.
2761 # This breaks if we want to detect stuff that
2762 # actually gets an extra space-separated word when
2763 # 'inflected'.
2764 and (
2765 len(titleparts) >= len(parts[i - 1 :])
2766 or "or" in parts[i - 1 :]
2767 )
2768 ):
2769 # print(f"Reached; {parts=}, {parts[i-1]=}")
2770 alt_related = related
2771 alt_tagsets = tagsets
2772 continue
2773 alt_related = None
2774 alt_tagsets = None
2775 break
2776 # for-else
2777 else:
2778 if alt_related is None and can_be_form:
2779 # Check if the parenthesized part is likely a
2780 # romanization
2781 if ( 2781 ↛ 2789line 2781 didn't jump to line 2789 because the condition on line 2781 was never true
2782 (have_ruby or classify_desc(base) == "other")
2783 and classify_desc(paren) == "romanization"
2784 and not (
2785 "categories" in data
2786 and desc in data["categories"]
2787 )
2788 ):
2789 for r in split_at_comma_semi(
2790 paren,
2791 extra=[" or "],
2792 skipped=link_words_not_alnum,
2793 ):
2794 add_romanization(
2795 wxr,
2796 data,
2797 r,
2798 text,
2799 is_reconstruction,
2800 head_group,
2801 ruby,
2802 )
2803 have_romanization = True
2804 continue
2805 tagsets = [("error-unrecognized-head-form",)]
2806 wxr.wtp.debug(
2807 "unrecognized head form: {}".format(desc),
2808 sortid="form_descriptions/1698",
2809 )
2810 continue
2812 if alt_related is not None: 2812 ↛ 2813line 2812 didn't jump to line 2813 because the condition on line 2812 was never true
2813 related = alt_related
2814 tagsets = alt_tagsets
2816 # print("FORM END: tagsets={} related={}".format(tagsets, related))
2817 # print("==================")
2819 if ( 2819 ↛ 2840line 2819 didn't jump to line 2840 because the condition on line 2819 was never true
2820 len(related) <= 0
2821 and wxr.wtp.section == "English"
2822 and tagsets is not None
2823 and len(tagsets) > 0
2824 and not any(
2825 s.startswith("error-") for tagset in tagsets for s in tagset
2826 )
2827 and any(
2828 s in FORM_ASSOCIATED_TAG_WORDS
2829 for tagset in tagsets
2830 for s in tagset
2831 )
2832 and (
2833 wxr.wtp.title not in FALSE_POSITIVE_MISSING_FORMS
2834 and not any(
2835 rel in FALSE_POSITIVE_MISSING_FORMS[wxr.wtp.title or ""]
2836 for rel in related
2837 )
2838 )
2839 ):
2840 wxr.wtp.debug(
2841 f"Form tags without form: {desc=}, {tagsets=}",
2842 sortid="form_description/20250107",
2843 )
2844 if not tagsets: 2844 ↛ 2845line 2844 didn't jump to line 2845 because the condition on line 2844 was never true
2845 continue
2847 # print(f"{alts=}, {related=}")
2849 assert isinstance(related, (list, tuple))
2850 related_str = " ".join(related)
2851 if "or" in titleparts:
2852 alts = [related_str]
2853 else:
2854 alts = split_at_comma_semi(
2855 related_str,
2856 separators=[r"\bor\b"],
2857 skipped=link_words_not_alnum,
2858 )
2859 # print(f"{related_str=}, {alts=}")
2860 if not alts:
2861 alts = [""]
2862 for related_str in alts:
2863 if related_str and can_be_form:
2864 if prev_tags and (
2865 all(
2866 all(
2867 t in ["nonstandard", "dialectal"]
2868 or valid_tags[t] == "dialect"
2869 for t in tags
2870 )
2871 for ts in tagsets
2872 )
2873 or (
2874 any("participle" in ts for ts in prev_tags)
2875 and all(
2876 "attributive" in ts
2877 or any(valid_tags[t] == "gender" for t in ts)
2878 for ts in tagsets
2879 )
2880 )
2881 ):
2882 # Merged with previous tags. Don't update previous
2883 # tags here; cf. burn/English/Verb
2884 for tags_l in tagsets:
2885 for ts in prev_tags:
2886 tags_l1 = sorted(set(tags_l) | set(ts))
2887 add_related(
2888 wxr,
2889 data,
2890 tags_l1,
2891 [related_str],
2892 text,
2893 True,
2894 is_reconstruction,
2895 head_group,
2896 ruby,
2897 links,
2898 link_dict,
2899 )
2900 else:
2901 # Not merged with previous tags
2902 for tags_l in tagsets:
2903 if following_tags is not None: 2903 ↛ 2904line 2903 didn't jump to line 2904 because the condition on line 2903 was never true
2904 for ts in following_tags:
2905 tags_l1 = list(
2906 sorted(set(tags_l) | set(ts))
2907 )
2908 add_related(
2909 wxr,
2910 data,
2911 tags_l1,
2912 [related_str],
2913 text,
2914 True,
2915 is_reconstruction,
2916 head_group,
2917 ruby,
2918 links,
2919 link_dict,
2920 )
2921 else:
2922 ret = add_related(
2923 wxr,
2924 data,
2925 tags_l,
2926 [related_str],
2927 text,
2928 True,
2929 is_reconstruction,
2930 head_group,
2931 ruby,
2932 links,
2933 link_dict,
2934 )
2935 if ret is not None: 2935 ↛ 2936line 2935 didn't jump to line 2936 because the condition on line 2935 was never true
2936 following_tags = ret
2937 prev_tags = tagsets
2938 else:
2939 if desc_i < len(new_desc) - 1 and all( 2939 ↛ 2946line 2939 didn't jump to line 2946 because the condition on line 2939 was never true
2940 "participle" in ts or "infinitive" in ts
2941 for ts in tagsets
2942 ):
2943 # Interpret it as a standalone form description
2944 # in the middle, probably followed by forms or
2945 # language-specific descriptors. cf. drikke/Danish
2946 new_prev_tags2 = []
2947 for ts1 in prev_tags or [()]:
2948 for ts2 in tagsets:
2949 ts = tuple(sorted(set(ts1) | set(ts2)))
2950 new_prev_tags2.append(ts)
2951 prev_tags = new_prev_tags2
2952 continue
2953 for tags in tagsets:
2954 data_extend(data, "tags", tags)
2955 prev_tags = tagsets
2956 following_tags = None
2958 # Finally, if we collected hirakana/katakana, add them now
2959 if hiragana: 2959 ↛ 2960line 2959 didn't jump to line 2960 because the condition on line 2959 was never true
2960 add_related(
2961 wxr,
2962 data,
2963 ["hiragana"],
2964 [hiragana],
2965 text,
2966 True,
2967 is_reconstruction,
2968 head_group,
2969 ruby,
2970 )
2971 if katakana: 2971 ↛ 2972line 2971 didn't jump to line 2972 because the condition on line 2971 was never true
2972 add_related(
2973 wxr,
2974 data,
2975 ["katakana"],
2976 [katakana],
2977 text,
2978 True,
2979 is_reconstruction,
2980 head_group,
2981 ruby,
2982 )
2984 # XXX check if this is actually relevant, tags in word root data
2985 # is extremely rare (not sure where they slip through).
2986 tags = data.get("tags", []) # type:ignore
2987 if len(tags) > 0:
2988 # wxr.wtp.debug(
2989 # f"Tags appear in word root data: {data['tags']=}", # type:ignore
2990 # sortid="form_descriptions/2620/20240606",
2991 # ) # Messes up tests.
2992 data["tags"] = sorted(set(tags)) # type:ignore
2995def parse_sense_qualifier(
2996 wxr: WiktextractContext, text: str, data: Union[SenseData, LinkageData]
2997) -> None:
2998 """Parses tags or topics for a sense or some other data. The values are
2999 added into the dictionary ``data``."""
3000 assert isinstance(wxr, WiktextractContext)
3001 assert isinstance(text, str)
3002 assert isinstance(data, dict)
3003 # print("parse_sense_qualifier:", text)
3004 if re.match(r"\([^()]+\)$", text): 3004 ↛ 3005line 3004 didn't jump to line 3005 because the condition on line 3004 was never true
3005 text = text[1:-1]
3006 if re.match(r'"[^"]+"$', text): 3006 ↛ 3007line 3006 didn't jump to line 3007 because the condition on line 3006 was never true
3007 text = text[1:-1]
3008 lst = map_with(xlat_descs_map, [text])
3009 sense_tags: list[str] = []
3010 for text in lst:
3011 for semi in split_at_comma_semi(text):
3012 if not semi: 3012 ↛ 3013line 3012 didn't jump to line 3013 because the condition on line 3012 was never true
3013 continue
3014 orig_semi = semi
3015 idx = semi.find(":")
3016 if idx >= 0: 3016 ↛ 3017line 3016 didn't jump to line 3017 because the condition on line 3016 was never true
3017 semi = semi[:idx]
3018 cls = classify_desc(semi, allow_unknown_tags=True)
3019 # print("parse_sense_qualifier: classify_desc: {} -> {}"
3020 # .format(semi, cls))
3021 if cls == "tags":
3022 tagsets, topics = decode_tags(semi)
3023 data_extend(data, "topics", topics)
3024 # XXX should think how to handle distinct options better,
3025 # e.g., "singular and plural genitive"; that can't really be
3026 # done with changing the calling convention of this function.
3027 # Should split sense if more than one category of tags differs.
3028 for tags in tagsets:
3029 sense_tags.extend(tags)
3030 elif cls == "taxonomic": 3030 ↛ 3031line 3030 didn't jump to line 3031 because the condition on line 3030 was never true
3031 if re.match(r"×[A-Z]", semi):
3032 sense_tags.append("extinct")
3033 semi = semi[1:]
3034 data["taxonomic"] = semi
3035 elif cls == "english":
3036 if "qualifier" in data and data["qualifier"] != orig_semi: 3036 ↛ 3037line 3036 didn't jump to line 3037 because the condition on line 3036 was never true
3037 data["qualifier"] += "; " + orig_semi
3038 else:
3039 data["qualifier"] = orig_semi
3040 else:
3041 wxr.wtp.debug(
3042 "unrecognized sense qualifier: {}".format(text),
3043 sortid="form_descriptions/1831",
3044 )
3045 sense_tags = sorted(set(sense_tags))
3046 data_extend(data, "tags", sense_tags)
3049def parse_pronunciation_tags(
3050 wxr: WiktextractContext, text: str, data: SoundData
3051) -> None:
3052 assert isinstance(wxr, WiktextractContext)
3053 assert isinstance(text, str)
3054 assert isinstance(data, dict)
3055 text = text.strip()
3056 if not text:
3057 return
3058 cls = classify_desc(text)
3059 found_tags = False
3060 got_notes_text = False
3061 if cls == "tags":
3062 tagsets, topics = decode_tags(text)
3063 data_extend(data, "topics", topics)
3064 for tagset in tagsets:
3065 for t in tagset:
3066 if " " in t:
3067 got_notes_text = True
3068 else:
3069 found_tags = True
3070 data_append(data, "tags", t)
3072 if not found_tags:
3073 # Try again to find tags, but keep notes text as above for context;
3074 # tags will be in the note text itself, but if we remove them, it'll
3075 # just be confusing.
3076 got_notes_text = False
3077 for s in split_at_comma_semi(text): 3077 ↛ 3091line 3077 didn't jump to line 3091 because the loop on line 3077 didn't complete
3078 if not (s := s.strip()): 3078 ↛ 3079line 3078 didn't jump to line 3079 because the condition on line 3078 was never true
3079 continue
3080 if classify_desc(s) != "tags":
3081 got_notes_text = True
3082 break
3083 tagsets, topics = decode_tags(s)
3084 data_extend(data, "topics", topics)
3085 for tagset in tagsets:
3086 for t in tagset:
3087 if " " not in t: 3087 ↛ 3090line 3087 didn't jump to line 3090 because the condition on line 3087 was always true
3088 data_append(data, "tags", t)
3089 else:
3090 got_notes_text = True
3091 if got_notes_text: 3091 ↛ 3096line 3091 didn't jump to line 3096 because the condition on line 3091 was always true
3092 # If we're mixing and matching tags and notes text, keep whole note
3093 # text
3094 data["note"] = text
3096 if got_notes_text is True:
3097 data["note"] = text
3100def parse_translation_desc(
3101 wxr: WiktextractContext, lang: str, text: str, tr: TranslationData
3102) -> None:
3103 assert isinstance(wxr, WiktextractContext)
3104 assert isinstance(lang, str) # The language of ``text``
3105 assert isinstance(text, str)
3106 assert isinstance(tr, dict)
3107 # print("parse_translation_desc:", text)
3109 # Process all parenthesized parts from the translation item
3110 note = None
3111 restore_beginning = ""
3112 restore_end = ""
3113 while True:
3114 beginning = False
3115 # See if we can find a parenthesized expression at the end
3116 m = re.search(r"\s*\((([^()]|\([^()]+\))+)\)\.?$", text)
3117 if m:
3118 par = m.group(1)
3119 text = text[: m.start()]
3120 if par.startswith(("literally ", "lit.")):
3121 continue # Not useful for disambiguation in many idioms
3122 else:
3123 # See if we can find a parenthesized expression at the start
3124 m = re.match(r"^\^?\((([^()]|\([^()]+\))+)\):?(\s+|$)", text)
3125 if m:
3126 par = m.group(1)
3127 text = text[m.end() :]
3128 beginning = True
3129 if re.match(r"^(\d|\s|,| or | and )+$", par): 3129 ↛ 3134line 3129 didn't jump to line 3134 because the condition on line 3129 was never true
3130 # Looks like this beginning parenthesized expression only
3131 # contains digits or their combinations. We assume such
3132 # to be sense descriptions if no sense has been selected,
3133 # or otherwise just ignore them.
3134 if not tr.get("sense"):
3135 tr["sense"] = par
3136 continue
3137 else:
3138 # See if we can find a parenthesized expression in the middle.
3139 # Romanizations are sometimes between word and gender marker,
3140 # e.g. wife/English/Tr/Yiddish.
3141 m = re.search(r"\s+\((([^()]|\([^()]+\))+)\)", text)
3142 if m:
3143 par = m.group(1)
3144 text = text[: m.start()] + text[m.end() :]
3145 else:
3146 # No more parenthesized expressions - break out of the loop
3147 break
3149 # Some cleanup of artifacts that may result from skipping some templates
3150 # in earlier stages
3151 if par.startswith(": "): 3151 ↛ 3152line 3151 didn't jump to line 3152 because the condition on line 3151 was never true
3152 par = par[2:]
3153 if par.endswith(","): 3153 ↛ 3154line 3153 didn't jump to line 3154 because the condition on line 3153 was never true
3154 par = par[:-1]
3155 if re.match(r'^[“"]([^“”"]*)[“”"]$', par): 3155 ↛ 3156line 3155 didn't jump to line 3156 because the condition on line 3155 was never true
3156 par = par[1:-1]
3157 par = par.strip()
3159 # Check for special script pronunciation followed by romanization,
3160 # used in many Asian languages.
3161 lst = par.split(", ")
3162 if len(lst) == 2:
3163 a, r = lst
3164 if classify_desc(a) == "other":
3165 cls = classify_desc(r)
3166 # print("parse_translation_desc: r={} cls={}".format(r, cls))
3167 if cls == "romanization" or (
3168 cls == "english" and len(r.split()) == 1 and r[0].islower()
3169 ):
3170 if tr.get("alt") and tr.get("alt") != a: 3170 ↛ 3171line 3170 didn't jump to line 3171 because the condition on line 3170 was never true
3171 wxr.wtp.debug(
3172 'more than one value in "alt": {} vs. {}'.format(
3173 tr["alt"], a
3174 ),
3175 sortid="form_descriptions/1930",
3176 )
3177 tr["alt"] = a
3178 if tr.get("roman") and tr.get("roman") != r: 3178 ↛ 3179line 3178 didn't jump to line 3179 because the condition on line 3178 was never true
3179 wxr.wtp.debug(
3180 'more than one value in "roman": {} vs. {}'.format(
3181 tr["roman"], r
3182 ),
3183 sortid="form_descriptions/1936",
3184 )
3185 tr["roman"] = r
3186 continue
3188 # Check for certain comma-separated tags combined with English text
3189 # at the beginning or end of a comma-separated parenthesized list
3190 while len(lst) > 1:
3191 cls = classify_desc(lst[0])
3192 if cls == "tags": 3192 ↛ 3193line 3192 didn't jump to line 3193 because the condition on line 3192 was never true
3193 tagsets, topics = decode_tags(lst[0])
3194 for t in tagsets:
3195 data_extend(tr, "tags", t)
3196 data_extend(tr, "topics", topics)
3197 lst = lst[1:]
3198 continue
3199 cls = classify_desc(lst[-1])
3200 if cls == "tags":
3201 tagsets, topics = decode_tags(lst[-1])
3202 for t in tagsets:
3203 data_extend(tr, "tags", t)
3204 data_extend(tr, "topics", topics)
3205 lst = lst[:-1]
3206 continue
3207 break
3208 par = ", ".join(lst)
3210 if not par: 3210 ↛ 3211line 3210 didn't jump to line 3211 because the condition on line 3210 was never true
3211 continue
3212 if re.search(tr_ignored_parens_re, par): 3212 ↛ 3213line 3212 didn't jump to line 3213 because the condition on line 3212 was never true
3213 continue
3214 if par.startswith("numeral:"):
3215 par = par[8:].strip()
3217 # Classify the part in parenthesis and process accordingly
3218 cls = classify_desc(par)
3219 # print("parse_translation_desc classify: {!r} -> {}"
3220 # .format(par, cls))
3221 if par == text:
3222 pass
3223 if par == "f": 3223 ↛ 3224line 3223 didn't jump to line 3224 because the condition on line 3223 was never true
3224 data_append(tr, "tags", "feminine")
3225 elif par == "m": 3225 ↛ 3226line 3225 didn't jump to line 3226 because the condition on line 3225 was never true
3226 data_append(tr, "tags", "masculine")
3227 elif cls == "tags":
3228 tagsets, topics = decode_tags(par)
3229 for tags in tagsets:
3230 data_extend(tr, "tags", tags)
3231 data_extend(tr, "topics", topics)
3232 elif cls == "english":
3233 # If the text contains any of certain grammatical words, treat it
3234 # as a "note" instead of "english"
3235 if re.search(tr_note_re, par):
3236 if par.endswith(":"): 3236 ↛ 3237line 3236 didn't jump to line 3237 because the condition on line 3236 was never true
3237 par = par[:-1]
3238 if par not in ("see entry for forms",): 3238 ↛ 3113line 3238 didn't jump to line 3113 because the condition on line 3238 was always true
3239 if note: 3239 ↛ 3240line 3239 didn't jump to line 3240 because the condition on line 3239 was never true
3240 note = note + ";" + par
3241 else:
3242 note = par
3243 else:
3244 # There can be more than one parenthesized english item, see
3245 # e.g. Aunt/English/Translations/Tamil
3246 if "translation" in tr and "english" in tr:
3247 tr["english"] += "; " + par # DEPRECATED for "translation"
3248 tr["translation"] += "; " + par
3249 else:
3250 tr["english"] = par # DEPRECATED for "translation"
3251 tr["translation"] = par
3252 elif cls == "romanization":
3253 # print("roman text={!r} text cls={}"
3254 # .format(text, classify_desc(text)))
3255 if classify_desc(text) in (
3256 "english",
3257 "romanization",
3258 ) and lang not in ("Egyptian",):
3259 if beginning:
3260 restore_beginning += "({}) ".format(par)
3261 else:
3262 restore_end = " ({})".format(par) + restore_end
3263 else:
3264 if tr.get("roman"): 3264 ↛ 3265line 3264 didn't jump to line 3265 because the condition on line 3264 was never true
3265 wxr.wtp.debug(
3266 'more than one value in "roman": {} vs. {}'.format(
3267 tr["roman"], par
3268 ),
3269 sortid="form_descriptions/2013",
3270 )
3271 tr["roman"] = par
3272 elif cls == "taxonomic": 3272 ↛ 3273line 3272 didn't jump to line 3273 because the condition on line 3272 was never true
3273 if tr.get("taxonomic"):
3274 wxr.wtp.debug(
3275 'more than one value in "taxonomic": {} vs. {}'.format(
3276 tr["taxonomic"], par
3277 ),
3278 sortid="form_descriptions/2019",
3279 )
3280 if re.match(r"×[A-Z]", par):
3281 data_append(tr, "tags", "extinct")
3282 par = par[1:]
3283 tr["taxonomic"] = par
3284 elif cls == "other": 3284 ↛ 3294line 3284 didn't jump to line 3294 because the condition on line 3284 was always true
3285 if tr.get("alt"): 3285 ↛ 3286line 3285 didn't jump to line 3286 because the condition on line 3285 was never true
3286 wxr.wtp.debug(
3287 'more than one value in "alt": {} vs. {}'.format(
3288 tr["alt"], par
3289 ),
3290 sortid="form_descriptions/2028",
3291 )
3292 tr["alt"] = par
3293 else:
3294 wxr.wtp.debug(
3295 "parse_translation_desc unimplemented cls {}: {}".format(
3296 cls, par
3297 ),
3298 sortid="form_descriptions/2033",
3299 )
3301 # Check for gender indications in suffix
3302 text, final_tags = parse_head_final_tags(wxr, lang, text)
3303 data_extend(tr, "tags", final_tags)
3305 # Restore those parts that we did not want to remove (they are often
3306 # optional words or words that are always used with the given translation)
3307 text = restore_beginning + text + restore_end
3309 if note:
3310 tr["note"] = note.strip()
3311 if text and text not in ignored_translations:
3312 tr["word"] = text.strip()
3314 # Sometimes gender seems to be at the end of "roman" field, see e.g.
3315 # fire/English/Noun/Translations/Egyptian (for "oxidation reaction")
3316 roman = tr.get("roman")
3317 if roman:
3318 if roman.endswith(" f"): 3318 ↛ 3319line 3318 didn't jump to line 3319 because the condition on line 3318 was never true
3319 data_append(tr, "tags", "feminine")
3320 tr["roman"] = roman[:-2].strip()
3321 elif roman.endswith(" m"): 3321 ↛ 3322line 3321 didn't jump to line 3322 because the condition on line 3321 was never true
3322 data_append(tr, "tags", "masculine")
3323 tr["roman"] = roman[:-2].strip()
3325 # If the word now has "translation" field but no "roman" field, and
3326 # the word would be classified "other" (generally non-latin
3327 # characters), and the value in "translation" is only one lowercase
3328 # word, move it to "roman". This happens semi-frequently when the
3329 # translation is transliterated the same as some English word.
3330 roman = tr.get("roman")
3331 english = tr.get("translation")
3332 if english and not roman and "word" in tr:
3333 cls = classify_desc(tr["word"])
3334 if cls == "other" and " " not in english and english[0].islower():
3335 del tr["translation"]
3336 if "english" in tr: # DEPRECATED for "translation" 3336 ↛ 3338line 3336 didn't jump to line 3338 because the condition on line 3336 was always true
3337 del tr["english"]
3338 tr["roman"] = english
3340 # If the entry now has both tr["roman"] and tr["word"] and they have
3341 # the same value, delete tr["roman"] (e.g., man/English/Translations
3342 # Evenki)
3343 if tr.get("word") and tr.get("roman") == tr.get("word"): 3343 ↛ 3344line 3343 didn't jump to line 3344 because the condition on line 3343 was never true
3344 del tr["roman"]
3347def parse_alt_or_inflection_of(
3348 wxr: WiktextractContext, gloss: str, gloss_template_args: set[str]
3349) -> Optional[tuple[list[str], Optional[list[AltOf]]]]:
3350 """Tries to parse an inflection-of or alt-of description. If successful,
3351 this returns (tags, alt-of/inflection-of-dict). If the description cannot
3352 be parsed, this returns None. This may also return (tags, None) when the
3353 gloss describes a form (or some other tags were extracted from it), but
3354 there was no alt-of/form-of/synonym-of word."""
3355 # print("parse_alt_or_inflection_of: {!r}".format(gloss))
3356 # Occasionally inflection_of/alt_of have "A(n) " etc. at the beginning.
3358 # Never interpret a gloss that is equal to the word itself as a tag
3359 # (e.g., instrumental/Romanian, instrumental/Spanish).
3360 if gloss.lower() == wxr.wtp.title.lower() or ( # type:ignore[union-attr]
3361 len(gloss) >= 5 and distw([gloss.lower()], wxr.wtp.title.lower()) < 0.2 # type:ignore[union-attr]
3362 ):
3363 return None
3365 # First try parsing it as-is
3366 parsed = parse_alt_or_inflection_of1(wxr, gloss, gloss_template_args)
3367 if parsed is not None:
3368 return parsed
3370 # Next try parsing it with the first character converted to lowercase if
3371 # it was previously uppercase.
3372 if gloss and gloss[0].isupper():
3373 gloss = gloss[0].lower() + gloss[1:]
3374 parsed = parse_alt_or_inflection_of1(wxr, gloss, gloss_template_args)
3375 if parsed is not None:
3376 return parsed
3378 return None
3381# These tags are not allowed in alt-or-inflection-of parsing
3382alt_infl_disallowed: set[str] = set(
3383 [
3384 "error-unknown-tag",
3385 "place", # Not in inflected forms and causes problems e.g. house/
3386 # English
3387 ]
3388)
3391def parse_alt_or_inflection_of1(
3392 wxr: WiktextractContext, gloss: str, gloss_template_args: set[str]
3393) -> Optional[tuple[list[str], Optional[list[AltOf]]]]:
3394 """Helper function for parse_alt_or_inflection_of. This handles a single
3395 capitalization."""
3396 if not gloss or not gloss.strip(): 3396 ↛ 3397line 3396 didn't jump to line 3397 because the condition on line 3396 was never true
3397 return None
3399 # Prevent some common errors where we would parse something we shouldn't
3400 if re.search(r"(?i)form of address ", gloss): 3400 ↛ 3401line 3400 didn't jump to line 3401 because the condition on line 3400 was never true
3401 return None
3403 gloss = re.sub(r"only used in [^,]+, ", "", gloss)
3405 # First try all formats ending with "of" (or other known last words that
3406 # can end a form description)
3407 matches = list(re.finditer(r"\b(of|for|by|as|letter|number) ", gloss))
3408 m: Optional[re.Match]
3409 for m in reversed(matches):
3410 desc = gloss[: m.end()].strip()
3411 base = gloss[m.end() :].strip()
3412 tagsets, topics = decode_tags(desc, no_unknown_starts=True)
3413 if not topics and any(
3414 not (alt_infl_disallowed & set(ts)) for ts in tagsets
3415 ):
3416 # Successfully parsed, including "of" etc.
3417 tags: list[str] = []
3418 # If you have ("Western-Armenian", ..., "form-of") as your
3419 # tag set, it's most probable that it's something like
3420 # "Western Armenian form of խոսել (xosel)", which should
3421 # get "alt-of" instead of "form-of" (inflection).
3422 # խօսիլ/Armenian
3423 for ts_t in tagsets:
3424 if "form-of" in ts_t and any(
3425 valid_tags.get(tk) == "dialect" for tk in ts_t
3426 ):
3427 ts_s = (set(ts_t) - {"form-of"}) | {"alt-of"}
3428 else:
3429 ts_s = set(ts_t)
3430 if not (alt_infl_disallowed & ts_s): 3430 ↛ 3423line 3430 didn't jump to line 3423 because the condition on line 3430 was always true
3431 tags.extend(ts_s)
3432 if (
3433 "alt-of" in tags
3434 or "form-of" in tags
3435 or "synonym-of" in tags
3436 or "compound-of" in tags
3437 ):
3438 break
3439 if m.group(1) == "of":
3440 # Try parsing without the final "of". This is commonly used in
3441 # various form-of expressions.
3442 desc = gloss[: m.start()]
3443 base = gloss[m.end() :]
3444 tagsets, topics = decode_tags(desc, no_unknown_starts=True)
3445 # print("ALT_OR_INFL: desc={!r} base={!r} tagsets={} topics={}"
3446 # .format(desc, base, tagsets, topics))
3447 if not topics and any(
3448 not (alt_infl_disallowed & set(t)) for t in tagsets
3449 ):
3450 tags = []
3451 for t in tagsets:
3452 if not (alt_infl_disallowed & set(t)): 3452 ↛ 3451line 3452 didn't jump to line 3451 because the condition on line 3452 was always true
3453 tags.extend(t)
3454 # It must have at least one tag from form_of_tags
3455 if set(tags) & form_of_tags:
3456 # Accept this as form-of
3457 tags.append("form-of")
3458 break
3459 if set(tags) & alt_of_tags:
3460 # Accept this as alt-of
3461 tags.append("alt-of")
3462 break
3464 else:
3465 # Did not find a form description based on last word; see if the
3466 # whole description is tags
3467 tagsets, topics = decode_tags(gloss, no_unknown_starts=True)
3468 if not topics and any(
3469 not (alt_infl_disallowed & set(ts)) and form_of_tags & set(ts)
3470 for ts in tagsets
3471 ):
3472 tags = []
3473 for ts in tagsets:
3474 if not (alt_infl_disallowed & set(ts)) and form_of_tags & set( 3474 ↛ 3473line 3474 didn't jump to line 3473 because the condition on line 3474 was always true
3475 ts
3476 ):
3477 tags.extend(ts)
3478 base = ""
3479 else:
3480 return None
3482 # kludge for Spanish (again): 'x of [word] combined with [clitic]'
3483 m = re.search(r"combined with \w+$", base)
3484 if m: 3484 ↛ 3485line 3484 didn't jump to line 3485 because the condition on line 3484 was never true
3485 tagsets, topics = decode_tags(m.group(0), no_unknown_starts=True)
3486 if not topics:
3487 for ts in tagsets:
3488 tags.extend(ts)
3489 base = base[: m.start()]
3491 # It is fairly common for form_of glosses to end with something like
3492 # "ablative case" or "in instructive case". Parse that ending.
3493 base = base.strip()
3494 lst = base.split()
3495 # print("parse_alt_or_inflection_of: lst={}".format(lst))
3496 if len(lst) >= 3 and lst[-1] in ("case", "case."): 3496 ↛ 3497line 3496 didn't jump to line 3497 because the condition on line 3496 was never true
3497 node = valid_sequences.children.get(lst[-2])
3498 if node and node.end:
3499 for s in node.tags:
3500 tags.extend(s.split(" "))
3501 lst = lst[:-2]
3502 if lst[-1] == "in" and len(lst) > 1:
3503 lst = lst[:-1]
3505 # Eliminate empty and duplicate tags
3506 tags = sorted(set(t for t in tags if t))
3508 # Clean up some extra stuff from the linked word, separating the text
3509 # into ``base`` (the linked word) and ``extra`` (additional information,
3510 # such as English translation or clarifying word sense information).
3511 orig_base = base
3512 base = re.sub(alt_of_form_of_clean_re, "", orig_base)
3513 base = re.sub(r" [(⟨][^()]*[)⟩]", "", base) # Remove all (...) groups
3514 extra = orig_base[len(base) :]
3515 extra = re.sub(r"^[- :;.,,—]+", "", extra)
3516 if extra.endswith(".") and extra.count(".") == 1:
3517 extra = extra[:-1].strip()
3518 m = re.match(r"^\(([^()]*)\)$", extra)
3519 if m: 3519 ↛ 3520line 3519 didn't jump to line 3520 because the condition on line 3519 was never true
3520 extra = m.group(1)
3521 else:
3522 # These weird backets used in "slash mark"
3523 m = re.match(r"^⟨([^()]*)⟩$", extra)
3524 if m: 3524 ↛ 3525line 3524 didn't jump to line 3525 because the condition on line 3524 was never true
3525 extra = m.group(1)
3526 m = re.match(r'^[“"]([^"“”]*)["”]$', extra)
3527 if m: 3527 ↛ 3528line 3527 didn't jump to line 3528 because the condition on line 3527 was never true
3528 extra = m.group(1)
3529 # Note: base might still contain comma-separated values and values
3530 # separated by "and"
3531 base = base.strip()
3532 if base.endswith(",") and len(base) > 2: 3532 ↛ 3533line 3532 didn't jump to line 3533 because the condition on line 3532 was never true
3533 base = base[:-1].strip()
3534 while (
3535 base.endswith(".")
3536 and not wxr.wtp.page_exists(base)
3537 and base not in gloss_template_args
3538 ):
3539 base = base[:-1].strip()
3540 if base.endswith('(\u201cconjecture")'): 3540 ↛ 3541line 3540 didn't jump to line 3541 because the condition on line 3540 was never true
3541 base = base[:-14].strip()
3542 tags.append("conjecture")
3543 while ( 3543 ↛ 3548line 3543 didn't jump to line 3548 because the condition on line 3543 was never true
3544 base.endswith(".")
3545 and not wxr.wtp.page_exists(base)
3546 and base not in gloss_template_args
3547 ):
3548 base = base[:-1].strip()
3549 if ( 3549 ↛ 3554line 3549 didn't jump to line 3554 because the condition on line 3549 was never true
3550 base.endswith(".")
3551 and base not in gloss_template_args
3552 and base[:-1] in gloss_template_args
3553 ):
3554 base = base[:-1]
3555 base = base.strip()
3556 if not base:
3557 return tags, None
3559 # Kludge: Spanish verb forms seem to have a dot added at the end.
3560 # Remove it; we know of no Spanish verbs ending with a dot.
3561 language = wxr.wtp.section
3562 pos = wxr.wtp.subsection
3563 # print("language={} pos={} base={}".format(language, pos, base))
3564 if ( 3564 ↛ 3570line 3564 didn't jump to line 3570 because the condition on line 3564 was never true
3565 base.endswith(".")
3566 and len(base) > 1
3567 and base[-2].isalpha()
3568 and (language == "Spanish" and pos == "Verb")
3569 ):
3570 base = base[:-1]
3572 # Split base to alternatives when multiple alternatives provided
3573 parts = split_at_comma_semi(base, extra=[" / ", "/", r" \+ "])
3574 titleword = re.sub(r"^Reconstruction:[^/]*/", "", wxr.wtp.title or "")
3575 if (
3576 len(parts) <= 1
3577 or base.startswith("/")
3578 or base.endswith("/")
3579 or "/" in titleword
3580 ):
3581 parts = [base]
3582 # Split base to alternatives when of form "a or b" and "a" and "b" are
3583 # similar (generally spelling variants of the same word or similar words)
3584 if len(parts) == 1:
3585 pp = base.split()
3586 if len(pp) == 3 and pp[1] == "or" and distw([pp[0]], pp[2]) < 0.4:
3587 parts = [pp[0], pp[2]]
3589 # Create form-of/alt-of entries based on the extracted data
3590 dt_lst: list[AltOf] = []
3591 for p in parts:
3592 # Check for some suspicious base forms
3593 m = re.search(r"[.,] |[{}()]", p)
3594 if m and not wxr.wtp.page_exists(p): 3594 ↛ 3595line 3594 didn't jump to line 3595 because the condition on line 3594 was never true
3595 wxr.wtp.debug(
3596 "suspicious alt_of/form_of with {!r}: {}".format(m.group(0), p),
3597 sortid="form_descriptions/2278",
3598 )
3599 if p.startswith("*") and len(p) >= 3 and p[1].isalpha(): 3599 ↛ 3600line 3599 didn't jump to line 3600 because the condition on line 3599 was never true
3600 p = p[1:]
3601 dt: AltOf = {"word": p}
3602 if extra:
3603 dt["extra"] = extra
3604 dt_lst.append(dt)
3605 # print("alt_or_infl_of returning tags={} lst={} base={!r}"
3606 # .format(tags, lst, base))
3607 return tags, dt_lst
3610@functools.lru_cache(maxsize=65536)
3611def classify_desc(
3612 desc: str,
3613 allow_unknown_tags=False,
3614 no_unknown_starts=False,
3615 accepted: Union[tuple[str, ...], frozenset[str]] = tuple(),
3616) -> str:
3617 """Determines whether the given description is most likely tags, english,
3618 a romanization, or something else. Returns one of: "tags", "english",
3619 "romanization", or "other". If ``allow_unknown_tags`` is True, then
3620 allow "tags" classification even when the only tags are those starting
3621 with a word in allowed_unknown_starts."""
3622 assert isinstance(desc, str)
3623 # Empty and whitespace-only strings are treated as "other"
3624 desc = desc.strip()
3625 if not desc:
3626 return "other"
3628 normalized_desc = unicodedata.normalize("NFKD", desc)
3630 # If it can be fully decoded as tags without errors, treat as tags
3631 tagsets, topics = decode_tags(desc, no_unknown_starts=no_unknown_starts)
3632 for tagset in tagsets:
3633 assert isinstance(tagset, (list, tuple, set))
3634 if "error-unknown-tag" not in tagset and (
3635 topics or allow_unknown_tags or any(" " not in x for x in tagset)
3636 ):
3637 return "tags"
3639 # Check if it looks like the taxonomic name of a species
3640 if desc in known_species:
3641 return "taxonomic"
3642 desc1 = re.sub(r"^×([A-Z])", r"\1", desc)
3643 desc1 = re.sub(r"\s*×.*", "", desc1)
3644 lst = desc1.split()
3645 if len(lst) > 1 and len(lst) <= 5 and lst[0] in known_firsts:
3646 have_non_english = 1 if lst[0].lower() not in english_words else 0
3647 for x in lst[1:]:
3648 if x in ("A", "B", "C", "D", "E", "F", "I", "II", "III", "IV", "V"):
3649 continue
3650 if x[0].isupper():
3651 break
3652 if x not in english_words:
3653 have_non_english += 1
3654 else:
3655 # Starts with known taxonomic term, does not contain uppercase
3656 # words (except allowed letters) and at least one word is not
3657 # English
3658 if have_non_english >= len(lst) - 1 and have_non_english > 0: 3658 ↛ 3664line 3658 didn't jump to line 3664 because the condition on line 3658 was always true
3659 return "taxonomic"
3661 # If all words are in our English dictionary, interpret as English.
3662 # [ -~] is regex black magic, "ALL CHARACTERS from space to tilde"
3663 # in ASCII. Took me a while to figure out.
3664 if re.match(r"[ -~―—“”…'‘’ʹ€]+$", normalized_desc) and len(desc) > 1:
3665 if desc in english_words and desc[0].isalpha():
3666 return "english" # Handles ones containing whitespace
3667 desc1 = re.sub(
3668 tokenizer_fixup_re, lambda m: tokenizer_fixup_map[m.group(0)], desc
3669 )
3670 tokens = tokenizer.tokenize(desc1)
3671 if not tokens: 3671 ↛ 3672line 3671 didn't jump to line 3672 because the condition on line 3671 was never true
3672 return "other"
3673 lst_bool = list(
3674 x not in not_english_words
3675 and
3676 # not x.isdecimal() and
3677 (
3678 x in english_words
3679 or x.lower() in english_words
3680 or x in known_firsts
3681 or x[0].isdigit()
3682 or x in accepted
3683 or
3684 # (x[0].isupper() and x.find("-") < 0 and x.isascii()) or
3685 (
3686 x.endswith("s") and len(x) >= 4 and x[:-1] in english_words
3687 ) # Plural
3688 or (
3689 x.endswith("ies")
3690 and len(x) >= 5
3691 and x[:-3] + "y" in english_words
3692 ) # E.g. lily - lilies
3693 or (
3694 x.endswith("ing")
3695 and len(x) >= 5
3696 and x[:-3] in english_words
3697 ) # E.g. bring - bringing
3698 or (
3699 x.endswith("ing")
3700 and len(x) >= 5
3701 and x[:-3] + "e" in english_words
3702 ) # E.g., tone - toning
3703 or (
3704 x.endswith("ed") and len(x) >= 5 and x[:-2] in english_words
3705 ) # E.g. hang - hanged
3706 or (
3707 x.endswith("ed")
3708 and len(x) >= 5
3709 and x[:-2] + "e" in english_words
3710 ) # E.g. atone - atoned
3711 or (x.endswith("'s") and x[:-2] in english_words)
3712 or (x.endswith("s'") and x[:-2] in english_words)
3713 or (
3714 x.endswith("ise")
3715 and len(x) >= 5
3716 and x[:-3] + "ize" in english_words
3717 )
3718 or (
3719 x.endswith("ised")
3720 and len(x) >= 6
3721 and x[:-4] + "ized" in english_words
3722 )
3723 or (
3724 x.endswith("ising")
3725 and len(x) >= 7
3726 and x[:-5] + "izing" in english_words
3727 )
3728 or (
3729 re.search(r"[-/]", x)
3730 and all(
3731 ((y in english_words and len(y) > 2) or not y)
3732 for y in re.split(r"[-/]", x)
3733 )
3734 )
3735 )
3736 for x in tokens
3737 )
3738 cnt = lst_bool.count(True)
3739 rejected_words = tuple(
3740 x for i, x in enumerate(tokens) if not lst_bool[i]
3741 )
3742 if (
3743 any(
3744 lst_bool[i] and x[0].isalpha() and len(x) > 1
3745 for i, x in enumerate(tokens)
3746 )
3747 and not desc.startswith("-")
3748 and not desc.endswith("-")
3749 and re.search(r"\w+", desc)
3750 and (
3751 cnt == len(lst_bool)
3752 or (
3753 any(
3754 lst_bool[i] and len(x) > 3 for i, x in enumerate(tokens)
3755 )
3756 and cnt >= len(lst_bool) - 1
3757 )
3758 or cnt / len(lst_bool) >= 0.8
3759 or (
3760 all(x in potentially_english_words for x in rejected_words)
3761 and cnt / len(lst_bool) >= 0.50
3762 )
3763 )
3764 ):
3765 return "english"
3766 # Some translations have apparent pronunciation descriptions in /.../
3767 # which we'll put in the romanization field (even though they probably are
3768 # not exactly romanizations).
3769 if desc.startswith("/") and desc.endswith("/"):
3770 return "romanization"
3771 # If all characters are in classes that could occur in romanizations,
3772 # treat as romanization
3773 classes = list(
3774 unicodedata.category(x) if x not in ("-", ",", ":", "/", '"') else "OK"
3775 for x in normalized_desc
3776 )
3777 classes1 = []
3778 num_latin = 0
3779 num_greek = 0
3780 # part = ""
3781 # for ch, cl in zip(normalized_desc, classes):
3782 # part += f"{ch}({cl})"
3783 # print(part)
3784 for ch, cl in zip(normalized_desc, classes):
3785 if ch in (
3786 "'", # ' in Arabic, / in IPA-like parenthesized forms
3787 ".", # e.g., "..." in translations
3788 ";",
3789 ":",
3790 "!",
3791 "‘",
3792 "’",
3793 '"',
3794 "“",
3795 "”",
3796 "/",
3797 "?",
3798 "…", # alternative to "..."
3799 "⁉", # 見る/Japanese automatic transcriptions...
3800 "?",
3801 "!",
3802 "⁻", # superscript -, used in some Cantonese roman, e.g. "we"
3803 "ʔ",
3804 "ʼ",
3805 "ʾ",
3806 "ʹ",
3807 ): # ʹ e.g. in understand/English/verb Russian transl
3808 classes1.append("OK")
3809 continue
3810 if cl not in ("Ll", "Lu"):
3811 classes1.append(cl)
3812 continue
3813 try:
3814 name = unicodedata.name(ch)
3815 first = name.split()[0]
3816 if first == "LATIN":
3817 num_latin += 1
3818 elif first == "GREEK":
3819 num_greek += 1
3820 elif first == "COMBINING": # Combining diacritic 3820 ↛ 3821line 3820 didn't jump to line 3821 because the condition on line 3820 was never true
3821 cl = "OK"
3822 elif re.match(non_latin_scripts_re, name): 3822 ↛ 3826line 3822 didn't jump to line 3826 because the condition on line 3822 was always true
3823 cl = "NO" # Not acceptable in romanizations
3824 except ValueError:
3825 cl = "NO" # Not acceptable in romanizations
3826 classes1.append(cl)
3827 # print("classify_desc: {!r} classes1: {}".format(desc, classes1))
3828 # print(set(classes1) )
3829 if all(
3830 x in ("Ll", "Lu", "Lt", "Lm", "Mn", "Mc", "Zs", "Nd", "OK")
3831 for x in classes1
3832 ):
3833 if (
3834 (num_latin >= num_greek + 2 or num_greek == 0)
3835 and classes1.count("OK") < len(classes1)
3836 and classes1.count("Nd") < len(classes1)
3837 ):
3838 return "romanization"
3839 # Otherwise it is something else, such as hanji version of the word
3840 return "other"
3843def remove_text_in_parentheses(text: str) -> str:
3844 parentheses = 0
3845 new_text = ""
3846 for c in text:
3847 if c == "(":
3848 parentheses += 1
3849 elif c == ")":
3850 parentheses -= 1
3851 elif parentheses == 0:
3852 new_text += c
3853 return new_text